1 //===- lib/MC/MCWin64EH.cpp - MCWin64EH implementation --------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/MC/MCWin64EH.h" 10 #include "llvm/ADT/Twine.h" 11 #include "llvm/MC/MCContext.h" 12 #include "llvm/MC/MCExpr.h" 13 #include "llvm/MC/MCObjectStreamer.h" 14 #include "llvm/MC/MCStreamer.h" 15 #include "llvm/MC/MCSymbol.h" 16 #include "llvm/Support/Win64EH.h" 17 namespace llvm { 18 class MCSection; 19 } 20 21 using namespace llvm; 22 23 // NOTE: All relocations generated here are 4-byte image-relative. 24 25 static uint8_t CountOfUnwindCodes(std::vector<WinEH::Instruction> &Insns) { 26 uint8_t Count = 0; 27 for (const auto &I : Insns) { 28 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 29 default: 30 llvm_unreachable("Unsupported unwind code"); 31 case Win64EH::UOP_PushNonVol: 32 case Win64EH::UOP_AllocSmall: 33 case Win64EH::UOP_SetFPReg: 34 case Win64EH::UOP_PushMachFrame: 35 Count += 1; 36 break; 37 case Win64EH::UOP_SaveNonVol: 38 case Win64EH::UOP_SaveXMM128: 39 Count += 2; 40 break; 41 case Win64EH::UOP_SaveNonVolBig: 42 case Win64EH::UOP_SaveXMM128Big: 43 Count += 3; 44 break; 45 case Win64EH::UOP_AllocLarge: 46 Count += (I.Offset > 512 * 1024 - 8) ? 3 : 2; 47 break; 48 } 49 } 50 return Count; 51 } 52 53 static void EmitAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS, 54 const MCSymbol *RHS) { 55 MCContext &Context = Streamer.getContext(); 56 const MCExpr *Diff = 57 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 58 MCSymbolRefExpr::create(RHS, Context), Context); 59 Streamer.emitValue(Diff, 1); 60 } 61 62 static void EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin, 63 WinEH::Instruction &inst) { 64 uint8_t b2; 65 uint16_t w; 66 b2 = (inst.Operation & 0x0F); 67 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 68 default: 69 llvm_unreachable("Unsupported unwind code"); 70 case Win64EH::UOP_PushNonVol: 71 EmitAbsDifference(streamer, inst.Label, begin); 72 b2 |= (inst.Register & 0x0F) << 4; 73 streamer.emitInt8(b2); 74 break; 75 case Win64EH::UOP_AllocLarge: 76 EmitAbsDifference(streamer, inst.Label, begin); 77 if (inst.Offset > 512 * 1024 - 8) { 78 b2 |= 0x10; 79 streamer.emitInt8(b2); 80 w = inst.Offset & 0xFFF8; 81 streamer.emitInt16(w); 82 w = inst.Offset >> 16; 83 } else { 84 streamer.emitInt8(b2); 85 w = inst.Offset >> 3; 86 } 87 streamer.emitInt16(w); 88 break; 89 case Win64EH::UOP_AllocSmall: 90 b2 |= (((inst.Offset - 8) >> 3) & 0x0F) << 4; 91 EmitAbsDifference(streamer, inst.Label, begin); 92 streamer.emitInt8(b2); 93 break; 94 case Win64EH::UOP_SetFPReg: 95 EmitAbsDifference(streamer, inst.Label, begin); 96 streamer.emitInt8(b2); 97 break; 98 case Win64EH::UOP_SaveNonVol: 99 case Win64EH::UOP_SaveXMM128: 100 b2 |= (inst.Register & 0x0F) << 4; 101 EmitAbsDifference(streamer, inst.Label, begin); 102 streamer.emitInt8(b2); 103 w = inst.Offset >> 3; 104 if (inst.Operation == Win64EH::UOP_SaveXMM128) 105 w >>= 1; 106 streamer.emitInt16(w); 107 break; 108 case Win64EH::UOP_SaveNonVolBig: 109 case Win64EH::UOP_SaveXMM128Big: 110 b2 |= (inst.Register & 0x0F) << 4; 111 EmitAbsDifference(streamer, inst.Label, begin); 112 streamer.emitInt8(b2); 113 if (inst.Operation == Win64EH::UOP_SaveXMM128Big) 114 w = inst.Offset & 0xFFF0; 115 else 116 w = inst.Offset & 0xFFF8; 117 streamer.emitInt16(w); 118 w = inst.Offset >> 16; 119 streamer.emitInt16(w); 120 break; 121 case Win64EH::UOP_PushMachFrame: 122 if (inst.Offset == 1) 123 b2 |= 0x10; 124 EmitAbsDifference(streamer, inst.Label, begin); 125 streamer.emitInt8(b2); 126 break; 127 } 128 } 129 130 static void EmitSymbolRefWithOfs(MCStreamer &streamer, 131 const MCSymbol *Base, 132 int64_t Offset) { 133 MCContext &Context = streamer.getContext(); 134 const MCConstantExpr *OffExpr = MCConstantExpr::create(Offset, Context); 135 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Base, 136 MCSymbolRefExpr::VK_COFF_IMGREL32, 137 Context); 138 streamer.emitValue(MCBinaryExpr::createAdd(BaseRefRel, OffExpr, Context), 4); 139 } 140 141 static void EmitSymbolRefWithOfs(MCStreamer &streamer, 142 const MCSymbol *Base, 143 const MCSymbol *Other) { 144 MCContext &Context = streamer.getContext(); 145 const MCSymbolRefExpr *BaseRef = MCSymbolRefExpr::create(Base, Context); 146 const MCSymbolRefExpr *OtherRef = MCSymbolRefExpr::create(Other, Context); 147 const MCExpr *Ofs = MCBinaryExpr::createSub(OtherRef, BaseRef, Context); 148 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Base, 149 MCSymbolRefExpr::VK_COFF_IMGREL32, 150 Context); 151 streamer.emitValue(MCBinaryExpr::createAdd(BaseRefRel, Ofs, Context), 4); 152 } 153 154 static void EmitRuntimeFunction(MCStreamer &streamer, 155 const WinEH::FrameInfo *info) { 156 MCContext &context = streamer.getContext(); 157 158 streamer.emitValueToAlignment(Align(4)); 159 EmitSymbolRefWithOfs(streamer, info->Begin, info->Begin); 160 EmitSymbolRefWithOfs(streamer, info->Begin, info->End); 161 streamer.emitValue(MCSymbolRefExpr::create(info->Symbol, 162 MCSymbolRefExpr::VK_COFF_IMGREL32, 163 context), 4); 164 } 165 166 static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) { 167 // If this UNWIND_INFO already has a symbol, it's already been emitted. 168 if (info->Symbol) 169 return; 170 171 MCContext &context = streamer.getContext(); 172 MCSymbol *Label = context.createTempSymbol(); 173 174 streamer.emitValueToAlignment(Align(4)); 175 streamer.emitLabel(Label); 176 info->Symbol = Label; 177 178 // Upper 3 bits are the version number (currently 1). 179 uint8_t flags = 0x01; 180 if (info->ChainedParent) 181 flags |= Win64EH::UNW_ChainInfo << 3; 182 else { 183 if (info->HandlesUnwind) 184 flags |= Win64EH::UNW_TerminateHandler << 3; 185 if (info->HandlesExceptions) 186 flags |= Win64EH::UNW_ExceptionHandler << 3; 187 } 188 streamer.emitInt8(flags); 189 190 if (info->PrologEnd) 191 EmitAbsDifference(streamer, info->PrologEnd, info->Begin); 192 else 193 streamer.emitInt8(0); 194 195 uint8_t numCodes = CountOfUnwindCodes(info->Instructions); 196 streamer.emitInt8(numCodes); 197 198 uint8_t frame = 0; 199 if (info->LastFrameInst >= 0) { 200 WinEH::Instruction &frameInst = info->Instructions[info->LastFrameInst]; 201 assert(frameInst.Operation == Win64EH::UOP_SetFPReg); 202 frame = (frameInst.Register & 0x0F) | (frameInst.Offset & 0xF0); 203 } 204 streamer.emitInt8(frame); 205 206 // Emit unwind instructions (in reverse order). 207 uint8_t numInst = info->Instructions.size(); 208 for (uint8_t c = 0; c < numInst; ++c) { 209 WinEH::Instruction inst = info->Instructions.back(); 210 info->Instructions.pop_back(); 211 EmitUnwindCode(streamer, info->Begin, inst); 212 } 213 214 // For alignment purposes, the instruction array will always have an even 215 // number of entries, with the final entry potentially unused (in which case 216 // the array will be one longer than indicated by the count of unwind codes 217 // field). 218 if (numCodes & 1) { 219 streamer.emitInt16(0); 220 } 221 222 if (flags & (Win64EH::UNW_ChainInfo << 3)) 223 EmitRuntimeFunction(streamer, info->ChainedParent); 224 else if (flags & 225 ((Win64EH::UNW_TerminateHandler|Win64EH::UNW_ExceptionHandler) << 3)) 226 streamer.emitValue(MCSymbolRefExpr::create(info->ExceptionHandler, 227 MCSymbolRefExpr::VK_COFF_IMGREL32, 228 context), 4); 229 else if (numCodes == 0) { 230 // The minimum size of an UNWIND_INFO struct is 8 bytes. If we're not 231 // a chained unwind info, if there is no handler, and if there are fewer 232 // than 2 slots used in the unwind code array, we have to pad to 8 bytes. 233 streamer.emitInt32(0); 234 } 235 } 236 237 void llvm::Win64EH::UnwindEmitter::Emit(MCStreamer &Streamer) const { 238 // Emit the unwind info structs first. 239 for (const auto &CFI : Streamer.getWinFrameInfos()) { 240 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 241 Streamer.switchSection(XData); 242 ::EmitUnwindInfo(Streamer, CFI.get()); 243 } 244 245 // Now emit RUNTIME_FUNCTION entries. 246 for (const auto &CFI : Streamer.getWinFrameInfos()) { 247 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 248 Streamer.switchSection(PData); 249 EmitRuntimeFunction(Streamer, CFI.get()); 250 } 251 } 252 253 void llvm::Win64EH::UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 254 WinEH::FrameInfo *info, 255 bool HandlerData) const { 256 // Switch sections (the static function above is meant to be called from 257 // here and from Emit(). 258 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 259 Streamer.switchSection(XData); 260 261 ::EmitUnwindInfo(Streamer, info); 262 } 263 264 static const MCExpr *GetSubDivExpr(MCStreamer &Streamer, const MCSymbol *LHS, 265 const MCSymbol *RHS, int Div) { 266 MCContext &Context = Streamer.getContext(); 267 const MCExpr *Expr = 268 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 269 MCSymbolRefExpr::create(RHS, Context), Context); 270 if (Div != 1) 271 Expr = MCBinaryExpr::createDiv(Expr, MCConstantExpr::create(Div, Context), 272 Context); 273 return Expr; 274 } 275 276 static std::optional<int64_t> GetOptionalAbsDifference(MCStreamer &Streamer, 277 const MCSymbol *LHS, 278 const MCSymbol *RHS) { 279 MCContext &Context = Streamer.getContext(); 280 const MCExpr *Diff = 281 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 282 MCSymbolRefExpr::create(RHS, Context), Context); 283 MCObjectStreamer *OS = (MCObjectStreamer *)(&Streamer); 284 // It should normally be possible to calculate the length of a function 285 // at this point, but it might not be possible in the presence of certain 286 // unusual constructs, like an inline asm with an alignment directive. 287 int64_t value; 288 if (!Diff->evaluateAsAbsolute(value, OS->getAssembler())) 289 return std::nullopt; 290 return value; 291 } 292 293 static int64_t GetAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS, 294 const MCSymbol *RHS) { 295 std::optional<int64_t> MaybeDiff = 296 GetOptionalAbsDifference(Streamer, LHS, RHS); 297 if (!MaybeDiff) 298 report_fatal_error("Failed to evaluate function length in SEH unwind info"); 299 return *MaybeDiff; 300 } 301 302 static void checkARM64Instructions(MCStreamer &Streamer, 303 ArrayRef<WinEH::Instruction> Insns, 304 const MCSymbol *Begin, const MCSymbol *End, 305 StringRef Name, StringRef Type) { 306 if (!End) 307 return; 308 std::optional<int64_t> MaybeDistance = 309 GetOptionalAbsDifference(Streamer, End, Begin); 310 if (!MaybeDistance) 311 return; 312 uint32_t Distance = (uint32_t)*MaybeDistance; 313 314 for (const auto &I : Insns) { 315 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 316 default: 317 break; 318 case Win64EH::UOP_TrapFrame: 319 case Win64EH::UOP_PushMachFrame: 320 case Win64EH::UOP_Context: 321 case Win64EH::UOP_ECContext: 322 case Win64EH::UOP_ClearUnwoundToCall: 323 // Can't reason about these opcodes and how they map to actual 324 // instructions. 325 return; 326 } 327 } 328 // Exclude the end opcode which doesn't map to an instruction. 329 uint32_t InstructionBytes = 4 * (Insns.size() - 1); 330 if (Distance != InstructionBytes) { 331 Streamer.getContext().reportError( 332 SMLoc(), "Incorrect size for " + Name + " " + Type + ": " + 333 Twine(Distance) + 334 " bytes of instructions in range, but .seh directives " 335 "corresponding to " + 336 Twine(InstructionBytes) + " bytes\n"); 337 } 338 } 339 340 static uint32_t ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) { 341 uint32_t Count = 0; 342 for (const auto &I : Insns) { 343 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 344 default: 345 llvm_unreachable("Unsupported ARM64 unwind code"); 346 case Win64EH::UOP_AllocSmall: 347 Count += 1; 348 break; 349 case Win64EH::UOP_AllocMedium: 350 Count += 2; 351 break; 352 case Win64EH::UOP_AllocLarge: 353 Count += 4; 354 break; 355 case Win64EH::UOP_SaveR19R20X: 356 Count += 1; 357 break; 358 case Win64EH::UOP_SaveFPLRX: 359 Count += 1; 360 break; 361 case Win64EH::UOP_SaveFPLR: 362 Count += 1; 363 break; 364 case Win64EH::UOP_SaveReg: 365 Count += 2; 366 break; 367 case Win64EH::UOP_SaveRegP: 368 Count += 2; 369 break; 370 case Win64EH::UOP_SaveRegPX: 371 Count += 2; 372 break; 373 case Win64EH::UOP_SaveRegX: 374 Count += 2; 375 break; 376 case Win64EH::UOP_SaveLRPair: 377 Count += 2; 378 break; 379 case Win64EH::UOP_SaveFReg: 380 Count += 2; 381 break; 382 case Win64EH::UOP_SaveFRegP: 383 Count += 2; 384 break; 385 case Win64EH::UOP_SaveFRegX: 386 Count += 2; 387 break; 388 case Win64EH::UOP_SaveFRegPX: 389 Count += 2; 390 break; 391 case Win64EH::UOP_SetFP: 392 Count += 1; 393 break; 394 case Win64EH::UOP_AddFP: 395 Count += 2; 396 break; 397 case Win64EH::UOP_Nop: 398 Count += 1; 399 break; 400 case Win64EH::UOP_End: 401 Count += 1; 402 break; 403 case Win64EH::UOP_SaveNext: 404 Count += 1; 405 break; 406 case Win64EH::UOP_TrapFrame: 407 Count += 1; 408 break; 409 case Win64EH::UOP_PushMachFrame: 410 Count += 1; 411 break; 412 case Win64EH::UOP_Context: 413 Count += 1; 414 break; 415 case Win64EH::UOP_ECContext: 416 Count += 1; 417 break; 418 case Win64EH::UOP_ClearUnwoundToCall: 419 Count += 1; 420 break; 421 case Win64EH::UOP_PACSignLR: 422 Count += 1; 423 break; 424 case Win64EH::UOP_SaveAnyRegI: 425 case Win64EH::UOP_SaveAnyRegIP: 426 case Win64EH::UOP_SaveAnyRegD: 427 case Win64EH::UOP_SaveAnyRegDP: 428 case Win64EH::UOP_SaveAnyRegQ: 429 case Win64EH::UOP_SaveAnyRegQP: 430 case Win64EH::UOP_SaveAnyRegIX: 431 case Win64EH::UOP_SaveAnyRegIPX: 432 case Win64EH::UOP_SaveAnyRegDX: 433 case Win64EH::UOP_SaveAnyRegDPX: 434 case Win64EH::UOP_SaveAnyRegQX: 435 case Win64EH::UOP_SaveAnyRegQPX: 436 Count += 3; 437 break; 438 } 439 } 440 return Count; 441 } 442 443 // Unwind opcode encodings and restrictions are documented at 444 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling 445 static void ARM64EmitUnwindCode(MCStreamer &streamer, 446 const WinEH::Instruction &inst) { 447 uint8_t b, reg; 448 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 449 default: 450 llvm_unreachable("Unsupported ARM64 unwind code"); 451 case Win64EH::UOP_AllocSmall: 452 b = (inst.Offset >> 4) & 0x1F; 453 streamer.emitInt8(b); 454 break; 455 case Win64EH::UOP_AllocMedium: { 456 uint16_t hw = (inst.Offset >> 4) & 0x7FF; 457 b = 0xC0; 458 b |= (hw >> 8); 459 streamer.emitInt8(b); 460 b = hw & 0xFF; 461 streamer.emitInt8(b); 462 break; 463 } 464 case Win64EH::UOP_AllocLarge: { 465 uint32_t w; 466 b = 0xE0; 467 streamer.emitInt8(b); 468 w = inst.Offset >> 4; 469 b = (w & 0x00FF0000) >> 16; 470 streamer.emitInt8(b); 471 b = (w & 0x0000FF00) >> 8; 472 streamer.emitInt8(b); 473 b = w & 0x000000FF; 474 streamer.emitInt8(b); 475 break; 476 } 477 case Win64EH::UOP_SetFP: 478 b = 0xE1; 479 streamer.emitInt8(b); 480 break; 481 case Win64EH::UOP_AddFP: 482 b = 0xE2; 483 streamer.emitInt8(b); 484 b = (inst.Offset >> 3); 485 streamer.emitInt8(b); 486 break; 487 case Win64EH::UOP_Nop: 488 b = 0xE3; 489 streamer.emitInt8(b); 490 break; 491 case Win64EH::UOP_SaveR19R20X: 492 b = 0x20; 493 b |= (inst.Offset >> 3) & 0x1F; 494 streamer.emitInt8(b); 495 break; 496 case Win64EH::UOP_SaveFPLRX: 497 b = 0x80; 498 b |= ((inst.Offset - 1) >> 3) & 0x3F; 499 streamer.emitInt8(b); 500 break; 501 case Win64EH::UOP_SaveFPLR: 502 b = 0x40; 503 b |= (inst.Offset >> 3) & 0x3F; 504 streamer.emitInt8(b); 505 break; 506 case Win64EH::UOP_SaveReg: 507 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 508 reg = inst.Register - 19; 509 b = 0xD0 | ((reg & 0xC) >> 2); 510 streamer.emitInt8(b); 511 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 512 streamer.emitInt8(b); 513 break; 514 case Win64EH::UOP_SaveRegX: 515 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 516 reg = inst.Register - 19; 517 b = 0xD4 | ((reg & 0x8) >> 3); 518 streamer.emitInt8(b); 519 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1); 520 streamer.emitInt8(b); 521 break; 522 case Win64EH::UOP_SaveRegP: 523 assert(inst.Register >= 19 && "Saved registers must be >= 19"); 524 reg = inst.Register - 19; 525 b = 0xC8 | ((reg & 0xC) >> 2); 526 streamer.emitInt8(b); 527 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 528 streamer.emitInt8(b); 529 break; 530 case Win64EH::UOP_SaveRegPX: 531 assert(inst.Register >= 19 && "Saved registers must be >= 19"); 532 reg = inst.Register - 19; 533 b = 0xCC | ((reg & 0xC) >> 2); 534 streamer.emitInt8(b); 535 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1); 536 streamer.emitInt8(b); 537 break; 538 case Win64EH::UOP_SaveLRPair: 539 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 540 reg = inst.Register - 19; 541 assert((reg % 2) == 0 && "Saved reg must be 19+2*X"); 542 reg /= 2; 543 b = 0xD6 | ((reg & 0x7) >> 2); 544 streamer.emitInt8(b); 545 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 546 streamer.emitInt8(b); 547 break; 548 case Win64EH::UOP_SaveFReg: 549 assert(inst.Register >= 8 && "Saved dreg must be >= 8"); 550 reg = inst.Register - 8; 551 b = 0xDC | ((reg & 0x4) >> 2); 552 streamer.emitInt8(b); 553 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 554 streamer.emitInt8(b); 555 break; 556 case Win64EH::UOP_SaveFRegX: 557 assert(inst.Register >= 8 && "Saved dreg must be >= 8"); 558 reg = inst.Register - 8; 559 b = 0xDE; 560 streamer.emitInt8(b); 561 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1); 562 streamer.emitInt8(b); 563 break; 564 case Win64EH::UOP_SaveFRegP: 565 assert(inst.Register >= 8 && "Saved dregs must be >= 8"); 566 reg = inst.Register - 8; 567 b = 0xD8 | ((reg & 0x4) >> 2); 568 streamer.emitInt8(b); 569 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 570 streamer.emitInt8(b); 571 break; 572 case Win64EH::UOP_SaveFRegPX: 573 assert(inst.Register >= 8 && "Saved dregs must be >= 8"); 574 reg = inst.Register - 8; 575 b = 0xDA | ((reg & 0x4) >> 2); 576 streamer.emitInt8(b); 577 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1); 578 streamer.emitInt8(b); 579 break; 580 case Win64EH::UOP_End: 581 b = 0xE4; 582 streamer.emitInt8(b); 583 break; 584 case Win64EH::UOP_SaveNext: 585 b = 0xE6; 586 streamer.emitInt8(b); 587 break; 588 case Win64EH::UOP_TrapFrame: 589 b = 0xE8; 590 streamer.emitInt8(b); 591 break; 592 case Win64EH::UOP_PushMachFrame: 593 b = 0xE9; 594 streamer.emitInt8(b); 595 break; 596 case Win64EH::UOP_Context: 597 b = 0xEA; 598 streamer.emitInt8(b); 599 break; 600 case Win64EH::UOP_ECContext: 601 b = 0xEB; 602 streamer.emitInt8(b); 603 break; 604 case Win64EH::UOP_ClearUnwoundToCall: 605 b = 0xEC; 606 streamer.emitInt8(b); 607 break; 608 case Win64EH::UOP_PACSignLR: 609 b = 0xFC; 610 streamer.emitInt8(b); 611 break; 612 case Win64EH::UOP_SaveAnyRegI: 613 case Win64EH::UOP_SaveAnyRegIP: 614 case Win64EH::UOP_SaveAnyRegD: 615 case Win64EH::UOP_SaveAnyRegDP: 616 case Win64EH::UOP_SaveAnyRegQ: 617 case Win64EH::UOP_SaveAnyRegQP: 618 case Win64EH::UOP_SaveAnyRegIX: 619 case Win64EH::UOP_SaveAnyRegIPX: 620 case Win64EH::UOP_SaveAnyRegDX: 621 case Win64EH::UOP_SaveAnyRegDPX: 622 case Win64EH::UOP_SaveAnyRegQX: 623 case Win64EH::UOP_SaveAnyRegQPX: { 624 // This assumes the opcodes are listed in the enum in a particular order. 625 int Op = inst.Operation - Win64EH::UOP_SaveAnyRegI; 626 int Writeback = Op / 6; 627 int Paired = Op % 2; 628 int Mode = (Op / 2) % 3; 629 int Offset = inst.Offset >> 3; 630 if (Writeback || Paired || Mode == 2) 631 Offset >>= 1; 632 if (Writeback) 633 --Offset; 634 b = 0xE7; 635 streamer.emitInt8(b); 636 assert(inst.Register < 32); 637 b = inst.Register | (Writeback << 5) | (Paired << 6); 638 streamer.emitInt8(b); 639 b = Offset | (Mode << 6); 640 streamer.emitInt8(b); 641 break; 642 } 643 } 644 } 645 646 // Returns the epilog symbol of an epilog with the exact same unwind code 647 // sequence, if it exists. Otherwise, returns nullptr. 648 // EpilogInstrs - Unwind codes for the current epilog. 649 // Epilogs - Epilogs that potentialy match the current epilog. 650 static MCSymbol* 651 FindMatchingEpilog(const std::vector<WinEH::Instruction>& EpilogInstrs, 652 const std::vector<MCSymbol *>& Epilogs, 653 const WinEH::FrameInfo *info) { 654 for (auto *EpilogStart : Epilogs) { 655 auto InstrsIter = info->EpilogMap.find(EpilogStart); 656 assert(InstrsIter != info->EpilogMap.end() && 657 "Epilog not found in EpilogMap"); 658 const auto &Instrs = InstrsIter->second.Instructions; 659 660 if (Instrs.size() != EpilogInstrs.size()) 661 continue; 662 663 bool Match = true; 664 for (unsigned i = 0; i < Instrs.size(); ++i) 665 if (Instrs[i] != EpilogInstrs[i]) { 666 Match = false; 667 break; 668 } 669 670 if (Match) 671 return EpilogStart; 672 } 673 return nullptr; 674 } 675 676 static void simplifyARM64Opcodes(std::vector<WinEH::Instruction> &Instructions, 677 bool Reverse) { 678 unsigned PrevOffset = -1; 679 unsigned PrevRegister = -1; 680 681 auto VisitInstruction = [&](WinEH::Instruction &Inst) { 682 // Convert 2-byte opcodes into equivalent 1-byte ones. 683 if (Inst.Operation == Win64EH::UOP_SaveRegP && Inst.Register == 29) { 684 Inst.Operation = Win64EH::UOP_SaveFPLR; 685 Inst.Register = -1; 686 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX && 687 Inst.Register == 29) { 688 Inst.Operation = Win64EH::UOP_SaveFPLRX; 689 Inst.Register = -1; 690 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX && 691 Inst.Register == 19 && Inst.Offset <= 248) { 692 Inst.Operation = Win64EH::UOP_SaveR19R20X; 693 Inst.Register = -1; 694 } else if (Inst.Operation == Win64EH::UOP_AddFP && Inst.Offset == 0) { 695 Inst.Operation = Win64EH::UOP_SetFP; 696 } else if (Inst.Operation == Win64EH::UOP_SaveRegP && 697 Inst.Register == PrevRegister + 2 && 698 Inst.Offset == PrevOffset + 16) { 699 Inst.Operation = Win64EH::UOP_SaveNext; 700 Inst.Register = -1; 701 Inst.Offset = 0; 702 // Intentionally not creating UOP_SaveNext for float register pairs, 703 // as current versions of Windows (up to at least 20.04) is buggy 704 // regarding SaveNext for float pairs. 705 } 706 // Update info about the previous instruction, for detecting if 707 // the next one can be made a UOP_SaveNext 708 if (Inst.Operation == Win64EH::UOP_SaveR19R20X) { 709 PrevOffset = 0; 710 PrevRegister = 19; 711 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX) { 712 PrevOffset = 0; 713 PrevRegister = Inst.Register; 714 } else if (Inst.Operation == Win64EH::UOP_SaveRegP) { 715 PrevOffset = Inst.Offset; 716 PrevRegister = Inst.Register; 717 } else if (Inst.Operation == Win64EH::UOP_SaveNext) { 718 PrevRegister += 2; 719 PrevOffset += 16; 720 } else { 721 PrevRegister = -1; 722 PrevOffset = -1; 723 } 724 }; 725 726 // Iterate over instructions in a forward order (for prologues), 727 // backwards for epilogues (i.e. always reverse compared to how the 728 // opcodes are stored). 729 if (Reverse) { 730 for (auto It = Instructions.rbegin(); It != Instructions.rend(); It++) 731 VisitInstruction(*It); 732 } else { 733 for (WinEH::Instruction &Inst : Instructions) 734 VisitInstruction(Inst); 735 } 736 } 737 738 // Check if an epilog exists as a subset of the end of a prolog (backwards). 739 static int 740 getARM64OffsetInProlog(const std::vector<WinEH::Instruction> &Prolog, 741 const std::vector<WinEH::Instruction> &Epilog) { 742 // Can't find an epilog as a subset if it is longer than the prolog. 743 if (Epilog.size() > Prolog.size()) 744 return -1; 745 746 // Check that the epilog actually is a perfect match for the end (backwrds) 747 // of the prolog. 748 for (int I = Epilog.size() - 1; I >= 0; I--) { 749 if (Prolog[I] != Epilog[Epilog.size() - 1 - I]) 750 return -1; 751 } 752 753 if (Epilog.size() == Prolog.size()) 754 return 0; 755 756 // If the epilog was a subset of the prolog, find its offset. 757 return ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction>( 758 &Prolog[Epilog.size()], Prolog.size() - Epilog.size())); 759 } 760 761 static int checkARM64PackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info, 762 WinEH::FrameInfo::Segment *Seg, 763 int PrologCodeBytes) { 764 // Can only pack if there's one single epilog 765 if (Seg->Epilogs.size() != 1) 766 return -1; 767 768 MCSymbol *Sym = Seg->Epilogs.begin()->first; 769 const std::vector<WinEH::Instruction> &Epilog = 770 info->EpilogMap[Sym].Instructions; 771 772 // Check that the epilog actually is at the very end of the function, 773 // otherwise it can't be packed. 774 uint32_t DistanceFromEnd = 775 (uint32_t)(Seg->Offset + Seg->Length - Seg->Epilogs.begin()->second); 776 if (DistanceFromEnd / 4 != Epilog.size()) 777 return -1; 778 779 int RetVal = -1; 780 // Even if we don't end up sharing opcodes with the prolog, we can still 781 // write the offset as a packed offset, if the single epilog is located at 782 // the end of the function and the offset (pointing after the prolog) fits 783 // as a packed offset. 784 if (PrologCodeBytes <= 31 && 785 PrologCodeBytes + ARM64CountOfUnwindCodes(Epilog) <= 124) 786 RetVal = PrologCodeBytes; 787 788 int Offset = getARM64OffsetInProlog(info->Instructions, Epilog); 789 if (Offset < 0) 790 return RetVal; 791 792 // Check that the offset and prolog size fits in the first word; it's 793 // unclear whether the epilog count in the extension word can be taken 794 // as packed epilog offset. 795 if (Offset > 31 || PrologCodeBytes > 124) 796 return RetVal; 797 798 // As we choose to express the epilog as part of the prolog, remove the 799 // epilog from the map, so we don't try to emit its opcodes. 800 info->EpilogMap.erase(Sym); 801 return Offset; 802 } 803 804 static bool tryARM64PackedUnwind(WinEH::FrameInfo *info, uint32_t FuncLength, 805 int PackedEpilogOffset) { 806 if (PackedEpilogOffset == 0) { 807 // Fully symmetric prolog and epilog, should be ok for packed format. 808 // For CR=3, the corresponding synthesized epilog actually lacks the 809 // SetFP opcode, but unwinding should work just fine despite that 810 // (if at the SetFP opcode, the unwinder considers it as part of the 811 // function body and just unwinds the full prolog instead). 812 } else if (PackedEpilogOffset == 1) { 813 // One single case of differences between prolog and epilog is allowed: 814 // The epilog can lack a single SetFP that is the last opcode in the 815 // prolog, for the CR=3 case. 816 if (info->Instructions.back().Operation != Win64EH::UOP_SetFP) 817 return false; 818 } else { 819 // Too much difference between prolog and epilog. 820 return false; 821 } 822 unsigned RegI = 0, RegF = 0; 823 int Predecrement = 0; 824 enum { 825 Start, 826 Start2, 827 Start3, 828 IntRegs, 829 FloatRegs, 830 InputArgs, 831 StackAdjust, 832 FrameRecord, 833 End 834 } Location = Start; 835 bool StandaloneLR = false, FPLRPair = false; 836 bool PAC = false; 837 int StackOffset = 0; 838 int Nops = 0; 839 // Iterate over the prolog and check that all opcodes exactly match 840 // the canonical order and form. A more lax check could verify that 841 // all saved registers are in the expected locations, but not enforce 842 // the order - that would work fine when unwinding from within 843 // functions, but not be exactly right if unwinding happens within 844 // prologs/epilogs. 845 for (const WinEH::Instruction &Inst : info->Instructions) { 846 switch (Inst.Operation) { 847 case Win64EH::UOP_End: 848 if (Location != Start) 849 return false; 850 Location = Start2; 851 break; 852 case Win64EH::UOP_PACSignLR: 853 if (Location != Start2) 854 return false; 855 PAC = true; 856 Location = Start3; 857 break; 858 case Win64EH::UOP_SaveR19R20X: 859 if (Location != Start2 && Location != Start3) 860 return false; 861 Predecrement = Inst.Offset; 862 RegI = 2; 863 Location = IntRegs; 864 break; 865 case Win64EH::UOP_SaveRegX: 866 if (Location != Start2 && Location != Start3) 867 return false; 868 Predecrement = Inst.Offset; 869 if (Inst.Register == 19) 870 RegI += 1; 871 else if (Inst.Register == 30) 872 StandaloneLR = true; 873 else 874 return false; 875 // Odd register; can't be any further int registers. 876 Location = FloatRegs; 877 break; 878 case Win64EH::UOP_SaveRegPX: 879 // Can't have this in a canonical prologue. Either this has been 880 // canonicalized into SaveR19R20X or SaveFPLRX, or it's an unsupported 881 // register pair. 882 // It can't be canonicalized into SaveR19R20X if the offset is 883 // larger than 248 bytes, but even with the maximum case with 884 // RegI=10/RegF=8/CR=1/H=1, we end up with SavSZ = 216, which should 885 // fit into SaveR19R20X. 886 // The unwinding opcodes can't describe the otherwise seemingly valid 887 // case for RegI=1 CR=1, that would start with a 888 // "stp x19, lr, [sp, #-...]!" as that fits neither SaveRegPX nor 889 // SaveLRPair. 890 return false; 891 case Win64EH::UOP_SaveRegP: 892 if (Location != IntRegs || Inst.Offset != 8 * RegI || 893 Inst.Register != 19 + RegI) 894 return false; 895 RegI += 2; 896 break; 897 case Win64EH::UOP_SaveReg: 898 if (Location != IntRegs || Inst.Offset != 8 * RegI) 899 return false; 900 if (Inst.Register == 19 + RegI) 901 RegI += 1; 902 else if (Inst.Register == 30) 903 StandaloneLR = true; 904 else 905 return false; 906 // Odd register; can't be any further int registers. 907 Location = FloatRegs; 908 break; 909 case Win64EH::UOP_SaveLRPair: 910 if (Location != IntRegs || Inst.Offset != 8 * RegI || 911 Inst.Register != 19 + RegI) 912 return false; 913 RegI += 1; 914 StandaloneLR = true; 915 Location = FloatRegs; 916 break; 917 case Win64EH::UOP_SaveFRegX: 918 // Packed unwind can't handle prologs that only save one single 919 // float register. 920 return false; 921 case Win64EH::UOP_SaveFReg: 922 if (Location != FloatRegs || RegF == 0 || Inst.Register != 8 + RegF || 923 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF)) 924 return false; 925 RegF += 1; 926 Location = InputArgs; 927 break; 928 case Win64EH::UOP_SaveFRegPX: 929 if ((Location != Start2 && Location != Start3) || Inst.Register != 8) 930 return false; 931 Predecrement = Inst.Offset; 932 RegF = 2; 933 Location = FloatRegs; 934 break; 935 case Win64EH::UOP_SaveFRegP: 936 if ((Location != IntRegs && Location != FloatRegs) || 937 Inst.Register != 8 + RegF || 938 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF)) 939 return false; 940 RegF += 2; 941 Location = FloatRegs; 942 break; 943 case Win64EH::UOP_SaveNext: 944 if (Location == IntRegs) 945 RegI += 2; 946 else if (Location == FloatRegs) 947 RegF += 2; 948 else 949 return false; 950 break; 951 case Win64EH::UOP_Nop: 952 if (Location != IntRegs && Location != FloatRegs && Location != InputArgs) 953 return false; 954 Location = InputArgs; 955 Nops++; 956 break; 957 case Win64EH::UOP_AllocSmall: 958 case Win64EH::UOP_AllocMedium: 959 if (Location != Start2 && Location != Start3 && Location != IntRegs && 960 Location != FloatRegs && Location != InputArgs && 961 Location != StackAdjust) 962 return false; 963 // Can have either a single decrement, or a pair of decrements with 964 // 4080 and another decrement. 965 if (StackOffset == 0) 966 StackOffset = Inst.Offset; 967 else if (StackOffset != 4080) 968 return false; 969 else 970 StackOffset += Inst.Offset; 971 Location = StackAdjust; 972 break; 973 case Win64EH::UOP_SaveFPLRX: 974 // Not allowing FPLRX after StackAdjust; if a StackAdjust is used, it 975 // should be followed by a FPLR instead. 976 if (Location != Start2 && Location != Start3 && Location != IntRegs && 977 Location != FloatRegs && Location != InputArgs) 978 return false; 979 StackOffset = Inst.Offset; 980 Location = FrameRecord; 981 FPLRPair = true; 982 break; 983 case Win64EH::UOP_SaveFPLR: 984 // This can only follow after a StackAdjust 985 if (Location != StackAdjust || Inst.Offset != 0) 986 return false; 987 Location = FrameRecord; 988 FPLRPair = true; 989 break; 990 case Win64EH::UOP_SetFP: 991 if (Location != FrameRecord) 992 return false; 993 Location = End; 994 break; 995 case Win64EH::UOP_SaveAnyRegI: 996 case Win64EH::UOP_SaveAnyRegIP: 997 case Win64EH::UOP_SaveAnyRegD: 998 case Win64EH::UOP_SaveAnyRegDP: 999 case Win64EH::UOP_SaveAnyRegQ: 1000 case Win64EH::UOP_SaveAnyRegQP: 1001 case Win64EH::UOP_SaveAnyRegIX: 1002 case Win64EH::UOP_SaveAnyRegIPX: 1003 case Win64EH::UOP_SaveAnyRegDX: 1004 case Win64EH::UOP_SaveAnyRegDPX: 1005 case Win64EH::UOP_SaveAnyRegQX: 1006 case Win64EH::UOP_SaveAnyRegQPX: 1007 // These are never canonical; they don't show up with the usual Arm64 1008 // calling convention. 1009 return false; 1010 case Win64EH::UOP_AllocLarge: 1011 // Allocations this large can't be represented in packed unwind (and 1012 // usually don't fit the canonical form anyway because we need to use 1013 // __chkstk to allocate the stack space). 1014 return false; 1015 case Win64EH::UOP_AddFP: 1016 // "add x29, sp, #N" doesn't show up in the canonical pattern (except for 1017 // N=0, which is UOP_SetFP). 1018 return false; 1019 case Win64EH::UOP_TrapFrame: 1020 case Win64EH::UOP_Context: 1021 case Win64EH::UOP_ECContext: 1022 case Win64EH::UOP_ClearUnwoundToCall: 1023 case Win64EH::UOP_PushMachFrame: 1024 // These are special opcodes that aren't normally generated. 1025 return false; 1026 default: 1027 report_fatal_error("Unknown Arm64 unwind opcode"); 1028 } 1029 } 1030 if (RegI > 10 || RegF > 8) 1031 return false; 1032 if (StandaloneLR && FPLRPair) 1033 return false; 1034 if (FPLRPair && Location != End) 1035 return false; 1036 if (Nops != 0 && Nops != 4) 1037 return false; 1038 if (PAC && !FPLRPair) 1039 return false; 1040 int H = Nops == 4; 1041 // There's an inconsistency regarding packed unwind info with homed 1042 // parameters; according to the documentation, the epilog shouldn't have 1043 // the same corresponding nops (and thus, to set the H bit, we should 1044 // require an epilog which isn't exactly symmetrical - we shouldn't accept 1045 // an exact mirrored epilog for those cases), but in practice, 1046 // RtlVirtualUnwind behaves as if it does expect the epilogue to contain 1047 // the same nops. See https://github.com/llvm/llvm-project/issues/54879. 1048 // To play it safe, don't produce packed unwind info with homed parameters. 1049 if (H) 1050 return false; 1051 int IntSZ = 8 * RegI; 1052 if (StandaloneLR) 1053 IntSZ += 8; 1054 int FpSZ = 8 * RegF; // RegF not yet decremented 1055 int SavSZ = (IntSZ + FpSZ + 8 * 8 * H + 0xF) & ~0xF; 1056 if (Predecrement != SavSZ) 1057 return false; 1058 if (FPLRPair && StackOffset < 16) 1059 return false; 1060 if (StackOffset % 16) 1061 return false; 1062 uint32_t FrameSize = (StackOffset + SavSZ) / 16; 1063 if (FrameSize > 0x1FF) 1064 return false; 1065 assert(RegF != 1 && "One single float reg not allowed"); 1066 if (RegF > 0) 1067 RegF--; // Convert from actual number of registers, to value stored 1068 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier"); 1069 int Flag = 0x01; // Function segments not supported yet 1070 int CR = PAC ? 2 : FPLRPair ? 3 : StandaloneLR ? 1 : 0; 1071 info->PackedInfo |= Flag << 0; 1072 info->PackedInfo |= (FuncLength & 0x7FF) << 2; 1073 info->PackedInfo |= (RegF & 0x7) << 13; 1074 info->PackedInfo |= (RegI & 0xF) << 16; 1075 info->PackedInfo |= (H & 0x1) << 20; 1076 info->PackedInfo |= (CR & 0x3) << 21; 1077 info->PackedInfo |= (FrameSize & 0x1FF) << 23; 1078 return true; 1079 } 1080 1081 static void ARM64ProcessEpilogs(WinEH::FrameInfo *info, 1082 WinEH::FrameInfo::Segment *Seg, 1083 uint32_t &TotalCodeBytes, 1084 MapVector<MCSymbol *, uint32_t> &EpilogInfo) { 1085 1086 std::vector<MCSymbol *> EpilogStarts; 1087 for (auto &I : Seg->Epilogs) 1088 EpilogStarts.push_back(I.first); 1089 1090 // Epilogs processed so far. 1091 std::vector<MCSymbol *> AddedEpilogs; 1092 for (auto *S : EpilogStarts) { 1093 MCSymbol *EpilogStart = S; 1094 auto &EpilogInstrs = info->EpilogMap[S].Instructions; 1095 uint32_t CodeBytes = ARM64CountOfUnwindCodes(EpilogInstrs); 1096 1097 MCSymbol* MatchingEpilog = 1098 FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info); 1099 int PrologOffset; 1100 if (MatchingEpilog) { 1101 assert(EpilogInfo.contains(MatchingEpilog) && 1102 "Duplicate epilog not found"); 1103 EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog); 1104 // Clear the unwind codes in the EpilogMap, so that they don't get output 1105 // in ARM64EmitUnwindInfoForSegment(). 1106 EpilogInstrs.clear(); 1107 } else if ((PrologOffset = getARM64OffsetInProlog(info->Instructions, 1108 EpilogInstrs)) >= 0) { 1109 EpilogInfo[EpilogStart] = PrologOffset; 1110 // If the segment doesn't have a prolog, an end_c will be emitted before 1111 // prolog opcodes. So epilog start index in opcodes array is moved by 1. 1112 if (!Seg->HasProlog) 1113 EpilogInfo[EpilogStart] += 1; 1114 // Clear the unwind codes in the EpilogMap, so that they don't get output 1115 // in ARM64EmitUnwindInfoForSegment(). 1116 EpilogInstrs.clear(); 1117 } else { 1118 EpilogInfo[EpilogStart] = TotalCodeBytes; 1119 TotalCodeBytes += CodeBytes; 1120 AddedEpilogs.push_back(EpilogStart); 1121 } 1122 } 1123 } 1124 1125 static void ARM64FindSegmentsInFunction(MCStreamer &streamer, 1126 WinEH::FrameInfo *info, 1127 int64_t RawFuncLength) { 1128 if (info->PrologEnd) 1129 checkARM64Instructions(streamer, info->Instructions, info->Begin, 1130 info->PrologEnd, info->Function->getName(), 1131 "prologue"); 1132 struct EpilogStartEnd { 1133 MCSymbol *Start; 1134 int64_t Offset; 1135 int64_t End; 1136 }; 1137 // Record Start and End of each epilog. 1138 SmallVector<struct EpilogStartEnd, 4> Epilogs; 1139 for (auto &I : info->EpilogMap) { 1140 MCSymbol *Start = I.first; 1141 auto &Instrs = I.second.Instructions; 1142 int64_t Offset = GetAbsDifference(streamer, Start, info->Begin); 1143 checkARM64Instructions(streamer, Instrs, Start, I.second.End, 1144 info->Function->getName(), "epilogue"); 1145 assert((Epilogs.size() == 0 || Offset >= Epilogs.back().End) && 1146 "Epilogs should be monotonically ordered"); 1147 // Exclue the end opcode from Instrs.size() when calculating the end of the 1148 // epilog. 1149 Epilogs.push_back({Start, Offset, Offset + (int64_t)(Instrs.size() - 1) * 4}); 1150 } 1151 1152 unsigned E = 0; 1153 int64_t SegLimit = 0xFFFFC; 1154 int64_t SegOffset = 0; 1155 1156 if (RawFuncLength > SegLimit) { 1157 1158 int64_t RemainingLength = RawFuncLength; 1159 1160 while (RemainingLength > SegLimit) { 1161 // Try divide the function into segments, requirements: 1162 // 1. Segment length <= 0xFFFFC; 1163 // 2. Each Prologue or Epilogue must be fully within a segment. 1164 int64_t SegLength = SegLimit; 1165 int64_t SegEnd = SegOffset + SegLength; 1166 // Keep record on symbols and offsets of epilogs in this segment. 1167 MapVector<MCSymbol *, int64_t> EpilogsInSegment; 1168 1169 while (E < Epilogs.size() && Epilogs[E].End < SegEnd) { 1170 // Epilogs within current segment. 1171 EpilogsInSegment[Epilogs[E].Start] = Epilogs[E].Offset; 1172 ++E; 1173 } 1174 1175 // At this point, we have: 1176 // 1. Put all epilogs in segments already. No action needed here; or 1177 // 2. Found an epilog that will cross segments boundry. We need to 1178 // move back current segment's end boundry, so the epilog is entirely 1179 // in the next segment; or 1180 // 3. Left at least one epilog that is entirely after this segment. 1181 // It'll be handled by the next iteration, or the last segment. 1182 if (E < Epilogs.size() && Epilogs[E].Offset <= SegEnd) 1183 // Move back current Segment's end boundry. 1184 SegLength = Epilogs[E].Offset - SegOffset; 1185 1186 auto Seg = WinEH::FrameInfo::Segment( 1187 SegOffset, SegLength, /* HasProlog */!SegOffset); 1188 Seg.Epilogs = std::move(EpilogsInSegment); 1189 info->Segments.push_back(Seg); 1190 1191 SegOffset += SegLength; 1192 RemainingLength -= SegLength; 1193 } 1194 } 1195 1196 // Add the last segment when RawFuncLength > 0xFFFFC, 1197 // or the only segment otherwise. 1198 auto LastSeg = 1199 WinEH::FrameInfo::Segment(SegOffset, RawFuncLength - SegOffset, 1200 /* HasProlog */!SegOffset); 1201 for (; E < Epilogs.size(); ++E) 1202 LastSeg.Epilogs[Epilogs[E].Start] = Epilogs[E].Offset; 1203 info->Segments.push_back(LastSeg); 1204 } 1205 1206 static void ARM64EmitUnwindInfoForSegment(MCStreamer &streamer, 1207 WinEH::FrameInfo *info, 1208 WinEH::FrameInfo::Segment &Seg, 1209 bool TryPacked = true) { 1210 MCContext &context = streamer.getContext(); 1211 MCSymbol *Label = context.createTempSymbol(); 1212 1213 streamer.emitValueToAlignment(Align(4)); 1214 streamer.emitLabel(Label); 1215 Seg.Symbol = Label; 1216 // Use the 1st segemnt's label as function's. 1217 if (Seg.Offset == 0) 1218 info->Symbol = Label; 1219 1220 bool HasProlog = Seg.HasProlog; 1221 bool HasEpilogs = (Seg.Epilogs.size() != 0); 1222 1223 uint32_t SegLength = (uint32_t)Seg.Length / 4; 1224 uint32_t PrologCodeBytes = info->PrologCodeBytes; 1225 1226 int PackedEpilogOffset = HasEpilogs ? 1227 checkARM64PackedEpilog(streamer, info, &Seg, PrologCodeBytes) : -1; 1228 1229 // TODO: 1230 // 1. Enable packed unwind info (.pdata only) for multi-segment functions. 1231 // 2. Emit packed unwind info (.pdata only) for segments that have neithor 1232 // prolog nor epilog. 1233 if (info->Segments.size() == 1 && PackedEpilogOffset >= 0 && 1234 uint32_t(PackedEpilogOffset) < PrologCodeBytes && 1235 !info->HandlesExceptions && SegLength <= 0x7ff && TryPacked) { 1236 // Matching prolog/epilog and no exception handlers; check if the 1237 // prolog matches the patterns that can be described by the packed 1238 // format. 1239 1240 // info->Symbol was already set even if we didn't actually write any 1241 // unwind info there. Keep using that as indicator that this unwind 1242 // info has been generated already. 1243 if (tryARM64PackedUnwind(info, SegLength, PackedEpilogOffset)) 1244 return; 1245 } 1246 1247 // If the prolog is not in this segment, we need to emit an end_c, which takes 1248 // 1 byte, before prolog unwind ops. 1249 if (!HasProlog) { 1250 PrologCodeBytes += 1; 1251 if (PackedEpilogOffset >= 0) 1252 PackedEpilogOffset += 1; 1253 // If a segment has neither prolog nor epilog, "With full .xdata record, 1254 // Epilog Count = 1. Epilog Start Index points to end_c." 1255 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#function-fragments 1256 // TODO: We can remove this if testing shows zero epilog scope is ok with 1257 // MS unwinder. 1258 if (!HasEpilogs) 1259 // Pack the fake epilog into phantom prolog. 1260 PackedEpilogOffset = 0; 1261 } 1262 1263 uint32_t TotalCodeBytes = PrologCodeBytes; 1264 1265 // Process epilogs. 1266 MapVector<MCSymbol *, uint32_t> EpilogInfo; 1267 ARM64ProcessEpilogs(info, &Seg, TotalCodeBytes, EpilogInfo); 1268 1269 // Code Words, Epilog count, E, X, Vers, Function Length 1270 uint32_t row1 = 0x0; 1271 uint32_t CodeWords = TotalCodeBytes / 4; 1272 uint32_t CodeWordsMod = TotalCodeBytes % 4; 1273 if (CodeWordsMod) 1274 CodeWords++; 1275 uint32_t EpilogCount = 1276 PackedEpilogOffset >= 0 ? PackedEpilogOffset : Seg.Epilogs.size(); 1277 bool ExtensionWord = EpilogCount > 31 || TotalCodeBytes > 124; 1278 if (!ExtensionWord) { 1279 row1 |= (EpilogCount & 0x1F) << 22; 1280 row1 |= (CodeWords & 0x1F) << 27; 1281 } 1282 if (info->HandlesExceptions) // X 1283 row1 |= 1 << 20; 1284 if (PackedEpilogOffset >= 0) // E 1285 row1 |= 1 << 21; 1286 row1 |= SegLength & 0x3FFFF; 1287 streamer.emitInt32(row1); 1288 1289 // Extended Code Words, Extended Epilog Count 1290 if (ExtensionWord) { 1291 // FIXME: We should be able to split unwind info into multiple sections. 1292 if (CodeWords > 0xFF || EpilogCount > 0xFFFF) 1293 report_fatal_error( 1294 "SEH unwind data splitting is only implemented for large functions, " 1295 "cases of too many code words or too many epilogs will be done " 1296 "later"); 1297 uint32_t row2 = 0x0; 1298 row2 |= (CodeWords & 0xFF) << 16; 1299 row2 |= (EpilogCount & 0xFFFF); 1300 streamer.emitInt32(row2); 1301 } 1302 1303 if (PackedEpilogOffset < 0) { 1304 // Epilog Start Index, Epilog Start Offset 1305 for (auto &I : EpilogInfo) { 1306 MCSymbol *EpilogStart = I.first; 1307 uint32_t EpilogIndex = I.second; 1308 // Epilog offset within the Segment. 1309 uint32_t EpilogOffset = (uint32_t)(Seg.Epilogs[EpilogStart] - Seg.Offset); 1310 if (EpilogOffset) 1311 EpilogOffset /= 4; 1312 uint32_t row3 = EpilogOffset; 1313 row3 |= (EpilogIndex & 0x3FF) << 22; 1314 streamer.emitInt32(row3); 1315 } 1316 } 1317 1318 // Note that even for segments that have no prolog, we still need to emit 1319 // prolog unwinding opcodes so that the unwinder knows how to unwind from 1320 // such a segment. 1321 // The end_c opcode at the start indicates to the unwinder that the actual 1322 // prolog is outside of the current segment, and the unwinder shouldn't try 1323 // to check for unwinding from a partial prolog. 1324 if (!HasProlog) 1325 // Emit an end_c. 1326 streamer.emitInt8((uint8_t)0xE5); 1327 1328 // Emit prolog unwind instructions (in reverse order). 1329 for (auto Inst : llvm::reverse(info->Instructions)) 1330 ARM64EmitUnwindCode(streamer, Inst); 1331 1332 // Emit epilog unwind instructions 1333 for (auto &I : Seg.Epilogs) { 1334 auto &EpilogInstrs = info->EpilogMap[I.first].Instructions; 1335 for (const WinEH::Instruction &inst : EpilogInstrs) 1336 ARM64EmitUnwindCode(streamer, inst); 1337 } 1338 1339 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes; 1340 assert(BytesMod >= 0); 1341 for (int i = 0; i < BytesMod; i++) 1342 streamer.emitInt8(0xE3); 1343 1344 if (info->HandlesExceptions) 1345 streamer.emitValue( 1346 MCSymbolRefExpr::create(info->ExceptionHandler, 1347 MCSymbolRefExpr::VK_COFF_IMGREL32, context), 1348 4); 1349 } 1350 1351 // Populate the .xdata section. The format of .xdata on ARM64 is documented at 1352 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling 1353 static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info, 1354 bool TryPacked = true) { 1355 // If this UNWIND_INFO already has a symbol, it's already been emitted. 1356 if (info->Symbol) 1357 return; 1358 // If there's no unwind info here (not even a terminating UOP_End), the 1359 // unwind info is considered bogus and skipped. If this was done in 1360 // response to an explicit .seh_handlerdata, the associated trailing 1361 // handler data is left orphaned in the xdata section. 1362 if (info->empty()) { 1363 info->EmitAttempted = true; 1364 return; 1365 } 1366 if (info->EmitAttempted) { 1367 // If we tried to emit unwind info before (due to an explicit 1368 // .seh_handlerdata directive), but skipped it (because there was no 1369 // valid information to emit at the time), and it later got valid unwind 1370 // opcodes, we can't emit it here, because the trailing handler data 1371 // was already emitted elsewhere in the xdata section. 1372 streamer.getContext().reportError( 1373 SMLoc(), "Earlier .seh_handlerdata for " + info->Function->getName() + 1374 " skipped due to no unwind info at the time " 1375 "(.seh_handlerdata too early?), but the function later " 1376 "did get unwind info that can't be emitted"); 1377 return; 1378 } 1379 1380 simplifyARM64Opcodes(info->Instructions, false); 1381 for (auto &I : info->EpilogMap) 1382 simplifyARM64Opcodes(I.second.Instructions, true); 1383 1384 int64_t RawFuncLength; 1385 if (!info->FuncletOrFuncEnd) { 1386 report_fatal_error("FuncletOrFuncEnd not set"); 1387 } else { 1388 // FIXME: GetAbsDifference tries to compute the length of the function 1389 // immediately, before the whole file is emitted, but in general 1390 // that's impossible: the size in bytes of certain assembler directives 1391 // like .align and .fill is not known until the whole file is parsed and 1392 // relaxations are applied. Currently, GetAbsDifference fails with a fatal 1393 // error in that case. (We mostly don't hit this because inline assembly 1394 // specifying those directives is rare, and we don't normally try to 1395 // align loops on AArch64.) 1396 // 1397 // There are two potential approaches to delaying the computation. One, 1398 // we could emit something like ".word (endfunc-beginfunc)/4+0x10800000", 1399 // as long as we have some conservative estimate we could use to prove 1400 // that we don't need to split the unwind data. Emitting the constant 1401 // is straightforward, but there's no existing code for estimating the 1402 // size of the function. 1403 // 1404 // The other approach would be to use a dedicated, relaxable fragment, 1405 // which could grow to accommodate splitting the unwind data if 1406 // necessary. This is more straightforward, since it automatically works 1407 // without any new infrastructure, and it's consistent with how we handle 1408 // relaxation in other contexts. But it would require some refactoring 1409 // to move parts of the pdata/xdata emission into the implementation of 1410 // a fragment. We could probably continue to encode the unwind codes 1411 // here, but we'd have to emit the pdata, the xdata header, and the 1412 // epilogue scopes later, since they depend on whether the we need to 1413 // split the unwind data. 1414 // 1415 // If this is fixed, remove code in AArch64ISelLowering.cpp that 1416 // disables loop alignment on Windows. 1417 RawFuncLength = GetAbsDifference(streamer, info->FuncletOrFuncEnd, 1418 info->Begin); 1419 } 1420 1421 ARM64FindSegmentsInFunction(streamer, info, RawFuncLength); 1422 1423 info->PrologCodeBytes = ARM64CountOfUnwindCodes(info->Instructions); 1424 for (auto &S : info->Segments) 1425 ARM64EmitUnwindInfoForSegment(streamer, info, S, TryPacked); 1426 1427 // Clear prolog instructions after unwind info is emitted for all segments. 1428 info->Instructions.clear(); 1429 } 1430 1431 static uint32_t ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) { 1432 uint32_t Count = 0; 1433 for (const auto &I : Insns) { 1434 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 1435 default: 1436 llvm_unreachable("Unsupported ARM unwind code"); 1437 case Win64EH::UOP_AllocSmall: 1438 Count += 1; 1439 break; 1440 case Win64EH::UOP_AllocLarge: 1441 Count += 3; 1442 break; 1443 case Win64EH::UOP_AllocHuge: 1444 Count += 4; 1445 break; 1446 case Win64EH::UOP_WideAllocMedium: 1447 Count += 2; 1448 break; 1449 case Win64EH::UOP_WideAllocLarge: 1450 Count += 3; 1451 break; 1452 case Win64EH::UOP_WideAllocHuge: 1453 Count += 4; 1454 break; 1455 case Win64EH::UOP_WideSaveRegMask: 1456 Count += 2; 1457 break; 1458 case Win64EH::UOP_SaveSP: 1459 Count += 1; 1460 break; 1461 case Win64EH::UOP_SaveRegsR4R7LR: 1462 Count += 1; 1463 break; 1464 case Win64EH::UOP_WideSaveRegsR4R11LR: 1465 Count += 1; 1466 break; 1467 case Win64EH::UOP_SaveFRegD8D15: 1468 Count += 1; 1469 break; 1470 case Win64EH::UOP_SaveRegMask: 1471 Count += 2; 1472 break; 1473 case Win64EH::UOP_SaveLR: 1474 Count += 2; 1475 break; 1476 case Win64EH::UOP_SaveFRegD0D15: 1477 Count += 2; 1478 break; 1479 case Win64EH::UOP_SaveFRegD16D31: 1480 Count += 2; 1481 break; 1482 case Win64EH::UOP_Nop: 1483 case Win64EH::UOP_WideNop: 1484 case Win64EH::UOP_End: 1485 case Win64EH::UOP_EndNop: 1486 case Win64EH::UOP_WideEndNop: 1487 Count += 1; 1488 break; 1489 case Win64EH::UOP_Custom: { 1490 int J; 1491 for (J = 3; J > 0; J--) 1492 if (I.Offset & (0xffu << (8 * J))) 1493 break; 1494 Count += J + 1; 1495 break; 1496 } 1497 } 1498 } 1499 return Count; 1500 } 1501 1502 static uint32_t ARMCountOfInstructionBytes(ArrayRef<WinEH::Instruction> Insns, 1503 bool *HasCustom = nullptr) { 1504 uint32_t Count = 0; 1505 for (const auto &I : Insns) { 1506 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 1507 default: 1508 llvm_unreachable("Unsupported ARM unwind code"); 1509 case Win64EH::UOP_AllocSmall: 1510 case Win64EH::UOP_AllocLarge: 1511 case Win64EH::UOP_AllocHuge: 1512 Count += 2; 1513 break; 1514 case Win64EH::UOP_WideAllocMedium: 1515 case Win64EH::UOP_WideAllocLarge: 1516 case Win64EH::UOP_WideAllocHuge: 1517 Count += 4; 1518 break; 1519 case Win64EH::UOP_WideSaveRegMask: 1520 case Win64EH::UOP_WideSaveRegsR4R11LR: 1521 Count += 4; 1522 break; 1523 case Win64EH::UOP_SaveSP: 1524 Count += 2; 1525 break; 1526 case Win64EH::UOP_SaveRegMask: 1527 case Win64EH::UOP_SaveRegsR4R7LR: 1528 Count += 2; 1529 break; 1530 case Win64EH::UOP_SaveFRegD8D15: 1531 case Win64EH::UOP_SaveFRegD0D15: 1532 case Win64EH::UOP_SaveFRegD16D31: 1533 Count += 4; 1534 break; 1535 case Win64EH::UOP_SaveLR: 1536 Count += 4; 1537 break; 1538 case Win64EH::UOP_Nop: 1539 case Win64EH::UOP_EndNop: 1540 Count += 2; 1541 break; 1542 case Win64EH::UOP_WideNop: 1543 case Win64EH::UOP_WideEndNop: 1544 Count += 4; 1545 break; 1546 case Win64EH::UOP_End: 1547 // This doesn't map to any instruction 1548 break; 1549 case Win64EH::UOP_Custom: 1550 // We can't reason about what instructions this maps to; return a 1551 // phony number to make sure we don't accidentally do epilog packing. 1552 Count += 1000; 1553 if (HasCustom) 1554 *HasCustom = true; 1555 break; 1556 } 1557 } 1558 return Count; 1559 } 1560 1561 static void checkARMInstructions(MCStreamer &Streamer, 1562 ArrayRef<WinEH::Instruction> Insns, 1563 const MCSymbol *Begin, const MCSymbol *End, 1564 StringRef Name, StringRef Type) { 1565 if (!End) 1566 return; 1567 std::optional<int64_t> MaybeDistance = 1568 GetOptionalAbsDifference(Streamer, End, Begin); 1569 if (!MaybeDistance) 1570 return; 1571 uint32_t Distance = (uint32_t)*MaybeDistance; 1572 bool HasCustom = false; 1573 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns, &HasCustom); 1574 if (HasCustom) 1575 return; 1576 if (Distance != InstructionBytes) { 1577 Streamer.getContext().reportError( 1578 SMLoc(), "Incorrect size for " + Name + " " + Type + ": " + 1579 Twine(Distance) + 1580 " bytes of instructions in range, but .seh directives " 1581 "corresponding to " + 1582 Twine(InstructionBytes) + " bytes\n"); 1583 } 1584 } 1585 1586 static bool isARMTerminator(const WinEH::Instruction &inst) { 1587 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 1588 case Win64EH::UOP_End: 1589 case Win64EH::UOP_EndNop: 1590 case Win64EH::UOP_WideEndNop: 1591 return true; 1592 default: 1593 return false; 1594 } 1595 } 1596 1597 // Unwind opcode encodings and restrictions are documented at 1598 // https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling 1599 static void ARMEmitUnwindCode(MCStreamer &streamer, 1600 const WinEH::Instruction &inst) { 1601 uint32_t w, lr; 1602 int i; 1603 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 1604 default: 1605 llvm_unreachable("Unsupported ARM unwind code"); 1606 case Win64EH::UOP_AllocSmall: 1607 assert((inst.Offset & 3) == 0); 1608 assert(inst.Offset / 4 <= 0x7f); 1609 streamer.emitInt8(inst.Offset / 4); 1610 break; 1611 case Win64EH::UOP_WideSaveRegMask: 1612 assert((inst.Register & ~0x5fff) == 0); 1613 lr = (inst.Register >> 14) & 1; 1614 w = 0x8000 | (inst.Register & 0x1fff) | (lr << 13); 1615 streamer.emitInt8((w >> 8) & 0xff); 1616 streamer.emitInt8((w >> 0) & 0xff); 1617 break; 1618 case Win64EH::UOP_SaveSP: 1619 assert(inst.Register <= 0x0f); 1620 streamer.emitInt8(0xc0 | inst.Register); 1621 break; 1622 case Win64EH::UOP_SaveRegsR4R7LR: 1623 assert(inst.Register >= 4 && inst.Register <= 7); 1624 assert(inst.Offset <= 1); 1625 streamer.emitInt8(0xd0 | (inst.Register - 4) | (inst.Offset << 2)); 1626 break; 1627 case Win64EH::UOP_WideSaveRegsR4R11LR: 1628 assert(inst.Register >= 8 && inst.Register <= 11); 1629 assert(inst.Offset <= 1); 1630 streamer.emitInt8(0xd8 | (inst.Register - 8) | (inst.Offset << 2)); 1631 break; 1632 case Win64EH::UOP_SaveFRegD8D15: 1633 assert(inst.Register >= 8 && inst.Register <= 15); 1634 streamer.emitInt8(0xe0 | (inst.Register - 8)); 1635 break; 1636 case Win64EH::UOP_WideAllocMedium: 1637 assert((inst.Offset & 3) == 0); 1638 assert(inst.Offset / 4 <= 0x3ff); 1639 w = 0xe800 | (inst.Offset / 4); 1640 streamer.emitInt8((w >> 8) & 0xff); 1641 streamer.emitInt8((w >> 0) & 0xff); 1642 break; 1643 case Win64EH::UOP_SaveRegMask: 1644 assert((inst.Register & ~0x40ff) == 0); 1645 lr = (inst.Register >> 14) & 1; 1646 w = 0xec00 | (inst.Register & 0x0ff) | (lr << 8); 1647 streamer.emitInt8((w >> 8) & 0xff); 1648 streamer.emitInt8((w >> 0) & 0xff); 1649 break; 1650 case Win64EH::UOP_SaveLR: 1651 assert((inst.Offset & 3) == 0); 1652 assert(inst.Offset / 4 <= 0x0f); 1653 streamer.emitInt8(0xef); 1654 streamer.emitInt8(inst.Offset / 4); 1655 break; 1656 case Win64EH::UOP_SaveFRegD0D15: 1657 assert(inst.Register <= 15); 1658 assert(inst.Offset <= 15); 1659 assert(inst.Register <= inst.Offset); 1660 streamer.emitInt8(0xf5); 1661 streamer.emitInt8((inst.Register << 4) | inst.Offset); 1662 break; 1663 case Win64EH::UOP_SaveFRegD16D31: 1664 assert(inst.Register >= 16 && inst.Register <= 31); 1665 assert(inst.Offset >= 16 && inst.Offset <= 31); 1666 assert(inst.Register <= inst.Offset); 1667 streamer.emitInt8(0xf6); 1668 streamer.emitInt8(((inst.Register - 16) << 4) | (inst.Offset - 16)); 1669 break; 1670 case Win64EH::UOP_AllocLarge: 1671 assert((inst.Offset & 3) == 0); 1672 assert(inst.Offset / 4 <= 0xffff); 1673 w = inst.Offset / 4; 1674 streamer.emitInt8(0xf7); 1675 streamer.emitInt8((w >> 8) & 0xff); 1676 streamer.emitInt8((w >> 0) & 0xff); 1677 break; 1678 case Win64EH::UOP_AllocHuge: 1679 assert((inst.Offset & 3) == 0); 1680 assert(inst.Offset / 4 <= 0xffffff); 1681 w = inst.Offset / 4; 1682 streamer.emitInt8(0xf8); 1683 streamer.emitInt8((w >> 16) & 0xff); 1684 streamer.emitInt8((w >> 8) & 0xff); 1685 streamer.emitInt8((w >> 0) & 0xff); 1686 break; 1687 case Win64EH::UOP_WideAllocLarge: 1688 assert((inst.Offset & 3) == 0); 1689 assert(inst.Offset / 4 <= 0xffff); 1690 w = inst.Offset / 4; 1691 streamer.emitInt8(0xf9); 1692 streamer.emitInt8((w >> 8) & 0xff); 1693 streamer.emitInt8((w >> 0) & 0xff); 1694 break; 1695 case Win64EH::UOP_WideAllocHuge: 1696 assert((inst.Offset & 3) == 0); 1697 assert(inst.Offset / 4 <= 0xffffff); 1698 w = inst.Offset / 4; 1699 streamer.emitInt8(0xfa); 1700 streamer.emitInt8((w >> 16) & 0xff); 1701 streamer.emitInt8((w >> 8) & 0xff); 1702 streamer.emitInt8((w >> 0) & 0xff); 1703 break; 1704 case Win64EH::UOP_Nop: 1705 streamer.emitInt8(0xfb); 1706 break; 1707 case Win64EH::UOP_WideNop: 1708 streamer.emitInt8(0xfc); 1709 break; 1710 case Win64EH::UOP_EndNop: 1711 streamer.emitInt8(0xfd); 1712 break; 1713 case Win64EH::UOP_WideEndNop: 1714 streamer.emitInt8(0xfe); 1715 break; 1716 case Win64EH::UOP_End: 1717 streamer.emitInt8(0xff); 1718 break; 1719 case Win64EH::UOP_Custom: 1720 for (i = 3; i > 0; i--) 1721 if (inst.Offset & (0xffu << (8 * i))) 1722 break; 1723 for (; i >= 0; i--) 1724 streamer.emitInt8((inst.Offset >> (8 * i)) & 0xff); 1725 break; 1726 } 1727 } 1728 1729 // Check if an epilog exists as a subset of the end of a prolog (backwards). 1730 // An epilog may end with one out of three different end opcodes; if this 1731 // is the first epilog that shares opcodes with the prolog, we can tolerate 1732 // that this opcode differs (and the caller will update the prolog to use 1733 // the same end opcode as the epilog). If another epilog already shares 1734 // opcodes with the prolog, the ending opcode must be a strict match. 1735 static int getARMOffsetInProlog(const std::vector<WinEH::Instruction> &Prolog, 1736 const std::vector<WinEH::Instruction> &Epilog, 1737 bool CanTweakProlog) { 1738 // Can't find an epilog as a subset if it is longer than the prolog. 1739 if (Epilog.size() > Prolog.size()) 1740 return -1; 1741 1742 // Check that the epilog actually is a perfect match for the end (backwrds) 1743 // of the prolog. 1744 // If we can adjust the prolog afterwards, don't check that the end opcodes 1745 // match. 1746 int EndIdx = CanTweakProlog ? 1 : 0; 1747 for (int I = Epilog.size() - 1; I >= EndIdx; I--) { 1748 // TODO: Could also allow minor mismatches, e.g. "add sp, #16" vs 1749 // "push {r0-r3}". 1750 if (Prolog[I] != Epilog[Epilog.size() - 1 - I]) 1751 return -1; 1752 } 1753 1754 if (CanTweakProlog) { 1755 // Check that both prolog and epilog end with an expected end opcode. 1756 if (Prolog.front().Operation != Win64EH::UOP_End) 1757 return -1; 1758 if (Epilog.back().Operation != Win64EH::UOP_End && 1759 Epilog.back().Operation != Win64EH::UOP_EndNop && 1760 Epilog.back().Operation != Win64EH::UOP_WideEndNop) 1761 return -1; 1762 } 1763 1764 // If the epilog was a subset of the prolog, find its offset. 1765 if (Epilog.size() == Prolog.size()) 1766 return 0; 1767 return ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction>( 1768 &Prolog[Epilog.size()], Prolog.size() - Epilog.size())); 1769 } 1770 1771 static int checkARMPackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info, 1772 int PrologCodeBytes) { 1773 // Can only pack if there's one single epilog 1774 if (info->EpilogMap.size() != 1) 1775 return -1; 1776 1777 const WinEH::FrameInfo::Epilog &EpilogInfo = info->EpilogMap.begin()->second; 1778 // Can only pack if the epilog is unconditional 1779 if (EpilogInfo.Condition != 0xe) // ARMCC::AL 1780 return -1; 1781 1782 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions; 1783 // Make sure we have at least the trailing end opcode 1784 if (info->Instructions.empty() || Epilog.empty()) 1785 return -1; 1786 1787 // Check that the epilog actually is at the very end of the function, 1788 // otherwise it can't be packed. 1789 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference( 1790 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first); 1791 if (!MaybeDistance) 1792 return -1; 1793 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance; 1794 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Epilog); 1795 if (DistanceFromEnd != InstructionBytes) 1796 return -1; 1797 1798 int RetVal = -1; 1799 // Even if we don't end up sharing opcodes with the prolog, we can still 1800 // write the offset as a packed offset, if the single epilog is located at 1801 // the end of the function and the offset (pointing after the prolog) fits 1802 // as a packed offset. 1803 if (PrologCodeBytes <= 31 && 1804 PrologCodeBytes + ARMCountOfUnwindCodes(Epilog) <= 63) 1805 RetVal = PrologCodeBytes; 1806 1807 int Offset = 1808 getARMOffsetInProlog(info->Instructions, Epilog, /*CanTweakProlog=*/true); 1809 if (Offset < 0) 1810 return RetVal; 1811 1812 // Check that the offset and prolog size fits in the first word; it's 1813 // unclear whether the epilog count in the extension word can be taken 1814 // as packed epilog offset. 1815 if (Offset > 31 || PrologCodeBytes > 63) 1816 return RetVal; 1817 1818 // Replace the regular end opcode of the prolog with the one from the 1819 // epilog. 1820 info->Instructions.front() = Epilog.back(); 1821 1822 // As we choose to express the epilog as part of the prolog, remove the 1823 // epilog from the map, so we don't try to emit its opcodes. 1824 info->EpilogMap.clear(); 1825 return Offset; 1826 } 1827 1828 static bool parseRegMask(unsigned Mask, bool &HasLR, bool &HasR11, 1829 unsigned &Folded, int &IntRegs) { 1830 if (Mask & (1 << 14)) { 1831 HasLR = true; 1832 Mask &= ~(1 << 14); 1833 } 1834 if (Mask & (1 << 11)) { 1835 HasR11 = true; 1836 Mask &= ~(1 << 11); 1837 } 1838 Folded = 0; 1839 IntRegs = -1; 1840 if (!Mask) 1841 return true; 1842 int First = 0; 1843 // Shift right until we have the bits at the bottom 1844 while ((Mask & 1) == 0) { 1845 First++; 1846 Mask >>= 1; 1847 } 1848 if ((Mask & (Mask + 1)) != 0) 1849 return false; // Not a consecutive series of bits? Can't be packed. 1850 // Count the bits 1851 int N = 0; 1852 while (Mask & (1 << N)) 1853 N++; 1854 if (First < 4) { 1855 if (First + N < 4) 1856 return false; 1857 Folded = 4 - First; 1858 N -= Folded; 1859 First = 4; 1860 } 1861 if (First > 4) 1862 return false; // Can't be packed 1863 if (N >= 1) 1864 IntRegs = N - 1; 1865 return true; 1866 } 1867 1868 static bool tryARMPackedUnwind(MCStreamer &streamer, WinEH::FrameInfo *info, 1869 uint32_t FuncLength) { 1870 int Step = 0; 1871 bool Homing = false; 1872 bool HasR11 = false; 1873 bool HasChain = false; 1874 bool HasLR = false; 1875 int IntRegs = -1; // r4 - r(4+N) 1876 int FloatRegs = -1; // d8 - d(8+N) 1877 unsigned PF = 0; // Number of extra pushed registers 1878 unsigned StackAdjust = 0; 1879 // Iterate over the prolog and check that all opcodes exactly match 1880 // the canonical order and form. 1881 for (const WinEH::Instruction &Inst : info->Instructions) { 1882 switch (Inst.Operation) { 1883 default: 1884 llvm_unreachable("Unsupported ARM unwind code"); 1885 case Win64EH::UOP_Custom: 1886 case Win64EH::UOP_AllocLarge: 1887 case Win64EH::UOP_AllocHuge: 1888 case Win64EH::UOP_WideAllocLarge: 1889 case Win64EH::UOP_WideAllocHuge: 1890 case Win64EH::UOP_SaveFRegD0D15: 1891 case Win64EH::UOP_SaveFRegD16D31: 1892 // Can't be packed 1893 return false; 1894 case Win64EH::UOP_SaveSP: 1895 // Can't be packed; we can't rely on restoring sp from r11 when 1896 // unwinding a packed prologue. 1897 return false; 1898 case Win64EH::UOP_SaveLR: 1899 // Can't be present in a packed prologue 1900 return false; 1901 1902 case Win64EH::UOP_End: 1903 case Win64EH::UOP_EndNop: 1904 case Win64EH::UOP_WideEndNop: 1905 if (Step != 0) 1906 return false; 1907 Step = 1; 1908 break; 1909 1910 case Win64EH::UOP_SaveRegsR4R7LR: 1911 case Win64EH::UOP_WideSaveRegsR4R11LR: 1912 // push {r4-r11,lr} 1913 if (Step != 1 && Step != 2) 1914 return false; 1915 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX 1916 assert(Inst.Offset <= 1); // Lr 1917 IntRegs = Inst.Register - 4; 1918 if (Inst.Register == 11) { 1919 HasR11 = true; 1920 IntRegs--; 1921 } 1922 if (Inst.Offset) 1923 HasLR = true; 1924 Step = 3; 1925 break; 1926 1927 case Win64EH::UOP_SaveRegMask: 1928 if (Step == 1 && Inst.Register == 0x0f) { 1929 // push {r0-r3} 1930 Homing = true; 1931 Step = 2; 1932 break; 1933 } 1934 [[fallthrough]]; 1935 case Win64EH::UOP_WideSaveRegMask: 1936 if (Step != 1 && Step != 2) 1937 return false; 1938 // push {r4-r9,r11,lr} 1939 // push {r11,lr} 1940 // push {r1-r5} 1941 if (!parseRegMask(Inst.Register, HasLR, HasR11, PF, IntRegs)) 1942 return false; 1943 Step = 3; 1944 break; 1945 1946 case Win64EH::UOP_Nop: 1947 // mov r11, sp 1948 if (Step != 3 || !HasR11 || IntRegs >= 0 || PF > 0) 1949 return false; 1950 HasChain = true; 1951 Step = 4; 1952 break; 1953 case Win64EH::UOP_WideNop: 1954 // add.w r11, sp, #xx 1955 if (Step != 3 || !HasR11 || (IntRegs < 0 && PF == 0)) 1956 return false; 1957 HasChain = true; 1958 Step = 4; 1959 break; 1960 1961 case Win64EH::UOP_SaveFRegD8D15: 1962 if (Step != 1 && Step != 2 && Step != 3 && Step != 4) 1963 return false; 1964 assert(Inst.Register >= 8 && Inst.Register <= 15); 1965 if (Inst.Register == 15) 1966 return false; // Can't pack this case, R==7 means no IntRegs 1967 if (IntRegs >= 0) 1968 return false; 1969 FloatRegs = Inst.Register - 8; 1970 Step = 5; 1971 break; 1972 1973 case Win64EH::UOP_AllocSmall: 1974 case Win64EH::UOP_WideAllocMedium: 1975 if (Step != 1 && Step != 2 && Step != 3 && Step != 4 && Step != 5) 1976 return false; 1977 if (PF > 0) // Can't have both folded and explicit stack allocation 1978 return false; 1979 if (Inst.Offset / 4 >= 0x3f4) 1980 return false; 1981 StackAdjust = Inst.Offset / 4; 1982 Step = 6; 1983 break; 1984 } 1985 } 1986 if (HasR11 && !HasChain) { 1987 if (IntRegs + 4 == 10) { 1988 // r11 stored, but not chaining; can be packed if already saving r4-r10 1989 // and we can fit r11 into this range. 1990 IntRegs++; 1991 HasR11 = false; 1992 } else 1993 return false; 1994 } 1995 if (HasChain && !HasLR) 1996 return false; 1997 1998 // Packed uneind info can't express multiple epilogues. 1999 if (info->EpilogMap.size() > 1) 2000 return false; 2001 2002 unsigned EF = 0; 2003 int Ret = 0; 2004 if (info->EpilogMap.size() == 0) { 2005 Ret = 3; // No epilogue 2006 } else { 2007 // As the prologue and epilogue aren't exact mirrors of each other, 2008 // we have to check the epilogue too and see if it matches what we've 2009 // concluded from the prologue. 2010 const WinEH::FrameInfo::Epilog &EpilogInfo = 2011 info->EpilogMap.begin()->second; 2012 if (EpilogInfo.Condition != 0xe) // ARMCC::AL 2013 return false; 2014 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions; 2015 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference( 2016 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first); 2017 if (!MaybeDistance) 2018 return false; 2019 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance; 2020 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Epilog); 2021 if (DistanceFromEnd != InstructionBytes) 2022 return false; 2023 2024 bool GotStackAdjust = false; 2025 bool GotFloatRegs = false; 2026 bool GotIntRegs = false; 2027 bool GotHomingRestore = false; 2028 bool GotLRRestore = false; 2029 bool NeedsReturn = false; 2030 bool GotReturn = false; 2031 2032 Step = 6; 2033 for (const WinEH::Instruction &Inst : Epilog) { 2034 switch (Inst.Operation) { 2035 default: 2036 llvm_unreachable("Unsupported ARM unwind code"); 2037 case Win64EH::UOP_Custom: 2038 case Win64EH::UOP_AllocLarge: 2039 case Win64EH::UOP_AllocHuge: 2040 case Win64EH::UOP_WideAllocLarge: 2041 case Win64EH::UOP_WideAllocHuge: 2042 case Win64EH::UOP_SaveFRegD0D15: 2043 case Win64EH::UOP_SaveFRegD16D31: 2044 case Win64EH::UOP_SaveSP: 2045 case Win64EH::UOP_Nop: 2046 case Win64EH::UOP_WideNop: 2047 // Can't be packed in an epilogue 2048 return false; 2049 2050 case Win64EH::UOP_AllocSmall: 2051 case Win64EH::UOP_WideAllocMedium: 2052 if (Inst.Offset / 4 >= 0x3f4) 2053 return false; 2054 if (Step == 6) { 2055 if (Homing && FloatRegs < 0 && IntRegs < 0 && StackAdjust == 0 && 2056 PF == 0 && Inst.Offset == 16) { 2057 GotHomingRestore = true; 2058 Step = 10; 2059 } else { 2060 if (StackAdjust > 0) { 2061 // Got stack adjust in prologue too; must match. 2062 if (StackAdjust != Inst.Offset / 4) 2063 return false; 2064 GotStackAdjust = true; 2065 } else if (PF == Inst.Offset / 4) { 2066 // Folded prologue, non-folded epilogue 2067 StackAdjust = Inst.Offset / 4; 2068 GotStackAdjust = true; 2069 } else { 2070 // StackAdjust == 0 in prologue, mismatch 2071 return false; 2072 } 2073 Step = 7; 2074 } 2075 } else if (Step == 7 || Step == 8 || Step == 9) { 2076 if (!Homing || Inst.Offset != 16) 2077 return false; 2078 GotHomingRestore = true; 2079 Step = 10; 2080 } else 2081 return false; 2082 break; 2083 2084 case Win64EH::UOP_SaveFRegD8D15: 2085 if (Step != 6 && Step != 7) 2086 return false; 2087 assert(Inst.Register >= 8 && Inst.Register <= 15); 2088 if (FloatRegs != (int)(Inst.Register - 8)) 2089 return false; 2090 GotFloatRegs = true; 2091 Step = 8; 2092 break; 2093 2094 case Win64EH::UOP_SaveRegsR4R7LR: 2095 case Win64EH::UOP_WideSaveRegsR4R11LR: { 2096 // push {r4-r11,lr} 2097 if (Step != 6 && Step != 7 && Step != 8) 2098 return false; 2099 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX 2100 assert(Inst.Offset <= 1); // Lr 2101 if (Homing && HasLR) { 2102 // If homing and LR is backed up, we can either restore LR here 2103 // and return with Ret == 1 or 2, or return with SaveLR below 2104 if (Inst.Offset) { 2105 GotLRRestore = true; 2106 NeedsReturn = true; 2107 } else { 2108 // Expecting a separate SaveLR below 2109 } 2110 } else { 2111 if (HasLR != (Inst.Offset == 1)) 2112 return false; 2113 } 2114 GotLRRestore = Inst.Offset == 1; 2115 if (IntRegs < 0) // This opcode must include r4 2116 return false; 2117 int Expected = IntRegs; 2118 if (HasChain) { 2119 // Can't express r11 here unless IntRegs describe r4-r10 2120 if (IntRegs != 6) 2121 return false; 2122 Expected++; 2123 } 2124 if (Expected != (int)(Inst.Register - 4)) 2125 return false; 2126 GotIntRegs = true; 2127 Step = 9; 2128 break; 2129 } 2130 2131 case Win64EH::UOP_SaveRegMask: 2132 case Win64EH::UOP_WideSaveRegMask: { 2133 if (Step != 6 && Step != 7 && Step != 8) 2134 return false; 2135 // push {r4-r9,r11,lr} 2136 // push {r11,lr} 2137 // push {r1-r5} 2138 bool CurHasLR = false, CurHasR11 = false; 2139 int Regs; 2140 if (!parseRegMask(Inst.Register, CurHasLR, CurHasR11, EF, Regs)) 2141 return false; 2142 if (EF > 0) { 2143 if (EF != PF && EF != StackAdjust) 2144 return false; 2145 } 2146 if (Homing && HasLR) { 2147 // If homing and LR is backed up, we can either restore LR here 2148 // and return with Ret == 1 or 2, or return with SaveLR below 2149 if (CurHasLR) { 2150 GotLRRestore = true; 2151 NeedsReturn = true; 2152 } else { 2153 // Expecting a separate SaveLR below 2154 } 2155 } else { 2156 if (CurHasLR != HasLR) 2157 return false; 2158 GotLRRestore = CurHasLR; 2159 } 2160 int Expected = IntRegs; 2161 if (HasChain) { 2162 // If we have chaining, the mask must have included r11. 2163 if (!CurHasR11) 2164 return false; 2165 } else if (Expected == 7) { 2166 // If we don't have chaining, the mask could still include r11, 2167 // expressed as part of IntRegs Instead. 2168 Expected--; 2169 if (!CurHasR11) 2170 return false; 2171 } else { 2172 // Neither HasChain nor r11 included in IntRegs, must not have r11 2173 // here either. 2174 if (CurHasR11) 2175 return false; 2176 } 2177 if (Expected != Regs) 2178 return false; 2179 GotIntRegs = true; 2180 Step = 9; 2181 break; 2182 } 2183 2184 case Win64EH::UOP_SaveLR: 2185 if (Step != 6 && Step != 7 && Step != 8 && Step != 9) 2186 return false; 2187 if (!Homing || Inst.Offset != 20 || GotLRRestore) 2188 return false; 2189 GotLRRestore = true; 2190 GotHomingRestore = true; 2191 Step = 10; 2192 break; 2193 2194 case Win64EH::UOP_EndNop: 2195 case Win64EH::UOP_WideEndNop: 2196 GotReturn = true; 2197 Ret = (Inst.Operation == Win64EH::UOP_EndNop) ? 1 : 2; 2198 [[fallthrough]]; 2199 case Win64EH::UOP_End: 2200 if (Step != 6 && Step != 7 && Step != 8 && Step != 9 && Step != 10) 2201 return false; 2202 Step = 11; 2203 break; 2204 } 2205 } 2206 2207 if (Step != 11) 2208 return false; 2209 if (StackAdjust > 0 && !GotStackAdjust && EF == 0) 2210 return false; 2211 if (FloatRegs >= 0 && !GotFloatRegs) 2212 return false; 2213 if (IntRegs >= 0 && !GotIntRegs) 2214 return false; 2215 if (Homing && !GotHomingRestore) 2216 return false; 2217 if (HasLR && !GotLRRestore) 2218 return false; 2219 if (NeedsReturn && !GotReturn) 2220 return false; 2221 } 2222 2223 assert(PF == 0 || EF == 0 || 2224 StackAdjust == 0); // Can't have adjust in all three 2225 if (PF > 0 || EF > 0) { 2226 StackAdjust = PF > 0 ? (PF - 1) : (EF - 1); 2227 assert(StackAdjust <= 3); 2228 StackAdjust |= 0x3f0; 2229 if (PF > 0) 2230 StackAdjust |= 1 << 2; 2231 if (EF > 0) 2232 StackAdjust |= 1 << 3; 2233 } 2234 2235 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier"); 2236 int Flag = info->Fragment ? 0x02 : 0x01; 2237 int H = Homing ? 1 : 0; 2238 int L = HasLR ? 1 : 0; 2239 int C = HasChain ? 1 : 0; 2240 assert(IntRegs < 0 || FloatRegs < 0); 2241 unsigned Reg, R; 2242 if (IntRegs >= 0) { 2243 Reg = IntRegs; 2244 assert(Reg <= 7); 2245 R = 0; 2246 } else if (FloatRegs >= 0) { 2247 Reg = FloatRegs; 2248 assert(Reg < 7); 2249 R = 1; 2250 } else { 2251 // No int or float regs stored (except possibly R11,LR) 2252 Reg = 7; 2253 R = 1; 2254 } 2255 info->PackedInfo |= Flag << 0; 2256 info->PackedInfo |= (FuncLength & 0x7FF) << 2; 2257 info->PackedInfo |= (Ret & 0x3) << 13; 2258 info->PackedInfo |= H << 15; 2259 info->PackedInfo |= Reg << 16; 2260 info->PackedInfo |= R << 19; 2261 info->PackedInfo |= L << 20; 2262 info->PackedInfo |= C << 21; 2263 assert(StackAdjust <= 0x3ff); 2264 info->PackedInfo |= StackAdjust << 22; 2265 return true; 2266 } 2267 2268 // Populate the .xdata section. The format of .xdata on ARM is documented at 2269 // https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling 2270 static void ARMEmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info, 2271 bool TryPacked = true) { 2272 // If this UNWIND_INFO already has a symbol, it's already been emitted. 2273 if (info->Symbol) 2274 return; 2275 // If there's no unwind info here (not even a terminating UOP_End), the 2276 // unwind info is considered bogus and skipped. If this was done in 2277 // response to an explicit .seh_handlerdata, the associated trailing 2278 // handler data is left orphaned in the xdata section. 2279 if (info->empty()) { 2280 info->EmitAttempted = true; 2281 return; 2282 } 2283 if (info->EmitAttempted) { 2284 // If we tried to emit unwind info before (due to an explicit 2285 // .seh_handlerdata directive), but skipped it (because there was no 2286 // valid information to emit at the time), and it later got valid unwind 2287 // opcodes, we can't emit it here, because the trailing handler data 2288 // was already emitted elsewhere in the xdata section. 2289 streamer.getContext().reportError( 2290 SMLoc(), "Earlier .seh_handlerdata for " + info->Function->getName() + 2291 " skipped due to no unwind info at the time " 2292 "(.seh_handlerdata too early?), but the function later " 2293 "did get unwind info that can't be emitted"); 2294 return; 2295 } 2296 2297 MCContext &context = streamer.getContext(); 2298 MCSymbol *Label = context.createTempSymbol(); 2299 2300 streamer.emitValueToAlignment(Align(4)); 2301 streamer.emitLabel(Label); 2302 info->Symbol = Label; 2303 2304 if (!info->PrologEnd) 2305 streamer.getContext().reportError(SMLoc(), "Prologue in " + 2306 info->Function->getName() + 2307 " not correctly terminated"); 2308 2309 if (info->PrologEnd && !info->Fragment) 2310 checkARMInstructions(streamer, info->Instructions, info->Begin, 2311 info->PrologEnd, info->Function->getName(), 2312 "prologue"); 2313 for (auto &I : info->EpilogMap) { 2314 MCSymbol *EpilogStart = I.first; 2315 auto &Epilog = I.second; 2316 checkARMInstructions(streamer, Epilog.Instructions, EpilogStart, Epilog.End, 2317 info->Function->getName(), "epilogue"); 2318 if (Epilog.Instructions.empty() || 2319 !isARMTerminator(Epilog.Instructions.back())) 2320 streamer.getContext().reportError( 2321 SMLoc(), "Epilogue in " + info->Function->getName() + 2322 " not correctly terminated"); 2323 } 2324 2325 std::optional<int64_t> RawFuncLength; 2326 const MCExpr *FuncLengthExpr = nullptr; 2327 if (!info->FuncletOrFuncEnd) { 2328 report_fatal_error("FuncletOrFuncEnd not set"); 2329 } else { 2330 // As the size of many thumb2 instructions isn't known until later, 2331 // we can't always rely on being able to calculate the absolute 2332 // length of the function here. If we can't calculate it, defer it 2333 // to a relocation. 2334 // 2335 // In such a case, we won't know if the function is too long so that 2336 // the unwind info would need to be split (but this isn't implemented 2337 // anyway). 2338 RawFuncLength = 2339 GetOptionalAbsDifference(streamer, info->FuncletOrFuncEnd, info->Begin); 2340 if (!RawFuncLength) 2341 FuncLengthExpr = 2342 GetSubDivExpr(streamer, info->FuncletOrFuncEnd, info->Begin, 2); 2343 } 2344 uint32_t FuncLength = 0; 2345 if (RawFuncLength) 2346 FuncLength = (uint32_t)*RawFuncLength / 2; 2347 if (FuncLength > 0x3FFFF) 2348 report_fatal_error("SEH unwind data splitting not yet implemented"); 2349 uint32_t PrologCodeBytes = ARMCountOfUnwindCodes(info->Instructions); 2350 uint32_t TotalCodeBytes = PrologCodeBytes; 2351 2352 if (!info->HandlesExceptions && RawFuncLength && FuncLength <= 0x7ff && 2353 TryPacked) { 2354 // No exception handlers; check if the prolog and epilog matches the 2355 // patterns that can be described by the packed format. If we don't 2356 // know the exact function length yet, we can't do this. 2357 2358 // info->Symbol was already set even if we didn't actually write any 2359 // unwind info there. Keep using that as indicator that this unwind 2360 // info has been generated already. 2361 2362 if (tryARMPackedUnwind(streamer, info, FuncLength)) 2363 return; 2364 } 2365 2366 int PackedEpilogOffset = 2367 checkARMPackedEpilog(streamer, info, PrologCodeBytes); 2368 2369 // Process epilogs. 2370 MapVector<MCSymbol *, uint32_t> EpilogInfo; 2371 // Epilogs processed so far. 2372 std::vector<MCSymbol *> AddedEpilogs; 2373 2374 bool CanTweakProlog = true; 2375 for (auto &I : info->EpilogMap) { 2376 MCSymbol *EpilogStart = I.first; 2377 auto &EpilogInstrs = I.second.Instructions; 2378 uint32_t CodeBytes = ARMCountOfUnwindCodes(EpilogInstrs); 2379 2380 MCSymbol *MatchingEpilog = 2381 FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info); 2382 int PrologOffset; 2383 if (MatchingEpilog) { 2384 assert(EpilogInfo.contains(MatchingEpilog) && 2385 "Duplicate epilog not found"); 2386 EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog); 2387 // Clear the unwind codes in the EpilogMap, so that they don't get output 2388 // in the logic below. 2389 EpilogInstrs.clear(); 2390 } else if ((PrologOffset = getARMOffsetInProlog( 2391 info->Instructions, EpilogInstrs, CanTweakProlog)) >= 0) { 2392 if (CanTweakProlog) { 2393 // Replace the regular end opcode of the prolog with the one from the 2394 // epilog. 2395 info->Instructions.front() = EpilogInstrs.back(); 2396 // Later epilogs need a strict match for the end opcode. 2397 CanTweakProlog = false; 2398 } 2399 EpilogInfo[EpilogStart] = PrologOffset; 2400 // Clear the unwind codes in the EpilogMap, so that they don't get output 2401 // in the logic below. 2402 EpilogInstrs.clear(); 2403 } else { 2404 EpilogInfo[EpilogStart] = TotalCodeBytes; 2405 TotalCodeBytes += CodeBytes; 2406 AddedEpilogs.push_back(EpilogStart); 2407 } 2408 } 2409 2410 // Code Words, Epilog count, F, E, X, Vers, Function Length 2411 uint32_t row1 = 0x0; 2412 uint32_t CodeWords = TotalCodeBytes / 4; 2413 uint32_t CodeWordsMod = TotalCodeBytes % 4; 2414 if (CodeWordsMod) 2415 CodeWords++; 2416 uint32_t EpilogCount = 2417 PackedEpilogOffset >= 0 ? PackedEpilogOffset : info->EpilogMap.size(); 2418 bool ExtensionWord = EpilogCount > 31 || CodeWords > 15; 2419 if (!ExtensionWord) { 2420 row1 |= (EpilogCount & 0x1F) << 23; 2421 row1 |= (CodeWords & 0x0F) << 28; 2422 } 2423 if (info->HandlesExceptions) // X 2424 row1 |= 1 << 20; 2425 if (PackedEpilogOffset >= 0) // E 2426 row1 |= 1 << 21; 2427 if (info->Fragment) // F 2428 row1 |= 1 << 22; 2429 row1 |= FuncLength & 0x3FFFF; 2430 if (RawFuncLength) 2431 streamer.emitInt32(row1); 2432 else 2433 streamer.emitValue( 2434 MCBinaryExpr::createOr(FuncLengthExpr, 2435 MCConstantExpr::create(row1, context), context), 2436 4); 2437 2438 // Extended Code Words, Extended Epilog Count 2439 if (ExtensionWord) { 2440 // FIXME: We should be able to split unwind info into multiple sections. 2441 if (CodeWords > 0xFF || EpilogCount > 0xFFFF) 2442 report_fatal_error("SEH unwind data splitting not yet implemented"); 2443 uint32_t row2 = 0x0; 2444 row2 |= (CodeWords & 0xFF) << 16; 2445 row2 |= (EpilogCount & 0xFFFF); 2446 streamer.emitInt32(row2); 2447 } 2448 2449 if (PackedEpilogOffset < 0) { 2450 // Epilog Start Index, Epilog Start Offset 2451 for (auto &I : EpilogInfo) { 2452 MCSymbol *EpilogStart = I.first; 2453 uint32_t EpilogIndex = I.second; 2454 2455 std::optional<int64_t> MaybeEpilogOffset = 2456 GetOptionalAbsDifference(streamer, EpilogStart, info->Begin); 2457 const MCExpr *OffsetExpr = nullptr; 2458 uint32_t EpilogOffset = 0; 2459 if (MaybeEpilogOffset) 2460 EpilogOffset = *MaybeEpilogOffset / 2; 2461 else 2462 OffsetExpr = GetSubDivExpr(streamer, EpilogStart, info->Begin, 2); 2463 2464 assert(info->EpilogMap.contains(EpilogStart)); 2465 unsigned Condition = info->EpilogMap[EpilogStart].Condition; 2466 assert(Condition <= 0xf); 2467 2468 uint32_t row3 = EpilogOffset; 2469 row3 |= Condition << 20; 2470 row3 |= (EpilogIndex & 0x3FF) << 24; 2471 if (MaybeEpilogOffset) 2472 streamer.emitInt32(row3); 2473 else 2474 streamer.emitValue( 2475 MCBinaryExpr::createOr( 2476 OffsetExpr, MCConstantExpr::create(row3, context), context), 2477 4); 2478 } 2479 } 2480 2481 // Emit prolog unwind instructions (in reverse order). 2482 uint8_t numInst = info->Instructions.size(); 2483 for (uint8_t c = 0; c < numInst; ++c) { 2484 WinEH::Instruction inst = info->Instructions.back(); 2485 info->Instructions.pop_back(); 2486 ARMEmitUnwindCode(streamer, inst); 2487 } 2488 2489 // Emit epilog unwind instructions 2490 for (auto &I : info->EpilogMap) { 2491 auto &EpilogInstrs = I.second.Instructions; 2492 for (const WinEH::Instruction &inst : EpilogInstrs) 2493 ARMEmitUnwindCode(streamer, inst); 2494 } 2495 2496 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes; 2497 assert(BytesMod >= 0); 2498 for (int i = 0; i < BytesMod; i++) 2499 streamer.emitInt8(0xFB); 2500 2501 if (info->HandlesExceptions) 2502 streamer.emitValue( 2503 MCSymbolRefExpr::create(info->ExceptionHandler, 2504 MCSymbolRefExpr::VK_COFF_IMGREL32, context), 2505 4); 2506 } 2507 2508 static void ARM64EmitRuntimeFunction(MCStreamer &streamer, 2509 const WinEH::FrameInfo *info) { 2510 MCContext &context = streamer.getContext(); 2511 2512 streamer.emitValueToAlignment(Align(4)); 2513 for (const auto &S : info->Segments) { 2514 EmitSymbolRefWithOfs(streamer, info->Begin, S.Offset); 2515 if (info->PackedInfo) 2516 streamer.emitInt32(info->PackedInfo); 2517 else 2518 streamer.emitValue( 2519 MCSymbolRefExpr::create(S.Symbol, MCSymbolRefExpr::VK_COFF_IMGREL32, 2520 context), 2521 4); 2522 } 2523 } 2524 2525 2526 static void ARMEmitRuntimeFunction(MCStreamer &streamer, 2527 const WinEH::FrameInfo *info) { 2528 MCContext &context = streamer.getContext(); 2529 2530 streamer.emitValueToAlignment(Align(4)); 2531 EmitSymbolRefWithOfs(streamer, info->Begin, info->Begin); 2532 if (info->PackedInfo) 2533 streamer.emitInt32(info->PackedInfo); 2534 else 2535 streamer.emitValue( 2536 MCSymbolRefExpr::create(info->Symbol, MCSymbolRefExpr::VK_COFF_IMGREL32, 2537 context), 2538 4); 2539 } 2540 2541 void llvm::Win64EH::ARM64UnwindEmitter::Emit(MCStreamer &Streamer) const { 2542 // Emit the unwind info structs first. 2543 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2544 WinEH::FrameInfo *Info = CFI.get(); 2545 if (Info->empty()) 2546 continue; 2547 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 2548 Streamer.switchSection(XData); 2549 ARM64EmitUnwindInfo(Streamer, Info); 2550 } 2551 2552 // Now emit RUNTIME_FUNCTION entries. 2553 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2554 WinEH::FrameInfo *Info = CFI.get(); 2555 // ARM64EmitUnwindInfo above clears the info struct, so we can't check 2556 // empty here. But if a Symbol is set, we should create the corresponding 2557 // pdata entry. 2558 if (!Info->Symbol) 2559 continue; 2560 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 2561 Streamer.switchSection(PData); 2562 ARM64EmitRuntimeFunction(Streamer, Info); 2563 } 2564 } 2565 2566 void llvm::Win64EH::ARM64UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 2567 WinEH::FrameInfo *info, 2568 bool HandlerData) const { 2569 // Called if there's an .seh_handlerdata directive before the end of the 2570 // function. This forces writing the xdata record already here - and 2571 // in this case, the function isn't actually ended already, but the xdata 2572 // record needs to know the function length. In these cases, if the funclet 2573 // end hasn't been marked yet, the xdata function length won't cover the 2574 // whole function, only up to this point. 2575 if (!info->FuncletOrFuncEnd) { 2576 Streamer.switchSection(info->TextSection); 2577 info->FuncletOrFuncEnd = Streamer.emitCFILabel(); 2578 } 2579 // Switch sections (the static function above is meant to be called from 2580 // here and from Emit(). 2581 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 2582 Streamer.switchSection(XData); 2583 ARM64EmitUnwindInfo(Streamer, info, /* TryPacked = */ !HandlerData); 2584 } 2585 2586 void llvm::Win64EH::ARMUnwindEmitter::Emit(MCStreamer &Streamer) const { 2587 // Emit the unwind info structs first. 2588 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2589 WinEH::FrameInfo *Info = CFI.get(); 2590 if (Info->empty()) 2591 continue; 2592 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 2593 Streamer.switchSection(XData); 2594 ARMEmitUnwindInfo(Streamer, Info); 2595 } 2596 2597 // Now emit RUNTIME_FUNCTION entries. 2598 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2599 WinEH::FrameInfo *Info = CFI.get(); 2600 // ARMEmitUnwindInfo above clears the info struct, so we can't check 2601 // empty here. But if a Symbol is set, we should create the corresponding 2602 // pdata entry. 2603 if (!Info->Symbol) 2604 continue; 2605 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 2606 Streamer.switchSection(PData); 2607 ARMEmitRuntimeFunction(Streamer, Info); 2608 } 2609 } 2610 2611 void llvm::Win64EH::ARMUnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 2612 WinEH::FrameInfo *info, 2613 bool HandlerData) const { 2614 // Called if there's an .seh_handlerdata directive before the end of the 2615 // function. This forces writing the xdata record already here - and 2616 // in this case, the function isn't actually ended already, but the xdata 2617 // record needs to know the function length. In these cases, if the funclet 2618 // end hasn't been marked yet, the xdata function length won't cover the 2619 // whole function, only up to this point. 2620 if (!info->FuncletOrFuncEnd) { 2621 Streamer.switchSection(info->TextSection); 2622 info->FuncletOrFuncEnd = Streamer.emitCFILabel(); 2623 } 2624 // Switch sections (the static function above is meant to be called from 2625 // here and from Emit(). 2626 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 2627 Streamer.switchSection(XData); 2628 ARMEmitUnwindInfo(Streamer, info, /* TryPacked = */ !HandlerData); 2629 } 2630