1 //===- lib/MC/MCWin64EH.cpp - MCWin64EH implementation --------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/MC/MCWin64EH.h" 10 #include "llvm/ADT/Twine.h" 11 #include "llvm/MC/MCContext.h" 12 #include "llvm/MC/MCExpr.h" 13 #include "llvm/MC/MCObjectStreamer.h" 14 #include "llvm/MC/MCStreamer.h" 15 #include "llvm/MC/MCSymbol.h" 16 #include "llvm/Support/Win64EH.h" 17 namespace llvm { 18 class MCSection; 19 } 20 21 using namespace llvm; 22 23 // NOTE: All relocations generated here are 4-byte image-relative. 24 25 static uint8_t CountOfUnwindCodes(std::vector<WinEH::Instruction> &Insns) { 26 uint8_t Count = 0; 27 for (const auto &I : Insns) { 28 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 29 default: 30 llvm_unreachable("Unsupported unwind code"); 31 case Win64EH::UOP_PushNonVol: 32 case Win64EH::UOP_AllocSmall: 33 case Win64EH::UOP_SetFPReg: 34 case Win64EH::UOP_PushMachFrame: 35 Count += 1; 36 break; 37 case Win64EH::UOP_SaveNonVol: 38 case Win64EH::UOP_SaveXMM128: 39 Count += 2; 40 break; 41 case Win64EH::UOP_SaveNonVolBig: 42 case Win64EH::UOP_SaveXMM128Big: 43 Count += 3; 44 break; 45 case Win64EH::UOP_AllocLarge: 46 Count += (I.Offset > 512 * 1024 - 8) ? 3 : 2; 47 break; 48 } 49 } 50 return Count; 51 } 52 53 static void EmitAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS, 54 const MCSymbol *RHS) { 55 MCContext &Context = Streamer.getContext(); 56 const MCExpr *Diff = 57 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 58 MCSymbolRefExpr::create(RHS, Context), Context); 59 Streamer.emitValue(Diff, 1); 60 } 61 62 static void EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin, 63 WinEH::Instruction &inst) { 64 uint8_t b2; 65 uint16_t w; 66 b2 = (inst.Operation & 0x0F); 67 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 68 default: 69 llvm_unreachable("Unsupported unwind code"); 70 case Win64EH::UOP_PushNonVol: 71 EmitAbsDifference(streamer, inst.Label, begin); 72 b2 |= (inst.Register & 0x0F) << 4; 73 streamer.emitInt8(b2); 74 break; 75 case Win64EH::UOP_AllocLarge: 76 EmitAbsDifference(streamer, inst.Label, begin); 77 if (inst.Offset > 512 * 1024 - 8) { 78 b2 |= 0x10; 79 streamer.emitInt8(b2); 80 w = inst.Offset & 0xFFF8; 81 streamer.emitInt16(w); 82 w = inst.Offset >> 16; 83 } else { 84 streamer.emitInt8(b2); 85 w = inst.Offset >> 3; 86 } 87 streamer.emitInt16(w); 88 break; 89 case Win64EH::UOP_AllocSmall: 90 b2 |= (((inst.Offset - 8) >> 3) & 0x0F) << 4; 91 EmitAbsDifference(streamer, inst.Label, begin); 92 streamer.emitInt8(b2); 93 break; 94 case Win64EH::UOP_SetFPReg: 95 EmitAbsDifference(streamer, inst.Label, begin); 96 streamer.emitInt8(b2); 97 break; 98 case Win64EH::UOP_SaveNonVol: 99 case Win64EH::UOP_SaveXMM128: 100 b2 |= (inst.Register & 0x0F) << 4; 101 EmitAbsDifference(streamer, inst.Label, begin); 102 streamer.emitInt8(b2); 103 w = inst.Offset >> 3; 104 if (inst.Operation == Win64EH::UOP_SaveXMM128) 105 w >>= 1; 106 streamer.emitInt16(w); 107 break; 108 case Win64EH::UOP_SaveNonVolBig: 109 case Win64EH::UOP_SaveXMM128Big: 110 b2 |= (inst.Register & 0x0F) << 4; 111 EmitAbsDifference(streamer, inst.Label, begin); 112 streamer.emitInt8(b2); 113 if (inst.Operation == Win64EH::UOP_SaveXMM128Big) 114 w = inst.Offset & 0xFFF0; 115 else 116 w = inst.Offset & 0xFFF8; 117 streamer.emitInt16(w); 118 w = inst.Offset >> 16; 119 streamer.emitInt16(w); 120 break; 121 case Win64EH::UOP_PushMachFrame: 122 if (inst.Offset == 1) 123 b2 |= 0x10; 124 EmitAbsDifference(streamer, inst.Label, begin); 125 streamer.emitInt8(b2); 126 break; 127 } 128 } 129 130 static void EmitSymbolRefWithOfs(MCStreamer &streamer, 131 const MCSymbol *Base, 132 int64_t Offset) { 133 MCContext &Context = streamer.getContext(); 134 const MCConstantExpr *OffExpr = MCConstantExpr::create(Offset, Context); 135 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Base, 136 MCSymbolRefExpr::VK_COFF_IMGREL32, 137 Context); 138 streamer.emitValue(MCBinaryExpr::createAdd(BaseRefRel, OffExpr, Context), 4); 139 } 140 141 static void EmitSymbolRefWithOfs(MCStreamer &streamer, 142 const MCSymbol *Base, 143 const MCSymbol *Other) { 144 MCContext &Context = streamer.getContext(); 145 const MCSymbolRefExpr *BaseRef = MCSymbolRefExpr::create(Base, Context); 146 const MCSymbolRefExpr *OtherRef = MCSymbolRefExpr::create(Other, Context); 147 const MCExpr *Ofs = MCBinaryExpr::createSub(OtherRef, BaseRef, Context); 148 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Base, 149 MCSymbolRefExpr::VK_COFF_IMGREL32, 150 Context); 151 streamer.emitValue(MCBinaryExpr::createAdd(BaseRefRel, Ofs, Context), 4); 152 } 153 154 static void EmitRuntimeFunction(MCStreamer &streamer, 155 const WinEH::FrameInfo *info) { 156 MCContext &context = streamer.getContext(); 157 158 streamer.emitValueToAlignment(Align(4)); 159 EmitSymbolRefWithOfs(streamer, info->Begin, info->Begin); 160 EmitSymbolRefWithOfs(streamer, info->Begin, info->End); 161 streamer.emitValue(MCSymbolRefExpr::create(info->Symbol, 162 MCSymbolRefExpr::VK_COFF_IMGREL32, 163 context), 4); 164 } 165 166 static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) { 167 // If this UNWIND_INFO already has a symbol, it's already been emitted. 168 if (info->Symbol) 169 return; 170 171 MCContext &context = streamer.getContext(); 172 MCSymbol *Label = context.createTempSymbol(); 173 174 streamer.emitValueToAlignment(Align(4)); 175 streamer.emitLabel(Label); 176 info->Symbol = Label; 177 178 // Upper 3 bits are the version number (currently 1). 179 uint8_t flags = 0x01; 180 if (info->ChainedParent) 181 flags |= Win64EH::UNW_ChainInfo << 3; 182 else { 183 if (info->HandlesUnwind) 184 flags |= Win64EH::UNW_TerminateHandler << 3; 185 if (info->HandlesExceptions) 186 flags |= Win64EH::UNW_ExceptionHandler << 3; 187 } 188 streamer.emitInt8(flags); 189 190 if (info->PrologEnd) 191 EmitAbsDifference(streamer, info->PrologEnd, info->Begin); 192 else 193 streamer.emitInt8(0); 194 195 uint8_t numCodes = CountOfUnwindCodes(info->Instructions); 196 streamer.emitInt8(numCodes); 197 198 uint8_t frame = 0; 199 if (info->LastFrameInst >= 0) { 200 WinEH::Instruction &frameInst = info->Instructions[info->LastFrameInst]; 201 assert(frameInst.Operation == Win64EH::UOP_SetFPReg); 202 frame = (frameInst.Register & 0x0F) | (frameInst.Offset & 0xF0); 203 } 204 streamer.emitInt8(frame); 205 206 // Emit unwind instructions (in reverse order). 207 uint8_t numInst = info->Instructions.size(); 208 for (uint8_t c = 0; c < numInst; ++c) { 209 WinEH::Instruction inst = info->Instructions.back(); 210 info->Instructions.pop_back(); 211 EmitUnwindCode(streamer, info->Begin, inst); 212 } 213 214 // For alignment purposes, the instruction array will always have an even 215 // number of entries, with the final entry potentially unused (in which case 216 // the array will be one longer than indicated by the count of unwind codes 217 // field). 218 if (numCodes & 1) { 219 streamer.emitInt16(0); 220 } 221 222 if (flags & (Win64EH::UNW_ChainInfo << 3)) 223 EmitRuntimeFunction(streamer, info->ChainedParent); 224 else if (flags & 225 ((Win64EH::UNW_TerminateHandler|Win64EH::UNW_ExceptionHandler) << 3)) 226 streamer.emitValue(MCSymbolRefExpr::create(info->ExceptionHandler, 227 MCSymbolRefExpr::VK_COFF_IMGREL32, 228 context), 4); 229 else if (numCodes == 0) { 230 // The minimum size of an UNWIND_INFO struct is 8 bytes. If we're not 231 // a chained unwind info, if there is no handler, and if there are fewer 232 // than 2 slots used in the unwind code array, we have to pad to 8 bytes. 233 streamer.emitInt32(0); 234 } 235 } 236 237 void llvm::Win64EH::UnwindEmitter::Emit(MCStreamer &Streamer) const { 238 // Emit the unwind info structs first. 239 for (const auto &CFI : Streamer.getWinFrameInfos()) { 240 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 241 Streamer.switchSection(XData); 242 ::EmitUnwindInfo(Streamer, CFI.get()); 243 } 244 245 // Now emit RUNTIME_FUNCTION entries. 246 for (const auto &CFI : Streamer.getWinFrameInfos()) { 247 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 248 Streamer.switchSection(PData); 249 EmitRuntimeFunction(Streamer, CFI.get()); 250 } 251 } 252 253 void llvm::Win64EH::UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 254 WinEH::FrameInfo *info, 255 bool HandlerData) const { 256 // Switch sections (the static function above is meant to be called from 257 // here and from Emit(). 258 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 259 Streamer.switchSection(XData); 260 261 ::EmitUnwindInfo(Streamer, info); 262 } 263 264 static const MCExpr *GetSubDivExpr(MCStreamer &Streamer, const MCSymbol *LHS, 265 const MCSymbol *RHS, int Div) { 266 MCContext &Context = Streamer.getContext(); 267 const MCExpr *Expr = 268 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 269 MCSymbolRefExpr::create(RHS, Context), Context); 270 if (Div != 1) 271 Expr = MCBinaryExpr::createDiv(Expr, MCConstantExpr::create(Div, Context), 272 Context); 273 return Expr; 274 } 275 276 static std::optional<int64_t> GetOptionalAbsDifference(MCStreamer &Streamer, 277 const MCSymbol *LHS, 278 const MCSymbol *RHS) { 279 MCContext &Context = Streamer.getContext(); 280 const MCExpr *Diff = 281 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 282 MCSymbolRefExpr::create(RHS, Context), Context); 283 MCObjectStreamer *OS = (MCObjectStreamer *)(&Streamer); 284 // It should normally be possible to calculate the length of a function 285 // at this point, but it might not be possible in the presence of certain 286 // unusual constructs, like an inline asm with an alignment directive. 287 int64_t value; 288 if (!Diff->evaluateAsAbsolute(value, OS->getAssembler())) 289 return std::nullopt; 290 return value; 291 } 292 293 static int64_t GetAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS, 294 const MCSymbol *RHS) { 295 std::optional<int64_t> MaybeDiff = 296 GetOptionalAbsDifference(Streamer, LHS, RHS); 297 if (!MaybeDiff) 298 report_fatal_error("Failed to evaluate function length in SEH unwind info"); 299 return *MaybeDiff; 300 } 301 302 static void checkARM64Instructions(MCStreamer &Streamer, 303 ArrayRef<WinEH::Instruction> Insns, 304 const MCSymbol *Begin, const MCSymbol *End, 305 StringRef Name, StringRef Type) { 306 if (!End) 307 return; 308 std::optional<int64_t> MaybeDistance = 309 GetOptionalAbsDifference(Streamer, End, Begin); 310 if (!MaybeDistance) 311 return; 312 uint32_t Distance = (uint32_t)*MaybeDistance; 313 314 for (const auto &I : Insns) { 315 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 316 default: 317 break; 318 case Win64EH::UOP_TrapFrame: 319 case Win64EH::UOP_PushMachFrame: 320 case Win64EH::UOP_Context: 321 case Win64EH::UOP_ClearUnwoundToCall: 322 // Can't reason about these opcodes and how they map to actual 323 // instructions. 324 return; 325 } 326 } 327 // Exclude the end opcode which doesn't map to an instruction. 328 uint32_t InstructionBytes = 4 * (Insns.size() - 1); 329 if (Distance != InstructionBytes) { 330 Streamer.getContext().reportError( 331 SMLoc(), "Incorrect size for " + Name + " " + Type + ": " + 332 Twine(Distance) + 333 " bytes of instructions in range, but .seh directives " 334 "corresponding to " + 335 Twine(InstructionBytes) + " bytes\n"); 336 } 337 } 338 339 static uint32_t ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) { 340 uint32_t Count = 0; 341 for (const auto &I : Insns) { 342 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 343 default: 344 llvm_unreachable("Unsupported ARM64 unwind code"); 345 case Win64EH::UOP_AllocSmall: 346 Count += 1; 347 break; 348 case Win64EH::UOP_AllocMedium: 349 Count += 2; 350 break; 351 case Win64EH::UOP_AllocLarge: 352 Count += 4; 353 break; 354 case Win64EH::UOP_SaveR19R20X: 355 Count += 1; 356 break; 357 case Win64EH::UOP_SaveFPLRX: 358 Count += 1; 359 break; 360 case Win64EH::UOP_SaveFPLR: 361 Count += 1; 362 break; 363 case Win64EH::UOP_SaveReg: 364 Count += 2; 365 break; 366 case Win64EH::UOP_SaveRegP: 367 Count += 2; 368 break; 369 case Win64EH::UOP_SaveRegPX: 370 Count += 2; 371 break; 372 case Win64EH::UOP_SaveRegX: 373 Count += 2; 374 break; 375 case Win64EH::UOP_SaveLRPair: 376 Count += 2; 377 break; 378 case Win64EH::UOP_SaveFReg: 379 Count += 2; 380 break; 381 case Win64EH::UOP_SaveFRegP: 382 Count += 2; 383 break; 384 case Win64EH::UOP_SaveFRegX: 385 Count += 2; 386 break; 387 case Win64EH::UOP_SaveFRegPX: 388 Count += 2; 389 break; 390 case Win64EH::UOP_SetFP: 391 Count += 1; 392 break; 393 case Win64EH::UOP_AddFP: 394 Count += 2; 395 break; 396 case Win64EH::UOP_Nop: 397 Count += 1; 398 break; 399 case Win64EH::UOP_End: 400 Count += 1; 401 break; 402 case Win64EH::UOP_SaveNext: 403 Count += 1; 404 break; 405 case Win64EH::UOP_TrapFrame: 406 Count += 1; 407 break; 408 case Win64EH::UOP_PushMachFrame: 409 Count += 1; 410 break; 411 case Win64EH::UOP_Context: 412 Count += 1; 413 break; 414 case Win64EH::UOP_ClearUnwoundToCall: 415 Count += 1; 416 break; 417 case Win64EH::UOP_PACSignLR: 418 Count += 1; 419 break; 420 case Win64EH::UOP_SaveAnyRegI: 421 case Win64EH::UOP_SaveAnyRegIP: 422 case Win64EH::UOP_SaveAnyRegD: 423 case Win64EH::UOP_SaveAnyRegDP: 424 case Win64EH::UOP_SaveAnyRegQ: 425 case Win64EH::UOP_SaveAnyRegQP: 426 case Win64EH::UOP_SaveAnyRegIX: 427 case Win64EH::UOP_SaveAnyRegIPX: 428 case Win64EH::UOP_SaveAnyRegDX: 429 case Win64EH::UOP_SaveAnyRegDPX: 430 case Win64EH::UOP_SaveAnyRegQX: 431 case Win64EH::UOP_SaveAnyRegQPX: 432 Count += 3; 433 break; 434 } 435 } 436 return Count; 437 } 438 439 // Unwind opcode encodings and restrictions are documented at 440 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling 441 static void ARM64EmitUnwindCode(MCStreamer &streamer, 442 const WinEH::Instruction &inst) { 443 uint8_t b, reg; 444 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 445 default: 446 llvm_unreachable("Unsupported ARM64 unwind code"); 447 case Win64EH::UOP_AllocSmall: 448 b = (inst.Offset >> 4) & 0x1F; 449 streamer.emitInt8(b); 450 break; 451 case Win64EH::UOP_AllocMedium: { 452 uint16_t hw = (inst.Offset >> 4) & 0x7FF; 453 b = 0xC0; 454 b |= (hw >> 8); 455 streamer.emitInt8(b); 456 b = hw & 0xFF; 457 streamer.emitInt8(b); 458 break; 459 } 460 case Win64EH::UOP_AllocLarge: { 461 uint32_t w; 462 b = 0xE0; 463 streamer.emitInt8(b); 464 w = inst.Offset >> 4; 465 b = (w & 0x00FF0000) >> 16; 466 streamer.emitInt8(b); 467 b = (w & 0x0000FF00) >> 8; 468 streamer.emitInt8(b); 469 b = w & 0x000000FF; 470 streamer.emitInt8(b); 471 break; 472 } 473 case Win64EH::UOP_SetFP: 474 b = 0xE1; 475 streamer.emitInt8(b); 476 break; 477 case Win64EH::UOP_AddFP: 478 b = 0xE2; 479 streamer.emitInt8(b); 480 b = (inst.Offset >> 3); 481 streamer.emitInt8(b); 482 break; 483 case Win64EH::UOP_Nop: 484 b = 0xE3; 485 streamer.emitInt8(b); 486 break; 487 case Win64EH::UOP_SaveR19R20X: 488 b = 0x20; 489 b |= (inst.Offset >> 3) & 0x1F; 490 streamer.emitInt8(b); 491 break; 492 case Win64EH::UOP_SaveFPLRX: 493 b = 0x80; 494 b |= ((inst.Offset - 1) >> 3) & 0x3F; 495 streamer.emitInt8(b); 496 break; 497 case Win64EH::UOP_SaveFPLR: 498 b = 0x40; 499 b |= (inst.Offset >> 3) & 0x3F; 500 streamer.emitInt8(b); 501 break; 502 case Win64EH::UOP_SaveReg: 503 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 504 reg = inst.Register - 19; 505 b = 0xD0 | ((reg & 0xC) >> 2); 506 streamer.emitInt8(b); 507 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 508 streamer.emitInt8(b); 509 break; 510 case Win64EH::UOP_SaveRegX: 511 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 512 reg = inst.Register - 19; 513 b = 0xD4 | ((reg & 0x8) >> 3); 514 streamer.emitInt8(b); 515 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1); 516 streamer.emitInt8(b); 517 break; 518 case Win64EH::UOP_SaveRegP: 519 assert(inst.Register >= 19 && "Saved registers must be >= 19"); 520 reg = inst.Register - 19; 521 b = 0xC8 | ((reg & 0xC) >> 2); 522 streamer.emitInt8(b); 523 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 524 streamer.emitInt8(b); 525 break; 526 case Win64EH::UOP_SaveRegPX: 527 assert(inst.Register >= 19 && "Saved registers must be >= 19"); 528 reg = inst.Register - 19; 529 b = 0xCC | ((reg & 0xC) >> 2); 530 streamer.emitInt8(b); 531 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1); 532 streamer.emitInt8(b); 533 break; 534 case Win64EH::UOP_SaveLRPair: 535 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 536 reg = inst.Register - 19; 537 assert((reg % 2) == 0 && "Saved reg must be 19+2*X"); 538 reg /= 2; 539 b = 0xD6 | ((reg & 0x7) >> 2); 540 streamer.emitInt8(b); 541 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 542 streamer.emitInt8(b); 543 break; 544 case Win64EH::UOP_SaveFReg: 545 assert(inst.Register >= 8 && "Saved dreg must be >= 8"); 546 reg = inst.Register - 8; 547 b = 0xDC | ((reg & 0x4) >> 2); 548 streamer.emitInt8(b); 549 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 550 streamer.emitInt8(b); 551 break; 552 case Win64EH::UOP_SaveFRegX: 553 assert(inst.Register >= 8 && "Saved dreg must be >= 8"); 554 reg = inst.Register - 8; 555 b = 0xDE; 556 streamer.emitInt8(b); 557 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1); 558 streamer.emitInt8(b); 559 break; 560 case Win64EH::UOP_SaveFRegP: 561 assert(inst.Register >= 8 && "Saved dregs must be >= 8"); 562 reg = inst.Register - 8; 563 b = 0xD8 | ((reg & 0x4) >> 2); 564 streamer.emitInt8(b); 565 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 566 streamer.emitInt8(b); 567 break; 568 case Win64EH::UOP_SaveFRegPX: 569 assert(inst.Register >= 8 && "Saved dregs must be >= 8"); 570 reg = inst.Register - 8; 571 b = 0xDA | ((reg & 0x4) >> 2); 572 streamer.emitInt8(b); 573 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1); 574 streamer.emitInt8(b); 575 break; 576 case Win64EH::UOP_End: 577 b = 0xE4; 578 streamer.emitInt8(b); 579 break; 580 case Win64EH::UOP_SaveNext: 581 b = 0xE6; 582 streamer.emitInt8(b); 583 break; 584 case Win64EH::UOP_TrapFrame: 585 b = 0xE8; 586 streamer.emitInt8(b); 587 break; 588 case Win64EH::UOP_PushMachFrame: 589 b = 0xE9; 590 streamer.emitInt8(b); 591 break; 592 case Win64EH::UOP_Context: 593 b = 0xEA; 594 streamer.emitInt8(b); 595 break; 596 case Win64EH::UOP_ClearUnwoundToCall: 597 b = 0xEC; 598 streamer.emitInt8(b); 599 break; 600 case Win64EH::UOP_PACSignLR: 601 b = 0xFC; 602 streamer.emitInt8(b); 603 break; 604 case Win64EH::UOP_SaveAnyRegI: 605 case Win64EH::UOP_SaveAnyRegIP: 606 case Win64EH::UOP_SaveAnyRegD: 607 case Win64EH::UOP_SaveAnyRegDP: 608 case Win64EH::UOP_SaveAnyRegQ: 609 case Win64EH::UOP_SaveAnyRegQP: 610 case Win64EH::UOP_SaveAnyRegIX: 611 case Win64EH::UOP_SaveAnyRegIPX: 612 case Win64EH::UOP_SaveAnyRegDX: 613 case Win64EH::UOP_SaveAnyRegDPX: 614 case Win64EH::UOP_SaveAnyRegQX: 615 case Win64EH::UOP_SaveAnyRegQPX: { 616 // This assumes the opcodes are listed in the enum in a particular order. 617 int Op = inst.Operation - Win64EH::UOP_SaveAnyRegI; 618 int Writeback = Op / 6; 619 int Paired = Op % 2; 620 int Mode = (Op / 2) % 3; 621 int Offset = inst.Offset >> 3; 622 if (Writeback || Paired || Mode == 2) 623 Offset >>= 1; 624 if (Writeback) 625 --Offset; 626 b = 0xE7; 627 streamer.emitInt8(b); 628 assert(inst.Register < 32); 629 b = inst.Register | (Writeback << 5) | (Paired << 6); 630 streamer.emitInt8(b); 631 b = Offset | (Mode << 6); 632 streamer.emitInt8(b); 633 break; 634 } 635 } 636 } 637 638 // Returns the epilog symbol of an epilog with the exact same unwind code 639 // sequence, if it exists. Otherwise, returns nullptr. 640 // EpilogInstrs - Unwind codes for the current epilog. 641 // Epilogs - Epilogs that potentialy match the current epilog. 642 static MCSymbol* 643 FindMatchingEpilog(const std::vector<WinEH::Instruction>& EpilogInstrs, 644 const std::vector<MCSymbol *>& Epilogs, 645 const WinEH::FrameInfo *info) { 646 for (auto *EpilogStart : Epilogs) { 647 auto InstrsIter = info->EpilogMap.find(EpilogStart); 648 assert(InstrsIter != info->EpilogMap.end() && 649 "Epilog not found in EpilogMap"); 650 const auto &Instrs = InstrsIter->second.Instructions; 651 652 if (Instrs.size() != EpilogInstrs.size()) 653 continue; 654 655 bool Match = true; 656 for (unsigned i = 0; i < Instrs.size(); ++i) 657 if (Instrs[i] != EpilogInstrs[i]) { 658 Match = false; 659 break; 660 } 661 662 if (Match) 663 return EpilogStart; 664 } 665 return nullptr; 666 } 667 668 static void simplifyARM64Opcodes(std::vector<WinEH::Instruction> &Instructions, 669 bool Reverse) { 670 unsigned PrevOffset = -1; 671 unsigned PrevRegister = -1; 672 673 auto VisitInstruction = [&](WinEH::Instruction &Inst) { 674 // Convert 2-byte opcodes into equivalent 1-byte ones. 675 if (Inst.Operation == Win64EH::UOP_SaveRegP && Inst.Register == 29) { 676 Inst.Operation = Win64EH::UOP_SaveFPLR; 677 Inst.Register = -1; 678 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX && 679 Inst.Register == 29) { 680 Inst.Operation = Win64EH::UOP_SaveFPLRX; 681 Inst.Register = -1; 682 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX && 683 Inst.Register == 19 && Inst.Offset <= 248) { 684 Inst.Operation = Win64EH::UOP_SaveR19R20X; 685 Inst.Register = -1; 686 } else if (Inst.Operation == Win64EH::UOP_AddFP && Inst.Offset == 0) { 687 Inst.Operation = Win64EH::UOP_SetFP; 688 } else if (Inst.Operation == Win64EH::UOP_SaveRegP && 689 Inst.Register == PrevRegister + 2 && 690 Inst.Offset == PrevOffset + 16) { 691 Inst.Operation = Win64EH::UOP_SaveNext; 692 Inst.Register = -1; 693 Inst.Offset = 0; 694 // Intentionally not creating UOP_SaveNext for float register pairs, 695 // as current versions of Windows (up to at least 20.04) is buggy 696 // regarding SaveNext for float pairs. 697 } 698 // Update info about the previous instruction, for detecting if 699 // the next one can be made a UOP_SaveNext 700 if (Inst.Operation == Win64EH::UOP_SaveR19R20X) { 701 PrevOffset = 0; 702 PrevRegister = 19; 703 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX) { 704 PrevOffset = 0; 705 PrevRegister = Inst.Register; 706 } else if (Inst.Operation == Win64EH::UOP_SaveRegP) { 707 PrevOffset = Inst.Offset; 708 PrevRegister = Inst.Register; 709 } else if (Inst.Operation == Win64EH::UOP_SaveNext) { 710 PrevRegister += 2; 711 PrevOffset += 16; 712 } else { 713 PrevRegister = -1; 714 PrevOffset = -1; 715 } 716 }; 717 718 // Iterate over instructions in a forward order (for prologues), 719 // backwards for epilogues (i.e. always reverse compared to how the 720 // opcodes are stored). 721 if (Reverse) { 722 for (auto It = Instructions.rbegin(); It != Instructions.rend(); It++) 723 VisitInstruction(*It); 724 } else { 725 for (WinEH::Instruction &Inst : Instructions) 726 VisitInstruction(Inst); 727 } 728 } 729 730 // Check if an epilog exists as a subset of the end of a prolog (backwards). 731 static int 732 getARM64OffsetInProlog(const std::vector<WinEH::Instruction> &Prolog, 733 const std::vector<WinEH::Instruction> &Epilog) { 734 // Can't find an epilog as a subset if it is longer than the prolog. 735 if (Epilog.size() > Prolog.size()) 736 return -1; 737 738 // Check that the epilog actually is a perfect match for the end (backwrds) 739 // of the prolog. 740 for (int I = Epilog.size() - 1; I >= 0; I--) { 741 if (Prolog[I] != Epilog[Epilog.size() - 1 - I]) 742 return -1; 743 } 744 745 if (Epilog.size() == Prolog.size()) 746 return 0; 747 748 // If the epilog was a subset of the prolog, find its offset. 749 return ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction>( 750 &Prolog[Epilog.size()], Prolog.size() - Epilog.size())); 751 } 752 753 static int checkARM64PackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info, 754 WinEH::FrameInfo::Segment *Seg, 755 int PrologCodeBytes) { 756 // Can only pack if there's one single epilog 757 if (Seg->Epilogs.size() != 1) 758 return -1; 759 760 MCSymbol *Sym = Seg->Epilogs.begin()->first; 761 const std::vector<WinEH::Instruction> &Epilog = 762 info->EpilogMap[Sym].Instructions; 763 764 // Check that the epilog actually is at the very end of the function, 765 // otherwise it can't be packed. 766 uint32_t DistanceFromEnd = 767 (uint32_t)(Seg->Offset + Seg->Length - Seg->Epilogs.begin()->second); 768 if (DistanceFromEnd / 4 != Epilog.size()) 769 return -1; 770 771 int RetVal = -1; 772 // Even if we don't end up sharing opcodes with the prolog, we can still 773 // write the offset as a packed offset, if the single epilog is located at 774 // the end of the function and the offset (pointing after the prolog) fits 775 // as a packed offset. 776 if (PrologCodeBytes <= 31 && 777 PrologCodeBytes + ARM64CountOfUnwindCodes(Epilog) <= 124) 778 RetVal = PrologCodeBytes; 779 780 int Offset = getARM64OffsetInProlog(info->Instructions, Epilog); 781 if (Offset < 0) 782 return RetVal; 783 784 // Check that the offset and prolog size fits in the first word; it's 785 // unclear whether the epilog count in the extension word can be taken 786 // as packed epilog offset. 787 if (Offset > 31 || PrologCodeBytes > 124) 788 return RetVal; 789 790 // As we choose to express the epilog as part of the prolog, remove the 791 // epilog from the map, so we don't try to emit its opcodes. 792 info->EpilogMap.erase(Sym); 793 return Offset; 794 } 795 796 static bool tryARM64PackedUnwind(WinEH::FrameInfo *info, uint32_t FuncLength, 797 int PackedEpilogOffset) { 798 if (PackedEpilogOffset == 0) { 799 // Fully symmetric prolog and epilog, should be ok for packed format. 800 // For CR=3, the corresponding synthesized epilog actually lacks the 801 // SetFP opcode, but unwinding should work just fine despite that 802 // (if at the SetFP opcode, the unwinder considers it as part of the 803 // function body and just unwinds the full prolog instead). 804 } else if (PackedEpilogOffset == 1) { 805 // One single case of differences between prolog and epilog is allowed: 806 // The epilog can lack a single SetFP that is the last opcode in the 807 // prolog, for the CR=3 case. 808 if (info->Instructions.back().Operation != Win64EH::UOP_SetFP) 809 return false; 810 } else { 811 // Too much difference between prolog and epilog. 812 return false; 813 } 814 unsigned RegI = 0, RegF = 0; 815 int Predecrement = 0; 816 enum { 817 Start, 818 Start2, 819 Start3, 820 IntRegs, 821 FloatRegs, 822 InputArgs, 823 StackAdjust, 824 FrameRecord, 825 End 826 } Location = Start; 827 bool StandaloneLR = false, FPLRPair = false; 828 bool PAC = false; 829 int StackOffset = 0; 830 int Nops = 0; 831 // Iterate over the prolog and check that all opcodes exactly match 832 // the canonical order and form. A more lax check could verify that 833 // all saved registers are in the expected locations, but not enforce 834 // the order - that would work fine when unwinding from within 835 // functions, but not be exactly right if unwinding happens within 836 // prologs/epilogs. 837 for (const WinEH::Instruction &Inst : info->Instructions) { 838 switch (Inst.Operation) { 839 case Win64EH::UOP_End: 840 if (Location != Start) 841 return false; 842 Location = Start2; 843 break; 844 case Win64EH::UOP_PACSignLR: 845 if (Location != Start2) 846 return false; 847 PAC = true; 848 Location = Start3; 849 break; 850 case Win64EH::UOP_SaveR19R20X: 851 if (Location != Start2 && Location != Start3) 852 return false; 853 Predecrement = Inst.Offset; 854 RegI = 2; 855 Location = IntRegs; 856 break; 857 case Win64EH::UOP_SaveRegX: 858 if (Location != Start2 && Location != Start3) 859 return false; 860 Predecrement = Inst.Offset; 861 if (Inst.Register == 19) 862 RegI += 1; 863 else if (Inst.Register == 30) 864 StandaloneLR = true; 865 else 866 return false; 867 // Odd register; can't be any further int registers. 868 Location = FloatRegs; 869 break; 870 case Win64EH::UOP_SaveRegPX: 871 // Can't have this in a canonical prologue. Either this has been 872 // canonicalized into SaveR19R20X or SaveFPLRX, or it's an unsupported 873 // register pair. 874 // It can't be canonicalized into SaveR19R20X if the offset is 875 // larger than 248 bytes, but even with the maximum case with 876 // RegI=10/RegF=8/CR=1/H=1, we end up with SavSZ = 216, which should 877 // fit into SaveR19R20X. 878 // The unwinding opcodes can't describe the otherwise seemingly valid 879 // case for RegI=1 CR=1, that would start with a 880 // "stp x19, lr, [sp, #-...]!" as that fits neither SaveRegPX nor 881 // SaveLRPair. 882 return false; 883 case Win64EH::UOP_SaveRegP: 884 if (Location != IntRegs || Inst.Offset != 8 * RegI || 885 Inst.Register != 19 + RegI) 886 return false; 887 RegI += 2; 888 break; 889 case Win64EH::UOP_SaveReg: 890 if (Location != IntRegs || Inst.Offset != 8 * RegI) 891 return false; 892 if (Inst.Register == 19 + RegI) 893 RegI += 1; 894 else if (Inst.Register == 30) 895 StandaloneLR = true; 896 else 897 return false; 898 // Odd register; can't be any further int registers. 899 Location = FloatRegs; 900 break; 901 case Win64EH::UOP_SaveLRPair: 902 if (Location != IntRegs || Inst.Offset != 8 * RegI || 903 Inst.Register != 19 + RegI) 904 return false; 905 RegI += 1; 906 StandaloneLR = true; 907 Location = FloatRegs; 908 break; 909 case Win64EH::UOP_SaveFRegX: 910 // Packed unwind can't handle prologs that only save one single 911 // float register. 912 return false; 913 case Win64EH::UOP_SaveFReg: 914 if (Location != FloatRegs || RegF == 0 || Inst.Register != 8 + RegF || 915 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF)) 916 return false; 917 RegF += 1; 918 Location = InputArgs; 919 break; 920 case Win64EH::UOP_SaveFRegPX: 921 if ((Location != Start2 && Location != Start3) || Inst.Register != 8) 922 return false; 923 Predecrement = Inst.Offset; 924 RegF = 2; 925 Location = FloatRegs; 926 break; 927 case Win64EH::UOP_SaveFRegP: 928 if ((Location != IntRegs && Location != FloatRegs) || 929 Inst.Register != 8 + RegF || 930 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF)) 931 return false; 932 RegF += 2; 933 Location = FloatRegs; 934 break; 935 case Win64EH::UOP_SaveNext: 936 if (Location == IntRegs) 937 RegI += 2; 938 else if (Location == FloatRegs) 939 RegF += 2; 940 else 941 return false; 942 break; 943 case Win64EH::UOP_Nop: 944 if (Location != IntRegs && Location != FloatRegs && Location != InputArgs) 945 return false; 946 Location = InputArgs; 947 Nops++; 948 break; 949 case Win64EH::UOP_AllocSmall: 950 case Win64EH::UOP_AllocMedium: 951 if (Location != Start2 && Location != Start3 && Location != IntRegs && 952 Location != FloatRegs && Location != InputArgs && 953 Location != StackAdjust) 954 return false; 955 // Can have either a single decrement, or a pair of decrements with 956 // 4080 and another decrement. 957 if (StackOffset == 0) 958 StackOffset = Inst.Offset; 959 else if (StackOffset != 4080) 960 return false; 961 else 962 StackOffset += Inst.Offset; 963 Location = StackAdjust; 964 break; 965 case Win64EH::UOP_SaveFPLRX: 966 // Not allowing FPLRX after StackAdjust; if a StackAdjust is used, it 967 // should be followed by a FPLR instead. 968 if (Location != Start2 && Location != Start3 && Location != IntRegs && 969 Location != FloatRegs && Location != InputArgs) 970 return false; 971 StackOffset = Inst.Offset; 972 Location = FrameRecord; 973 FPLRPair = true; 974 break; 975 case Win64EH::UOP_SaveFPLR: 976 // This can only follow after a StackAdjust 977 if (Location != StackAdjust || Inst.Offset != 0) 978 return false; 979 Location = FrameRecord; 980 FPLRPair = true; 981 break; 982 case Win64EH::UOP_SetFP: 983 if (Location != FrameRecord) 984 return false; 985 Location = End; 986 break; 987 case Win64EH::UOP_SaveAnyRegI: 988 case Win64EH::UOP_SaveAnyRegIP: 989 case Win64EH::UOP_SaveAnyRegD: 990 case Win64EH::UOP_SaveAnyRegDP: 991 case Win64EH::UOP_SaveAnyRegQ: 992 case Win64EH::UOP_SaveAnyRegQP: 993 case Win64EH::UOP_SaveAnyRegIX: 994 case Win64EH::UOP_SaveAnyRegIPX: 995 case Win64EH::UOP_SaveAnyRegDX: 996 case Win64EH::UOP_SaveAnyRegDPX: 997 case Win64EH::UOP_SaveAnyRegQX: 998 case Win64EH::UOP_SaveAnyRegQPX: 999 // These are never canonical; they don't show up with the usual Arm64 1000 // calling convention. 1001 return false; 1002 case Win64EH::UOP_AllocLarge: 1003 // Allocations this large can't be represented in packed unwind (and 1004 // usually don't fit the canonical form anyway because we need to use 1005 // __chkstk to allocate the stack space). 1006 return false; 1007 case Win64EH::UOP_AddFP: 1008 // "add x29, sp, #N" doesn't show up in the canonical pattern (except for 1009 // N=0, which is UOP_SetFP). 1010 return false; 1011 case Win64EH::UOP_TrapFrame: 1012 case Win64EH::UOP_Context: 1013 case Win64EH::UOP_ClearUnwoundToCall: 1014 case Win64EH::UOP_PushMachFrame: 1015 // These are special opcodes that aren't normally generated. 1016 return false; 1017 default: 1018 report_fatal_error("Unknown Arm64 unwind opcode"); 1019 } 1020 } 1021 if (RegI > 10 || RegF > 8) 1022 return false; 1023 if (StandaloneLR && FPLRPair) 1024 return false; 1025 if (FPLRPair && Location != End) 1026 return false; 1027 if (Nops != 0 && Nops != 4) 1028 return false; 1029 if (PAC && !FPLRPair) 1030 return false; 1031 int H = Nops == 4; 1032 // There's an inconsistency regarding packed unwind info with homed 1033 // parameters; according to the documentation, the epilog shouldn't have 1034 // the same corresponding nops (and thus, to set the H bit, we should 1035 // require an epilog which isn't exactly symmetrical - we shouldn't accept 1036 // an exact mirrored epilog for those cases), but in practice, 1037 // RtlVirtualUnwind behaves as if it does expect the epilogue to contain 1038 // the same nops. See https://github.com/llvm/llvm-project/issues/54879. 1039 // To play it safe, don't produce packed unwind info with homed parameters. 1040 if (H) 1041 return false; 1042 int IntSZ = 8 * RegI; 1043 if (StandaloneLR) 1044 IntSZ += 8; 1045 int FpSZ = 8 * RegF; // RegF not yet decremented 1046 int SavSZ = (IntSZ + FpSZ + 8 * 8 * H + 0xF) & ~0xF; 1047 if (Predecrement != SavSZ) 1048 return false; 1049 if (FPLRPair && StackOffset < 16) 1050 return false; 1051 if (StackOffset % 16) 1052 return false; 1053 uint32_t FrameSize = (StackOffset + SavSZ) / 16; 1054 if (FrameSize > 0x1FF) 1055 return false; 1056 assert(RegF != 1 && "One single float reg not allowed"); 1057 if (RegF > 0) 1058 RegF--; // Convert from actual number of registers, to value stored 1059 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier"); 1060 int Flag = 0x01; // Function segments not supported yet 1061 int CR = PAC ? 2 : FPLRPair ? 3 : StandaloneLR ? 1 : 0; 1062 info->PackedInfo |= Flag << 0; 1063 info->PackedInfo |= (FuncLength & 0x7FF) << 2; 1064 info->PackedInfo |= (RegF & 0x7) << 13; 1065 info->PackedInfo |= (RegI & 0xF) << 16; 1066 info->PackedInfo |= (H & 0x1) << 20; 1067 info->PackedInfo |= (CR & 0x3) << 21; 1068 info->PackedInfo |= (FrameSize & 0x1FF) << 23; 1069 return true; 1070 } 1071 1072 static void ARM64ProcessEpilogs(WinEH::FrameInfo *info, 1073 WinEH::FrameInfo::Segment *Seg, 1074 uint32_t &TotalCodeBytes, 1075 MapVector<MCSymbol *, uint32_t> &EpilogInfo) { 1076 1077 std::vector<MCSymbol *> EpilogStarts; 1078 for (auto &I : Seg->Epilogs) 1079 EpilogStarts.push_back(I.first); 1080 1081 // Epilogs processed so far. 1082 std::vector<MCSymbol *> AddedEpilogs; 1083 for (auto *S : EpilogStarts) { 1084 MCSymbol *EpilogStart = S; 1085 auto &EpilogInstrs = info->EpilogMap[S].Instructions; 1086 uint32_t CodeBytes = ARM64CountOfUnwindCodes(EpilogInstrs); 1087 1088 MCSymbol* MatchingEpilog = 1089 FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info); 1090 int PrologOffset; 1091 if (MatchingEpilog) { 1092 assert(EpilogInfo.contains(MatchingEpilog) && 1093 "Duplicate epilog not found"); 1094 EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog); 1095 // Clear the unwind codes in the EpilogMap, so that they don't get output 1096 // in ARM64EmitUnwindInfoForSegment(). 1097 EpilogInstrs.clear(); 1098 } else if ((PrologOffset = getARM64OffsetInProlog(info->Instructions, 1099 EpilogInstrs)) >= 0) { 1100 EpilogInfo[EpilogStart] = PrologOffset; 1101 // If the segment doesn't have a prolog, an end_c will be emitted before 1102 // prolog opcodes. So epilog start index in opcodes array is moved by 1. 1103 if (!Seg->HasProlog) 1104 EpilogInfo[EpilogStart] += 1; 1105 // Clear the unwind codes in the EpilogMap, so that they don't get output 1106 // in ARM64EmitUnwindInfoForSegment(). 1107 EpilogInstrs.clear(); 1108 } else { 1109 EpilogInfo[EpilogStart] = TotalCodeBytes; 1110 TotalCodeBytes += CodeBytes; 1111 AddedEpilogs.push_back(EpilogStart); 1112 } 1113 } 1114 } 1115 1116 static void ARM64FindSegmentsInFunction(MCStreamer &streamer, 1117 WinEH::FrameInfo *info, 1118 int64_t RawFuncLength) { 1119 if (info->PrologEnd) 1120 checkARM64Instructions(streamer, info->Instructions, info->Begin, 1121 info->PrologEnd, info->Function->getName(), 1122 "prologue"); 1123 struct EpilogStartEnd { 1124 MCSymbol *Start; 1125 int64_t Offset; 1126 int64_t End; 1127 }; 1128 // Record Start and End of each epilog. 1129 SmallVector<struct EpilogStartEnd, 4> Epilogs; 1130 for (auto &I : info->EpilogMap) { 1131 MCSymbol *Start = I.first; 1132 auto &Instrs = I.second.Instructions; 1133 int64_t Offset = GetAbsDifference(streamer, Start, info->Begin); 1134 checkARM64Instructions(streamer, Instrs, Start, I.second.End, 1135 info->Function->getName(), "epilogue"); 1136 assert((Epilogs.size() == 0 || Offset >= Epilogs.back().End) && 1137 "Epilogs should be monotonically ordered"); 1138 // Exclue the end opcode from Instrs.size() when calculating the end of the 1139 // epilog. 1140 Epilogs.push_back({Start, Offset, Offset + (int64_t)(Instrs.size() - 1) * 4}); 1141 } 1142 1143 unsigned E = 0; 1144 int64_t SegLimit = 0xFFFFC; 1145 int64_t SegOffset = 0; 1146 1147 if (RawFuncLength > SegLimit) { 1148 1149 int64_t RemainingLength = RawFuncLength; 1150 1151 while (RemainingLength > SegLimit) { 1152 // Try divide the function into segments, requirements: 1153 // 1. Segment length <= 0xFFFFC; 1154 // 2. Each Prologue or Epilogue must be fully within a segment. 1155 int64_t SegLength = SegLimit; 1156 int64_t SegEnd = SegOffset + SegLength; 1157 // Keep record on symbols and offsets of epilogs in this segment. 1158 MapVector<MCSymbol *, int64_t> EpilogsInSegment; 1159 1160 while (E < Epilogs.size() && Epilogs[E].End < SegEnd) { 1161 // Epilogs within current segment. 1162 EpilogsInSegment[Epilogs[E].Start] = Epilogs[E].Offset; 1163 ++E; 1164 } 1165 1166 // At this point, we have: 1167 // 1. Put all epilogs in segments already. No action needed here; or 1168 // 2. Found an epilog that will cross segments boundry. We need to 1169 // move back current segment's end boundry, so the epilog is entirely 1170 // in the next segment; or 1171 // 3. Left at least one epilog that is entirely after this segment. 1172 // It'll be handled by the next iteration, or the last segment. 1173 if (E < Epilogs.size() && Epilogs[E].Offset <= SegEnd) 1174 // Move back current Segment's end boundry. 1175 SegLength = Epilogs[E].Offset - SegOffset; 1176 1177 auto Seg = WinEH::FrameInfo::Segment( 1178 SegOffset, SegLength, /* HasProlog */!SegOffset); 1179 Seg.Epilogs = std::move(EpilogsInSegment); 1180 info->Segments.push_back(Seg); 1181 1182 SegOffset += SegLength; 1183 RemainingLength -= SegLength; 1184 } 1185 } 1186 1187 // Add the last segment when RawFuncLength > 0xFFFFC, 1188 // or the only segment otherwise. 1189 auto LastSeg = 1190 WinEH::FrameInfo::Segment(SegOffset, RawFuncLength - SegOffset, 1191 /* HasProlog */!SegOffset); 1192 for (; E < Epilogs.size(); ++E) 1193 LastSeg.Epilogs[Epilogs[E].Start] = Epilogs[E].Offset; 1194 info->Segments.push_back(LastSeg); 1195 } 1196 1197 static void ARM64EmitUnwindInfoForSegment(MCStreamer &streamer, 1198 WinEH::FrameInfo *info, 1199 WinEH::FrameInfo::Segment &Seg, 1200 bool TryPacked = true) { 1201 MCContext &context = streamer.getContext(); 1202 MCSymbol *Label = context.createTempSymbol(); 1203 1204 streamer.emitValueToAlignment(Align(4)); 1205 streamer.emitLabel(Label); 1206 Seg.Symbol = Label; 1207 // Use the 1st segemnt's label as function's. 1208 if (Seg.Offset == 0) 1209 info->Symbol = Label; 1210 1211 bool HasProlog = Seg.HasProlog; 1212 bool HasEpilogs = (Seg.Epilogs.size() != 0); 1213 1214 uint32_t SegLength = (uint32_t)Seg.Length / 4; 1215 uint32_t PrologCodeBytes = info->PrologCodeBytes; 1216 1217 int PackedEpilogOffset = HasEpilogs ? 1218 checkARM64PackedEpilog(streamer, info, &Seg, PrologCodeBytes) : -1; 1219 1220 // TODO: 1221 // 1. Enable packed unwind info (.pdata only) for multi-segment functions. 1222 // 2. Emit packed unwind info (.pdata only) for segments that have neithor 1223 // prolog nor epilog. 1224 if (info->Segments.size() == 1 && PackedEpilogOffset >= 0 && 1225 uint32_t(PackedEpilogOffset) < PrologCodeBytes && 1226 !info->HandlesExceptions && SegLength <= 0x7ff && TryPacked) { 1227 // Matching prolog/epilog and no exception handlers; check if the 1228 // prolog matches the patterns that can be described by the packed 1229 // format. 1230 1231 // info->Symbol was already set even if we didn't actually write any 1232 // unwind info there. Keep using that as indicator that this unwind 1233 // info has been generated already. 1234 if (tryARM64PackedUnwind(info, SegLength, PackedEpilogOffset)) 1235 return; 1236 } 1237 1238 // If the prolog is not in this segment, we need to emit an end_c, which takes 1239 // 1 byte, before prolog unwind ops. 1240 if (!HasProlog) { 1241 PrologCodeBytes += 1; 1242 if (PackedEpilogOffset >= 0) 1243 PackedEpilogOffset += 1; 1244 // If a segment has neither prolog nor epilog, "With full .xdata record, 1245 // Epilog Count = 1. Epilog Start Index points to end_c." 1246 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling#function-fragments 1247 // TODO: We can remove this if testing shows zero epilog scope is ok with 1248 // MS unwinder. 1249 if (!HasEpilogs) 1250 // Pack the fake epilog into phantom prolog. 1251 PackedEpilogOffset = 0; 1252 } 1253 1254 uint32_t TotalCodeBytes = PrologCodeBytes; 1255 1256 // Process epilogs. 1257 MapVector<MCSymbol *, uint32_t> EpilogInfo; 1258 ARM64ProcessEpilogs(info, &Seg, TotalCodeBytes, EpilogInfo); 1259 1260 // Code Words, Epilog count, E, X, Vers, Function Length 1261 uint32_t row1 = 0x0; 1262 uint32_t CodeWords = TotalCodeBytes / 4; 1263 uint32_t CodeWordsMod = TotalCodeBytes % 4; 1264 if (CodeWordsMod) 1265 CodeWords++; 1266 uint32_t EpilogCount = 1267 PackedEpilogOffset >= 0 ? PackedEpilogOffset : Seg.Epilogs.size(); 1268 bool ExtensionWord = EpilogCount > 31 || TotalCodeBytes > 124; 1269 if (!ExtensionWord) { 1270 row1 |= (EpilogCount & 0x1F) << 22; 1271 row1 |= (CodeWords & 0x1F) << 27; 1272 } 1273 if (info->HandlesExceptions) // X 1274 row1 |= 1 << 20; 1275 if (PackedEpilogOffset >= 0) // E 1276 row1 |= 1 << 21; 1277 row1 |= SegLength & 0x3FFFF; 1278 streamer.emitInt32(row1); 1279 1280 // Extended Code Words, Extended Epilog Count 1281 if (ExtensionWord) { 1282 // FIXME: We should be able to split unwind info into multiple sections. 1283 if (CodeWords > 0xFF || EpilogCount > 0xFFFF) 1284 report_fatal_error( 1285 "SEH unwind data splitting is only implemented for large functions, " 1286 "cases of too many code words or too many epilogs will be done " 1287 "later"); 1288 uint32_t row2 = 0x0; 1289 row2 |= (CodeWords & 0xFF) << 16; 1290 row2 |= (EpilogCount & 0xFFFF); 1291 streamer.emitInt32(row2); 1292 } 1293 1294 if (PackedEpilogOffset < 0) { 1295 // Epilog Start Index, Epilog Start Offset 1296 for (auto &I : EpilogInfo) { 1297 MCSymbol *EpilogStart = I.first; 1298 uint32_t EpilogIndex = I.second; 1299 // Epilog offset within the Segment. 1300 uint32_t EpilogOffset = (uint32_t)(Seg.Epilogs[EpilogStart] - Seg.Offset); 1301 if (EpilogOffset) 1302 EpilogOffset /= 4; 1303 uint32_t row3 = EpilogOffset; 1304 row3 |= (EpilogIndex & 0x3FF) << 22; 1305 streamer.emitInt32(row3); 1306 } 1307 } 1308 1309 // Note that even for segments that have no prolog, we still need to emit 1310 // prolog unwinding opcodes so that the unwinder knows how to unwind from 1311 // such a segment. 1312 // The end_c opcode at the start indicates to the unwinder that the actual 1313 // prolog is outside of the current segment, and the unwinder shouldn't try 1314 // to check for unwinding from a partial prolog. 1315 if (!HasProlog) 1316 // Emit an end_c. 1317 streamer.emitInt8((uint8_t)0xE5); 1318 1319 // Emit prolog unwind instructions (in reverse order). 1320 for (auto Inst : llvm::reverse(info->Instructions)) 1321 ARM64EmitUnwindCode(streamer, Inst); 1322 1323 // Emit epilog unwind instructions 1324 for (auto &I : Seg.Epilogs) { 1325 auto &EpilogInstrs = info->EpilogMap[I.first].Instructions; 1326 for (const WinEH::Instruction &inst : EpilogInstrs) 1327 ARM64EmitUnwindCode(streamer, inst); 1328 } 1329 1330 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes; 1331 assert(BytesMod >= 0); 1332 for (int i = 0; i < BytesMod; i++) 1333 streamer.emitInt8(0xE3); 1334 1335 if (info->HandlesExceptions) 1336 streamer.emitValue( 1337 MCSymbolRefExpr::create(info->ExceptionHandler, 1338 MCSymbolRefExpr::VK_COFF_IMGREL32, context), 1339 4); 1340 } 1341 1342 // Populate the .xdata section. The format of .xdata on ARM64 is documented at 1343 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling 1344 static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info, 1345 bool TryPacked = true) { 1346 // If this UNWIND_INFO already has a symbol, it's already been emitted. 1347 if (info->Symbol) 1348 return; 1349 // If there's no unwind info here (not even a terminating UOP_End), the 1350 // unwind info is considered bogus and skipped. If this was done in 1351 // response to an explicit .seh_handlerdata, the associated trailing 1352 // handler data is left orphaned in the xdata section. 1353 if (info->empty()) { 1354 info->EmitAttempted = true; 1355 return; 1356 } 1357 if (info->EmitAttempted) { 1358 // If we tried to emit unwind info before (due to an explicit 1359 // .seh_handlerdata directive), but skipped it (because there was no 1360 // valid information to emit at the time), and it later got valid unwind 1361 // opcodes, we can't emit it here, because the trailing handler data 1362 // was already emitted elsewhere in the xdata section. 1363 streamer.getContext().reportError( 1364 SMLoc(), "Earlier .seh_handlerdata for " + info->Function->getName() + 1365 " skipped due to no unwind info at the time " 1366 "(.seh_handlerdata too early?), but the function later " 1367 "did get unwind info that can't be emitted"); 1368 return; 1369 } 1370 1371 simplifyARM64Opcodes(info->Instructions, false); 1372 for (auto &I : info->EpilogMap) 1373 simplifyARM64Opcodes(I.second.Instructions, true); 1374 1375 int64_t RawFuncLength; 1376 if (!info->FuncletOrFuncEnd) { 1377 report_fatal_error("FuncletOrFuncEnd not set"); 1378 } else { 1379 // FIXME: GetAbsDifference tries to compute the length of the function 1380 // immediately, before the whole file is emitted, but in general 1381 // that's impossible: the size in bytes of certain assembler directives 1382 // like .align and .fill is not known until the whole file is parsed and 1383 // relaxations are applied. Currently, GetAbsDifference fails with a fatal 1384 // error in that case. (We mostly don't hit this because inline assembly 1385 // specifying those directives is rare, and we don't normally try to 1386 // align loops on AArch64.) 1387 // 1388 // There are two potential approaches to delaying the computation. One, 1389 // we could emit something like ".word (endfunc-beginfunc)/4+0x10800000", 1390 // as long as we have some conservative estimate we could use to prove 1391 // that we don't need to split the unwind data. Emitting the constant 1392 // is straightforward, but there's no existing code for estimating the 1393 // size of the function. 1394 // 1395 // The other approach would be to use a dedicated, relaxable fragment, 1396 // which could grow to accommodate splitting the unwind data if 1397 // necessary. This is more straightforward, since it automatically works 1398 // without any new infrastructure, and it's consistent with how we handle 1399 // relaxation in other contexts. But it would require some refactoring 1400 // to move parts of the pdata/xdata emission into the implementation of 1401 // a fragment. We could probably continue to encode the unwind codes 1402 // here, but we'd have to emit the pdata, the xdata header, and the 1403 // epilogue scopes later, since they depend on whether the we need to 1404 // split the unwind data. 1405 // 1406 // If this is fixed, remove code in AArch64ISelLowering.cpp that 1407 // disables loop alignment on Windows. 1408 RawFuncLength = GetAbsDifference(streamer, info->FuncletOrFuncEnd, 1409 info->Begin); 1410 } 1411 1412 ARM64FindSegmentsInFunction(streamer, info, RawFuncLength); 1413 1414 info->PrologCodeBytes = ARM64CountOfUnwindCodes(info->Instructions); 1415 for (auto &S : info->Segments) 1416 ARM64EmitUnwindInfoForSegment(streamer, info, S, TryPacked); 1417 1418 // Clear prolog instructions after unwind info is emitted for all segments. 1419 info->Instructions.clear(); 1420 } 1421 1422 static uint32_t ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) { 1423 uint32_t Count = 0; 1424 for (const auto &I : Insns) { 1425 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 1426 default: 1427 llvm_unreachable("Unsupported ARM unwind code"); 1428 case Win64EH::UOP_AllocSmall: 1429 Count += 1; 1430 break; 1431 case Win64EH::UOP_AllocLarge: 1432 Count += 3; 1433 break; 1434 case Win64EH::UOP_AllocHuge: 1435 Count += 4; 1436 break; 1437 case Win64EH::UOP_WideAllocMedium: 1438 Count += 2; 1439 break; 1440 case Win64EH::UOP_WideAllocLarge: 1441 Count += 3; 1442 break; 1443 case Win64EH::UOP_WideAllocHuge: 1444 Count += 4; 1445 break; 1446 case Win64EH::UOP_WideSaveRegMask: 1447 Count += 2; 1448 break; 1449 case Win64EH::UOP_SaveSP: 1450 Count += 1; 1451 break; 1452 case Win64EH::UOP_SaveRegsR4R7LR: 1453 Count += 1; 1454 break; 1455 case Win64EH::UOP_WideSaveRegsR4R11LR: 1456 Count += 1; 1457 break; 1458 case Win64EH::UOP_SaveFRegD8D15: 1459 Count += 1; 1460 break; 1461 case Win64EH::UOP_SaveRegMask: 1462 Count += 2; 1463 break; 1464 case Win64EH::UOP_SaveLR: 1465 Count += 2; 1466 break; 1467 case Win64EH::UOP_SaveFRegD0D15: 1468 Count += 2; 1469 break; 1470 case Win64EH::UOP_SaveFRegD16D31: 1471 Count += 2; 1472 break; 1473 case Win64EH::UOP_Nop: 1474 case Win64EH::UOP_WideNop: 1475 case Win64EH::UOP_End: 1476 case Win64EH::UOP_EndNop: 1477 case Win64EH::UOP_WideEndNop: 1478 Count += 1; 1479 break; 1480 case Win64EH::UOP_Custom: { 1481 int J; 1482 for (J = 3; J > 0; J--) 1483 if (I.Offset & (0xffu << (8 * J))) 1484 break; 1485 Count += J + 1; 1486 break; 1487 } 1488 } 1489 } 1490 return Count; 1491 } 1492 1493 static uint32_t ARMCountOfInstructionBytes(ArrayRef<WinEH::Instruction> Insns, 1494 bool *HasCustom = nullptr) { 1495 uint32_t Count = 0; 1496 for (const auto &I : Insns) { 1497 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 1498 default: 1499 llvm_unreachable("Unsupported ARM unwind code"); 1500 case Win64EH::UOP_AllocSmall: 1501 case Win64EH::UOP_AllocLarge: 1502 case Win64EH::UOP_AllocHuge: 1503 Count += 2; 1504 break; 1505 case Win64EH::UOP_WideAllocMedium: 1506 case Win64EH::UOP_WideAllocLarge: 1507 case Win64EH::UOP_WideAllocHuge: 1508 Count += 4; 1509 break; 1510 case Win64EH::UOP_WideSaveRegMask: 1511 case Win64EH::UOP_WideSaveRegsR4R11LR: 1512 Count += 4; 1513 break; 1514 case Win64EH::UOP_SaveSP: 1515 Count += 2; 1516 break; 1517 case Win64EH::UOP_SaveRegMask: 1518 case Win64EH::UOP_SaveRegsR4R7LR: 1519 Count += 2; 1520 break; 1521 case Win64EH::UOP_SaveFRegD8D15: 1522 case Win64EH::UOP_SaveFRegD0D15: 1523 case Win64EH::UOP_SaveFRegD16D31: 1524 Count += 4; 1525 break; 1526 case Win64EH::UOP_SaveLR: 1527 Count += 4; 1528 break; 1529 case Win64EH::UOP_Nop: 1530 case Win64EH::UOP_EndNop: 1531 Count += 2; 1532 break; 1533 case Win64EH::UOP_WideNop: 1534 case Win64EH::UOP_WideEndNop: 1535 Count += 4; 1536 break; 1537 case Win64EH::UOP_End: 1538 // This doesn't map to any instruction 1539 break; 1540 case Win64EH::UOP_Custom: 1541 // We can't reason about what instructions this maps to; return a 1542 // phony number to make sure we don't accidentally do epilog packing. 1543 Count += 1000; 1544 if (HasCustom) 1545 *HasCustom = true; 1546 break; 1547 } 1548 } 1549 return Count; 1550 } 1551 1552 static void checkARMInstructions(MCStreamer &Streamer, 1553 ArrayRef<WinEH::Instruction> Insns, 1554 const MCSymbol *Begin, const MCSymbol *End, 1555 StringRef Name, StringRef Type) { 1556 if (!End) 1557 return; 1558 std::optional<int64_t> MaybeDistance = 1559 GetOptionalAbsDifference(Streamer, End, Begin); 1560 if (!MaybeDistance) 1561 return; 1562 uint32_t Distance = (uint32_t)*MaybeDistance; 1563 bool HasCustom = false; 1564 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns, &HasCustom); 1565 if (HasCustom) 1566 return; 1567 if (Distance != InstructionBytes) { 1568 Streamer.getContext().reportError( 1569 SMLoc(), "Incorrect size for " + Name + " " + Type + ": " + 1570 Twine(Distance) + 1571 " bytes of instructions in range, but .seh directives " 1572 "corresponding to " + 1573 Twine(InstructionBytes) + " bytes\n"); 1574 } 1575 } 1576 1577 static bool isARMTerminator(const WinEH::Instruction &inst) { 1578 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 1579 case Win64EH::UOP_End: 1580 case Win64EH::UOP_EndNop: 1581 case Win64EH::UOP_WideEndNop: 1582 return true; 1583 default: 1584 return false; 1585 } 1586 } 1587 1588 // Unwind opcode encodings and restrictions are documented at 1589 // https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling 1590 static void ARMEmitUnwindCode(MCStreamer &streamer, 1591 const WinEH::Instruction &inst) { 1592 uint32_t w, lr; 1593 int i; 1594 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 1595 default: 1596 llvm_unreachable("Unsupported ARM unwind code"); 1597 case Win64EH::UOP_AllocSmall: 1598 assert((inst.Offset & 3) == 0); 1599 assert(inst.Offset / 4 <= 0x7f); 1600 streamer.emitInt8(inst.Offset / 4); 1601 break; 1602 case Win64EH::UOP_WideSaveRegMask: 1603 assert((inst.Register & ~0x5fff) == 0); 1604 lr = (inst.Register >> 14) & 1; 1605 w = 0x8000 | (inst.Register & 0x1fff) | (lr << 13); 1606 streamer.emitInt8((w >> 8) & 0xff); 1607 streamer.emitInt8((w >> 0) & 0xff); 1608 break; 1609 case Win64EH::UOP_SaveSP: 1610 assert(inst.Register <= 0x0f); 1611 streamer.emitInt8(0xc0 | inst.Register); 1612 break; 1613 case Win64EH::UOP_SaveRegsR4R7LR: 1614 assert(inst.Register >= 4 && inst.Register <= 7); 1615 assert(inst.Offset <= 1); 1616 streamer.emitInt8(0xd0 | (inst.Register - 4) | (inst.Offset << 2)); 1617 break; 1618 case Win64EH::UOP_WideSaveRegsR4R11LR: 1619 assert(inst.Register >= 8 && inst.Register <= 11); 1620 assert(inst.Offset <= 1); 1621 streamer.emitInt8(0xd8 | (inst.Register - 8) | (inst.Offset << 2)); 1622 break; 1623 case Win64EH::UOP_SaveFRegD8D15: 1624 assert(inst.Register >= 8 && inst.Register <= 15); 1625 streamer.emitInt8(0xe0 | (inst.Register - 8)); 1626 break; 1627 case Win64EH::UOP_WideAllocMedium: 1628 assert((inst.Offset & 3) == 0); 1629 assert(inst.Offset / 4 <= 0x3ff); 1630 w = 0xe800 | (inst.Offset / 4); 1631 streamer.emitInt8((w >> 8) & 0xff); 1632 streamer.emitInt8((w >> 0) & 0xff); 1633 break; 1634 case Win64EH::UOP_SaveRegMask: 1635 assert((inst.Register & ~0x40ff) == 0); 1636 lr = (inst.Register >> 14) & 1; 1637 w = 0xec00 | (inst.Register & 0x0ff) | (lr << 8); 1638 streamer.emitInt8((w >> 8) & 0xff); 1639 streamer.emitInt8((w >> 0) & 0xff); 1640 break; 1641 case Win64EH::UOP_SaveLR: 1642 assert((inst.Offset & 3) == 0); 1643 assert(inst.Offset / 4 <= 0x0f); 1644 streamer.emitInt8(0xef); 1645 streamer.emitInt8(inst.Offset / 4); 1646 break; 1647 case Win64EH::UOP_SaveFRegD0D15: 1648 assert(inst.Register <= 15); 1649 assert(inst.Offset <= 15); 1650 assert(inst.Register <= inst.Offset); 1651 streamer.emitInt8(0xf5); 1652 streamer.emitInt8((inst.Register << 4) | inst.Offset); 1653 break; 1654 case Win64EH::UOP_SaveFRegD16D31: 1655 assert(inst.Register >= 16 && inst.Register <= 31); 1656 assert(inst.Offset >= 16 && inst.Offset <= 31); 1657 assert(inst.Register <= inst.Offset); 1658 streamer.emitInt8(0xf6); 1659 streamer.emitInt8(((inst.Register - 16) << 4) | (inst.Offset - 16)); 1660 break; 1661 case Win64EH::UOP_AllocLarge: 1662 assert((inst.Offset & 3) == 0); 1663 assert(inst.Offset / 4 <= 0xffff); 1664 w = inst.Offset / 4; 1665 streamer.emitInt8(0xf7); 1666 streamer.emitInt8((w >> 8) & 0xff); 1667 streamer.emitInt8((w >> 0) & 0xff); 1668 break; 1669 case Win64EH::UOP_AllocHuge: 1670 assert((inst.Offset & 3) == 0); 1671 assert(inst.Offset / 4 <= 0xffffff); 1672 w = inst.Offset / 4; 1673 streamer.emitInt8(0xf8); 1674 streamer.emitInt8((w >> 16) & 0xff); 1675 streamer.emitInt8((w >> 8) & 0xff); 1676 streamer.emitInt8((w >> 0) & 0xff); 1677 break; 1678 case Win64EH::UOP_WideAllocLarge: 1679 assert((inst.Offset & 3) == 0); 1680 assert(inst.Offset / 4 <= 0xffff); 1681 w = inst.Offset / 4; 1682 streamer.emitInt8(0xf9); 1683 streamer.emitInt8((w >> 8) & 0xff); 1684 streamer.emitInt8((w >> 0) & 0xff); 1685 break; 1686 case Win64EH::UOP_WideAllocHuge: 1687 assert((inst.Offset & 3) == 0); 1688 assert(inst.Offset / 4 <= 0xffffff); 1689 w = inst.Offset / 4; 1690 streamer.emitInt8(0xfa); 1691 streamer.emitInt8((w >> 16) & 0xff); 1692 streamer.emitInt8((w >> 8) & 0xff); 1693 streamer.emitInt8((w >> 0) & 0xff); 1694 break; 1695 case Win64EH::UOP_Nop: 1696 streamer.emitInt8(0xfb); 1697 break; 1698 case Win64EH::UOP_WideNop: 1699 streamer.emitInt8(0xfc); 1700 break; 1701 case Win64EH::UOP_EndNop: 1702 streamer.emitInt8(0xfd); 1703 break; 1704 case Win64EH::UOP_WideEndNop: 1705 streamer.emitInt8(0xfe); 1706 break; 1707 case Win64EH::UOP_End: 1708 streamer.emitInt8(0xff); 1709 break; 1710 case Win64EH::UOP_Custom: 1711 for (i = 3; i > 0; i--) 1712 if (inst.Offset & (0xffu << (8 * i))) 1713 break; 1714 for (; i >= 0; i--) 1715 streamer.emitInt8((inst.Offset >> (8 * i)) & 0xff); 1716 break; 1717 } 1718 } 1719 1720 // Check if an epilog exists as a subset of the end of a prolog (backwards). 1721 // An epilog may end with one out of three different end opcodes; if this 1722 // is the first epilog that shares opcodes with the prolog, we can tolerate 1723 // that this opcode differs (and the caller will update the prolog to use 1724 // the same end opcode as the epilog). If another epilog already shares 1725 // opcodes with the prolog, the ending opcode must be a strict match. 1726 static int getARMOffsetInProlog(const std::vector<WinEH::Instruction> &Prolog, 1727 const std::vector<WinEH::Instruction> &Epilog, 1728 bool CanTweakProlog) { 1729 // Can't find an epilog as a subset if it is longer than the prolog. 1730 if (Epilog.size() > Prolog.size()) 1731 return -1; 1732 1733 // Check that the epilog actually is a perfect match for the end (backwrds) 1734 // of the prolog. 1735 // If we can adjust the prolog afterwards, don't check that the end opcodes 1736 // match. 1737 int EndIdx = CanTweakProlog ? 1 : 0; 1738 for (int I = Epilog.size() - 1; I >= EndIdx; I--) { 1739 // TODO: Could also allow minor mismatches, e.g. "add sp, #16" vs 1740 // "push {r0-r3}". 1741 if (Prolog[I] != Epilog[Epilog.size() - 1 - I]) 1742 return -1; 1743 } 1744 1745 if (CanTweakProlog) { 1746 // Check that both prolog and epilog end with an expected end opcode. 1747 if (Prolog.front().Operation != Win64EH::UOP_End) 1748 return -1; 1749 if (Epilog.back().Operation != Win64EH::UOP_End && 1750 Epilog.back().Operation != Win64EH::UOP_EndNop && 1751 Epilog.back().Operation != Win64EH::UOP_WideEndNop) 1752 return -1; 1753 } 1754 1755 // If the epilog was a subset of the prolog, find its offset. 1756 if (Epilog.size() == Prolog.size()) 1757 return 0; 1758 return ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction>( 1759 &Prolog[Epilog.size()], Prolog.size() - Epilog.size())); 1760 } 1761 1762 static int checkARMPackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info, 1763 int PrologCodeBytes) { 1764 // Can only pack if there's one single epilog 1765 if (info->EpilogMap.size() != 1) 1766 return -1; 1767 1768 const WinEH::FrameInfo::Epilog &EpilogInfo = info->EpilogMap.begin()->second; 1769 // Can only pack if the epilog is unconditional 1770 if (EpilogInfo.Condition != 0xe) // ARMCC::AL 1771 return -1; 1772 1773 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions; 1774 // Make sure we have at least the trailing end opcode 1775 if (info->Instructions.empty() || Epilog.empty()) 1776 return -1; 1777 1778 // Check that the epilog actually is at the very end of the function, 1779 // otherwise it can't be packed. 1780 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference( 1781 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first); 1782 if (!MaybeDistance) 1783 return -1; 1784 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance; 1785 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Epilog); 1786 if (DistanceFromEnd != InstructionBytes) 1787 return -1; 1788 1789 int RetVal = -1; 1790 // Even if we don't end up sharing opcodes with the prolog, we can still 1791 // write the offset as a packed offset, if the single epilog is located at 1792 // the end of the function and the offset (pointing after the prolog) fits 1793 // as a packed offset. 1794 if (PrologCodeBytes <= 31 && 1795 PrologCodeBytes + ARMCountOfUnwindCodes(Epilog) <= 63) 1796 RetVal = PrologCodeBytes; 1797 1798 int Offset = 1799 getARMOffsetInProlog(info->Instructions, Epilog, /*CanTweakProlog=*/true); 1800 if (Offset < 0) 1801 return RetVal; 1802 1803 // Check that the offset and prolog size fits in the first word; it's 1804 // unclear whether the epilog count in the extension word can be taken 1805 // as packed epilog offset. 1806 if (Offset > 31 || PrologCodeBytes > 63) 1807 return RetVal; 1808 1809 // Replace the regular end opcode of the prolog with the one from the 1810 // epilog. 1811 info->Instructions.front() = Epilog.back(); 1812 1813 // As we choose to express the epilog as part of the prolog, remove the 1814 // epilog from the map, so we don't try to emit its opcodes. 1815 info->EpilogMap.clear(); 1816 return Offset; 1817 } 1818 1819 static bool parseRegMask(unsigned Mask, bool &HasLR, bool &HasR11, 1820 unsigned &Folded, int &IntRegs) { 1821 if (Mask & (1 << 14)) { 1822 HasLR = true; 1823 Mask &= ~(1 << 14); 1824 } 1825 if (Mask & (1 << 11)) { 1826 HasR11 = true; 1827 Mask &= ~(1 << 11); 1828 } 1829 Folded = 0; 1830 IntRegs = -1; 1831 if (!Mask) 1832 return true; 1833 int First = 0; 1834 // Shift right until we have the bits at the bottom 1835 while ((Mask & 1) == 0) { 1836 First++; 1837 Mask >>= 1; 1838 } 1839 if ((Mask & (Mask + 1)) != 0) 1840 return false; // Not a consecutive series of bits? Can't be packed. 1841 // Count the bits 1842 int N = 0; 1843 while (Mask & (1 << N)) 1844 N++; 1845 if (First < 4) { 1846 if (First + N < 4) 1847 return false; 1848 Folded = 4 - First; 1849 N -= Folded; 1850 First = 4; 1851 } 1852 if (First > 4) 1853 return false; // Can't be packed 1854 if (N >= 1) 1855 IntRegs = N - 1; 1856 return true; 1857 } 1858 1859 static bool tryARMPackedUnwind(MCStreamer &streamer, WinEH::FrameInfo *info, 1860 uint32_t FuncLength) { 1861 int Step = 0; 1862 bool Homing = false; 1863 bool HasR11 = false; 1864 bool HasChain = false; 1865 bool HasLR = false; 1866 int IntRegs = -1; // r4 - r(4+N) 1867 int FloatRegs = -1; // d8 - d(8+N) 1868 unsigned PF = 0; // Number of extra pushed registers 1869 unsigned StackAdjust = 0; 1870 // Iterate over the prolog and check that all opcodes exactly match 1871 // the canonical order and form. 1872 for (const WinEH::Instruction &Inst : info->Instructions) { 1873 switch (Inst.Operation) { 1874 default: 1875 llvm_unreachable("Unsupported ARM unwind code"); 1876 case Win64EH::UOP_Custom: 1877 case Win64EH::UOP_AllocLarge: 1878 case Win64EH::UOP_AllocHuge: 1879 case Win64EH::UOP_WideAllocLarge: 1880 case Win64EH::UOP_WideAllocHuge: 1881 case Win64EH::UOP_SaveFRegD0D15: 1882 case Win64EH::UOP_SaveFRegD16D31: 1883 // Can't be packed 1884 return false; 1885 case Win64EH::UOP_SaveSP: 1886 // Can't be packed; we can't rely on restoring sp from r11 when 1887 // unwinding a packed prologue. 1888 return false; 1889 case Win64EH::UOP_SaveLR: 1890 // Can't be present in a packed prologue 1891 return false; 1892 1893 case Win64EH::UOP_End: 1894 case Win64EH::UOP_EndNop: 1895 case Win64EH::UOP_WideEndNop: 1896 if (Step != 0) 1897 return false; 1898 Step = 1; 1899 break; 1900 1901 case Win64EH::UOP_SaveRegsR4R7LR: 1902 case Win64EH::UOP_WideSaveRegsR4R11LR: 1903 // push {r4-r11,lr} 1904 if (Step != 1 && Step != 2) 1905 return false; 1906 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX 1907 assert(Inst.Offset <= 1); // Lr 1908 IntRegs = Inst.Register - 4; 1909 if (Inst.Register == 11) { 1910 HasR11 = true; 1911 IntRegs--; 1912 } 1913 if (Inst.Offset) 1914 HasLR = true; 1915 Step = 3; 1916 break; 1917 1918 case Win64EH::UOP_SaveRegMask: 1919 if (Step == 1 && Inst.Register == 0x0f) { 1920 // push {r0-r3} 1921 Homing = true; 1922 Step = 2; 1923 break; 1924 } 1925 [[fallthrough]]; 1926 case Win64EH::UOP_WideSaveRegMask: 1927 if (Step != 1 && Step != 2) 1928 return false; 1929 // push {r4-r9,r11,lr} 1930 // push {r11,lr} 1931 // push {r1-r5} 1932 if (!parseRegMask(Inst.Register, HasLR, HasR11, PF, IntRegs)) 1933 return false; 1934 Step = 3; 1935 break; 1936 1937 case Win64EH::UOP_Nop: 1938 // mov r11, sp 1939 if (Step != 3 || !HasR11 || IntRegs >= 0 || PF > 0) 1940 return false; 1941 HasChain = true; 1942 Step = 4; 1943 break; 1944 case Win64EH::UOP_WideNop: 1945 // add.w r11, sp, #xx 1946 if (Step != 3 || !HasR11 || (IntRegs < 0 && PF == 0)) 1947 return false; 1948 HasChain = true; 1949 Step = 4; 1950 break; 1951 1952 case Win64EH::UOP_SaveFRegD8D15: 1953 if (Step != 1 && Step != 2 && Step != 3 && Step != 4) 1954 return false; 1955 assert(Inst.Register >= 8 && Inst.Register <= 15); 1956 if (Inst.Register == 15) 1957 return false; // Can't pack this case, R==7 means no IntRegs 1958 if (IntRegs >= 0) 1959 return false; 1960 FloatRegs = Inst.Register - 8; 1961 Step = 5; 1962 break; 1963 1964 case Win64EH::UOP_AllocSmall: 1965 case Win64EH::UOP_WideAllocMedium: 1966 if (Step != 1 && Step != 2 && Step != 3 && Step != 4 && Step != 5) 1967 return false; 1968 if (PF > 0) // Can't have both folded and explicit stack allocation 1969 return false; 1970 if (Inst.Offset / 4 >= 0x3f4) 1971 return false; 1972 StackAdjust = Inst.Offset / 4; 1973 Step = 6; 1974 break; 1975 } 1976 } 1977 if (HasR11 && !HasChain) { 1978 if (IntRegs + 4 == 10) { 1979 // r11 stored, but not chaining; can be packed if already saving r4-r10 1980 // and we can fit r11 into this range. 1981 IntRegs++; 1982 HasR11 = false; 1983 } else 1984 return false; 1985 } 1986 if (HasChain && !HasLR) 1987 return false; 1988 1989 // Packed uneind info can't express multiple epilogues. 1990 if (info->EpilogMap.size() > 1) 1991 return false; 1992 1993 unsigned EF = 0; 1994 int Ret = 0; 1995 if (info->EpilogMap.size() == 0) { 1996 Ret = 3; // No epilogue 1997 } else { 1998 // As the prologue and epilogue aren't exact mirrors of each other, 1999 // we have to check the epilogue too and see if it matches what we've 2000 // concluded from the prologue. 2001 const WinEH::FrameInfo::Epilog &EpilogInfo = 2002 info->EpilogMap.begin()->second; 2003 if (EpilogInfo.Condition != 0xe) // ARMCC::AL 2004 return false; 2005 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions; 2006 std::optional<int64_t> MaybeDistance = GetOptionalAbsDifference( 2007 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first); 2008 if (!MaybeDistance) 2009 return false; 2010 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance; 2011 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Epilog); 2012 if (DistanceFromEnd != InstructionBytes) 2013 return false; 2014 2015 bool GotStackAdjust = false; 2016 bool GotFloatRegs = false; 2017 bool GotIntRegs = false; 2018 bool GotHomingRestore = false; 2019 bool GotLRRestore = false; 2020 bool NeedsReturn = false; 2021 bool GotReturn = false; 2022 2023 Step = 6; 2024 for (const WinEH::Instruction &Inst : Epilog) { 2025 switch (Inst.Operation) { 2026 default: 2027 llvm_unreachable("Unsupported ARM unwind code"); 2028 case Win64EH::UOP_Custom: 2029 case Win64EH::UOP_AllocLarge: 2030 case Win64EH::UOP_AllocHuge: 2031 case Win64EH::UOP_WideAllocLarge: 2032 case Win64EH::UOP_WideAllocHuge: 2033 case Win64EH::UOP_SaveFRegD0D15: 2034 case Win64EH::UOP_SaveFRegD16D31: 2035 case Win64EH::UOP_SaveSP: 2036 case Win64EH::UOP_Nop: 2037 case Win64EH::UOP_WideNop: 2038 // Can't be packed in an epilogue 2039 return false; 2040 2041 case Win64EH::UOP_AllocSmall: 2042 case Win64EH::UOP_WideAllocMedium: 2043 if (Inst.Offset / 4 >= 0x3f4) 2044 return false; 2045 if (Step == 6) { 2046 if (Homing && FloatRegs < 0 && IntRegs < 0 && StackAdjust == 0 && 2047 PF == 0 && Inst.Offset == 16) { 2048 GotHomingRestore = true; 2049 Step = 10; 2050 } else { 2051 if (StackAdjust > 0) { 2052 // Got stack adjust in prologue too; must match. 2053 if (StackAdjust != Inst.Offset / 4) 2054 return false; 2055 GotStackAdjust = true; 2056 } else if (PF == Inst.Offset / 4) { 2057 // Folded prologue, non-folded epilogue 2058 StackAdjust = Inst.Offset / 4; 2059 GotStackAdjust = true; 2060 } else { 2061 // StackAdjust == 0 in prologue, mismatch 2062 return false; 2063 } 2064 Step = 7; 2065 } 2066 } else if (Step == 7 || Step == 8 || Step == 9) { 2067 if (!Homing || Inst.Offset != 16) 2068 return false; 2069 GotHomingRestore = true; 2070 Step = 10; 2071 } else 2072 return false; 2073 break; 2074 2075 case Win64EH::UOP_SaveFRegD8D15: 2076 if (Step != 6 && Step != 7) 2077 return false; 2078 assert(Inst.Register >= 8 && Inst.Register <= 15); 2079 if (FloatRegs != (int)(Inst.Register - 8)) 2080 return false; 2081 GotFloatRegs = true; 2082 Step = 8; 2083 break; 2084 2085 case Win64EH::UOP_SaveRegsR4R7LR: 2086 case Win64EH::UOP_WideSaveRegsR4R11LR: { 2087 // push {r4-r11,lr} 2088 if (Step != 6 && Step != 7 && Step != 8) 2089 return false; 2090 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX 2091 assert(Inst.Offset <= 1); // Lr 2092 if (Homing && HasLR) { 2093 // If homing and LR is backed up, we can either restore LR here 2094 // and return with Ret == 1 or 2, or return with SaveLR below 2095 if (Inst.Offset) { 2096 GotLRRestore = true; 2097 NeedsReturn = true; 2098 } else { 2099 // Expecting a separate SaveLR below 2100 } 2101 } else { 2102 if (HasLR != (Inst.Offset == 1)) 2103 return false; 2104 } 2105 GotLRRestore = Inst.Offset == 1; 2106 if (IntRegs < 0) // This opcode must include r4 2107 return false; 2108 int Expected = IntRegs; 2109 if (HasChain) { 2110 // Can't express r11 here unless IntRegs describe r4-r10 2111 if (IntRegs != 6) 2112 return false; 2113 Expected++; 2114 } 2115 if (Expected != (int)(Inst.Register - 4)) 2116 return false; 2117 GotIntRegs = true; 2118 Step = 9; 2119 break; 2120 } 2121 2122 case Win64EH::UOP_SaveRegMask: 2123 case Win64EH::UOP_WideSaveRegMask: { 2124 if (Step != 6 && Step != 7 && Step != 8) 2125 return false; 2126 // push {r4-r9,r11,lr} 2127 // push {r11,lr} 2128 // push {r1-r5} 2129 bool CurHasLR = false, CurHasR11 = false; 2130 int Regs; 2131 if (!parseRegMask(Inst.Register, CurHasLR, CurHasR11, EF, Regs)) 2132 return false; 2133 if (EF > 0) { 2134 if (EF != PF && EF != StackAdjust) 2135 return false; 2136 } 2137 if (Homing && HasLR) { 2138 // If homing and LR is backed up, we can either restore LR here 2139 // and return with Ret == 1 or 2, or return with SaveLR below 2140 if (CurHasLR) { 2141 GotLRRestore = true; 2142 NeedsReturn = true; 2143 } else { 2144 // Expecting a separate SaveLR below 2145 } 2146 } else { 2147 if (CurHasLR != HasLR) 2148 return false; 2149 GotLRRestore = CurHasLR; 2150 } 2151 int Expected = IntRegs; 2152 if (HasChain) { 2153 // If we have chaining, the mask must have included r11. 2154 if (!CurHasR11) 2155 return false; 2156 } else if (Expected == 7) { 2157 // If we don't have chaining, the mask could still include r11, 2158 // expressed as part of IntRegs Instead. 2159 Expected--; 2160 if (!CurHasR11) 2161 return false; 2162 } else { 2163 // Neither HasChain nor r11 included in IntRegs, must not have r11 2164 // here either. 2165 if (CurHasR11) 2166 return false; 2167 } 2168 if (Expected != Regs) 2169 return false; 2170 GotIntRegs = true; 2171 Step = 9; 2172 break; 2173 } 2174 2175 case Win64EH::UOP_SaveLR: 2176 if (Step != 6 && Step != 7 && Step != 8 && Step != 9) 2177 return false; 2178 if (!Homing || Inst.Offset != 20 || GotLRRestore) 2179 return false; 2180 GotLRRestore = true; 2181 GotHomingRestore = true; 2182 Step = 10; 2183 break; 2184 2185 case Win64EH::UOP_EndNop: 2186 case Win64EH::UOP_WideEndNop: 2187 GotReturn = true; 2188 Ret = (Inst.Operation == Win64EH::UOP_EndNop) ? 1 : 2; 2189 [[fallthrough]]; 2190 case Win64EH::UOP_End: 2191 if (Step != 6 && Step != 7 && Step != 8 && Step != 9 && Step != 10) 2192 return false; 2193 Step = 11; 2194 break; 2195 } 2196 } 2197 2198 if (Step != 11) 2199 return false; 2200 if (StackAdjust > 0 && !GotStackAdjust && EF == 0) 2201 return false; 2202 if (FloatRegs >= 0 && !GotFloatRegs) 2203 return false; 2204 if (IntRegs >= 0 && !GotIntRegs) 2205 return false; 2206 if (Homing && !GotHomingRestore) 2207 return false; 2208 if (HasLR && !GotLRRestore) 2209 return false; 2210 if (NeedsReturn && !GotReturn) 2211 return false; 2212 } 2213 2214 assert(PF == 0 || EF == 0 || 2215 StackAdjust == 0); // Can't have adjust in all three 2216 if (PF > 0 || EF > 0) { 2217 StackAdjust = PF > 0 ? (PF - 1) : (EF - 1); 2218 assert(StackAdjust <= 3); 2219 StackAdjust |= 0x3f0; 2220 if (PF > 0) 2221 StackAdjust |= 1 << 2; 2222 if (EF > 0) 2223 StackAdjust |= 1 << 3; 2224 } 2225 2226 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier"); 2227 int Flag = info->Fragment ? 0x02 : 0x01; 2228 int H = Homing ? 1 : 0; 2229 int L = HasLR ? 1 : 0; 2230 int C = HasChain ? 1 : 0; 2231 assert(IntRegs < 0 || FloatRegs < 0); 2232 unsigned Reg, R; 2233 if (IntRegs >= 0) { 2234 Reg = IntRegs; 2235 assert(Reg <= 7); 2236 R = 0; 2237 } else if (FloatRegs >= 0) { 2238 Reg = FloatRegs; 2239 assert(Reg < 7); 2240 R = 1; 2241 } else { 2242 // No int or float regs stored (except possibly R11,LR) 2243 Reg = 7; 2244 R = 1; 2245 } 2246 info->PackedInfo |= Flag << 0; 2247 info->PackedInfo |= (FuncLength & 0x7FF) << 2; 2248 info->PackedInfo |= (Ret & 0x3) << 13; 2249 info->PackedInfo |= H << 15; 2250 info->PackedInfo |= Reg << 16; 2251 info->PackedInfo |= R << 19; 2252 info->PackedInfo |= L << 20; 2253 info->PackedInfo |= C << 21; 2254 assert(StackAdjust <= 0x3ff); 2255 info->PackedInfo |= StackAdjust << 22; 2256 return true; 2257 } 2258 2259 // Populate the .xdata section. The format of .xdata on ARM is documented at 2260 // https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling 2261 static void ARMEmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info, 2262 bool TryPacked = true) { 2263 // If this UNWIND_INFO already has a symbol, it's already been emitted. 2264 if (info->Symbol) 2265 return; 2266 // If there's no unwind info here (not even a terminating UOP_End), the 2267 // unwind info is considered bogus and skipped. If this was done in 2268 // response to an explicit .seh_handlerdata, the associated trailing 2269 // handler data is left orphaned in the xdata section. 2270 if (info->empty()) { 2271 info->EmitAttempted = true; 2272 return; 2273 } 2274 if (info->EmitAttempted) { 2275 // If we tried to emit unwind info before (due to an explicit 2276 // .seh_handlerdata directive), but skipped it (because there was no 2277 // valid information to emit at the time), and it later got valid unwind 2278 // opcodes, we can't emit it here, because the trailing handler data 2279 // was already emitted elsewhere in the xdata section. 2280 streamer.getContext().reportError( 2281 SMLoc(), "Earlier .seh_handlerdata for " + info->Function->getName() + 2282 " skipped due to no unwind info at the time " 2283 "(.seh_handlerdata too early?), but the function later " 2284 "did get unwind info that can't be emitted"); 2285 return; 2286 } 2287 2288 MCContext &context = streamer.getContext(); 2289 MCSymbol *Label = context.createTempSymbol(); 2290 2291 streamer.emitValueToAlignment(Align(4)); 2292 streamer.emitLabel(Label); 2293 info->Symbol = Label; 2294 2295 if (!info->PrologEnd) 2296 streamer.getContext().reportError(SMLoc(), "Prologue in " + 2297 info->Function->getName() + 2298 " not correctly terminated"); 2299 2300 if (info->PrologEnd && !info->Fragment) 2301 checkARMInstructions(streamer, info->Instructions, info->Begin, 2302 info->PrologEnd, info->Function->getName(), 2303 "prologue"); 2304 for (auto &I : info->EpilogMap) { 2305 MCSymbol *EpilogStart = I.first; 2306 auto &Epilog = I.second; 2307 checkARMInstructions(streamer, Epilog.Instructions, EpilogStart, Epilog.End, 2308 info->Function->getName(), "epilogue"); 2309 if (Epilog.Instructions.empty() || 2310 !isARMTerminator(Epilog.Instructions.back())) 2311 streamer.getContext().reportError( 2312 SMLoc(), "Epilogue in " + info->Function->getName() + 2313 " not correctly terminated"); 2314 } 2315 2316 std::optional<int64_t> RawFuncLength; 2317 const MCExpr *FuncLengthExpr = nullptr; 2318 if (!info->FuncletOrFuncEnd) { 2319 report_fatal_error("FuncletOrFuncEnd not set"); 2320 } else { 2321 // As the size of many thumb2 instructions isn't known until later, 2322 // we can't always rely on being able to calculate the absolute 2323 // length of the function here. If we can't calculate it, defer it 2324 // to a relocation. 2325 // 2326 // In such a case, we won't know if the function is too long so that 2327 // the unwind info would need to be split (but this isn't implemented 2328 // anyway). 2329 RawFuncLength = 2330 GetOptionalAbsDifference(streamer, info->FuncletOrFuncEnd, info->Begin); 2331 if (!RawFuncLength) 2332 FuncLengthExpr = 2333 GetSubDivExpr(streamer, info->FuncletOrFuncEnd, info->Begin, 2); 2334 } 2335 uint32_t FuncLength = 0; 2336 if (RawFuncLength) 2337 FuncLength = (uint32_t)*RawFuncLength / 2; 2338 if (FuncLength > 0x3FFFF) 2339 report_fatal_error("SEH unwind data splitting not yet implemented"); 2340 uint32_t PrologCodeBytes = ARMCountOfUnwindCodes(info->Instructions); 2341 uint32_t TotalCodeBytes = PrologCodeBytes; 2342 2343 if (!info->HandlesExceptions && RawFuncLength && FuncLength <= 0x7ff && 2344 TryPacked) { 2345 // No exception handlers; check if the prolog and epilog matches the 2346 // patterns that can be described by the packed format. If we don't 2347 // know the exact function length yet, we can't do this. 2348 2349 // info->Symbol was already set even if we didn't actually write any 2350 // unwind info there. Keep using that as indicator that this unwind 2351 // info has been generated already. 2352 2353 if (tryARMPackedUnwind(streamer, info, FuncLength)) 2354 return; 2355 } 2356 2357 int PackedEpilogOffset = 2358 checkARMPackedEpilog(streamer, info, PrologCodeBytes); 2359 2360 // Process epilogs. 2361 MapVector<MCSymbol *, uint32_t> EpilogInfo; 2362 // Epilogs processed so far. 2363 std::vector<MCSymbol *> AddedEpilogs; 2364 2365 bool CanTweakProlog = true; 2366 for (auto &I : info->EpilogMap) { 2367 MCSymbol *EpilogStart = I.first; 2368 auto &EpilogInstrs = I.second.Instructions; 2369 uint32_t CodeBytes = ARMCountOfUnwindCodes(EpilogInstrs); 2370 2371 MCSymbol *MatchingEpilog = 2372 FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info); 2373 int PrologOffset; 2374 if (MatchingEpilog) { 2375 assert(EpilogInfo.contains(MatchingEpilog) && 2376 "Duplicate epilog not found"); 2377 EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog); 2378 // Clear the unwind codes in the EpilogMap, so that they don't get output 2379 // in the logic below. 2380 EpilogInstrs.clear(); 2381 } else if ((PrologOffset = getARMOffsetInProlog( 2382 info->Instructions, EpilogInstrs, CanTweakProlog)) >= 0) { 2383 if (CanTweakProlog) { 2384 // Replace the regular end opcode of the prolog with the one from the 2385 // epilog. 2386 info->Instructions.front() = EpilogInstrs.back(); 2387 // Later epilogs need a strict match for the end opcode. 2388 CanTweakProlog = false; 2389 } 2390 EpilogInfo[EpilogStart] = PrologOffset; 2391 // Clear the unwind codes in the EpilogMap, so that they don't get output 2392 // in the logic below. 2393 EpilogInstrs.clear(); 2394 } else { 2395 EpilogInfo[EpilogStart] = TotalCodeBytes; 2396 TotalCodeBytes += CodeBytes; 2397 AddedEpilogs.push_back(EpilogStart); 2398 } 2399 } 2400 2401 // Code Words, Epilog count, F, E, X, Vers, Function Length 2402 uint32_t row1 = 0x0; 2403 uint32_t CodeWords = TotalCodeBytes / 4; 2404 uint32_t CodeWordsMod = TotalCodeBytes % 4; 2405 if (CodeWordsMod) 2406 CodeWords++; 2407 uint32_t EpilogCount = 2408 PackedEpilogOffset >= 0 ? PackedEpilogOffset : info->EpilogMap.size(); 2409 bool ExtensionWord = EpilogCount > 31 || CodeWords > 15; 2410 if (!ExtensionWord) { 2411 row1 |= (EpilogCount & 0x1F) << 23; 2412 row1 |= (CodeWords & 0x0F) << 28; 2413 } 2414 if (info->HandlesExceptions) // X 2415 row1 |= 1 << 20; 2416 if (PackedEpilogOffset >= 0) // E 2417 row1 |= 1 << 21; 2418 if (info->Fragment) // F 2419 row1 |= 1 << 22; 2420 row1 |= FuncLength & 0x3FFFF; 2421 if (RawFuncLength) 2422 streamer.emitInt32(row1); 2423 else 2424 streamer.emitValue( 2425 MCBinaryExpr::createOr(FuncLengthExpr, 2426 MCConstantExpr::create(row1, context), context), 2427 4); 2428 2429 // Extended Code Words, Extended Epilog Count 2430 if (ExtensionWord) { 2431 // FIXME: We should be able to split unwind info into multiple sections. 2432 if (CodeWords > 0xFF || EpilogCount > 0xFFFF) 2433 report_fatal_error("SEH unwind data splitting not yet implemented"); 2434 uint32_t row2 = 0x0; 2435 row2 |= (CodeWords & 0xFF) << 16; 2436 row2 |= (EpilogCount & 0xFFFF); 2437 streamer.emitInt32(row2); 2438 } 2439 2440 if (PackedEpilogOffset < 0) { 2441 // Epilog Start Index, Epilog Start Offset 2442 for (auto &I : EpilogInfo) { 2443 MCSymbol *EpilogStart = I.first; 2444 uint32_t EpilogIndex = I.second; 2445 2446 std::optional<int64_t> MaybeEpilogOffset = 2447 GetOptionalAbsDifference(streamer, EpilogStart, info->Begin); 2448 const MCExpr *OffsetExpr = nullptr; 2449 uint32_t EpilogOffset = 0; 2450 if (MaybeEpilogOffset) 2451 EpilogOffset = *MaybeEpilogOffset / 2; 2452 else 2453 OffsetExpr = GetSubDivExpr(streamer, EpilogStart, info->Begin, 2); 2454 2455 assert(info->EpilogMap.contains(EpilogStart)); 2456 unsigned Condition = info->EpilogMap[EpilogStart].Condition; 2457 assert(Condition <= 0xf); 2458 2459 uint32_t row3 = EpilogOffset; 2460 row3 |= Condition << 20; 2461 row3 |= (EpilogIndex & 0x3FF) << 24; 2462 if (MaybeEpilogOffset) 2463 streamer.emitInt32(row3); 2464 else 2465 streamer.emitValue( 2466 MCBinaryExpr::createOr( 2467 OffsetExpr, MCConstantExpr::create(row3, context), context), 2468 4); 2469 } 2470 } 2471 2472 // Emit prolog unwind instructions (in reverse order). 2473 uint8_t numInst = info->Instructions.size(); 2474 for (uint8_t c = 0; c < numInst; ++c) { 2475 WinEH::Instruction inst = info->Instructions.back(); 2476 info->Instructions.pop_back(); 2477 ARMEmitUnwindCode(streamer, inst); 2478 } 2479 2480 // Emit epilog unwind instructions 2481 for (auto &I : info->EpilogMap) { 2482 auto &EpilogInstrs = I.second.Instructions; 2483 for (const WinEH::Instruction &inst : EpilogInstrs) 2484 ARMEmitUnwindCode(streamer, inst); 2485 } 2486 2487 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes; 2488 assert(BytesMod >= 0); 2489 for (int i = 0; i < BytesMod; i++) 2490 streamer.emitInt8(0xFB); 2491 2492 if (info->HandlesExceptions) 2493 streamer.emitValue( 2494 MCSymbolRefExpr::create(info->ExceptionHandler, 2495 MCSymbolRefExpr::VK_COFF_IMGREL32, context), 2496 4); 2497 } 2498 2499 static void ARM64EmitRuntimeFunction(MCStreamer &streamer, 2500 const WinEH::FrameInfo *info) { 2501 MCContext &context = streamer.getContext(); 2502 2503 streamer.emitValueToAlignment(Align(4)); 2504 for (const auto &S : info->Segments) { 2505 EmitSymbolRefWithOfs(streamer, info->Begin, S.Offset); 2506 if (info->PackedInfo) 2507 streamer.emitInt32(info->PackedInfo); 2508 else 2509 streamer.emitValue( 2510 MCSymbolRefExpr::create(S.Symbol, MCSymbolRefExpr::VK_COFF_IMGREL32, 2511 context), 2512 4); 2513 } 2514 } 2515 2516 2517 static void ARMEmitRuntimeFunction(MCStreamer &streamer, 2518 const WinEH::FrameInfo *info) { 2519 MCContext &context = streamer.getContext(); 2520 2521 streamer.emitValueToAlignment(Align(4)); 2522 EmitSymbolRefWithOfs(streamer, info->Begin, info->Begin); 2523 if (info->PackedInfo) 2524 streamer.emitInt32(info->PackedInfo); 2525 else 2526 streamer.emitValue( 2527 MCSymbolRefExpr::create(info->Symbol, MCSymbolRefExpr::VK_COFF_IMGREL32, 2528 context), 2529 4); 2530 } 2531 2532 void llvm::Win64EH::ARM64UnwindEmitter::Emit(MCStreamer &Streamer) const { 2533 // Emit the unwind info structs first. 2534 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2535 WinEH::FrameInfo *Info = CFI.get(); 2536 if (Info->empty()) 2537 continue; 2538 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 2539 Streamer.switchSection(XData); 2540 ARM64EmitUnwindInfo(Streamer, Info); 2541 } 2542 2543 // Now emit RUNTIME_FUNCTION entries. 2544 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2545 WinEH::FrameInfo *Info = CFI.get(); 2546 // ARM64EmitUnwindInfo above clears the info struct, so we can't check 2547 // empty here. But if a Symbol is set, we should create the corresponding 2548 // pdata entry. 2549 if (!Info->Symbol) 2550 continue; 2551 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 2552 Streamer.switchSection(PData); 2553 ARM64EmitRuntimeFunction(Streamer, Info); 2554 } 2555 } 2556 2557 void llvm::Win64EH::ARM64UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 2558 WinEH::FrameInfo *info, 2559 bool HandlerData) const { 2560 // Called if there's an .seh_handlerdata directive before the end of the 2561 // function. This forces writing the xdata record already here - and 2562 // in this case, the function isn't actually ended already, but the xdata 2563 // record needs to know the function length. In these cases, if the funclet 2564 // end hasn't been marked yet, the xdata function length won't cover the 2565 // whole function, only up to this point. 2566 if (!info->FuncletOrFuncEnd) { 2567 Streamer.switchSection(info->TextSection); 2568 info->FuncletOrFuncEnd = Streamer.emitCFILabel(); 2569 } 2570 // Switch sections (the static function above is meant to be called from 2571 // here and from Emit(). 2572 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 2573 Streamer.switchSection(XData); 2574 ARM64EmitUnwindInfo(Streamer, info, /* TryPacked = */ !HandlerData); 2575 } 2576 2577 void llvm::Win64EH::ARMUnwindEmitter::Emit(MCStreamer &Streamer) const { 2578 // Emit the unwind info structs first. 2579 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2580 WinEH::FrameInfo *Info = CFI.get(); 2581 if (Info->empty()) 2582 continue; 2583 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 2584 Streamer.switchSection(XData); 2585 ARMEmitUnwindInfo(Streamer, Info); 2586 } 2587 2588 // Now emit RUNTIME_FUNCTION entries. 2589 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2590 WinEH::FrameInfo *Info = CFI.get(); 2591 // ARMEmitUnwindInfo above clears the info struct, so we can't check 2592 // empty here. But if a Symbol is set, we should create the corresponding 2593 // pdata entry. 2594 if (!Info->Symbol) 2595 continue; 2596 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 2597 Streamer.switchSection(PData); 2598 ARMEmitRuntimeFunction(Streamer, Info); 2599 } 2600 } 2601 2602 void llvm::Win64EH::ARMUnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 2603 WinEH::FrameInfo *info, 2604 bool HandlerData) const { 2605 // Called if there's an .seh_handlerdata directive before the end of the 2606 // function. This forces writing the xdata record already here - and 2607 // in this case, the function isn't actually ended already, but the xdata 2608 // record needs to know the function length. In these cases, if the funclet 2609 // end hasn't been marked yet, the xdata function length won't cover the 2610 // whole function, only up to this point. 2611 if (!info->FuncletOrFuncEnd) { 2612 Streamer.switchSection(info->TextSection); 2613 info->FuncletOrFuncEnd = Streamer.emitCFILabel(); 2614 } 2615 // Switch sections (the static function above is meant to be called from 2616 // here and from Emit(). 2617 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 2618 Streamer.switchSection(XData); 2619 ARMEmitUnwindInfo(Streamer, info, /* TryPacked = */ !HandlerData); 2620 } 2621