1 //===- lib/MC/MCWin64EH.cpp - MCWin64EH implementation --------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/MC/MCWin64EH.h" 10 #include "llvm/ADT/Optional.h" 11 #include "llvm/ADT/Twine.h" 12 #include "llvm/MC/MCContext.h" 13 #include "llvm/MC/MCExpr.h" 14 #include "llvm/MC/MCObjectStreamer.h" 15 #include "llvm/MC/MCStreamer.h" 16 #include "llvm/MC/MCSymbol.h" 17 #include "llvm/Support/Win64EH.h" 18 namespace llvm { 19 class MCSection; 20 } 21 22 using namespace llvm; 23 24 // NOTE: All relocations generated here are 4-byte image-relative. 25 26 static uint8_t CountOfUnwindCodes(std::vector<WinEH::Instruction> &Insns) { 27 uint8_t Count = 0; 28 for (const auto &I : Insns) { 29 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 30 default: 31 llvm_unreachable("Unsupported unwind code"); 32 case Win64EH::UOP_PushNonVol: 33 case Win64EH::UOP_AllocSmall: 34 case Win64EH::UOP_SetFPReg: 35 case Win64EH::UOP_PushMachFrame: 36 Count += 1; 37 break; 38 case Win64EH::UOP_SaveNonVol: 39 case Win64EH::UOP_SaveXMM128: 40 Count += 2; 41 break; 42 case Win64EH::UOP_SaveNonVolBig: 43 case Win64EH::UOP_SaveXMM128Big: 44 Count += 3; 45 break; 46 case Win64EH::UOP_AllocLarge: 47 Count += (I.Offset > 512 * 1024 - 8) ? 3 : 2; 48 break; 49 } 50 } 51 return Count; 52 } 53 54 static void EmitAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS, 55 const MCSymbol *RHS) { 56 MCContext &Context = Streamer.getContext(); 57 const MCExpr *Diff = 58 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 59 MCSymbolRefExpr::create(RHS, Context), Context); 60 Streamer.emitValue(Diff, 1); 61 } 62 63 static void EmitUnwindCode(MCStreamer &streamer, const MCSymbol *begin, 64 WinEH::Instruction &inst) { 65 uint8_t b2; 66 uint16_t w; 67 b2 = (inst.Operation & 0x0F); 68 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 69 default: 70 llvm_unreachable("Unsupported unwind code"); 71 case Win64EH::UOP_PushNonVol: 72 EmitAbsDifference(streamer, inst.Label, begin); 73 b2 |= (inst.Register & 0x0F) << 4; 74 streamer.emitInt8(b2); 75 break; 76 case Win64EH::UOP_AllocLarge: 77 EmitAbsDifference(streamer, inst.Label, begin); 78 if (inst.Offset > 512 * 1024 - 8) { 79 b2 |= 0x10; 80 streamer.emitInt8(b2); 81 w = inst.Offset & 0xFFF8; 82 streamer.emitInt16(w); 83 w = inst.Offset >> 16; 84 } else { 85 streamer.emitInt8(b2); 86 w = inst.Offset >> 3; 87 } 88 streamer.emitInt16(w); 89 break; 90 case Win64EH::UOP_AllocSmall: 91 b2 |= (((inst.Offset - 8) >> 3) & 0x0F) << 4; 92 EmitAbsDifference(streamer, inst.Label, begin); 93 streamer.emitInt8(b2); 94 break; 95 case Win64EH::UOP_SetFPReg: 96 EmitAbsDifference(streamer, inst.Label, begin); 97 streamer.emitInt8(b2); 98 break; 99 case Win64EH::UOP_SaveNonVol: 100 case Win64EH::UOP_SaveXMM128: 101 b2 |= (inst.Register & 0x0F) << 4; 102 EmitAbsDifference(streamer, inst.Label, begin); 103 streamer.emitInt8(b2); 104 w = inst.Offset >> 3; 105 if (inst.Operation == Win64EH::UOP_SaveXMM128) 106 w >>= 1; 107 streamer.emitInt16(w); 108 break; 109 case Win64EH::UOP_SaveNonVolBig: 110 case Win64EH::UOP_SaveXMM128Big: 111 b2 |= (inst.Register & 0x0F) << 4; 112 EmitAbsDifference(streamer, inst.Label, begin); 113 streamer.emitInt8(b2); 114 if (inst.Operation == Win64EH::UOP_SaveXMM128Big) 115 w = inst.Offset & 0xFFF0; 116 else 117 w = inst.Offset & 0xFFF8; 118 streamer.emitInt16(w); 119 w = inst.Offset >> 16; 120 streamer.emitInt16(w); 121 break; 122 case Win64EH::UOP_PushMachFrame: 123 if (inst.Offset == 1) 124 b2 |= 0x10; 125 EmitAbsDifference(streamer, inst.Label, begin); 126 streamer.emitInt8(b2); 127 break; 128 } 129 } 130 131 static void EmitSymbolRefWithOfs(MCStreamer &streamer, 132 const MCSymbol *Base, 133 const MCSymbol *Other) { 134 MCContext &Context = streamer.getContext(); 135 const MCSymbolRefExpr *BaseRef = MCSymbolRefExpr::create(Base, Context); 136 const MCSymbolRefExpr *OtherRef = MCSymbolRefExpr::create(Other, Context); 137 const MCExpr *Ofs = MCBinaryExpr::createSub(OtherRef, BaseRef, Context); 138 const MCSymbolRefExpr *BaseRefRel = MCSymbolRefExpr::create(Base, 139 MCSymbolRefExpr::VK_COFF_IMGREL32, 140 Context); 141 streamer.emitValue(MCBinaryExpr::createAdd(BaseRefRel, Ofs, Context), 4); 142 } 143 144 static void EmitRuntimeFunction(MCStreamer &streamer, 145 const WinEH::FrameInfo *info) { 146 MCContext &context = streamer.getContext(); 147 148 streamer.emitValueToAlignment(4); 149 EmitSymbolRefWithOfs(streamer, info->Begin, info->Begin); 150 EmitSymbolRefWithOfs(streamer, info->Begin, info->End); 151 streamer.emitValue(MCSymbolRefExpr::create(info->Symbol, 152 MCSymbolRefExpr::VK_COFF_IMGREL32, 153 context), 4); 154 } 155 156 static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) { 157 // If this UNWIND_INFO already has a symbol, it's already been emitted. 158 if (info->Symbol) 159 return; 160 161 MCContext &context = streamer.getContext(); 162 MCSymbol *Label = context.createTempSymbol(); 163 164 streamer.emitValueToAlignment(4); 165 streamer.emitLabel(Label); 166 info->Symbol = Label; 167 168 // Upper 3 bits are the version number (currently 1). 169 uint8_t flags = 0x01; 170 if (info->ChainedParent) 171 flags |= Win64EH::UNW_ChainInfo << 3; 172 else { 173 if (info->HandlesUnwind) 174 flags |= Win64EH::UNW_TerminateHandler << 3; 175 if (info->HandlesExceptions) 176 flags |= Win64EH::UNW_ExceptionHandler << 3; 177 } 178 streamer.emitInt8(flags); 179 180 if (info->PrologEnd) 181 EmitAbsDifference(streamer, info->PrologEnd, info->Begin); 182 else 183 streamer.emitInt8(0); 184 185 uint8_t numCodes = CountOfUnwindCodes(info->Instructions); 186 streamer.emitInt8(numCodes); 187 188 uint8_t frame = 0; 189 if (info->LastFrameInst >= 0) { 190 WinEH::Instruction &frameInst = info->Instructions[info->LastFrameInst]; 191 assert(frameInst.Operation == Win64EH::UOP_SetFPReg); 192 frame = (frameInst.Register & 0x0F) | (frameInst.Offset & 0xF0); 193 } 194 streamer.emitInt8(frame); 195 196 // Emit unwind instructions (in reverse order). 197 uint8_t numInst = info->Instructions.size(); 198 for (uint8_t c = 0; c < numInst; ++c) { 199 WinEH::Instruction inst = info->Instructions.back(); 200 info->Instructions.pop_back(); 201 EmitUnwindCode(streamer, info->Begin, inst); 202 } 203 204 // For alignment purposes, the instruction array will always have an even 205 // number of entries, with the final entry potentially unused (in which case 206 // the array will be one longer than indicated by the count of unwind codes 207 // field). 208 if (numCodes & 1) { 209 streamer.emitInt16(0); 210 } 211 212 if (flags & (Win64EH::UNW_ChainInfo << 3)) 213 EmitRuntimeFunction(streamer, info->ChainedParent); 214 else if (flags & 215 ((Win64EH::UNW_TerminateHandler|Win64EH::UNW_ExceptionHandler) << 3)) 216 streamer.emitValue(MCSymbolRefExpr::create(info->ExceptionHandler, 217 MCSymbolRefExpr::VK_COFF_IMGREL32, 218 context), 4); 219 else if (numCodes == 0) { 220 // The minimum size of an UNWIND_INFO struct is 8 bytes. If we're not 221 // a chained unwind info, if there is no handler, and if there are fewer 222 // than 2 slots used in the unwind code array, we have to pad to 8 bytes. 223 streamer.emitInt32(0); 224 } 225 } 226 227 void llvm::Win64EH::UnwindEmitter::Emit(MCStreamer &Streamer) const { 228 // Emit the unwind info structs first. 229 for (const auto &CFI : Streamer.getWinFrameInfos()) { 230 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 231 Streamer.switchSection(XData); 232 ::EmitUnwindInfo(Streamer, CFI.get()); 233 } 234 235 // Now emit RUNTIME_FUNCTION entries. 236 for (const auto &CFI : Streamer.getWinFrameInfos()) { 237 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 238 Streamer.switchSection(PData); 239 EmitRuntimeFunction(Streamer, CFI.get()); 240 } 241 } 242 243 void llvm::Win64EH::UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 244 WinEH::FrameInfo *info, 245 bool HandlerData) const { 246 // Switch sections (the static function above is meant to be called from 247 // here and from Emit(). 248 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 249 Streamer.switchSection(XData); 250 251 ::EmitUnwindInfo(Streamer, info); 252 } 253 254 static const MCExpr *GetSubDivExpr(MCStreamer &Streamer, const MCSymbol *LHS, 255 const MCSymbol *RHS, int Div) { 256 MCContext &Context = Streamer.getContext(); 257 const MCExpr *Expr = 258 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 259 MCSymbolRefExpr::create(RHS, Context), Context); 260 if (Div != 1) 261 Expr = MCBinaryExpr::createDiv(Expr, MCConstantExpr::create(Div, Context), 262 Context); 263 return Expr; 264 } 265 266 static Optional<int64_t> GetOptionalAbsDifference(MCStreamer &Streamer, 267 const MCSymbol *LHS, 268 const MCSymbol *RHS) { 269 MCContext &Context = Streamer.getContext(); 270 const MCExpr *Diff = 271 MCBinaryExpr::createSub(MCSymbolRefExpr::create(LHS, Context), 272 MCSymbolRefExpr::create(RHS, Context), Context); 273 MCObjectStreamer *OS = (MCObjectStreamer *)(&Streamer); 274 // It should normally be possible to calculate the length of a function 275 // at this point, but it might not be possible in the presence of certain 276 // unusual constructs, like an inline asm with an alignment directive. 277 int64_t value; 278 if (!Diff->evaluateAsAbsolute(value, OS->getAssembler())) 279 return None; 280 return value; 281 } 282 283 static int64_t GetAbsDifference(MCStreamer &Streamer, const MCSymbol *LHS, 284 const MCSymbol *RHS) { 285 Optional<int64_t> MaybeDiff = GetOptionalAbsDifference(Streamer, LHS, RHS); 286 if (!MaybeDiff) 287 report_fatal_error("Failed to evaluate function length in SEH unwind info"); 288 return *MaybeDiff; 289 } 290 291 static uint32_t ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) { 292 uint32_t Count = 0; 293 for (const auto &I : Insns) { 294 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 295 default: 296 llvm_unreachable("Unsupported ARM64 unwind code"); 297 case Win64EH::UOP_AllocSmall: 298 Count += 1; 299 break; 300 case Win64EH::UOP_AllocMedium: 301 Count += 2; 302 break; 303 case Win64EH::UOP_AllocLarge: 304 Count += 4; 305 break; 306 case Win64EH::UOP_SaveR19R20X: 307 Count += 1; 308 break; 309 case Win64EH::UOP_SaveFPLRX: 310 Count += 1; 311 break; 312 case Win64EH::UOP_SaveFPLR: 313 Count += 1; 314 break; 315 case Win64EH::UOP_SaveReg: 316 Count += 2; 317 break; 318 case Win64EH::UOP_SaveRegP: 319 Count += 2; 320 break; 321 case Win64EH::UOP_SaveRegPX: 322 Count += 2; 323 break; 324 case Win64EH::UOP_SaveRegX: 325 Count += 2; 326 break; 327 case Win64EH::UOP_SaveLRPair: 328 Count += 2; 329 break; 330 case Win64EH::UOP_SaveFReg: 331 Count += 2; 332 break; 333 case Win64EH::UOP_SaveFRegP: 334 Count += 2; 335 break; 336 case Win64EH::UOP_SaveFRegX: 337 Count += 2; 338 break; 339 case Win64EH::UOP_SaveFRegPX: 340 Count += 2; 341 break; 342 case Win64EH::UOP_SetFP: 343 Count += 1; 344 break; 345 case Win64EH::UOP_AddFP: 346 Count += 2; 347 break; 348 case Win64EH::UOP_Nop: 349 Count += 1; 350 break; 351 case Win64EH::UOP_End: 352 Count += 1; 353 break; 354 case Win64EH::UOP_SaveNext: 355 Count += 1; 356 break; 357 case Win64EH::UOP_TrapFrame: 358 Count += 1; 359 break; 360 case Win64EH::UOP_PushMachFrame: 361 Count += 1; 362 break; 363 case Win64EH::UOP_Context: 364 Count += 1; 365 break; 366 case Win64EH::UOP_ClearUnwoundToCall: 367 Count += 1; 368 break; 369 } 370 } 371 return Count; 372 } 373 374 // Unwind opcode encodings and restrictions are documented at 375 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling 376 static void ARM64EmitUnwindCode(MCStreamer &streamer, 377 const WinEH::Instruction &inst) { 378 uint8_t b, reg; 379 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 380 default: 381 llvm_unreachable("Unsupported ARM64 unwind code"); 382 case Win64EH::UOP_AllocSmall: 383 b = (inst.Offset >> 4) & 0x1F; 384 streamer.emitInt8(b); 385 break; 386 case Win64EH::UOP_AllocMedium: { 387 uint16_t hw = (inst.Offset >> 4) & 0x7FF; 388 b = 0xC0; 389 b |= (hw >> 8); 390 streamer.emitInt8(b); 391 b = hw & 0xFF; 392 streamer.emitInt8(b); 393 break; 394 } 395 case Win64EH::UOP_AllocLarge: { 396 uint32_t w; 397 b = 0xE0; 398 streamer.emitInt8(b); 399 w = inst.Offset >> 4; 400 b = (w & 0x00FF0000) >> 16; 401 streamer.emitInt8(b); 402 b = (w & 0x0000FF00) >> 8; 403 streamer.emitInt8(b); 404 b = w & 0x000000FF; 405 streamer.emitInt8(b); 406 break; 407 } 408 case Win64EH::UOP_SetFP: 409 b = 0xE1; 410 streamer.emitInt8(b); 411 break; 412 case Win64EH::UOP_AddFP: 413 b = 0xE2; 414 streamer.emitInt8(b); 415 b = (inst.Offset >> 3); 416 streamer.emitInt8(b); 417 break; 418 case Win64EH::UOP_Nop: 419 b = 0xE3; 420 streamer.emitInt8(b); 421 break; 422 case Win64EH::UOP_SaveR19R20X: 423 b = 0x20; 424 b |= (inst.Offset >> 3) & 0x1F; 425 streamer.emitInt8(b); 426 break; 427 case Win64EH::UOP_SaveFPLRX: 428 b = 0x80; 429 b |= ((inst.Offset - 1) >> 3) & 0x3F; 430 streamer.emitInt8(b); 431 break; 432 case Win64EH::UOP_SaveFPLR: 433 b = 0x40; 434 b |= (inst.Offset >> 3) & 0x3F; 435 streamer.emitInt8(b); 436 break; 437 case Win64EH::UOP_SaveReg: 438 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 439 reg = inst.Register - 19; 440 b = 0xD0 | ((reg & 0xC) >> 2); 441 streamer.emitInt8(b); 442 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 443 streamer.emitInt8(b); 444 break; 445 case Win64EH::UOP_SaveRegX: 446 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 447 reg = inst.Register - 19; 448 b = 0xD4 | ((reg & 0x8) >> 3); 449 streamer.emitInt8(b); 450 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1); 451 streamer.emitInt8(b); 452 break; 453 case Win64EH::UOP_SaveRegP: 454 assert(inst.Register >= 19 && "Saved registers must be >= 19"); 455 reg = inst.Register - 19; 456 b = 0xC8 | ((reg & 0xC) >> 2); 457 streamer.emitInt8(b); 458 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 459 streamer.emitInt8(b); 460 break; 461 case Win64EH::UOP_SaveRegPX: 462 assert(inst.Register >= 19 && "Saved registers must be >= 19"); 463 reg = inst.Register - 19; 464 b = 0xCC | ((reg & 0xC) >> 2); 465 streamer.emitInt8(b); 466 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1); 467 streamer.emitInt8(b); 468 break; 469 case Win64EH::UOP_SaveLRPair: 470 assert(inst.Register >= 19 && "Saved reg must be >= 19"); 471 reg = inst.Register - 19; 472 assert((reg % 2) == 0 && "Saved reg must be 19+2*X"); 473 reg /= 2; 474 b = 0xD6 | ((reg & 0x7) >> 2); 475 streamer.emitInt8(b); 476 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 477 streamer.emitInt8(b); 478 break; 479 case Win64EH::UOP_SaveFReg: 480 assert(inst.Register >= 8 && "Saved dreg must be >= 8"); 481 reg = inst.Register - 8; 482 b = 0xDC | ((reg & 0x4) >> 2); 483 streamer.emitInt8(b); 484 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 485 streamer.emitInt8(b); 486 break; 487 case Win64EH::UOP_SaveFRegX: 488 assert(inst.Register >= 8 && "Saved dreg must be >= 8"); 489 reg = inst.Register - 8; 490 b = 0xDE; 491 streamer.emitInt8(b); 492 b = ((reg & 0x7) << 5) | ((inst.Offset >> 3) - 1); 493 streamer.emitInt8(b); 494 break; 495 case Win64EH::UOP_SaveFRegP: 496 assert(inst.Register >= 8 && "Saved dregs must be >= 8"); 497 reg = inst.Register - 8; 498 b = 0xD8 | ((reg & 0x4) >> 2); 499 streamer.emitInt8(b); 500 b = ((reg & 0x3) << 6) | (inst.Offset >> 3); 501 streamer.emitInt8(b); 502 break; 503 case Win64EH::UOP_SaveFRegPX: 504 assert(inst.Register >= 8 && "Saved dregs must be >= 8"); 505 reg = inst.Register - 8; 506 b = 0xDA | ((reg & 0x4) >> 2); 507 streamer.emitInt8(b); 508 b = ((reg & 0x3) << 6) | ((inst.Offset >> 3) - 1); 509 streamer.emitInt8(b); 510 break; 511 case Win64EH::UOP_End: 512 b = 0xE4; 513 streamer.emitInt8(b); 514 break; 515 case Win64EH::UOP_SaveNext: 516 b = 0xE6; 517 streamer.emitInt8(b); 518 break; 519 case Win64EH::UOP_TrapFrame: 520 b = 0xE8; 521 streamer.emitInt8(b); 522 break; 523 case Win64EH::UOP_PushMachFrame: 524 b = 0xE9; 525 streamer.emitInt8(b); 526 break; 527 case Win64EH::UOP_Context: 528 b = 0xEA; 529 streamer.emitInt8(b); 530 break; 531 case Win64EH::UOP_ClearUnwoundToCall: 532 b = 0xEC; 533 streamer.emitInt8(b); 534 break; 535 } 536 } 537 538 // Returns the epilog symbol of an epilog with the exact same unwind code 539 // sequence, if it exists. Otherwise, returns nullptr. 540 // EpilogInstrs - Unwind codes for the current epilog. 541 // Epilogs - Epilogs that potentialy match the current epilog. 542 static MCSymbol* 543 FindMatchingEpilog(const std::vector<WinEH::Instruction>& EpilogInstrs, 544 const std::vector<MCSymbol *>& Epilogs, 545 const WinEH::FrameInfo *info) { 546 for (auto *EpilogStart : Epilogs) { 547 auto InstrsIter = info->EpilogMap.find(EpilogStart); 548 assert(InstrsIter != info->EpilogMap.end() && 549 "Epilog not found in EpilogMap"); 550 const auto &Instrs = InstrsIter->second.Instructions; 551 552 if (Instrs.size() != EpilogInstrs.size()) 553 continue; 554 555 bool Match = true; 556 for (unsigned i = 0; i < Instrs.size(); ++i) 557 if (Instrs[i] != EpilogInstrs[i]) { 558 Match = false; 559 break; 560 } 561 562 if (Match) 563 return EpilogStart; 564 } 565 return nullptr; 566 } 567 568 static void simplifyARM64Opcodes(std::vector<WinEH::Instruction> &Instructions, 569 bool Reverse) { 570 unsigned PrevOffset = -1; 571 unsigned PrevRegister = -1; 572 573 auto VisitInstruction = [&](WinEH::Instruction &Inst) { 574 // Convert 2-byte opcodes into equivalent 1-byte ones. 575 if (Inst.Operation == Win64EH::UOP_SaveRegP && Inst.Register == 29) { 576 Inst.Operation = Win64EH::UOP_SaveFPLR; 577 Inst.Register = -1; 578 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX && 579 Inst.Register == 29) { 580 Inst.Operation = Win64EH::UOP_SaveFPLRX; 581 Inst.Register = -1; 582 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX && 583 Inst.Register == 19 && Inst.Offset <= 248) { 584 Inst.Operation = Win64EH::UOP_SaveR19R20X; 585 Inst.Register = -1; 586 } else if (Inst.Operation == Win64EH::UOP_AddFP && Inst.Offset == 0) { 587 Inst.Operation = Win64EH::UOP_SetFP; 588 } else if (Inst.Operation == Win64EH::UOP_SaveRegP && 589 Inst.Register == PrevRegister + 2 && 590 Inst.Offset == PrevOffset + 16) { 591 Inst.Operation = Win64EH::UOP_SaveNext; 592 Inst.Register = -1; 593 Inst.Offset = 0; 594 // Intentionally not creating UOP_SaveNext for float register pairs, 595 // as current versions of Windows (up to at least 20.04) is buggy 596 // regarding SaveNext for float pairs. 597 } 598 // Update info about the previous instruction, for detecting if 599 // the next one can be made a UOP_SaveNext 600 if (Inst.Operation == Win64EH::UOP_SaveR19R20X) { 601 PrevOffset = 0; 602 PrevRegister = 19; 603 } else if (Inst.Operation == Win64EH::UOP_SaveRegPX) { 604 PrevOffset = 0; 605 PrevRegister = Inst.Register; 606 } else if (Inst.Operation == Win64EH::UOP_SaveRegP) { 607 PrevOffset = Inst.Offset; 608 PrevRegister = Inst.Register; 609 } else if (Inst.Operation == Win64EH::UOP_SaveNext) { 610 PrevRegister += 2; 611 PrevOffset += 16; 612 } else { 613 PrevRegister = -1; 614 PrevOffset = -1; 615 } 616 }; 617 618 // Iterate over instructions in a forward order (for prologues), 619 // backwards for epilogues (i.e. always reverse compared to how the 620 // opcodes are stored). 621 if (Reverse) { 622 for (auto It = Instructions.rbegin(); It != Instructions.rend(); It++) 623 VisitInstruction(*It); 624 } else { 625 for (WinEH::Instruction &Inst : Instructions) 626 VisitInstruction(Inst); 627 } 628 } 629 630 // Check if an epilog exists as a subset of the end of a prolog (backwards). 631 static int 632 getARM64OffsetInProlog(const std::vector<WinEH::Instruction> &Prolog, 633 const std::vector<WinEH::Instruction> &Epilog) { 634 // Can't find an epilog as a subset if it is longer than the prolog. 635 if (Epilog.size() > Prolog.size()) 636 return -1; 637 638 // Check that the epilog actually is a perfect match for the end (backwrds) 639 // of the prolog. 640 for (int I = Epilog.size() - 1; I >= 0; I--) { 641 if (Prolog[I] != Epilog[Epilog.size() - 1 - I]) 642 return -1; 643 } 644 645 // If the epilog was a subset of the prolog, find its offset. 646 if (Epilog.size() == Prolog.size()) 647 return 0; 648 return ARM64CountOfUnwindCodes(ArrayRef<WinEH::Instruction>( 649 &Prolog[Epilog.size()], Prolog.size() - Epilog.size())); 650 } 651 652 static int checkARM64PackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info, 653 int PrologCodeBytes) { 654 // Can only pack if there's one single epilog 655 if (info->EpilogMap.size() != 1) 656 return -1; 657 658 const std::vector<WinEH::Instruction> &Epilog = 659 info->EpilogMap.begin()->second.Instructions; 660 661 // Check that the epilog actually is at the very end of the function, 662 // otherwise it can't be packed. 663 uint32_t DistanceFromEnd = (uint32_t)GetAbsDifference( 664 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first); 665 if (DistanceFromEnd / 4 != Epilog.size()) 666 return -1; 667 668 int RetVal = -1; 669 // Even if we don't end up sharing opcodes with the prolog, we can still 670 // write the offset as a packed offset, if the single epilog is located at 671 // the end of the function and the offset (pointing after the prolog) fits 672 // as a packed offset. 673 if (PrologCodeBytes <= 31 && 674 PrologCodeBytes + ARM64CountOfUnwindCodes(Epilog) <= 124) 675 RetVal = PrologCodeBytes; 676 677 int Offset = getARM64OffsetInProlog(info->Instructions, Epilog); 678 if (Offset < 0) 679 return RetVal; 680 681 // Check that the offset and prolog size fits in the first word; it's 682 // unclear whether the epilog count in the extension word can be taken 683 // as packed epilog offset. 684 if (Offset > 31 || PrologCodeBytes > 124) 685 return RetVal; 686 687 // As we choose to express the epilog as part of the prolog, remove the 688 // epilog from the map, so we don't try to emit its opcodes. 689 info->EpilogMap.clear(); 690 return Offset; 691 } 692 693 static bool tryARM64PackedUnwind(WinEH::FrameInfo *info, uint32_t FuncLength, 694 int PackedEpilogOffset) { 695 if (PackedEpilogOffset == 0) { 696 // Fully symmetric prolog and epilog, should be ok for packed format. 697 // For CR=3, the corresponding synthesized epilog actually lacks the 698 // SetFP opcode, but unwinding should work just fine despite that 699 // (if at the SetFP opcode, the unwinder considers it as part of the 700 // function body and just unwinds the full prolog instead). 701 } else if (PackedEpilogOffset == 1) { 702 // One single case of differences between prolog and epilog is allowed: 703 // The epilog can lack a single SetFP that is the last opcode in the 704 // prolog, for the CR=3 case. 705 if (info->Instructions.back().Operation != Win64EH::UOP_SetFP) 706 return false; 707 } else { 708 // Too much difference between prolog and epilog. 709 return false; 710 } 711 unsigned RegI = 0, RegF = 0; 712 int Predecrement = 0; 713 enum { 714 Start, 715 Start2, 716 IntRegs, 717 FloatRegs, 718 InputArgs, 719 StackAdjust, 720 FrameRecord, 721 End 722 } Location = Start; 723 bool StandaloneLR = false, FPLRPair = false; 724 int StackOffset = 0; 725 int Nops = 0; 726 // Iterate over the prolog and check that all opcodes exactly match 727 // the canonical order and form. A more lax check could verify that 728 // all saved registers are in the expected locations, but not enforce 729 // the order - that would work fine when unwinding from within 730 // functions, but not be exactly right if unwinding happens within 731 // prologs/epilogs. 732 for (const WinEH::Instruction &Inst : info->Instructions) { 733 switch (Inst.Operation) { 734 case Win64EH::UOP_End: 735 if (Location != Start) 736 return false; 737 Location = Start2; 738 break; 739 case Win64EH::UOP_SaveR19R20X: 740 if (Location != Start2) 741 return false; 742 Predecrement = Inst.Offset; 743 RegI = 2; 744 Location = IntRegs; 745 break; 746 case Win64EH::UOP_SaveRegX: 747 if (Location != Start2) 748 return false; 749 Predecrement = Inst.Offset; 750 if (Inst.Register == 19) 751 RegI += 1; 752 else if (Inst.Register == 30) 753 StandaloneLR = true; 754 else 755 return false; 756 // Odd register; can't be any further int registers. 757 Location = FloatRegs; 758 break; 759 case Win64EH::UOP_SaveRegPX: 760 // Can't have this in a canonical prologue. Either this has been 761 // canonicalized into SaveR19R20X or SaveFPLRX, or it's an unsupported 762 // register pair. 763 // It can't be canonicalized into SaveR19R20X if the offset is 764 // larger than 248 bytes, but even with the maximum case with 765 // RegI=10/RegF=8/CR=1/H=1, we end up with SavSZ = 216, which should 766 // fit into SaveR19R20X. 767 // The unwinding opcodes can't describe the otherwise seemingly valid 768 // case for RegI=1 CR=1, that would start with a 769 // "stp x19, lr, [sp, #-...]!" as that fits neither SaveRegPX nor 770 // SaveLRPair. 771 return false; 772 case Win64EH::UOP_SaveRegP: 773 if (Location != IntRegs || Inst.Offset != 8 * RegI || 774 Inst.Register != 19 + RegI) 775 return false; 776 RegI += 2; 777 break; 778 case Win64EH::UOP_SaveReg: 779 if (Location != IntRegs || Inst.Offset != 8 * RegI) 780 return false; 781 if (Inst.Register == 19 + RegI) 782 RegI += 1; 783 else if (Inst.Register == 30) 784 StandaloneLR = true; 785 else 786 return false; 787 // Odd register; can't be any further int registers. 788 Location = FloatRegs; 789 break; 790 case Win64EH::UOP_SaveLRPair: 791 if (Location != IntRegs || Inst.Offset != 8 * RegI || 792 Inst.Register != 19 + RegI) 793 return false; 794 RegI += 1; 795 StandaloneLR = true; 796 Location = FloatRegs; 797 break; 798 case Win64EH::UOP_SaveFRegX: 799 // Packed unwind can't handle prologs that only save one single 800 // float register. 801 return false; 802 case Win64EH::UOP_SaveFReg: 803 if (Location != FloatRegs || RegF == 0 || Inst.Register != 8 + RegF || 804 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF)) 805 return false; 806 RegF += 1; 807 Location = InputArgs; 808 break; 809 case Win64EH::UOP_SaveFRegPX: 810 if (Location != Start2 || Inst.Register != 8) 811 return false; 812 Predecrement = Inst.Offset; 813 RegF = 2; 814 Location = FloatRegs; 815 break; 816 case Win64EH::UOP_SaveFRegP: 817 if ((Location != IntRegs && Location != FloatRegs) || 818 Inst.Register != 8 + RegF || 819 Inst.Offset != 8 * (RegI + (StandaloneLR ? 1 : 0) + RegF)) 820 return false; 821 RegF += 2; 822 Location = FloatRegs; 823 break; 824 case Win64EH::UOP_SaveNext: 825 if (Location == IntRegs) 826 RegI += 2; 827 else if (Location == FloatRegs) 828 RegF += 2; 829 else 830 return false; 831 break; 832 case Win64EH::UOP_Nop: 833 if (Location != IntRegs && Location != FloatRegs && Location != InputArgs) 834 return false; 835 Location = InputArgs; 836 Nops++; 837 break; 838 case Win64EH::UOP_AllocSmall: 839 case Win64EH::UOP_AllocMedium: 840 if (Location != Start2 && Location != IntRegs && Location != FloatRegs && 841 Location != InputArgs && Location != StackAdjust) 842 return false; 843 // Can have either a single decrement, or a pair of decrements with 844 // 4080 and another decrement. 845 if (StackOffset == 0) 846 StackOffset = Inst.Offset; 847 else if (StackOffset != 4080) 848 return false; 849 else 850 StackOffset += Inst.Offset; 851 Location = StackAdjust; 852 break; 853 case Win64EH::UOP_SaveFPLRX: 854 // Not allowing FPLRX after StackAdjust; if a StackAdjust is used, it 855 // should be followed by a FPLR instead. 856 if (Location != Start2 && Location != IntRegs && Location != FloatRegs && 857 Location != InputArgs) 858 return false; 859 StackOffset = Inst.Offset; 860 Location = FrameRecord; 861 FPLRPair = true; 862 break; 863 case Win64EH::UOP_SaveFPLR: 864 // This can only follow after a StackAdjust 865 if (Location != StackAdjust || Inst.Offset != 0) 866 return false; 867 Location = FrameRecord; 868 FPLRPair = true; 869 break; 870 case Win64EH::UOP_SetFP: 871 if (Location != FrameRecord) 872 return false; 873 Location = End; 874 break; 875 } 876 } 877 if (RegI > 10 || RegF > 8) 878 return false; 879 if (StandaloneLR && FPLRPair) 880 return false; 881 if (FPLRPair && Location != End) 882 return false; 883 if (Nops != 0 && Nops != 4) 884 return false; 885 int H = Nops == 4; 886 // There's an inconsistency regarding packed unwind info with homed 887 // parameters; according to the documentation, the epilog shouldn't have 888 // the same corresponding nops (and thus, to set the H bit, we should 889 // require an epilog which isn't exactly symmetrical - we shouldn't accept 890 // an exact mirrored epilog for those cases), but in practice, 891 // RtlVirtualUnwind behaves as if it does expect the epilogue to contain 892 // the same nops. See https://github.com/llvm/llvm-project/issues/54879. 893 // To play it safe, don't produce packed unwind info with homed parameters. 894 if (H) 895 return false; 896 int IntSZ = 8 * RegI; 897 if (StandaloneLR) 898 IntSZ += 8; 899 int FpSZ = 8 * RegF; // RegF not yet decremented 900 int SavSZ = (IntSZ + FpSZ + 8 * 8 * H + 0xF) & ~0xF; 901 if (Predecrement != SavSZ) 902 return false; 903 if (FPLRPair && StackOffset < 16) 904 return false; 905 if (StackOffset % 16) 906 return false; 907 uint32_t FrameSize = (StackOffset + SavSZ) / 16; 908 if (FrameSize > 0x1FF) 909 return false; 910 assert(RegF != 1 && "One single float reg not allowed"); 911 if (RegF > 0) 912 RegF--; // Convert from actual number of registers, to value stored 913 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier"); 914 int Flag = 0x01; // Function segments not supported yet 915 int CR = FPLRPair ? 3 : StandaloneLR ? 1 : 0; 916 info->PackedInfo |= Flag << 0; 917 info->PackedInfo |= (FuncLength & 0x7FF) << 2; 918 info->PackedInfo |= (RegF & 0x7) << 13; 919 info->PackedInfo |= (RegI & 0xF) << 16; 920 info->PackedInfo |= (H & 0x1) << 20; 921 info->PackedInfo |= (CR & 0x3) << 21; 922 info->PackedInfo |= (FrameSize & 0x1FF) << 23; 923 return true; 924 } 925 926 // Populate the .xdata section. The format of .xdata on ARM64 is documented at 927 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling 928 static void ARM64EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info, 929 bool TryPacked = true) { 930 // If this UNWIND_INFO already has a symbol, it's already been emitted. 931 if (info->Symbol) 932 return; 933 // If there's no unwind info here (not even a terminating UOP_End), the 934 // unwind info is considered bogus and skipped. If this was done in 935 // response to an explicit .seh_handlerdata, the associated trailing 936 // handler data is left orphaned in the xdata section. 937 if (info->empty()) { 938 info->EmitAttempted = true; 939 return; 940 } 941 if (info->EmitAttempted) { 942 // If we tried to emit unwind info before (due to an explicit 943 // .seh_handlerdata directive), but skipped it (because there was no 944 // valid information to emit at the time), and it later got valid unwind 945 // opcodes, we can't emit it here, because the trailing handler data 946 // was already emitted elsewhere in the xdata section. 947 streamer.getContext().reportError( 948 SMLoc(), "Earlier .seh_handlerdata for " + info->Function->getName() + 949 " skipped due to no unwind info at the time " 950 "(.seh_handlerdata too early?), but the function later " 951 "did get unwind info that can't be emitted"); 952 return; 953 } 954 955 simplifyARM64Opcodes(info->Instructions, false); 956 for (auto &I : info->EpilogMap) 957 simplifyARM64Opcodes(I.second.Instructions, true); 958 959 MCContext &context = streamer.getContext(); 960 MCSymbol *Label = context.createTempSymbol(); 961 962 streamer.emitValueToAlignment(4); 963 streamer.emitLabel(Label); 964 info->Symbol = Label; 965 966 int64_t RawFuncLength; 967 if (!info->FuncletOrFuncEnd) { 968 report_fatal_error("FuncletOrFuncEnd not set"); 969 } else { 970 // FIXME: GetAbsDifference tries to compute the length of the function 971 // immediately, before the whole file is emitted, but in general 972 // that's impossible: the size in bytes of certain assembler directives 973 // like .align and .fill is not known until the whole file is parsed and 974 // relaxations are applied. Currently, GetAbsDifference fails with a fatal 975 // error in that case. (We mostly don't hit this because inline assembly 976 // specifying those directives is rare, and we don't normally try to 977 // align loops on AArch64.) 978 // 979 // There are two potential approaches to delaying the computation. One, 980 // we could emit something like ".word (endfunc-beginfunc)/4+0x10800000", 981 // as long as we have some conservative estimate we could use to prove 982 // that we don't need to split the unwind data. Emitting the constant 983 // is straightforward, but there's no existing code for estimating the 984 // size of the function. 985 // 986 // The other approach would be to use a dedicated, relaxable fragment, 987 // which could grow to accommodate splitting the unwind data if 988 // necessary. This is more straightforward, since it automatically works 989 // without any new infrastructure, and it's consistent with how we handle 990 // relaxation in other contexts. But it would require some refactoring 991 // to move parts of the pdata/xdata emission into the implementation of 992 // a fragment. We could probably continue to encode the unwind codes 993 // here, but we'd have to emit the pdata, the xdata header, and the 994 // epilogue scopes later, since they depend on whether the we need to 995 // split the unwind data. 996 RawFuncLength = GetAbsDifference(streamer, info->FuncletOrFuncEnd, 997 info->Begin); 998 } 999 if (RawFuncLength > 0xFFFFF) 1000 report_fatal_error("SEH unwind data splitting not yet implemented"); 1001 uint32_t FuncLength = (uint32_t)RawFuncLength / 4; 1002 uint32_t PrologCodeBytes = ARM64CountOfUnwindCodes(info->Instructions); 1003 uint32_t TotalCodeBytes = PrologCodeBytes; 1004 1005 int PackedEpilogOffset = 1006 checkARM64PackedEpilog(streamer, info, PrologCodeBytes); 1007 1008 if (PackedEpilogOffset >= 0 && 1009 uint32_t(PackedEpilogOffset) < PrologCodeBytes && 1010 !info->HandlesExceptions && FuncLength <= 0x7ff && TryPacked) { 1011 // Matching prolog/epilog and no exception handlers; check if the 1012 // prolog matches the patterns that can be described by the packed 1013 // format. 1014 1015 // info->Symbol was already set even if we didn't actually write any 1016 // unwind info there. Keep using that as indicator that this unwind 1017 // info has been generated already. 1018 1019 if (tryARM64PackedUnwind(info, FuncLength, PackedEpilogOffset)) 1020 return; 1021 } 1022 1023 // Process epilogs. 1024 MapVector<MCSymbol *, uint32_t> EpilogInfo; 1025 // Epilogs processed so far. 1026 std::vector<MCSymbol *> AddedEpilogs; 1027 1028 for (auto &I : info->EpilogMap) { 1029 MCSymbol *EpilogStart = I.first; 1030 auto &EpilogInstrs = I.second.Instructions; 1031 uint32_t CodeBytes = ARM64CountOfUnwindCodes(EpilogInstrs); 1032 1033 MCSymbol* MatchingEpilog = 1034 FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info); 1035 int PrologOffset; 1036 if (MatchingEpilog) { 1037 assert(EpilogInfo.find(MatchingEpilog) != EpilogInfo.end() && 1038 "Duplicate epilog not found"); 1039 EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog); 1040 // Clear the unwind codes in the EpilogMap, so that they don't get output 1041 // in the logic below. 1042 EpilogInstrs.clear(); 1043 } else if ((PrologOffset = getARM64OffsetInProlog(info->Instructions, 1044 EpilogInstrs)) >= 0) { 1045 EpilogInfo[EpilogStart] = PrologOffset; 1046 // Clear the unwind codes in the EpilogMap, so that they don't get output 1047 // in the logic below. 1048 EpilogInstrs.clear(); 1049 } else { 1050 EpilogInfo[EpilogStart] = TotalCodeBytes; 1051 TotalCodeBytes += CodeBytes; 1052 AddedEpilogs.push_back(EpilogStart); 1053 } 1054 } 1055 1056 // Code Words, Epilog count, E, X, Vers, Function Length 1057 uint32_t row1 = 0x0; 1058 uint32_t CodeWords = TotalCodeBytes / 4; 1059 uint32_t CodeWordsMod = TotalCodeBytes % 4; 1060 if (CodeWordsMod) 1061 CodeWords++; 1062 uint32_t EpilogCount = 1063 PackedEpilogOffset >= 0 ? PackedEpilogOffset : info->EpilogMap.size(); 1064 bool ExtensionWord = EpilogCount > 31 || TotalCodeBytes > 124; 1065 if (!ExtensionWord) { 1066 row1 |= (EpilogCount & 0x1F) << 22; 1067 row1 |= (CodeWords & 0x1F) << 27; 1068 } 1069 if (info->HandlesExceptions) // X 1070 row1 |= 1 << 20; 1071 if (PackedEpilogOffset >= 0) // E 1072 row1 |= 1 << 21; 1073 row1 |= FuncLength & 0x3FFFF; 1074 streamer.emitInt32(row1); 1075 1076 // Extended Code Words, Extended Epilog Count 1077 if (ExtensionWord) { 1078 // FIXME: We should be able to split unwind info into multiple sections. 1079 if (CodeWords > 0xFF || EpilogCount > 0xFFFF) 1080 report_fatal_error("SEH unwind data splitting not yet implemented"); 1081 uint32_t row2 = 0x0; 1082 row2 |= (CodeWords & 0xFF) << 16; 1083 row2 |= (EpilogCount & 0xFFFF); 1084 streamer.emitInt32(row2); 1085 } 1086 1087 if (PackedEpilogOffset < 0) { 1088 // Epilog Start Index, Epilog Start Offset 1089 for (auto &I : EpilogInfo) { 1090 MCSymbol *EpilogStart = I.first; 1091 uint32_t EpilogIndex = I.second; 1092 uint32_t EpilogOffset = 1093 (uint32_t)GetAbsDifference(streamer, EpilogStart, info->Begin); 1094 if (EpilogOffset) 1095 EpilogOffset /= 4; 1096 uint32_t row3 = EpilogOffset; 1097 row3 |= (EpilogIndex & 0x3FF) << 22; 1098 streamer.emitInt32(row3); 1099 } 1100 } 1101 1102 // Emit prolog unwind instructions (in reverse order). 1103 uint8_t numInst = info->Instructions.size(); 1104 for (uint8_t c = 0; c < numInst; ++c) { 1105 WinEH::Instruction inst = info->Instructions.back(); 1106 info->Instructions.pop_back(); 1107 ARM64EmitUnwindCode(streamer, inst); 1108 } 1109 1110 // Emit epilog unwind instructions 1111 for (auto &I : info->EpilogMap) { 1112 auto &EpilogInstrs = I.second.Instructions; 1113 for (const WinEH::Instruction &inst : EpilogInstrs) 1114 ARM64EmitUnwindCode(streamer, inst); 1115 } 1116 1117 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes; 1118 assert(BytesMod >= 0); 1119 for (int i = 0; i < BytesMod; i++) 1120 streamer.emitInt8(0xE3); 1121 1122 if (info->HandlesExceptions) 1123 streamer.emitValue( 1124 MCSymbolRefExpr::create(info->ExceptionHandler, 1125 MCSymbolRefExpr::VK_COFF_IMGREL32, context), 1126 4); 1127 } 1128 1129 static uint32_t ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction> Insns) { 1130 uint32_t Count = 0; 1131 for (const auto &I : Insns) { 1132 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 1133 default: 1134 llvm_unreachable("Unsupported ARM unwind code"); 1135 case Win64EH::UOP_AllocSmall: 1136 Count += 1; 1137 break; 1138 case Win64EH::UOP_AllocLarge: 1139 Count += 3; 1140 break; 1141 case Win64EH::UOP_AllocHuge: 1142 Count += 4; 1143 break; 1144 case Win64EH::UOP_WideAllocMedium: 1145 Count += 2; 1146 break; 1147 case Win64EH::UOP_WideAllocLarge: 1148 Count += 3; 1149 break; 1150 case Win64EH::UOP_WideAllocHuge: 1151 Count += 4; 1152 break; 1153 case Win64EH::UOP_WideSaveRegMask: 1154 Count += 2; 1155 break; 1156 case Win64EH::UOP_SaveSP: 1157 Count += 1; 1158 break; 1159 case Win64EH::UOP_SaveRegsR4R7LR: 1160 Count += 1; 1161 break; 1162 case Win64EH::UOP_WideSaveRegsR4R11LR: 1163 Count += 1; 1164 break; 1165 case Win64EH::UOP_SaveFRegD8D15: 1166 Count += 1; 1167 break; 1168 case Win64EH::UOP_SaveRegMask: 1169 Count += 2; 1170 break; 1171 case Win64EH::UOP_SaveLR: 1172 Count += 2; 1173 break; 1174 case Win64EH::UOP_SaveFRegD0D15: 1175 Count += 2; 1176 break; 1177 case Win64EH::UOP_SaveFRegD16D31: 1178 Count += 2; 1179 break; 1180 case Win64EH::UOP_Nop: 1181 case Win64EH::UOP_WideNop: 1182 case Win64EH::UOP_End: 1183 case Win64EH::UOP_EndNop: 1184 case Win64EH::UOP_WideEndNop: 1185 Count += 1; 1186 break; 1187 case Win64EH::UOP_Custom: { 1188 int J; 1189 for (J = 3; J > 0; J--) 1190 if (I.Offset & (0xffu << (8 * J))) 1191 break; 1192 Count += J + 1; 1193 break; 1194 } 1195 } 1196 } 1197 return Count; 1198 } 1199 1200 static uint32_t ARMCountOfInstructionBytes(ArrayRef<WinEH::Instruction> Insns, 1201 bool *HasCustom = nullptr) { 1202 uint32_t Count = 0; 1203 for (const auto &I : Insns) { 1204 switch (static_cast<Win64EH::UnwindOpcodes>(I.Operation)) { 1205 default: 1206 llvm_unreachable("Unsupported ARM unwind code"); 1207 case Win64EH::UOP_AllocSmall: 1208 case Win64EH::UOP_AllocLarge: 1209 case Win64EH::UOP_AllocHuge: 1210 Count += 2; 1211 break; 1212 case Win64EH::UOP_WideAllocMedium: 1213 case Win64EH::UOP_WideAllocLarge: 1214 case Win64EH::UOP_WideAllocHuge: 1215 Count += 4; 1216 break; 1217 case Win64EH::UOP_WideSaveRegMask: 1218 case Win64EH::UOP_WideSaveRegsR4R11LR: 1219 Count += 4; 1220 break; 1221 case Win64EH::UOP_SaveSP: 1222 Count += 2; 1223 break; 1224 case Win64EH::UOP_SaveRegMask: 1225 case Win64EH::UOP_SaveRegsR4R7LR: 1226 Count += 2; 1227 break; 1228 case Win64EH::UOP_SaveFRegD8D15: 1229 case Win64EH::UOP_SaveFRegD0D15: 1230 case Win64EH::UOP_SaveFRegD16D31: 1231 Count += 4; 1232 break; 1233 case Win64EH::UOP_SaveLR: 1234 Count += 4; 1235 break; 1236 case Win64EH::UOP_Nop: 1237 case Win64EH::UOP_EndNop: 1238 Count += 2; 1239 break; 1240 case Win64EH::UOP_WideNop: 1241 case Win64EH::UOP_WideEndNop: 1242 Count += 4; 1243 break; 1244 case Win64EH::UOP_End: 1245 // This doesn't map to any instruction 1246 break; 1247 case Win64EH::UOP_Custom: 1248 // We can't reason about what instructions this maps to; return a 1249 // phony number to make sure we don't accidentally do epilog packing. 1250 Count += 1000; 1251 if (HasCustom) 1252 *HasCustom = true; 1253 break; 1254 } 1255 } 1256 return Count; 1257 } 1258 1259 static void checkARMInstructions(MCStreamer &Streamer, 1260 ArrayRef<WinEH::Instruction> Insns, 1261 const MCSymbol *Begin, const MCSymbol *End, 1262 StringRef Name, StringRef Type) { 1263 if (!End) 1264 return; 1265 Optional<int64_t> MaybeDistance = 1266 GetOptionalAbsDifference(Streamer, End, Begin); 1267 if (!MaybeDistance) 1268 return; 1269 uint32_t Distance = (uint32_t)*MaybeDistance; 1270 bool HasCustom = false; 1271 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Insns, &HasCustom); 1272 if (HasCustom) 1273 return; 1274 if (Distance != InstructionBytes) { 1275 Streamer.getContext().reportError( 1276 SMLoc(), "Incorrect size for " + Name + " " + Type + ": " + 1277 Twine(Distance) + 1278 " bytes of instructions in range, but .seh directives " 1279 "corresponding to " + 1280 Twine(InstructionBytes) + " bytes\n"); 1281 } 1282 } 1283 1284 static bool isARMTerminator(const WinEH::Instruction &inst) { 1285 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 1286 case Win64EH::UOP_End: 1287 case Win64EH::UOP_EndNop: 1288 case Win64EH::UOP_WideEndNop: 1289 return true; 1290 default: 1291 return false; 1292 } 1293 } 1294 1295 // Unwind opcode encodings and restrictions are documented at 1296 // https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling 1297 static void ARMEmitUnwindCode(MCStreamer &streamer, 1298 const WinEH::Instruction &inst) { 1299 uint32_t w, lr; 1300 int i; 1301 switch (static_cast<Win64EH::UnwindOpcodes>(inst.Operation)) { 1302 default: 1303 llvm_unreachable("Unsupported ARM unwind code"); 1304 case Win64EH::UOP_AllocSmall: 1305 assert((inst.Offset & 3) == 0); 1306 assert(inst.Offset / 4 <= 0x7f); 1307 streamer.emitInt8(inst.Offset / 4); 1308 break; 1309 case Win64EH::UOP_WideSaveRegMask: 1310 assert((inst.Register & ~0x5fff) == 0); 1311 lr = (inst.Register >> 14) & 1; 1312 w = 0x8000 | (inst.Register & 0x1fff) | (lr << 13); 1313 streamer.emitInt8((w >> 8) & 0xff); 1314 streamer.emitInt8((w >> 0) & 0xff); 1315 break; 1316 case Win64EH::UOP_SaveSP: 1317 assert(inst.Register <= 0x0f); 1318 streamer.emitInt8(0xc0 | inst.Register); 1319 break; 1320 case Win64EH::UOP_SaveRegsR4R7LR: 1321 assert(inst.Register >= 4 && inst.Register <= 7); 1322 assert(inst.Offset <= 1); 1323 streamer.emitInt8(0xd0 | (inst.Register - 4) | (inst.Offset << 2)); 1324 break; 1325 case Win64EH::UOP_WideSaveRegsR4R11LR: 1326 assert(inst.Register >= 8 && inst.Register <= 11); 1327 assert(inst.Offset <= 1); 1328 streamer.emitInt8(0xd8 | (inst.Register - 8) | (inst.Offset << 2)); 1329 break; 1330 case Win64EH::UOP_SaveFRegD8D15: 1331 assert(inst.Register >= 8 && inst.Register <= 15); 1332 streamer.emitInt8(0xe0 | (inst.Register - 8)); 1333 break; 1334 case Win64EH::UOP_WideAllocMedium: 1335 assert((inst.Offset & 3) == 0); 1336 assert(inst.Offset / 4 <= 0x3ff); 1337 w = 0xe800 | (inst.Offset / 4); 1338 streamer.emitInt8((w >> 8) & 0xff); 1339 streamer.emitInt8((w >> 0) & 0xff); 1340 break; 1341 case Win64EH::UOP_SaveRegMask: 1342 assert((inst.Register & ~0x40ff) == 0); 1343 lr = (inst.Register >> 14) & 1; 1344 w = 0xec00 | (inst.Register & 0x0ff) | (lr << 8); 1345 streamer.emitInt8((w >> 8) & 0xff); 1346 streamer.emitInt8((w >> 0) & 0xff); 1347 break; 1348 case Win64EH::UOP_SaveLR: 1349 assert((inst.Offset & 3) == 0); 1350 assert(inst.Offset / 4 <= 0x0f); 1351 streamer.emitInt8(0xef); 1352 streamer.emitInt8(inst.Offset / 4); 1353 break; 1354 case Win64EH::UOP_SaveFRegD0D15: 1355 assert(inst.Register <= 15); 1356 assert(inst.Offset <= 15); 1357 assert(inst.Register <= inst.Offset); 1358 streamer.emitInt8(0xf5); 1359 streamer.emitInt8((inst.Register << 4) | inst.Offset); 1360 break; 1361 case Win64EH::UOP_SaveFRegD16D31: 1362 assert(inst.Register >= 16 && inst.Register <= 31); 1363 assert(inst.Offset >= 16 && inst.Offset <= 31); 1364 assert(inst.Register <= inst.Offset); 1365 streamer.emitInt8(0xf6); 1366 streamer.emitInt8(((inst.Register - 16) << 4) | (inst.Offset - 16)); 1367 break; 1368 case Win64EH::UOP_AllocLarge: 1369 assert((inst.Offset & 3) == 0); 1370 assert(inst.Offset / 4 <= 0xffff); 1371 w = inst.Offset / 4; 1372 streamer.emitInt8(0xf7); 1373 streamer.emitInt8((w >> 8) & 0xff); 1374 streamer.emitInt8((w >> 0) & 0xff); 1375 break; 1376 case Win64EH::UOP_AllocHuge: 1377 assert((inst.Offset & 3) == 0); 1378 assert(inst.Offset / 4 <= 0xffffff); 1379 w = inst.Offset / 4; 1380 streamer.emitInt8(0xf8); 1381 streamer.emitInt8((w >> 16) & 0xff); 1382 streamer.emitInt8((w >> 8) & 0xff); 1383 streamer.emitInt8((w >> 0) & 0xff); 1384 break; 1385 case Win64EH::UOP_WideAllocLarge: 1386 assert((inst.Offset & 3) == 0); 1387 assert(inst.Offset / 4 <= 0xffff); 1388 w = inst.Offset / 4; 1389 streamer.emitInt8(0xf9); 1390 streamer.emitInt8((w >> 8) & 0xff); 1391 streamer.emitInt8((w >> 0) & 0xff); 1392 break; 1393 case Win64EH::UOP_WideAllocHuge: 1394 assert((inst.Offset & 3) == 0); 1395 assert(inst.Offset / 4 <= 0xffffff); 1396 w = inst.Offset / 4; 1397 streamer.emitInt8(0xfa); 1398 streamer.emitInt8((w >> 16) & 0xff); 1399 streamer.emitInt8((w >> 8) & 0xff); 1400 streamer.emitInt8((w >> 0) & 0xff); 1401 break; 1402 case Win64EH::UOP_Nop: 1403 streamer.emitInt8(0xfb); 1404 break; 1405 case Win64EH::UOP_WideNop: 1406 streamer.emitInt8(0xfc); 1407 break; 1408 case Win64EH::UOP_EndNop: 1409 streamer.emitInt8(0xfd); 1410 break; 1411 case Win64EH::UOP_WideEndNop: 1412 streamer.emitInt8(0xfe); 1413 break; 1414 case Win64EH::UOP_End: 1415 streamer.emitInt8(0xff); 1416 break; 1417 case Win64EH::UOP_Custom: 1418 for (i = 3; i > 0; i--) 1419 if (inst.Offset & (0xffu << (8 * i))) 1420 break; 1421 for (; i >= 0; i--) 1422 streamer.emitInt8((inst.Offset >> (8 * i)) & 0xff); 1423 break; 1424 } 1425 } 1426 1427 // Check if an epilog exists as a subset of the end of a prolog (backwards). 1428 // An epilog may end with one out of three different end opcodes; if this 1429 // is the first epilog that shares opcodes with the prolog, we can tolerate 1430 // that this opcode differs (and the caller will update the prolog to use 1431 // the same end opcode as the epilog). If another epilog already shares 1432 // opcodes with the prolog, the ending opcode must be a strict match. 1433 static int getARMOffsetInProlog(const std::vector<WinEH::Instruction> &Prolog, 1434 const std::vector<WinEH::Instruction> &Epilog, 1435 bool CanTweakProlog) { 1436 // Can't find an epilog as a subset if it is longer than the prolog. 1437 if (Epilog.size() > Prolog.size()) 1438 return -1; 1439 1440 // Check that the epilog actually is a perfect match for the end (backwrds) 1441 // of the prolog. 1442 // If we can adjust the prolog afterwards, don't check that the end opcodes 1443 // match. 1444 int EndIdx = CanTweakProlog ? 1 : 0; 1445 for (int I = Epilog.size() - 1; I >= EndIdx; I--) { 1446 // TODO: Could also allow minor mismatches, e.g. "add sp, #16" vs 1447 // "push {r0-r3}". 1448 if (Prolog[I] != Epilog[Epilog.size() - 1 - I]) 1449 return -1; 1450 } 1451 1452 if (CanTweakProlog) { 1453 // Check that both prolog and epilog end with an expected end opcode. 1454 if (Prolog.front().Operation != Win64EH::UOP_End) 1455 return -1; 1456 if (Epilog.back().Operation != Win64EH::UOP_End && 1457 Epilog.back().Operation != Win64EH::UOP_EndNop && 1458 Epilog.back().Operation != Win64EH::UOP_WideEndNop) 1459 return -1; 1460 } 1461 1462 // If the epilog was a subset of the prolog, find its offset. 1463 if (Epilog.size() == Prolog.size()) 1464 return 0; 1465 return ARMCountOfUnwindCodes(ArrayRef<WinEH::Instruction>( 1466 &Prolog[Epilog.size()], Prolog.size() - Epilog.size())); 1467 } 1468 1469 static int checkARMPackedEpilog(MCStreamer &streamer, WinEH::FrameInfo *info, 1470 int PrologCodeBytes) { 1471 // Can only pack if there's one single epilog 1472 if (info->EpilogMap.size() != 1) 1473 return -1; 1474 1475 const WinEH::FrameInfo::Epilog &EpilogInfo = info->EpilogMap.begin()->second; 1476 // Can only pack if the epilog is unconditional 1477 if (EpilogInfo.Condition != 0xe) // ARMCC::AL 1478 return -1; 1479 1480 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions; 1481 // Make sure we have at least the trailing end opcode 1482 if (info->Instructions.empty() || Epilog.empty()) 1483 return -1; 1484 1485 // Check that the epilog actually is at the very end of the function, 1486 // otherwise it can't be packed. 1487 Optional<int64_t> MaybeDistance = GetOptionalAbsDifference( 1488 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first); 1489 if (!MaybeDistance) 1490 return -1; 1491 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance; 1492 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Epilog); 1493 if (DistanceFromEnd != InstructionBytes) 1494 return -1; 1495 1496 int RetVal = -1; 1497 // Even if we don't end up sharing opcodes with the prolog, we can still 1498 // write the offset as a packed offset, if the single epilog is located at 1499 // the end of the function and the offset (pointing after the prolog) fits 1500 // as a packed offset. 1501 if (PrologCodeBytes <= 31 && 1502 PrologCodeBytes + ARMCountOfUnwindCodes(Epilog) <= 63) 1503 RetVal = PrologCodeBytes; 1504 1505 int Offset = 1506 getARMOffsetInProlog(info->Instructions, Epilog, /*CanTweakProlog=*/true); 1507 if (Offset < 0) 1508 return RetVal; 1509 1510 // Check that the offset and prolog size fits in the first word; it's 1511 // unclear whether the epilog count in the extension word can be taken 1512 // as packed epilog offset. 1513 if (Offset > 31 || PrologCodeBytes > 63) 1514 return RetVal; 1515 1516 // Replace the regular end opcode of the prolog with the one from the 1517 // epilog. 1518 info->Instructions.front() = Epilog.back(); 1519 1520 // As we choose to express the epilog as part of the prolog, remove the 1521 // epilog from the map, so we don't try to emit its opcodes. 1522 info->EpilogMap.clear(); 1523 return Offset; 1524 } 1525 1526 static bool parseRegMask(unsigned Mask, bool &HasLR, bool &HasR11, 1527 unsigned &Folded, int &IntRegs) { 1528 if (Mask & (1 << 14)) { 1529 HasLR = true; 1530 Mask &= ~(1 << 14); 1531 } 1532 if (Mask & (1 << 11)) { 1533 HasR11 = true; 1534 Mask &= ~(1 << 11); 1535 } 1536 Folded = 0; 1537 IntRegs = -1; 1538 if (!Mask) 1539 return true; 1540 int First = 0; 1541 // Shift right until we have the bits at the bottom 1542 while ((Mask & 1) == 0) { 1543 First++; 1544 Mask >>= 1; 1545 } 1546 if ((Mask & (Mask + 1)) != 0) 1547 return false; // Not a consecutive series of bits? Can't be packed. 1548 // Count the bits 1549 int N = 0; 1550 while (Mask & (1 << N)) 1551 N++; 1552 if (First < 4) { 1553 if (First + N < 4) 1554 return false; 1555 Folded = 4 - First; 1556 N -= Folded; 1557 First = 4; 1558 } 1559 if (First > 4) 1560 return false; // Can't be packed 1561 if (N >= 1) 1562 IntRegs = N - 1; 1563 return true; 1564 } 1565 1566 static bool tryARMPackedUnwind(MCStreamer &streamer, WinEH::FrameInfo *info, 1567 uint32_t FuncLength) { 1568 int Step = 0; 1569 bool Homing = false; 1570 bool HasR11 = false; 1571 bool HasChain = false; 1572 bool HasLR = false; 1573 int IntRegs = -1; // r4 - r(4+N) 1574 int FloatRegs = -1; // d8 - d(8+N) 1575 unsigned PF = 0; // Number of extra pushed registers 1576 unsigned StackAdjust = 0; 1577 // Iterate over the prolog and check that all opcodes exactly match 1578 // the canonical order and form. 1579 for (const WinEH::Instruction &Inst : info->Instructions) { 1580 switch (Inst.Operation) { 1581 default: 1582 llvm_unreachable("Unsupported ARM unwind code"); 1583 case Win64EH::UOP_Custom: 1584 case Win64EH::UOP_AllocLarge: 1585 case Win64EH::UOP_AllocHuge: 1586 case Win64EH::UOP_WideAllocLarge: 1587 case Win64EH::UOP_WideAllocHuge: 1588 case Win64EH::UOP_SaveFRegD0D15: 1589 case Win64EH::UOP_SaveFRegD16D31: 1590 // Can't be packed 1591 return false; 1592 case Win64EH::UOP_SaveSP: 1593 // Can't be packed; we can't rely on restoring sp from r11 when 1594 // unwinding a packed prologue. 1595 return false; 1596 case Win64EH::UOP_SaveLR: 1597 // Can't be present in a packed prologue 1598 return false; 1599 1600 case Win64EH::UOP_End: 1601 case Win64EH::UOP_EndNop: 1602 case Win64EH::UOP_WideEndNop: 1603 if (Step != 0) 1604 return false; 1605 Step = 1; 1606 break; 1607 1608 case Win64EH::UOP_SaveRegsR4R7LR: 1609 case Win64EH::UOP_WideSaveRegsR4R11LR: 1610 // push {r4-r11,lr} 1611 if (Step != 1 && Step != 2) 1612 return false; 1613 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX 1614 assert(Inst.Offset <= 1); // Lr 1615 IntRegs = Inst.Register - 4; 1616 if (Inst.Register == 11) { 1617 HasR11 = true; 1618 IntRegs--; 1619 } 1620 if (Inst.Offset) 1621 HasLR = true; 1622 Step = 3; 1623 break; 1624 1625 case Win64EH::UOP_SaveRegMask: 1626 if (Step == 1 && Inst.Register == 0x0f) { 1627 // push {r0-r3} 1628 Homing = true; 1629 Step = 2; 1630 break; 1631 } 1632 LLVM_FALLTHROUGH; 1633 case Win64EH::UOP_WideSaveRegMask: 1634 if (Step != 1 && Step != 2) 1635 return false; 1636 // push {r4-r9,r11,lr} 1637 // push {r11,lr} 1638 // push {r1-r5} 1639 if (!parseRegMask(Inst.Register, HasLR, HasR11, PF, IntRegs)) 1640 return false; 1641 Step = 3; 1642 break; 1643 1644 case Win64EH::UOP_Nop: 1645 // mov r11, sp 1646 if (Step != 3 || !HasR11 || IntRegs >= 0 || PF > 0) 1647 return false; 1648 HasChain = true; 1649 Step = 4; 1650 break; 1651 case Win64EH::UOP_WideNop: 1652 // add.w r11, sp, #xx 1653 if (Step != 3 || !HasR11 || (IntRegs < 0 && PF == 0)) 1654 return false; 1655 HasChain = true; 1656 Step = 4; 1657 break; 1658 1659 case Win64EH::UOP_SaveFRegD8D15: 1660 if (Step != 1 && Step != 2 && Step != 3 && Step != 4) 1661 return false; 1662 assert(Inst.Register >= 8 && Inst.Register <= 15); 1663 if (Inst.Register == 15) 1664 return false; // Can't pack this case, R==7 means no IntRegs 1665 if (IntRegs >= 0) 1666 return false; 1667 FloatRegs = Inst.Register - 8; 1668 Step = 5; 1669 break; 1670 1671 case Win64EH::UOP_AllocSmall: 1672 case Win64EH::UOP_WideAllocMedium: 1673 if (Step != 1 && Step != 2 && Step != 3 && Step != 4 && Step != 5) 1674 return false; 1675 if (PF > 0) // Can't have both folded and explicit stack allocation 1676 return false; 1677 if (Inst.Offset / 4 >= 0x3f4) 1678 return false; 1679 StackAdjust = Inst.Offset / 4; 1680 Step = 6; 1681 break; 1682 } 1683 } 1684 if (HasR11 && !HasChain) { 1685 if (IntRegs + 4 == 10) { 1686 // r11 stored, but not chaining; can be packed if already saving r4-r10 1687 // and we can fit r11 into this range. 1688 IntRegs++; 1689 HasR11 = false; 1690 } else 1691 return false; 1692 } 1693 if (HasChain && !HasLR) 1694 return false; 1695 1696 // Packed uneind info can't express multiple epilogues. 1697 if (info->EpilogMap.size() > 1) 1698 return false; 1699 1700 unsigned EF = 0; 1701 int Ret = 0; 1702 if (info->EpilogMap.size() == 0) { 1703 Ret = 3; // No epilogue 1704 } else { 1705 // As the prologue and epilogue aren't exact mirrors of each other, 1706 // we have to check the epilogue too and see if it matches what we've 1707 // concluded from the prologue. 1708 const WinEH::FrameInfo::Epilog &EpilogInfo = 1709 info->EpilogMap.begin()->second; 1710 if (EpilogInfo.Condition != 0xe) // ARMCC::AL 1711 return false; 1712 const std::vector<WinEH::Instruction> &Epilog = EpilogInfo.Instructions; 1713 Optional<int64_t> MaybeDistance = GetOptionalAbsDifference( 1714 streamer, info->FuncletOrFuncEnd, info->EpilogMap.begin()->first); 1715 if (!MaybeDistance) 1716 return false; 1717 uint32_t DistanceFromEnd = (uint32_t)*MaybeDistance; 1718 uint32_t InstructionBytes = ARMCountOfInstructionBytes(Epilog); 1719 if (DistanceFromEnd != InstructionBytes) 1720 return false; 1721 1722 bool GotStackAdjust = false; 1723 bool GotFloatRegs = false; 1724 bool GotIntRegs = false; 1725 bool GotHomingRestore = false; 1726 bool GotLRRestore = false; 1727 bool NeedsReturn = false; 1728 bool GotReturn = false; 1729 1730 Step = 6; 1731 for (const WinEH::Instruction &Inst : Epilog) { 1732 switch (Inst.Operation) { 1733 default: 1734 llvm_unreachable("Unsupported ARM unwind code"); 1735 case Win64EH::UOP_Custom: 1736 case Win64EH::UOP_AllocLarge: 1737 case Win64EH::UOP_AllocHuge: 1738 case Win64EH::UOP_WideAllocLarge: 1739 case Win64EH::UOP_WideAllocHuge: 1740 case Win64EH::UOP_SaveFRegD0D15: 1741 case Win64EH::UOP_SaveFRegD16D31: 1742 case Win64EH::UOP_SaveSP: 1743 case Win64EH::UOP_Nop: 1744 case Win64EH::UOP_WideNop: 1745 // Can't be packed in an epilogue 1746 return false; 1747 1748 case Win64EH::UOP_AllocSmall: 1749 case Win64EH::UOP_WideAllocMedium: 1750 if (Inst.Offset / 4 >= 0x3f4) 1751 return false; 1752 if (Step == 6) { 1753 if (Homing && FloatRegs < 0 && IntRegs < 0 && StackAdjust == 0 && 1754 PF == 0 && Inst.Offset == 16) { 1755 GotHomingRestore = true; 1756 Step = 10; 1757 } else { 1758 if (StackAdjust > 0) { 1759 // Got stack adjust in prologue too; must match. 1760 if (StackAdjust != Inst.Offset / 4) 1761 return false; 1762 GotStackAdjust = true; 1763 } else if (PF == Inst.Offset / 4) { 1764 // Folded prologue, non-folded epilogue 1765 StackAdjust = Inst.Offset / 4; 1766 GotStackAdjust = true; 1767 } else { 1768 // StackAdjust == 0 in prologue, mismatch 1769 return false; 1770 } 1771 Step = 7; 1772 } 1773 } else if (Step == 7 || Step == 8 || Step == 9) { 1774 if (!Homing || Inst.Offset != 16) 1775 return false; 1776 GotHomingRestore = true; 1777 Step = 10; 1778 } else 1779 return false; 1780 break; 1781 1782 case Win64EH::UOP_SaveFRegD8D15: 1783 if (Step != 6 && Step != 7) 1784 return false; 1785 assert(Inst.Register >= 8 && Inst.Register <= 15); 1786 if (FloatRegs != (int)(Inst.Register - 8)) 1787 return false; 1788 GotFloatRegs = true; 1789 Step = 8; 1790 break; 1791 1792 case Win64EH::UOP_SaveRegsR4R7LR: 1793 case Win64EH::UOP_WideSaveRegsR4R11LR: { 1794 // push {r4-r11,lr} 1795 if (Step != 6 && Step != 7 && Step != 8) 1796 return false; 1797 assert(Inst.Register >= 4 && Inst.Register <= 11); // r4-rX 1798 assert(Inst.Offset <= 1); // Lr 1799 if (Homing && HasLR) { 1800 // If homing and LR is backed up, we can either restore LR here 1801 // and return with Ret == 1 or 2, or return with SaveLR below 1802 if (Inst.Offset) { 1803 GotLRRestore = true; 1804 NeedsReturn = true; 1805 } else { 1806 // Expecting a separate SaveLR below 1807 } 1808 } else { 1809 if (HasLR != (Inst.Offset == 1)) 1810 return false; 1811 } 1812 GotLRRestore = Inst.Offset == 1; 1813 if (IntRegs < 0) // This opcode must include r4 1814 return false; 1815 int Expected = IntRegs; 1816 if (HasChain) { 1817 // Can't express r11 here unless IntRegs describe r4-r10 1818 if (IntRegs != 6) 1819 return false; 1820 Expected++; 1821 } 1822 if (Expected != (int)(Inst.Register - 4)) 1823 return false; 1824 GotIntRegs = true; 1825 Step = 9; 1826 break; 1827 } 1828 1829 case Win64EH::UOP_SaveRegMask: 1830 case Win64EH::UOP_WideSaveRegMask: { 1831 if (Step != 6 && Step != 7 && Step != 8) 1832 return false; 1833 // push {r4-r9,r11,lr} 1834 // push {r11,lr} 1835 // push {r1-r5} 1836 bool CurHasLR = false, CurHasR11 = false; 1837 int Regs; 1838 if (!parseRegMask(Inst.Register, CurHasLR, CurHasR11, EF, Regs)) 1839 return false; 1840 if (EF > 0) { 1841 if (EF != PF && EF != StackAdjust) 1842 return false; 1843 } 1844 if (Homing && HasLR) { 1845 // If homing and LR is backed up, we can either restore LR here 1846 // and return with Ret == 1 or 2, or return with SaveLR below 1847 if (CurHasLR) { 1848 GotLRRestore = true; 1849 NeedsReturn = true; 1850 } else { 1851 // Expecting a separate SaveLR below 1852 } 1853 } else { 1854 if (CurHasLR != HasLR) 1855 return false; 1856 GotLRRestore = CurHasLR; 1857 } 1858 int Expected = IntRegs; 1859 if (HasChain) { 1860 // If we have chaining, the mask must have included r11. 1861 if (!CurHasR11) 1862 return false; 1863 } else if (Expected == 7) { 1864 // If we don't have chaining, the mask could still include r11, 1865 // expressed as part of IntRegs Instead. 1866 Expected--; 1867 if (!CurHasR11) 1868 return false; 1869 } else { 1870 // Neither HasChain nor r11 included in IntRegs, must not have r11 1871 // here either. 1872 if (CurHasR11) 1873 return false; 1874 } 1875 if (Expected != Regs) 1876 return false; 1877 GotIntRegs = true; 1878 Step = 9; 1879 break; 1880 } 1881 1882 case Win64EH::UOP_SaveLR: 1883 if (Step != 6 && Step != 7 && Step != 8 && Step != 9) 1884 return false; 1885 if (!Homing || Inst.Offset != 20 || GotLRRestore) 1886 return false; 1887 GotLRRestore = true; 1888 GotHomingRestore = true; 1889 Step = 10; 1890 break; 1891 1892 case Win64EH::UOP_EndNop: 1893 case Win64EH::UOP_WideEndNop: 1894 GotReturn = true; 1895 Ret = (Inst.Operation == Win64EH::UOP_EndNop) ? 1 : 2; 1896 LLVM_FALLTHROUGH; 1897 case Win64EH::UOP_End: 1898 if (Step != 6 && Step != 7 && Step != 8 && Step != 9 && Step != 10) 1899 return false; 1900 Step = 11; 1901 break; 1902 } 1903 } 1904 1905 if (Step != 11) 1906 return false; 1907 if (StackAdjust > 0 && !GotStackAdjust && EF == 0) 1908 return false; 1909 if (FloatRegs >= 0 && !GotFloatRegs) 1910 return false; 1911 if (IntRegs >= 0 && !GotIntRegs) 1912 return false; 1913 if (Homing && !GotHomingRestore) 1914 return false; 1915 if (HasLR && !GotLRRestore) 1916 return false; 1917 if (NeedsReturn && !GotReturn) 1918 return false; 1919 } 1920 1921 assert(PF == 0 || EF == 0 || 1922 StackAdjust == 0); // Can't have adjust in all three 1923 if (PF > 0 || EF > 0) { 1924 StackAdjust = PF > 0 ? (PF - 1) : (EF - 1); 1925 assert(StackAdjust <= 3); 1926 StackAdjust |= 0x3f0; 1927 if (PF > 0) 1928 StackAdjust |= 1 << 2; 1929 if (EF > 0) 1930 StackAdjust |= 1 << 3; 1931 } 1932 1933 assert(FuncLength <= 0x7FF && "FuncLength should have been checked earlier"); 1934 int Flag = info->Fragment ? 0x02 : 0x01; 1935 int H = Homing ? 1 : 0; 1936 int L = HasLR ? 1 : 0; 1937 int C = HasChain ? 1 : 0; 1938 assert(IntRegs < 0 || FloatRegs < 0); 1939 unsigned Reg, R; 1940 if (IntRegs >= 0) { 1941 Reg = IntRegs; 1942 assert(Reg <= 7); 1943 R = 0; 1944 } else if (FloatRegs >= 0) { 1945 Reg = FloatRegs; 1946 assert(Reg < 7); 1947 R = 1; 1948 } else { 1949 // No int or float regs stored (except possibly R11,LR) 1950 Reg = 7; 1951 R = 1; 1952 } 1953 info->PackedInfo |= Flag << 0; 1954 info->PackedInfo |= (FuncLength & 0x7FF) << 2; 1955 info->PackedInfo |= (Ret & 0x3) << 13; 1956 info->PackedInfo |= H << 15; 1957 info->PackedInfo |= Reg << 16; 1958 info->PackedInfo |= R << 19; 1959 info->PackedInfo |= L << 20; 1960 info->PackedInfo |= C << 21; 1961 assert(StackAdjust <= 0x3ff); 1962 info->PackedInfo |= StackAdjust << 22; 1963 return true; 1964 } 1965 1966 // Populate the .xdata section. The format of .xdata on ARM is documented at 1967 // https://docs.microsoft.com/en-us/cpp/build/arm-exception-handling 1968 static void ARMEmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info, 1969 bool TryPacked = true) { 1970 // If this UNWIND_INFO already has a symbol, it's already been emitted. 1971 if (info->Symbol) 1972 return; 1973 // If there's no unwind info here (not even a terminating UOP_End), the 1974 // unwind info is considered bogus and skipped. If this was done in 1975 // response to an explicit .seh_handlerdata, the associated trailing 1976 // handler data is left orphaned in the xdata section. 1977 if (info->empty()) { 1978 info->EmitAttempted = true; 1979 return; 1980 } 1981 if (info->EmitAttempted) { 1982 // If we tried to emit unwind info before (due to an explicit 1983 // .seh_handlerdata directive), but skipped it (because there was no 1984 // valid information to emit at the time), and it later got valid unwind 1985 // opcodes, we can't emit it here, because the trailing handler data 1986 // was already emitted elsewhere in the xdata section. 1987 streamer.getContext().reportError( 1988 SMLoc(), "Earlier .seh_handlerdata for " + info->Function->getName() + 1989 " skipped due to no unwind info at the time " 1990 "(.seh_handlerdata too early?), but the function later " 1991 "did get unwind info that can't be emitted"); 1992 return; 1993 } 1994 1995 MCContext &context = streamer.getContext(); 1996 MCSymbol *Label = context.createTempSymbol(); 1997 1998 streamer.emitValueToAlignment(4); 1999 streamer.emitLabel(Label); 2000 info->Symbol = Label; 2001 2002 if (!info->PrologEnd) 2003 streamer.getContext().reportError(SMLoc(), "Prologue in " + 2004 info->Function->getName() + 2005 " not correctly terminated"); 2006 2007 if (info->PrologEnd && !info->Fragment) 2008 checkARMInstructions(streamer, info->Instructions, info->Begin, 2009 info->PrologEnd, info->Function->getName(), 2010 "prologue"); 2011 for (auto &I : info->EpilogMap) { 2012 MCSymbol *EpilogStart = I.first; 2013 auto &Epilog = I.second; 2014 checkARMInstructions(streamer, Epilog.Instructions, EpilogStart, Epilog.End, 2015 info->Function->getName(), "epilogue"); 2016 if (Epilog.Instructions.empty() || 2017 !isARMTerminator(Epilog.Instructions.back())) 2018 streamer.getContext().reportError( 2019 SMLoc(), "Epilogue in " + info->Function->getName() + 2020 " not correctly terminated"); 2021 } 2022 2023 Optional<int64_t> RawFuncLength; 2024 const MCExpr *FuncLengthExpr = nullptr; 2025 if (!info->FuncletOrFuncEnd) { 2026 report_fatal_error("FuncletOrFuncEnd not set"); 2027 } else { 2028 // As the size of many thumb2 instructions isn't known until later, 2029 // we can't always rely on being able to calculate the absolute 2030 // length of the function here. If we can't calculate it, defer it 2031 // to a relocation. 2032 // 2033 // In such a case, we won't know if the function is too long so that 2034 // the unwind info would need to be split (but this isn't implemented 2035 // anyway). 2036 RawFuncLength = 2037 GetOptionalAbsDifference(streamer, info->FuncletOrFuncEnd, info->Begin); 2038 if (!RawFuncLength) 2039 FuncLengthExpr = 2040 GetSubDivExpr(streamer, info->FuncletOrFuncEnd, info->Begin, 2); 2041 } 2042 uint32_t FuncLength = 0; 2043 if (RawFuncLength) 2044 FuncLength = (uint32_t)*RawFuncLength / 2; 2045 if (FuncLength > 0x3FFFF) 2046 report_fatal_error("SEH unwind data splitting not yet implemented"); 2047 uint32_t PrologCodeBytes = ARMCountOfUnwindCodes(info->Instructions); 2048 uint32_t TotalCodeBytes = PrologCodeBytes; 2049 2050 if (!info->HandlesExceptions && RawFuncLength && FuncLength <= 0x7ff && 2051 TryPacked) { 2052 // No exception handlers; check if the prolog and epilog matches the 2053 // patterns that can be described by the packed format. If we don't 2054 // know the exact function length yet, we can't do this. 2055 2056 // info->Symbol was already set even if we didn't actually write any 2057 // unwind info there. Keep using that as indicator that this unwind 2058 // info has been generated already. 2059 2060 if (tryARMPackedUnwind(streamer, info, FuncLength)) 2061 return; 2062 } 2063 2064 int PackedEpilogOffset = 2065 checkARMPackedEpilog(streamer, info, PrologCodeBytes); 2066 2067 // Process epilogs. 2068 MapVector<MCSymbol *, uint32_t> EpilogInfo; 2069 // Epilogs processed so far. 2070 std::vector<MCSymbol *> AddedEpilogs; 2071 2072 bool CanTweakProlog = true; 2073 for (auto &I : info->EpilogMap) { 2074 MCSymbol *EpilogStart = I.first; 2075 auto &EpilogInstrs = I.second.Instructions; 2076 uint32_t CodeBytes = ARMCountOfUnwindCodes(EpilogInstrs); 2077 2078 MCSymbol *MatchingEpilog = 2079 FindMatchingEpilog(EpilogInstrs, AddedEpilogs, info); 2080 int PrologOffset; 2081 if (MatchingEpilog) { 2082 assert(EpilogInfo.find(MatchingEpilog) != EpilogInfo.end() && 2083 "Duplicate epilog not found"); 2084 EpilogInfo[EpilogStart] = EpilogInfo.lookup(MatchingEpilog); 2085 // Clear the unwind codes in the EpilogMap, so that they don't get output 2086 // in the logic below. 2087 EpilogInstrs.clear(); 2088 } else if ((PrologOffset = getARMOffsetInProlog( 2089 info->Instructions, EpilogInstrs, CanTweakProlog)) >= 0) { 2090 if (CanTweakProlog) { 2091 // Replace the regular end opcode of the prolog with the one from the 2092 // epilog. 2093 info->Instructions.front() = EpilogInstrs.back(); 2094 // Later epilogs need a strict match for the end opcode. 2095 CanTweakProlog = false; 2096 } 2097 EpilogInfo[EpilogStart] = PrologOffset; 2098 // Clear the unwind codes in the EpilogMap, so that they don't get output 2099 // in the logic below. 2100 EpilogInstrs.clear(); 2101 } else { 2102 EpilogInfo[EpilogStart] = TotalCodeBytes; 2103 TotalCodeBytes += CodeBytes; 2104 AddedEpilogs.push_back(EpilogStart); 2105 } 2106 } 2107 2108 // Code Words, Epilog count, F, E, X, Vers, Function Length 2109 uint32_t row1 = 0x0; 2110 uint32_t CodeWords = TotalCodeBytes / 4; 2111 uint32_t CodeWordsMod = TotalCodeBytes % 4; 2112 if (CodeWordsMod) 2113 CodeWords++; 2114 uint32_t EpilogCount = 2115 PackedEpilogOffset >= 0 ? PackedEpilogOffset : info->EpilogMap.size(); 2116 bool ExtensionWord = EpilogCount > 31 || CodeWords > 15; 2117 if (!ExtensionWord) { 2118 row1 |= (EpilogCount & 0x1F) << 23; 2119 row1 |= (CodeWords & 0x0F) << 28; 2120 } 2121 if (info->HandlesExceptions) // X 2122 row1 |= 1 << 20; 2123 if (PackedEpilogOffset >= 0) // E 2124 row1 |= 1 << 21; 2125 if (info->Fragment) // F 2126 row1 |= 1 << 22; 2127 row1 |= FuncLength & 0x3FFFF; 2128 if (RawFuncLength) 2129 streamer.emitInt32(row1); 2130 else 2131 streamer.emitValue( 2132 MCBinaryExpr::createOr(FuncLengthExpr, 2133 MCConstantExpr::create(row1, context), context), 2134 4); 2135 2136 // Extended Code Words, Extended Epilog Count 2137 if (ExtensionWord) { 2138 // FIXME: We should be able to split unwind info into multiple sections. 2139 if (CodeWords > 0xFF || EpilogCount > 0xFFFF) 2140 report_fatal_error("SEH unwind data splitting not yet implemented"); 2141 uint32_t row2 = 0x0; 2142 row2 |= (CodeWords & 0xFF) << 16; 2143 row2 |= (EpilogCount & 0xFFFF); 2144 streamer.emitInt32(row2); 2145 } 2146 2147 if (PackedEpilogOffset < 0) { 2148 // Epilog Start Index, Epilog Start Offset 2149 for (auto &I : EpilogInfo) { 2150 MCSymbol *EpilogStart = I.first; 2151 uint32_t EpilogIndex = I.second; 2152 2153 Optional<int64_t> MaybeEpilogOffset = 2154 GetOptionalAbsDifference(streamer, EpilogStart, info->Begin); 2155 const MCExpr *OffsetExpr = nullptr; 2156 uint32_t EpilogOffset = 0; 2157 if (MaybeEpilogOffset) 2158 EpilogOffset = *MaybeEpilogOffset / 2; 2159 else 2160 OffsetExpr = GetSubDivExpr(streamer, EpilogStart, info->Begin, 2); 2161 2162 assert(info->EpilogMap.find(EpilogStart) != info->EpilogMap.end()); 2163 unsigned Condition = info->EpilogMap[EpilogStart].Condition; 2164 assert(Condition <= 0xf); 2165 2166 uint32_t row3 = EpilogOffset; 2167 row3 |= Condition << 20; 2168 row3 |= (EpilogIndex & 0x3FF) << 24; 2169 if (MaybeEpilogOffset) 2170 streamer.emitInt32(row3); 2171 else 2172 streamer.emitValue( 2173 MCBinaryExpr::createOr( 2174 OffsetExpr, MCConstantExpr::create(row3, context), context), 2175 4); 2176 } 2177 } 2178 2179 // Emit prolog unwind instructions (in reverse order). 2180 uint8_t numInst = info->Instructions.size(); 2181 for (uint8_t c = 0; c < numInst; ++c) { 2182 WinEH::Instruction inst = info->Instructions.back(); 2183 info->Instructions.pop_back(); 2184 ARMEmitUnwindCode(streamer, inst); 2185 } 2186 2187 // Emit epilog unwind instructions 2188 for (auto &I : info->EpilogMap) { 2189 auto &EpilogInstrs = I.second.Instructions; 2190 for (uint32_t i = 0; i < EpilogInstrs.size(); i++) { 2191 WinEH::Instruction inst = EpilogInstrs[i]; 2192 ARMEmitUnwindCode(streamer, inst); 2193 } 2194 } 2195 2196 int32_t BytesMod = CodeWords * 4 - TotalCodeBytes; 2197 assert(BytesMod >= 0); 2198 for (int i = 0; i < BytesMod; i++) 2199 streamer.emitInt8(0xFB); 2200 2201 if (info->HandlesExceptions) 2202 streamer.emitValue( 2203 MCSymbolRefExpr::create(info->ExceptionHandler, 2204 MCSymbolRefExpr::VK_COFF_IMGREL32, context), 2205 4); 2206 } 2207 2208 static void ARMEmitRuntimeFunction(MCStreamer &streamer, 2209 const WinEH::FrameInfo *info) { 2210 MCContext &context = streamer.getContext(); 2211 2212 streamer.emitValueToAlignment(4); 2213 EmitSymbolRefWithOfs(streamer, info->Begin, info->Begin); 2214 if (info->PackedInfo) 2215 streamer.emitInt32(info->PackedInfo); 2216 else 2217 streamer.emitValue( 2218 MCSymbolRefExpr::create(info->Symbol, MCSymbolRefExpr::VK_COFF_IMGREL32, 2219 context), 2220 4); 2221 } 2222 2223 void llvm::Win64EH::ARM64UnwindEmitter::Emit(MCStreamer &Streamer) const { 2224 // Emit the unwind info structs first. 2225 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2226 WinEH::FrameInfo *Info = CFI.get(); 2227 if (Info->empty()) 2228 continue; 2229 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 2230 Streamer.switchSection(XData); 2231 ARM64EmitUnwindInfo(Streamer, Info); 2232 } 2233 2234 // Now emit RUNTIME_FUNCTION entries. 2235 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2236 WinEH::FrameInfo *Info = CFI.get(); 2237 // ARM64EmitUnwindInfo above clears the info struct, so we can't check 2238 // empty here. But if a Symbol is set, we should create the corresponding 2239 // pdata entry. 2240 if (!Info->Symbol) 2241 continue; 2242 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 2243 Streamer.switchSection(PData); 2244 ARMEmitRuntimeFunction(Streamer, Info); 2245 } 2246 } 2247 2248 void llvm::Win64EH::ARM64UnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 2249 WinEH::FrameInfo *info, 2250 bool HandlerData) const { 2251 // Called if there's an .seh_handlerdata directive before the end of the 2252 // function. This forces writing the xdata record already here - and 2253 // in this case, the function isn't actually ended already, but the xdata 2254 // record needs to know the function length. In these cases, if the funclet 2255 // end hasn't been marked yet, the xdata function length won't cover the 2256 // whole function, only up to this point. 2257 if (!info->FuncletOrFuncEnd) { 2258 Streamer.switchSection(info->TextSection); 2259 info->FuncletOrFuncEnd = Streamer.emitCFILabel(); 2260 } 2261 // Switch sections (the static function above is meant to be called from 2262 // here and from Emit(). 2263 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 2264 Streamer.switchSection(XData); 2265 ARM64EmitUnwindInfo(Streamer, info, /* TryPacked = */ !HandlerData); 2266 } 2267 2268 void llvm::Win64EH::ARMUnwindEmitter::Emit(MCStreamer &Streamer) const { 2269 // Emit the unwind info structs first. 2270 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2271 WinEH::FrameInfo *Info = CFI.get(); 2272 if (Info->empty()) 2273 continue; 2274 MCSection *XData = Streamer.getAssociatedXDataSection(CFI->TextSection); 2275 Streamer.switchSection(XData); 2276 ARMEmitUnwindInfo(Streamer, Info); 2277 } 2278 2279 // Now emit RUNTIME_FUNCTION entries. 2280 for (const auto &CFI : Streamer.getWinFrameInfos()) { 2281 WinEH::FrameInfo *Info = CFI.get(); 2282 // ARMEmitUnwindInfo above clears the info struct, so we can't check 2283 // empty here. But if a Symbol is set, we should create the corresponding 2284 // pdata entry. 2285 if (!Info->Symbol) 2286 continue; 2287 MCSection *PData = Streamer.getAssociatedPDataSection(CFI->TextSection); 2288 Streamer.switchSection(PData); 2289 ARMEmitRuntimeFunction(Streamer, Info); 2290 } 2291 } 2292 2293 void llvm::Win64EH::ARMUnwindEmitter::EmitUnwindInfo(MCStreamer &Streamer, 2294 WinEH::FrameInfo *info, 2295 bool HandlerData) const { 2296 // Called if there's an .seh_handlerdata directive before the end of the 2297 // function. This forces writing the xdata record already here - and 2298 // in this case, the function isn't actually ended already, but the xdata 2299 // record needs to know the function length. In these cases, if the funclet 2300 // end hasn't been marked yet, the xdata function length won't cover the 2301 // whole function, only up to this point. 2302 if (!info->FuncletOrFuncEnd) { 2303 Streamer.switchSection(info->TextSection); 2304 info->FuncletOrFuncEnd = Streamer.emitCFILabel(); 2305 } 2306 // Switch sections (the static function above is meant to be called from 2307 // here and from Emit(). 2308 MCSection *XData = Streamer.getAssociatedXDataSection(info->TextSection); 2309 Streamer.switchSection(XData); 2310 ARMEmitUnwindInfo(Streamer, info, /* TryPacked = */ !HandlerData); 2311 } 2312