Lines Matching +full:odd +full:- +full:parity
1 //===-- AArch64A57FPLoadBalancing.cpp - Balance FP ops statically on A57---===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
8 // For best-case performance on Cortex-A57, we should try to use a balanced
9 // mix of odd and even D-registers when performing a critical sequence of
10 // independent, non-quadword FP/ASIMD floating-point multiply or
11 // multiply-accumulate operations.
17 // Ideally we'd just take each multiply or multiply-accumulate in turn and
18 // allocate it alternating even or odd registers. However, multiply-accumulates
21 // ("Chains") of multiply-accumulates linked via their accumulation operand,
24 // This optimization affects S-register and D-register floating point
26 // FMADD/FMA. Q register instructions (and 128-bit vector instructions) are
28 //===----------------------------------------------------------------------===//
46 #define DEBUG_TYPE "aarch64-a57-fp-load-balancing"
51 TransformAll("aarch64-a57-fp-load-balancing-force-all",
55 // Never use the balance information obtained from chains - return a specific
58 OverrideBalance("aarch64-a57-fp-load-balancing-override",
60 "(1: Even, 2: Odd)."),
63 //===----------------------------------------------------------------------===//
66 // Is the instruction a type of multiply on 64-bit (or 32-bit) FPRs?
68 switch (MI->getOpcode()) {
79 // Is the instruction a type of FP multiply-accumulate on 64-bit (or 32-bit) FPRs?
81 switch (MI->getOpcode()) {
96 //===----------------------------------------------------------------------===//
99 /// A "color", which is either even or odd. Yes, these aren't really colors
100 /// but the algorithm is conceptually doing two-color graph coloring.
101 enum class Color { Even, Odd };
103 static const char *ColorNames[2] = { "Even", "Odd" };
127 return "A57 FP Anti-dependency breaker";
154 "AArch64 A57 FP Load-Balancing", false, false)
156 "AArch64 A57 FP Load-Balancing", false, false)
172 /// A chain has three important instructions - Start, Last and Kill.
260 return OverrideBalance == 1 ? Color::Even : Color::Odd;
275 return StartInstIdx < Other->StartInstIdx;
289 StartInst->print(OS, /* SkipOpers= */true);
290 OS << " -> ";
291 LastInst->print(OS, /* SkipOpers= */true);
294 KillInst->print(OS, /* SkipOpers= */true);
306 //===----------------------------------------------------------------------===//
332 << " - scanning instructions...\n");
336 // The currently "active" chains - chains that can be added to and haven't
337 // been killed yet. This is keyed by register - all chains can only have one
349 // a poor-man's version of graph coloring. Ideally we'd create an interference
350 // graph and perform full-on graph coloring on that, but;
352 // (b) We expect multiple disjoint interference regions - in practice the live
361 if (I != J && I->rangeOverlapsWith(*J))
380 return A.front()->startsBefore(B.front());
383 // As we only have two colors, we can track the global (BB-level) balance of
386 // Positive means we're even-heavy, negative we're odd-heavy.
394 int Parity = 0;
397 Changed |= colorChainSet(std::move(I), MBB, Parity);
416 unsigned MinSize = L.front()->size() - SizeFuzz;
418 if ((*I)->size() <= MinSize) {
420 Chain *Ch = *--I;
425 if ((*I)->getPreferredColor() == PreferredColor) {
432 // Bailout case - just return the first item.
440 int &Parity) {
446 // Tie-break equivalent sizes by sorting chains requiring fixups before
449 // so the parity counter is updated and we know what color we should
451 // Final tie-break with instruction order so pass output is stable (i.e. not
454 if (G1->size() != G2->size())
455 return G1->size() > G2->size();
456 if (G1->requiresFixup() != G2->requiresFixup())
457 return G1->requiresFixup() > G2->requiresFixup();
459 assert((G1 == G2 || (G1->startsBefore(G2) ^ G2->startsBefore(G1))) &&
461 return G1->startsBefore(G2);
464 Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
468 if (Parity == 0)
470 C = G->getPreferredColor();
472 LLVM_DEBUG(dbgs() << " - Parity=" << Parity
478 if (G->requiresFixup() && C != G->getPreferredColor()) {
479 C = G->getPreferredColor();
480 LLVM_DEBUG(dbgs() << " - " << G->str()
481 << " - not worthwhile changing; "
488 Parity += (C == Color::Even) ? G->size() : -G->size();
489 PreferredColor = Parity < 0 ? Color::Even : Color::Odd;
502 MachineBasicBlock::iterator ChainEnd = G->end();
504 --I;
509 MachineBasicBlock::iterator ChainBegin = G->begin();
512 --I;
516 // Make sure we allocate in-order, to get the cheapest registers first.
517 unsigned RegClassID = ChainBegin->getDesc().operands()[0].RegClass;
518 auto Ord = RCI.getOrder(TRI->getRegClass(RegClassID));
526 return -1;
532 LLVM_DEBUG(dbgs() << " - colorChain(" << G->str() << ", "
538 if (Reg == -1) {
542 LLVM_DEBUG(dbgs() << " - Scavenged register: " << printReg(Reg, TRI) << "\n");
546 if (!G->contains(I) && (&I != G->getKill() || G->isKillImmutable()))
572 if (&I != G->getKill()) {
576 if (G->requiresFixup() && &I == G->getLast())
589 if (G->getKill()) {
590 LLVM_DEBUG(dbgs() << " - Kill instruction seen.\n");
594 LLVM_DEBUG(dbgs() << " - Destination register not changed.\n");
606 for (auto &I : MI->uses())
608 for (auto &I : MI->defs())
613 Register DestReg = MI->getOperand(0).getReg();
626 Register DestReg = MI->getOperand(0).getReg();
627 Register AccumReg = MI->getOperand(3).getReg();
629 maybeKillChain(MI->getOperand(1), Idx, ActiveChains);
630 maybeKillChain(MI->getOperand(2), Idx, ActiveChains);
632 maybeKillChain(MI->getOperand(0), Idx, ActiveChains);
642 // FIXME: We could extend to handle the non-kill cases for more coverage.
643 if (MI->getOperand(3).isKill()) {
646 ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg));
658 maybeKillChain(MI->getOperand(3), Idx, ActiveChains);
669 // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs
671 for (auto &I : MI->uses())
673 for (auto &I : MI->defs())
692 ActiveChains[MO.getReg()]->setKill(MI, Idx, /*Immutable=*/MO.isTied());
700 if (MO.clobbersPhysReg(I->first)) {
702 << printReg(I->first, TRI) << "\n");
703 I->second->setKill(MI, Idx, /*Immutable=*/true);
713 if ((TRI->getEncodingValue(Reg) % 2) == 0)
716 return Color::Odd;