File Manager

003 File Manager

Current Path: /usr/src/contrib/llvm-project/llvm/lib/Target/AArch64

usr / src / contrib / llvm-project / llvm / lib / Target / AArch64 /

📁 ..
📄 AArch64.h(4.09 KB)
📄 AArch64.td(49.85 KB)
📄 AArch64A53Fix835769.cpp(8.3 KB)
📄 AArch64A57FPLoadBalancing.cpp(25.73 KB)
📄 AArch64AdvSIMDScalarPass.cpp(16.1 KB)
📄 AArch64AsmPrinter.cpp(49.26 KB)
📄 AArch64BranchTargets.cpp(4.91 KB)
📄 AArch64CallingConvention.cpp(6.63 KB)
📄 AArch64CallingConvention.h(2.62 KB)
📄 AArch64CallingConvention.td(23.84 KB)
📄 AArch64CleanupLocalDynamicTLSPass.cpp(5.53 KB)
📄 AArch64CollectLOH.cpp(20.07 KB)
📄 AArch64Combine.td(3.27 KB)
📄 AArch64CompressJumpTables.cpp(5.06 KB)
📄 AArch64CondBrTuning.cpp(10.19 KB)
📄 AArch64ConditionOptimizer.cpp(15.26 KB)
📄 AArch64ConditionalCompares.cpp(33.26 KB)
📄 AArch64DeadRegisterDefinitionsPass.cpp(7.76 KB)
📄 AArch64ExpandImm.cpp(14.25 KB)
📄 AArch64ExpandImm.h(959 B)
📄 AArch64ExpandPseudoInsts.cpp(38.08 KB)
📄 AArch64FalkorHWPFFix.cpp(23.3 KB)
📄 AArch64FastISel.cpp(171.76 KB)
📄 AArch64FrameLowering.cpp(124.13 KB)
📄 AArch64FrameLowering.h(5.54 KB)
📄 AArch64GenRegisterBankInfo.def(11 KB)
📄 AArch64ISelDAGToDAG.cpp(180.26 KB)
📄 AArch64ISelLowering.cpp(578.95 KB)
📄 AArch64ISelLowering.h(33.88 KB)
📄 AArch64InstrAtomics.td(20.33 KB)
📄 AArch64InstrFormats.td(430.93 KB)
📄 AArch64InstrGISel.td(4.29 KB)
📄 AArch64InstrInfo.cpp(243.23 KB)
📄 AArch64InstrInfo.h(19.92 KB)
📄 AArch64InstrInfo.td(374.84 KB)
📄 AArch64LoadStoreOptimizer.cpp(77.04 KB)
📄 AArch64MCInstLower.cpp(11.72 KB)
📄 AArch64MCInstLower.h(1.69 KB)
📄 AArch64MachineFunctionInfo.cpp(1.02 KB)
📄 AArch64MachineFunctionInfo.h(12.9 KB)
📄 AArch64MacroFusion.cpp(11.47 KB)
📄 AArch64MacroFusion.h(891 B)
📄 AArch64PBQPRegAlloc.cpp(11.35 KB)
📄 AArch64PBQPRegAlloc.h(1.3 KB)
📄 AArch64PerfectShuffle.h(382.04 KB)
📄 AArch64PfmCounters.td(713 B)
📄 AArch64PromoteConstant.cpp(22.43 KB)
📄 AArch64RedundantCopyElimination.cpp(17.09 KB)
📄 AArch64RegisterBanks.td(719 B)
📄 AArch64RegisterInfo.cpp(29.6 KB)
📄 AArch64RegisterInfo.h(5.5 KB)
📄 AArch64RegisterInfo.td(51 KB)
📄 AArch64SIMDInstrOpt.cpp(26.07 KB)
📄 AArch64SLSHardening.cpp(15.92 KB)
📄 AArch64SVEInstrInfo.td(169.43 KB)
📄 AArch64SchedA53.td(15.28 KB)
📄 AArch64SchedA57.td(34.69 KB)
📄 AArch64SchedA57WriteRes.td(19.87 KB)
📄 AArch64SchedCyclone.td(29.82 KB)
📄 AArch64SchedExynosM3.td(42.57 KB)
📄 AArch64SchedExynosM4.td(49.81 KB)
📄 AArch64SchedExynosM5.td(50.74 KB)
📄 AArch64SchedFalkor.td(5.3 KB)
📄 AArch64SchedFalkorDetails.td(67.66 KB)
📄 AArch64SchedKryo.td(6.21 KB)
📄 AArch64SchedKryoDetails.td(82.63 KB)
📄 AArch64SchedPredExynos.td(7.5 KB)
📄 AArch64SchedPredicates.td(27.86 KB)
📄 AArch64SchedThunderX.td(14.99 KB)
📄 AArch64SchedThunderX2T99.td(68.58 KB)
📄 AArch64SchedThunderX3T110.td(68.77 KB)
📄 AArch64Schedule.td(3.86 KB)
📄 AArch64SelectionDAGInfo.cpp(5.54 KB)
📄 AArch64SelectionDAGInfo.h(1.46 KB)
📄 AArch64SpeculationHardening.cpp(29.6 KB)
📄 AArch64StackOffset.h(5.01 KB)
📄 AArch64StackTagging.cpp(24.31 KB)
📄 AArch64StackTaggingPreRA.cpp(7.25 KB)
📄 AArch64StorePairSuppress.cpp(6.26 KB)
📄 AArch64Subtarget.cpp(12.38 KB)
📄 AArch64Subtarget.h(18.49 KB)
📄 AArch64SystemOperands.td(81.75 KB)
📄 AArch64TargetMachine.cpp(26.65 KB)
📄 AArch64TargetMachine.h(3.25 KB)
📄 AArch64TargetObjectFile.cpp(3.35 KB)
📄 AArch64TargetObjectFile.h(2.31 KB)
📄 AArch64TargetTransformInfo.cpp(42.6 KB)
📄 AArch64TargetTransformInfo.h(8.68 KB)
📁 AsmParser
📁 Disassembler
📁 GISel
📁 MCTargetDesc
📄 SVEInstrFormats.td(304.03 KB)
📄 SVEIntrinsicOpts.cpp(8.13 KB)
📁 TargetInfo
📁 Utils

Editing: AArch64A53Fix835769.cpp

//===-- AArch64A53Fix835769.cpp -------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
// This pass changes code to work around Cortex-A53 erratum 835769.
// It works around it by inserting a nop instruction in code sequences that
// in some circumstances may trigger the erratum.
// It inserts a nop instruction between a sequence of the following 2 classes
// of instructions:
// instr 1: mem-instr (including loads, stores and prefetches).
// instr 2: non-SIMD integer multiply-accumulate writing 64-bit X registers.
//===----------------------------------------------------------------------===//

#include "AArch64.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"

using namespace llvm;

#define DEBUG_TYPE "aarch64-fix-cortex-a53-835769"

STATISTIC(NumNopsAdded, "Number of Nops added to work around erratum 835769");

//===----------------------------------------------------------------------===//
// Helper functions

// Is the instruction a match for the instruction that comes first in the
// sequence of instructions that can trigger the erratum?
static bool isFirstInstructionInSequence(MachineInstr *MI) {
  // Must return true if this instruction is a load, a store or a prefetch.
  switch (MI->getOpcode()) {
  case AArch64::PRFMl:
  case AArch64::PRFMroW:
  case AArch64::PRFMroX:
  case AArch64::PRFMui:
  case AArch64::PRFUMi:
    return true;
  default:
    return MI->mayLoadOrStore();
  }
}

// Is the instruction a match for the instruction that comes second in the
// sequence that can trigger the erratum?
static bool isSecondInstructionInSequence(MachineInstr *MI) {
  // Must return true for non-SIMD integer multiply-accumulates, writing
  // to a 64-bit register.
  switch (MI->getOpcode()) {
  // Erratum cannot be triggered when the destination register is 32 bits,
  // therefore only include the following.
  case AArch64::MSUBXrrr:
  case AArch64::MADDXrrr:
  case AArch64::SMADDLrrr:
  case AArch64::SMSUBLrrr:
  case AArch64::UMADDLrrr:
  case AArch64::UMSUBLrrr:
    // Erratum can only be triggered by multiply-adds, not by regular
    // non-accumulating multiplies, i.e. when Ra=XZR='11111'
    return MI->getOperand(3).getReg() != AArch64::XZR;
  default:
    return false;
  }
}

//===----------------------------------------------------------------------===//

namespace {
class AArch64A53Fix835769 : public MachineFunctionPass {
  const TargetInstrInfo *TII;

public:
  static char ID;
  explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {
    initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry());
  }

bool runOnMachineFunction(MachineFunction &F) override;

MachineFunctionProperties getRequiredProperties() const override {
    return MachineFunctionProperties().set(
        MachineFunctionProperties::Property::NoVRegs);
  }

StringRef getPassName() const override {
    return "Workaround A53 erratum 835769 pass";
  }

void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.setPreservesCFG();
    MachineFunctionPass::getAnalysisUsage(AU);
  }

private:
  bool runOnBasicBlock(MachineBasicBlock &MBB);
};
char AArch64A53Fix835769::ID = 0;

} // end anonymous namespace

INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass",
                "AArch64 fix for A53 erratum 835769", false, false)

//===----------------------------------------------------------------------===//

bool
AArch64A53Fix835769::runOnMachineFunction(MachineFunction &F) {
  LLVM_DEBUG(dbgs() << "***** AArch64A53Fix835769 *****\n");
  bool Changed = false;
  TII = F.getSubtarget().getInstrInfo();

for (auto &MBB : F) {
    Changed |= runOnBasicBlock(MBB);
  }
  return Changed;
}

// Return the block that was fallen through to get to MBB, if any,
// otherwise nullptr.
static MachineBasicBlock *getBBFallenThrough(MachineBasicBlock *MBB,
                                             const TargetInstrInfo *TII) {
  // Get the previous machine basic block in the function.
  MachineFunction::iterator MBBI(MBB);

// Can't go off top of function.
  if (MBBI == MBB->getParent()->begin())
    return nullptr;

MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
  SmallVector<MachineOperand, 2> Cond;

MachineBasicBlock *PrevBB = &*std::prev(MBBI);
  for (MachineBasicBlock *S : MBB->predecessors())
    if (S == PrevBB && !TII->analyzeBranch(*PrevBB, TBB, FBB, Cond) && !TBB &&
        !FBB)
      return S;

return nullptr;
}

// Iterate through fallen through blocks trying to find a previous non-pseudo if
// there is one, otherwise return nullptr. Only look for instructions in
// previous blocks, not the current block, since we only use this to look at
// previous blocks.
static MachineInstr *getLastNonPseudo(MachineBasicBlock &MBB,
                                      const TargetInstrInfo *TII) {
  MachineBasicBlock *FMBB = &MBB;

// If there is no non-pseudo in the current block, loop back around and try
  // the previous block (if there is one).
  while ((FMBB = getBBFallenThrough(FMBB, TII))) {
    for (MachineInstr &I : make_range(FMBB->rbegin(), FMBB->rend()))
      if (!I.isPseudo())
        return &I;
  }

// There was no previous non-pseudo in the fallen through blocks
  return nullptr;
}

static void insertNopBeforeInstruction(MachineBasicBlock &MBB, MachineInstr* MI,
                                       const TargetInstrInfo *TII) {
  // If we are the first instruction of the block, put the NOP at the end of
  // the previous fallthrough block
  if (MI == &MBB.front()) {
    MachineInstr *I = getLastNonPseudo(MBB, TII);
    assert(I && "Expected instruction");
    DebugLoc DL = I->getDebugLoc();
    BuildMI(I->getParent(), DL, TII->get(AArch64::HINT)).addImm(0);
  }
  else {
    DebugLoc DL = MI->getDebugLoc();
    BuildMI(MBB, MI, DL, TII->get(AArch64::HINT)).addImm(0);
  }

++NumNopsAdded;
}

bool
AArch64A53Fix835769::runOnBasicBlock(MachineBasicBlock &MBB) {
  bool Changed = false;
  LLVM_DEBUG(dbgs() << "Running on MBB: " << MBB
                    << " - scanning instructions...\n");

// First, scan the basic block, looking for a sequence of 2 instructions
  // that match the conditions under which the erratum may trigger.

// List of terminating instructions in matching sequences
  std::vector<MachineInstr*> Sequences;
  unsigned Idx = 0;
  MachineInstr *PrevInstr = nullptr;

// Try and find the last non-pseudo instruction in any fallen through blocks,
  // if there isn't one, then we use nullptr to represent that.
  PrevInstr = getLastNonPseudo(MBB, TII);

for (auto &MI : MBB) {
    MachineInstr *CurrInstr = &MI;
    LLVM_DEBUG(dbgs() << "  Examining: " << MI);
    if (PrevInstr) {
      LLVM_DEBUG(dbgs() << "    PrevInstr: " << *PrevInstr
                        << "    CurrInstr: " << *CurrInstr
                        << "    isFirstInstructionInSequence(PrevInstr): "
                        << isFirstInstructionInSequence(PrevInstr) << "\n"
                        << "    isSecondInstructionInSequence(CurrInstr): "
                        << isSecondInstructionInSequence(CurrInstr) << "\n");
      if (isFirstInstructionInSequence(PrevInstr) &&
          isSecondInstructionInSequence(CurrInstr)) {
        LLVM_DEBUG(dbgs() << "   ** pattern found at Idx " << Idx << "!\n");
        Sequences.push_back(CurrInstr);
      }
    }
    if (!CurrInstr->isPseudo())
      PrevInstr = CurrInstr;
    ++Idx;
  }

LLVM_DEBUG(dbgs() << "Scan complete, " << Sequences.size()
                    << " occurrences of pattern found.\n");

// Then update the basic block, inserting nops between the detected sequences.
  for (auto &MI : Sequences) {
    Changed = true;
    insertNopBeforeInstruction(MBB, MI, TII);
  }

return Changed;
}

// Factory function used by AArch64TargetMachine to add the pass to
// the passmanager.
FunctionPass *llvm::createAArch64A53Fix835769() {
  return new AArch64A53Fix835769();
}

003 File Manager

Editing: AArch64A53Fix835769.cpp

Upload File

Create Folder