File Manager

003 File Manager

Current Path: /usr/src/contrib/llvm-project/llvm/lib/Target/AArch64

usr / src / contrib / llvm-project / llvm / lib / Target / AArch64 /

📁 ..
📄 AArch64.h(4.09 KB)
📄 AArch64.td(49.85 KB)
📄 AArch64A53Fix835769.cpp(8.3 KB)
📄 AArch64A57FPLoadBalancing.cpp(25.73 KB)
📄 AArch64AdvSIMDScalarPass.cpp(16.1 KB)
📄 AArch64AsmPrinter.cpp(49.26 KB)
📄 AArch64BranchTargets.cpp(4.91 KB)
📄 AArch64CallingConvention.cpp(6.63 KB)
📄 AArch64CallingConvention.h(2.62 KB)
📄 AArch64CallingConvention.td(23.84 KB)
📄 AArch64CleanupLocalDynamicTLSPass.cpp(5.53 KB)
📄 AArch64CollectLOH.cpp(20.07 KB)
📄 AArch64Combine.td(3.27 KB)
📄 AArch64CompressJumpTables.cpp(5.06 KB)
📄 AArch64CondBrTuning.cpp(10.19 KB)
📄 AArch64ConditionOptimizer.cpp(15.26 KB)
📄 AArch64ConditionalCompares.cpp(33.26 KB)
📄 AArch64DeadRegisterDefinitionsPass.cpp(7.76 KB)
📄 AArch64ExpandImm.cpp(14.25 KB)
📄 AArch64ExpandImm.h(959 B)
📄 AArch64ExpandPseudoInsts.cpp(38.08 KB)
📄 AArch64FalkorHWPFFix.cpp(23.3 KB)
📄 AArch64FastISel.cpp(171.76 KB)
📄 AArch64FrameLowering.cpp(124.13 KB)
📄 AArch64FrameLowering.h(5.54 KB)
📄 AArch64GenRegisterBankInfo.def(11 KB)
📄 AArch64ISelDAGToDAG.cpp(180.26 KB)
📄 AArch64ISelLowering.cpp(578.95 KB)
📄 AArch64ISelLowering.h(33.88 KB)
📄 AArch64InstrAtomics.td(20.33 KB)
📄 AArch64InstrFormats.td(430.93 KB)
📄 AArch64InstrGISel.td(4.29 KB)
📄 AArch64InstrInfo.cpp(243.23 KB)
📄 AArch64InstrInfo.h(19.92 KB)
📄 AArch64InstrInfo.td(374.84 KB)
📄 AArch64LoadStoreOptimizer.cpp(77.04 KB)
📄 AArch64MCInstLower.cpp(11.72 KB)
📄 AArch64MCInstLower.h(1.69 KB)
📄 AArch64MachineFunctionInfo.cpp(1.02 KB)
📄 AArch64MachineFunctionInfo.h(12.9 KB)
📄 AArch64MacroFusion.cpp(11.47 KB)
📄 AArch64MacroFusion.h(891 B)
📄 AArch64PBQPRegAlloc.cpp(11.35 KB)
📄 AArch64PBQPRegAlloc.h(1.3 KB)
📄 AArch64PerfectShuffle.h(382.04 KB)
📄 AArch64PfmCounters.td(713 B)
📄 AArch64PromoteConstant.cpp(22.43 KB)
📄 AArch64RedundantCopyElimination.cpp(17.09 KB)
📄 AArch64RegisterBanks.td(719 B)
📄 AArch64RegisterInfo.cpp(29.6 KB)
📄 AArch64RegisterInfo.h(5.5 KB)
📄 AArch64RegisterInfo.td(51 KB)
📄 AArch64SIMDInstrOpt.cpp(26.07 KB)
📄 AArch64SLSHardening.cpp(15.92 KB)
📄 AArch64SVEInstrInfo.td(169.43 KB)
📄 AArch64SchedA53.td(15.28 KB)
📄 AArch64SchedA57.td(34.69 KB)
📄 AArch64SchedA57WriteRes.td(19.87 KB)
📄 AArch64SchedCyclone.td(29.82 KB)
📄 AArch64SchedExynosM3.td(42.57 KB)
📄 AArch64SchedExynosM4.td(49.81 KB)
📄 AArch64SchedExynosM5.td(50.74 KB)
📄 AArch64SchedFalkor.td(5.3 KB)
📄 AArch64SchedFalkorDetails.td(67.66 KB)
📄 AArch64SchedKryo.td(6.21 KB)
📄 AArch64SchedKryoDetails.td(82.63 KB)
📄 AArch64SchedPredExynos.td(7.5 KB)
📄 AArch64SchedPredicates.td(27.86 KB)
📄 AArch64SchedThunderX.td(14.99 KB)
📄 AArch64SchedThunderX2T99.td(68.58 KB)
📄 AArch64SchedThunderX3T110.td(68.77 KB)
📄 AArch64Schedule.td(3.86 KB)
📄 AArch64SelectionDAGInfo.cpp(5.54 KB)
📄 AArch64SelectionDAGInfo.h(1.46 KB)
📄 AArch64SpeculationHardening.cpp(29.6 KB)
📄 AArch64StackOffset.h(5.01 KB)
📄 AArch64StackTagging.cpp(24.31 KB)
📄 AArch64StackTaggingPreRA.cpp(7.25 KB)
📄 AArch64StorePairSuppress.cpp(6.26 KB)
📄 AArch64Subtarget.cpp(12.38 KB)
📄 AArch64Subtarget.h(18.49 KB)
📄 AArch64SystemOperands.td(81.75 KB)
📄 AArch64TargetMachine.cpp(26.65 KB)
📄 AArch64TargetMachine.h(3.25 KB)
📄 AArch64TargetObjectFile.cpp(3.35 KB)
📄 AArch64TargetObjectFile.h(2.31 KB)
📄 AArch64TargetTransformInfo.cpp(42.6 KB)
📄 AArch64TargetTransformInfo.h(8.68 KB)
📁 AsmParser
📁 Disassembler
📁 GISel
📁 MCTargetDesc
📄 SVEInstrFormats.td(304.03 KB)
📄 SVEIntrinsicOpts.cpp(8.13 KB)
📁 TargetInfo
📁 Utils

Editing: SVEIntrinsicOpts.cpp

//===----- SVEIntrinsicOpts - SVE ACLE Intrinsics Opts --------------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
//
// Performs general IR level optimizations on SVE intrinsics.
//
// The main goal of this pass is to remove unnecessary reinterpret
// intrinsics (llvm.aarch64.sve.convert.[to|from].svbool), e.g:
//
//   %1 = @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1> %a)
//   %2 = @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> %1)
//
// This pass also looks for ptest intrinsics & phi instructions where the
// operands are being needlessly converted to and from svbool_t.
//
//===----------------------------------------------------------------------===//

#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/InitializePasses.h"
#include "llvm/Support/Debug.h"

using namespace llvm;
using namespace llvm::PatternMatch;

#define DEBUG_TYPE "sve-intrinsic-opts"

namespace llvm {
void initializeSVEIntrinsicOptsPass(PassRegistry &);
}

namespace {
struct SVEIntrinsicOpts : public ModulePass {
  static char ID; // Pass identification, replacement for typeid
  SVEIntrinsicOpts() : ModulePass(ID) {
    initializeSVEIntrinsicOptsPass(*PassRegistry::getPassRegistry());
  }

bool runOnModule(Module &M) override;
  void getAnalysisUsage(AnalysisUsage &AU) const override;

private:
  static IntrinsicInst *isReinterpretToSVBool(Value *V);

static bool optimizeIntrinsic(Instruction *I);

bool optimizeFunctions(SmallSetVector<Function *, 4> &Functions);

static bool optimizeConvertFromSVBool(IntrinsicInst *I);
  static bool optimizePTest(IntrinsicInst *I);

static bool processPhiNode(IntrinsicInst *I);
};
} // end anonymous namespace

void SVEIntrinsicOpts::getAnalysisUsage(AnalysisUsage &AU) const {
  AU.addRequired<DominatorTreeWrapperPass>();
  AU.setPreservesCFG();
}

char SVEIntrinsicOpts::ID = 0;
static const char *name = "SVE intrinsics optimizations";
INITIALIZE_PASS_BEGIN(SVEIntrinsicOpts, DEBUG_TYPE, name, false, false)
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass);
INITIALIZE_PASS_END(SVEIntrinsicOpts, DEBUG_TYPE, name, false, false)

namespace llvm {
ModulePass *createSVEIntrinsicOptsPass() { return new SVEIntrinsicOpts(); }
} // namespace llvm

/// Returns V if it's a cast from <n x 16 x i1> (aka svbool_t), nullptr
/// otherwise.
IntrinsicInst *SVEIntrinsicOpts::isReinterpretToSVBool(Value *V) {
  IntrinsicInst *I = dyn_cast<IntrinsicInst>(V);
  if (!I)
    return nullptr;

if (I->getIntrinsicID() != Intrinsic::aarch64_sve_convert_to_svbool)
    return nullptr;

return I;
}

/// The function will remove redundant reinterprets casting in the presence
/// of the control flow
bool SVEIntrinsicOpts::processPhiNode(IntrinsicInst *X) {

SmallVector<Instruction *, 32> Worklist;
  auto RequiredType = X->getType();

auto *PN = dyn_cast<PHINode>(X->getArgOperand(0));
  assert(PN && "Expected Phi Node!");

// Don't create a new Phi unless we can remove the old one.
  if (!PN->hasOneUse())
    return false;

for (Value *IncValPhi : PN->incoming_values()) {
    auto *Reinterpret = isReinterpretToSVBool(IncValPhi);
    if (!Reinterpret ||
        RequiredType != Reinterpret->getArgOperand(0)->getType())
      return false;
  }

// Create the new Phi
  LLVMContext &Ctx = PN->getContext();
  IRBuilder<> Builder(Ctx);
  Builder.SetInsertPoint(PN);
  PHINode *NPN = Builder.CreatePHI(RequiredType, PN->getNumIncomingValues());
  Worklist.push_back(PN);

for (unsigned I = 0; I < PN->getNumIncomingValues(); I++) {
    auto *Reinterpret = cast<Instruction>(PN->getIncomingValue(I));
    NPN->addIncoming(Reinterpret->getOperand(0), PN->getIncomingBlock(I));
    Worklist.push_back(Reinterpret);
  }

// Cleanup Phi Node and reinterprets
  X->replaceAllUsesWith(NPN);
  X->eraseFromParent();

for (auto &I : Worklist)
    if (I->use_empty())
      I->eraseFromParent();

return true;
}

bool SVEIntrinsicOpts::optimizePTest(IntrinsicInst *I) {
  IntrinsicInst *Op1 = dyn_cast<IntrinsicInst>(I->getArgOperand(0));
  IntrinsicInst *Op2 = dyn_cast<IntrinsicInst>(I->getArgOperand(1));

if (Op1 && Op2 &&
      Op1->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool &&
      Op2->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool &&
      Op1->getArgOperand(0)->getType() == Op2->getArgOperand(0)->getType()) {

Value *Ops[] = {Op1->getArgOperand(0), Op2->getArgOperand(0)};
    Type *Tys[] = {Op1->getArgOperand(0)->getType()};
    Module *M = I->getParent()->getParent()->getParent();

auto Fn = Intrinsic::getDeclaration(M, I->getIntrinsicID(), Tys);
    auto CI = CallInst::Create(Fn, Ops, I->getName(), I);

I->replaceAllUsesWith(CI);
    I->eraseFromParent();
    if (Op1->use_empty())
      Op1->eraseFromParent();
    if (Op1 != Op2 && Op2->use_empty())
      Op2->eraseFromParent();

return true;
  }

return false;
}

bool SVEIntrinsicOpts::optimizeConvertFromSVBool(IntrinsicInst *I) {
  assert(I->getIntrinsicID() == Intrinsic::aarch64_sve_convert_from_svbool &&
         "Unexpected opcode");

// If the reinterpret instruction operand is a PHI Node
  if (isa<PHINode>(I->getArgOperand(0)))
    return processPhiNode(I);

// If we have a reinterpret intrinsic I of type A which is converting from
  // another reinterpret Y of type B, and the source type of Y is A, then we can
  // elide away both reinterprets if there are no other users of Y.
  auto *Y = isReinterpretToSVBool(I->getArgOperand(0));
  if (!Y)
    return false;

Value *SourceVal = Y->getArgOperand(0);
  if (I->getType() != SourceVal->getType())
    return false;

I->replaceAllUsesWith(SourceVal);
  I->eraseFromParent();
  if (Y->use_empty())
    Y->eraseFromParent();

return true;
}

bool SVEIntrinsicOpts::optimizeIntrinsic(Instruction *I) {
  IntrinsicInst *IntrI = dyn_cast<IntrinsicInst>(I);
  if (!IntrI)
    return false;

switch (IntrI->getIntrinsicID()) {
  case Intrinsic::aarch64_sve_convert_from_svbool:
    return optimizeConvertFromSVBool(IntrI);
  case Intrinsic::aarch64_sve_ptest_any:
  case Intrinsic::aarch64_sve_ptest_first:
  case Intrinsic::aarch64_sve_ptest_last:
    return optimizePTest(IntrI);
  default:
    return false;
  }

return true;
}

bool SVEIntrinsicOpts::optimizeFunctions(
    SmallSetVector<Function *, 4> &Functions) {
  bool Changed = false;
  for (auto *F : Functions) {
    DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>(*F).getDomTree();

// Traverse the DT with an rpo walk so we see defs before uses, allowing
    // simplification to be done incrementally.
    BasicBlock *Root = DT->getRoot();
    ReversePostOrderTraversal<BasicBlock *> RPOT(Root);
    for (auto *BB : RPOT)
      for (Instruction &I : make_early_inc_range(*BB))
        Changed |= optimizeIntrinsic(&I);
  }
  return Changed;
}

bool SVEIntrinsicOpts::runOnModule(Module &M) {
  bool Changed = false;
  SmallSetVector<Function *, 4> Functions;

// Check for SVE intrinsic declarations first so that we only iterate over
  // relevant functions. Where an appropriate declaration is found, store the
  // function(s) where it is used so we can target these only.
  for (auto &F : M.getFunctionList()) {
    if (!F.isDeclaration())
      continue;

switch (F.getIntrinsicID()) {
    case Intrinsic::aarch64_sve_convert_from_svbool:
    case Intrinsic::aarch64_sve_ptest_any:
    case Intrinsic::aarch64_sve_ptest_first:
    case Intrinsic::aarch64_sve_ptest_last:
      for (auto I = F.user_begin(), E = F.user_end(); I != E;) {
        auto *Inst = dyn_cast<Instruction>(*I++);
        Functions.insert(Inst->getFunction());
      }
      break;
    default:
      break;
    }
  }

if (!Functions.empty())
    Changed |= optimizeFunctions(Functions);

return Changed;
}

003 File Manager

Editing: SVEIntrinsicOpts.cpp

Upload File

Create Folder