File Manager

003 File Manager

Current Path: /usr/src/contrib/llvm-project/llvm/lib/Target/AMDGPU

usr / src / contrib / llvm-project / llvm / lib / Target / AMDGPU /

📁 ..
📄 AMDGPU.h(11.46 KB)
📄 AMDGPU.td(36.97 KB)
📄 AMDGPUAliasAnalysis.cpp(5.58 KB)
📄 AMDGPUAliasAnalysis.h(3.32 KB)
📄 AMDGPUAlwaysInlinePass.cpp(4.83 KB)
📄 AMDGPUAnnotateKernelFeatures.cpp(11.94 KB)
📄 AMDGPUAnnotateUniformValues.cpp(6.13 KB)
📄 AMDGPUArgumentUsageInfo.cpp(7.66 KB)
📄 AMDGPUArgumentUsageInfo.h(4.81 KB)
📄 AMDGPUAsmPrinter.cpp(50.42 KB)
📄 AMDGPUAsmPrinter.h(5.13 KB)
📄 AMDGPUAtomicOptimizer.cpp(23.79 KB)
📄 AMDGPUCallLowering.cpp(28.66 KB)
📄 AMDGPUCallLowering.h(2.37 KB)
📄 AMDGPUCallingConv.td(7.33 KB)
📄 AMDGPUCodeGenPrepare.cpp(46.42 KB)
📄 AMDGPUCombine.td(2.79 KB)
📄 AMDGPUExportClustering.cpp(4.52 KB)
📄 AMDGPUExportClustering.h(533 B)
📄 AMDGPUFeatures.td(1.81 KB)
📄 AMDGPUFixFunctionBitcasts.cpp(1.87 KB)
📄 AMDGPUFrameLowering.cpp(1.98 KB)
📄 AMDGPUFrameLowering.h(1.39 KB)
📄 AMDGPUGISel.td(11.57 KB)
📄 AMDGPUGenRegisterBankInfo.def(5.83 KB)
📄 AMDGPUGlobalISelUtils.cpp(1.77 KB)
📄 AMDGPUGlobalISelUtils.h(2.07 KB)
📄 AMDGPUHSAMetadataStreamer.cpp(31.21 KB)
📄 AMDGPUHSAMetadataStreamer.h(5.46 KB)
📄 AMDGPUISelDAGToDAG.cpp(101.59 KB)
📄 AMDGPUISelLowering.cpp(168.65 KB)
📄 AMDGPUISelLowering.h(19.23 KB)
📄 AMDGPUInline.cpp(7.97 KB)
📄 AMDGPUInstrInfo.cpp(1.71 KB)
📄 AMDGPUInstrInfo.h(1.66 KB)
📄 AMDGPUInstrInfo.td(17.18 KB)
📄 AMDGPUInstructionSelector.cpp(128.53 KB)
📄 AMDGPUInstructionSelector.h(11.04 KB)
📄 AMDGPUInstructions.td(25.36 KB)
📄 AMDGPULegalizerInfo.cpp(149.32 KB)
📄 AMDGPULegalizerInfo.h(8.49 KB)
📄 AMDGPULibCalls.cpp(53.89 KB)
📄 AMDGPULibFunc.cpp(37.85 KB)
📄 AMDGPULibFunc.h(10.99 KB)
📄 AMDGPULowerIntrinsics.cpp(4.55 KB)
📄 AMDGPULowerKernelArguments.cpp(8.89 KB)
📄 AMDGPULowerKernelAttributes.cpp(7.78 KB)
📄 AMDGPUMCInstLower.cpp(14.27 KB)
📄 AMDGPUMachineCFGStructurizer.cpp(101.97 KB)
📄 AMDGPUMachineFunction.cpp(2.24 KB)
📄 AMDGPUMachineFunction.h(2.13 KB)
📄 AMDGPUMachineModuleInfo.cpp(1.34 KB)
📄 AMDGPUMachineModuleInfo.h(5.46 KB)
📄 AMDGPUMacroFusion.cpp(2.28 KB)
📄 AMDGPUMacroFusion.h(679 B)
📄 AMDGPUOpenCLEnqueuedBlockLowering.cpp(5.31 KB)
📄 AMDGPUPTNote.h(1.29 KB)
📄 AMDGPUPerfHintAnalysis.cpp(12.17 KB)
📄 AMDGPUPerfHintAnalysis.h(1.67 KB)
📄 AMDGPUPostLegalizerCombiner.cpp(12.02 KB)
📄 AMDGPUPreLegalizerCombiner.cpp(5.45 KB)
📄 AMDGPUPrintfRuntimeBinding.cpp(21.7 KB)
📄 AMDGPUPromoteAlloca.cpp(35.24 KB)
📄 AMDGPUPropagateAttributes.cpp(11.76 KB)
📄 AMDGPURegBankCombiner.cpp(5.36 KB)
📄 AMDGPURegisterBankInfo.cpp(161.67 KB)
📄 AMDGPURegisterBankInfo.h(7.41 KB)
📄 AMDGPURegisterBanks.td(921 B)
📄 AMDGPURewriteOutArguments.cpp(15.82 KB)
📄 AMDGPUSearchableTables.td(21.04 KB)
📄 AMDGPUSubtarget.cpp(29.62 KB)
📄 AMDGPUSubtarget.h(35.82 KB)
📄 AMDGPUTargetMachine.cpp(42.67 KB)
📄 AMDGPUTargetMachine.h(4.52 KB)
📄 AMDGPUTargetObjectFile.cpp(1.54 KB)
📄 AMDGPUTargetObjectFile.h(1.14 KB)
📄 AMDGPUTargetTransformInfo.cpp(39.07 KB)
📄 AMDGPUTargetTransformInfo.h(11.11 KB)
📄 AMDGPUUnifyDivergentExitNodes.cpp(13.84 KB)
📄 AMDGPUUnifyMetadata.cpp(4.46 KB)
📄 AMDILCFGStructurizer.cpp(56.32 KB)
📄 AMDKernelCodeT.h(32.84 KB)
📁 AsmParser
📄 BUFInstructions.td(110.75 KB)
📄 CaymanInstructions.td(7.93 KB)
📄 DSInstructions.td(52.37 KB)
📁 Disassembler
📄 EvergreenInstructions.td(28.24 KB)
📄 FLATInstructions.td(66.93 KB)
📄 GCNDPPCombine.cpp(19.92 KB)
📄 GCNHazardRecognizer.cpp(45.3 KB)
📄 GCNHazardRecognizer.h(3.96 KB)
📄 GCNILPSched.cpp(11.3 KB)
📄 GCNIterativeScheduler.cpp(20.62 KB)
📄 GCNIterativeScheduler.h(4.16 KB)
📄 GCNMinRegStrategy.cpp(8.47 KB)
📄 GCNNSAReassign.cpp(10.92 KB)
📄 GCNProcessors.td(4.84 KB)
📄 GCNRegBankReassign.cpp(26.68 KB)
📄 GCNRegPressure.cpp(16.27 KB)
📄 GCNRegPressure.h(9.15 KB)
📄 GCNSchedStrategy.cpp(21.67 KB)
📄 GCNSchedStrategy.h(3.77 KB)
📁 MCTargetDesc
📄 MIMGInstructions.td(39.85 KB)
📄 R600.td(1.51 KB)
📄 R600AsmPrinter.cpp(4.46 KB)
📄 R600AsmPrinter.h(1.5 KB)
📄 R600ClauseMergePass.cpp(7.38 KB)
📄 R600ControlFlowFinalizer.cpp(23.4 KB)
📄 R600Defines.h(4.25 KB)
📄 R600EmitClauseMarkers.cpp(12.1 KB)
📄 R600ExpandSpecialInstrs.cpp(10.11 KB)
📄 R600FrameLowering.cpp(1.83 KB)
📄 R600FrameLowering.h(1.25 KB)
📄 R600ISelLowering.cpp(81.88 KB)
📄 R600ISelLowering.h(4.8 KB)
📄 R600InstrFormats.td(11.58 KB)
📄 R600InstrInfo.cpp(49.47 KB)
📄 R600InstrInfo.h(13.7 KB)
📄 R600Instructions.td(55.13 KB)
📄 R600MachineFunctionInfo.cpp(551 B)
📄 R600MachineFunctionInfo.h(824 B)
📄 R600MachineScheduler.cpp(13.57 KB)
📄 R600MachineScheduler.h(2.53 KB)
📄 R600OpenCLImageTypeLoweringPass.cpp(11.75 KB)
📄 R600OptimizeVectorRegisters.cpp(13.4 KB)
📄 R600Packetizer.cpp(13.4 KB)
📄 R600Processors.td(4.42 KB)
📄 R600RegisterInfo.cpp(3.95 KB)
📄 R600RegisterInfo.h(2 KB)
📄 R600RegisterInfo.td(9.75 KB)
📄 R600Schedule.td(1.62 KB)
📄 R700Instructions.td(783 B)
📄 SIAddIMGInit.cpp(6.24 KB)
📄 SIAnnotateControlFlow.cpp(11.18 KB)
📄 SIDefines.h(20.86 KB)
📄 SIFixSGPRCopies.cpp(29.46 KB)
📄 SIFixVGPRCopies.cpp(2 KB)
📄 SIFixupVectorISel.cpp(8.75 KB)
📄 SIFoldOperands.cpp(54.56 KB)
📄 SIFormMemoryClauses.cpp(12.76 KB)
📄 SIFrameLowering.cpp(48.08 KB)
📄 SIFrameLowering.h(2.98 KB)
📄 SIISelLowering.cpp(423.43 KB)
📄 SIISelLowering.h(22.13 KB)
📄 SIInsertHardClauses.cpp(7.01 KB)
📄 SIInsertSkips.cpp(15.29 KB)
📄 SIInsertWaitcnts.cpp(58.33 KB)
📄 SIInstrFormats.td(9.44 KB)
📄 SIInstrInfo.cpp(247.15 KB)
📄 SIInstrInfo.h(41.24 KB)
📄 SIInstrInfo.td(90.7 KB)
📄 SIInstructions.td(77.7 KB)
📄 SILoadStoreOptimizer.cpp(76.21 KB)
📄 SILowerControlFlow.cpp(22.66 KB)
📄 SILowerI1Copies.cpp(27.83 KB)
📄 SILowerSGPRSpills.cpp(12.68 KB)
📄 SIMachineFunctionInfo.cpp(20.01 KB)
📄 SIMachineFunctionInfo.h(26.91 KB)
📄 SIMachineScheduler.cpp(69.44 KB)
📄 SIMachineScheduler.h(15.65 KB)
📄 SIMemoryLegalizer.cpp(45.84 KB)
📄 SIModeRegister.cpp(17.43 KB)
📄 SIOptimizeExecMasking.cpp(12.81 KB)
📄 SIOptimizeExecMaskingPreRA.cpp(11.13 KB)
📄 SIPeepholeSDWA.cpp(42.84 KB)
📄 SIPostRABundler.cpp(3.6 KB)
📄 SIPreAllocateWWMRegs.cpp(6.09 KB)
📄 SIPreEmitPeephole.cpp(10.51 KB)
📄 SIProgramInfo.h(2.04 KB)
📄 SIRegisterInfo.cpp(71.51 KB)
📄 SIRegisterInfo.h(13.04 KB)
📄 SIRegisterInfo.td(37.28 KB)
📄 SIRemoveShortExecBranches.cpp(4.96 KB)
📄 SISchedule.td(7.58 KB)
📄 SIShrinkInstructions.cpp(26.86 KB)
📄 SIWholeQuadMode.cpp(30.22 KB)
📄 SMInstructions.td(48.14 KB)
📄 SOPInstructions.td(60.51 KB)
📁 TargetInfo
📁 Utils
📄 VIInstrFormats.td(645 B)
📄 VOP1Instructions.td(35.53 KB)
📄 VOP2Instructions.td(65.04 KB)
📄 VOP3Instructions.td(53.14 KB)
📄 VOP3PInstructions.td(26.47 KB)
📄 VOPCInstructions.td(63.31 KB)
📄 VOPInstructions.td(23.76 KB)

Editing: SIFixupVectorISel.cpp

//===-- SIFixupVectorISel.cpp - Fixup post ISel vector issues -------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
/// \file
/// SIFixupVectorISel pass cleans up post ISEL Vector issues.
/// Currently this will convert GLOBAL_{LOAD|STORE}_*
/// and GLOBAL_Atomic_* instructions into their _SADDR variants,
/// feeding the sreg into the saddr field of the new instruction.
/// We currently handle a REG_SEQUENCE feeding the vaddr
/// and decompose it into a base and index.
///
/// Transform:
/// %17:vgpr_32, %19:sreg_64_xexec = V_ADD_I32_e64 %21:sgpr_32, %22:vgpr_32
/// %18:vgpr_32, %20:sreg_64_xexec = V_ADDC_U32_e64 %25:vgpr_32,
///                                    %24:vgpr_32, %19:sreg_64_xexec
/// %16:vreg_64 = REG_SEQUENCE %17:vgpr_32, %sub0, %18:vgpr_32, %sub1
/// %11:vreg_64 = COPY %16:vreg_64
/// %10:vgpr_32 = GLOBAL_LOAD_DWORD killed %11:vreg_64, 16, 0, 0
/// Into:
/// %4:sreg_64_xexec = S_LOAD_DWORDX2_IMM %1:sgpr_64, 36, 0
/// %14:vreg_64 = REG_SEQUENCE %6:vgpr_32, %sub0, %15:vgpr_32, %sub1
/// %10:vgpr_32 = GLOBAL_LOAD_DWORD_SADDR %14:vreg_64, %4:sreg_64_xexec,16...
///
//===----------------------------------------------------------------------===//
//

#include "AMDGPU.h"
#include "AMDGPUSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/Support/Debug.h"
#include "llvm/Target/TargetMachine.h"
#define DEBUG_TYPE "si-fixup-vector-isel"

using namespace llvm;

static cl::opt<bool> EnableGlobalSGPRAddr(
  "amdgpu-enable-global-sgpr-addr",
  cl::desc("Enable use of SGPR regs for GLOBAL LOAD/STORE instructions"),
  cl::init(false));

STATISTIC(NumSGPRGlobalOccurs, "Number of global ld/st opportunities");
STATISTIC(NumSGPRGlobalSaddrs, "Number of global sgpr instructions converted");

namespace {

class SIFixupVectorISel : public MachineFunctionPass {
public:
  static char ID;

public:
  SIFixupVectorISel() : MachineFunctionPass(ID) {
    initializeSIFixupVectorISelPass(*PassRegistry::getPassRegistry());
  }

bool runOnMachineFunction(MachineFunction &MF) override;

void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.setPreservesCFG();
    MachineFunctionPass::getAnalysisUsage(AU);
  }
};

} // End anonymous namespace.

INITIALIZE_PASS(SIFixupVectorISel, DEBUG_TYPE,
                "SI Fixup Vector ISel", false, false)

char SIFixupVectorISel::ID = 0;

char &llvm::SIFixupVectorISelID = SIFixupVectorISel::ID;

FunctionPass *llvm::createSIFixupVectorISelPass() {
  return new SIFixupVectorISel();
}

static bool findSRegBaseAndIndex(MachineOperand *Op,
                                 unsigned &BaseReg,
                                 unsigned &IndexReg,
                                 MachineRegisterInfo &MRI,
                                 const SIRegisterInfo *TRI) {
  SmallVector<MachineOperand *, 8> Worklist;
  Worklist.push_back(Op);
  while (!Worklist.empty()) {
    MachineOperand *WOp = Worklist.pop_back_val();
    if (!WOp->isReg() || !Register::isVirtualRegister(WOp->getReg()))
      continue;
    MachineInstr *DefInst = MRI.getUniqueVRegDef(WOp->getReg());
    switch (DefInst->getOpcode()) {
    default:
      continue;
    case AMDGPU::COPY:
      Worklist.push_back(&DefInst->getOperand(1));
      break;
    case AMDGPU::REG_SEQUENCE:
      if (DefInst->getNumOperands() != 5)
        continue;
      Worklist.push_back(&DefInst->getOperand(1));
      Worklist.push_back(&DefInst->getOperand(3));
      break;
    case AMDGPU::V_ADD_I32_e64:
      // The V_ADD_* and its analogous V_ADDCV_* are generated by
      // a previous pass which lowered from an ADD_64_PSEUDO,
      // which generates subregs to break up the 64 bit args.
      if (DefInst->getOperand(2).getSubReg() != AMDGPU::NoSubRegister)
        continue;
      BaseReg = DefInst->getOperand(2).getReg();
      if (DefInst->getOperand(3).getSubReg() != AMDGPU::NoSubRegister)
        continue;
      IndexReg = DefInst->getOperand(3).getReg();
      // Chase the IndexReg.
      MachineInstr *MI = MRI.getUniqueVRegDef(IndexReg);
      if (!MI || !MI->isCopy())
        continue;
      // Make sure the reg class is 64 bit for Index.
      // If the Index register is a subreg, we want it to reference
      // a 64 bit register which we will use as the Index reg.
      const TargetRegisterClass *IdxRC, *BaseRC;
      IdxRC = MRI.getRegClass(MI->getOperand(1).getReg());
      if (AMDGPU::getRegBitWidth(IdxRC->getID()) != 64)
        continue;
      IndexReg = MI->getOperand(1).getReg();
      // Chase the BaseReg.
      MI = MRI.getUniqueVRegDef(BaseReg);
      if (!MI || !MI->isCopy())
        continue;
      // Make sure the register class is 64 bit for Base.
      BaseReg = MI->getOperand(1).getReg();
      BaseRC = MRI.getRegClass(BaseReg);
      if (AMDGPU::getRegBitWidth(BaseRC->getID()) != 64)
        continue;
      // Make sure Base is SReg and Index is VReg.
      if (!TRI->isSGPRReg(MRI, BaseReg))
        return false;
      if (!TRI->hasVGPRs(MRI.getRegClass(IndexReg)))
        return false;
      // clear any killed flags on Index and Base regs, used later.
      MRI.clearKillFlags(IndexReg);
      MRI.clearKillFlags(BaseReg);
      return true;
    }
  }
  return false;
}

// Identify Global LOAD|STORE/ATOMIC and try to convert to _SADDR.
static bool fixupGlobalSaddr(MachineBasicBlock &MBB,
                             MachineFunction &MF,
                             MachineRegisterInfo &MRI,
                             const GCNSubtarget &ST,
                             const SIInstrInfo *TII,
                             const SIRegisterInfo *TRI) {
  if (!EnableGlobalSGPRAddr)
    return false;
  bool FuncModified = false;
  MachineBasicBlock::iterator I, Next;
  for (I = MBB.begin(); I != MBB.end(); I = Next) {
    Next = std::next(I);
    MachineInstr &MI = *I;
    int NewOpcd = AMDGPU::getGlobalSaddrOp(MI.getOpcode());
    if (NewOpcd < 0)
      continue;
    // Update our statistics on opportunities seen.
    ++NumSGPRGlobalOccurs;
    LLVM_DEBUG(dbgs() << "Global Mem opp " << MI << '\n');
    // Need a Base and Index or we cant transform to _SADDR.
    unsigned BaseReg = 0;
    unsigned IndexReg = 0;
    MachineOperand *Op = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr);
    if (!findSRegBaseAndIndex(Op, BaseReg, IndexReg, MRI, TRI))
      continue;
    ++NumSGPRGlobalSaddrs;
    FuncModified = true;
    // Create the new _SADDR Memory instruction.
    bool HasVdst = TII->getNamedOperand(MI, AMDGPU::OpName::vdst) != nullptr;
    MachineOperand *VData = TII->getNamedOperand(MI, AMDGPU::OpName::vdata);
    MachineInstr *NewGlob = nullptr;
    NewGlob = BuildMI(MBB, I, MI.getDebugLoc(), TII->get(NewOpcd));
    if (HasVdst)
      NewGlob->addOperand(MF, MI.getOperand(0));
    NewGlob->addOperand(MF, MachineOperand::CreateReg(IndexReg, false));
    if (VData)
      NewGlob->addOperand(MF, *VData);
    NewGlob->addOperand(MF, MachineOperand::CreateReg(BaseReg, false));
    NewGlob->addOperand(*TII->getNamedOperand(MI, AMDGPU::OpName::offset));

MachineOperand *Glc = TII->getNamedOperand(MI, AMDGPU::OpName::glc);
    // Atomics dont have a GLC, so omit the field if not there.
    if (Glc)
      NewGlob->addOperand(MF, *Glc);

MachineOperand *DLC = TII->getNamedOperand(MI, AMDGPU::OpName::dlc);
    if (DLC)
      NewGlob->addOperand(MF, *DLC);

NewGlob->addOperand(*TII->getNamedOperand(MI, AMDGPU::OpName::slc));
    // _D16 have an vdst_in operand, copy it in.
    MachineOperand *VDstInOp = TII->getNamedOperand(MI,
                                      AMDGPU::OpName::vdst_in);
    if (VDstInOp)
      NewGlob->addOperand(MF, *VDstInOp);
    NewGlob->copyImplicitOps(MF, MI);
    NewGlob->cloneMemRefs(MF, MI);
    // Remove the old Global Memop instruction.
    MI.eraseFromParent();
    LLVM_DEBUG(dbgs() << "New Global Mem " << *NewGlob << '\n');
  }
  return FuncModified;
}

bool SIFixupVectorISel::runOnMachineFunction(MachineFunction &MF) {
  // Only need to run this in SelectionDAG path.
  if (MF.getProperties().hasProperty(
        MachineFunctionProperties::Property::Selected))
    return false;

if (skipFunction(MF.getFunction()))
    return false;

MachineRegisterInfo &MRI = MF.getRegInfo();
  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
  const SIInstrInfo *TII = ST.getInstrInfo();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();

bool FuncModified = false;
  for (MachineBasicBlock &MBB : MF) {
    // Cleanup missed Saddr opportunites from ISel.
    FuncModified |= fixupGlobalSaddr(MBB, MF, MRI, ST, TII, TRI);
  }
  return FuncModified;
}

003 File Manager

Editing: SIFixupVectorISel.cpp

Upload File

Create Folder