003 File Manager
Current Path:
/usr/src/contrib/llvm-project/llvm/lib/Target/AMDGPU
usr
/
src
/
contrib
/
llvm-project
/
llvm
/
lib
/
Target
/
AMDGPU
/
📁
..
📄
AMDGPU.h
(11.46 KB)
📄
AMDGPU.td
(36.97 KB)
📄
AMDGPUAliasAnalysis.cpp
(5.58 KB)
📄
AMDGPUAliasAnalysis.h
(3.32 KB)
📄
AMDGPUAlwaysInlinePass.cpp
(4.83 KB)
📄
AMDGPUAnnotateKernelFeatures.cpp
(11.94 KB)
📄
AMDGPUAnnotateUniformValues.cpp
(6.13 KB)
📄
AMDGPUArgumentUsageInfo.cpp
(7.66 KB)
📄
AMDGPUArgumentUsageInfo.h
(4.81 KB)
📄
AMDGPUAsmPrinter.cpp
(50.42 KB)
📄
AMDGPUAsmPrinter.h
(5.13 KB)
📄
AMDGPUAtomicOptimizer.cpp
(23.79 KB)
📄
AMDGPUCallLowering.cpp
(28.66 KB)
📄
AMDGPUCallLowering.h
(2.37 KB)
📄
AMDGPUCallingConv.td
(7.33 KB)
📄
AMDGPUCodeGenPrepare.cpp
(46.42 KB)
📄
AMDGPUCombine.td
(2.79 KB)
📄
AMDGPUExportClustering.cpp
(4.52 KB)
📄
AMDGPUExportClustering.h
(533 B)
📄
AMDGPUFeatures.td
(1.81 KB)
📄
AMDGPUFixFunctionBitcasts.cpp
(1.87 KB)
📄
AMDGPUFrameLowering.cpp
(1.98 KB)
📄
AMDGPUFrameLowering.h
(1.39 KB)
📄
AMDGPUGISel.td
(11.57 KB)
📄
AMDGPUGenRegisterBankInfo.def
(5.83 KB)
📄
AMDGPUGlobalISelUtils.cpp
(1.77 KB)
📄
AMDGPUGlobalISelUtils.h
(2.07 KB)
📄
AMDGPUHSAMetadataStreamer.cpp
(31.21 KB)
📄
AMDGPUHSAMetadataStreamer.h
(5.46 KB)
📄
AMDGPUISelDAGToDAG.cpp
(101.59 KB)
📄
AMDGPUISelLowering.cpp
(168.65 KB)
📄
AMDGPUISelLowering.h
(19.23 KB)
📄
AMDGPUInline.cpp
(7.97 KB)
📄
AMDGPUInstrInfo.cpp
(1.71 KB)
📄
AMDGPUInstrInfo.h
(1.66 KB)
📄
AMDGPUInstrInfo.td
(17.18 KB)
📄
AMDGPUInstructionSelector.cpp
(128.53 KB)
📄
AMDGPUInstructionSelector.h
(11.04 KB)
📄
AMDGPUInstructions.td
(25.36 KB)
📄
AMDGPULegalizerInfo.cpp
(149.32 KB)
📄
AMDGPULegalizerInfo.h
(8.49 KB)
📄
AMDGPULibCalls.cpp
(53.89 KB)
📄
AMDGPULibFunc.cpp
(37.85 KB)
📄
AMDGPULibFunc.h
(10.99 KB)
📄
AMDGPULowerIntrinsics.cpp
(4.55 KB)
📄
AMDGPULowerKernelArguments.cpp
(8.89 KB)
📄
AMDGPULowerKernelAttributes.cpp
(7.78 KB)
📄
AMDGPUMCInstLower.cpp
(14.27 KB)
📄
AMDGPUMachineCFGStructurizer.cpp
(101.97 KB)
📄
AMDGPUMachineFunction.cpp
(2.24 KB)
📄
AMDGPUMachineFunction.h
(2.13 KB)
📄
AMDGPUMachineModuleInfo.cpp
(1.34 KB)
📄
AMDGPUMachineModuleInfo.h
(5.46 KB)
📄
AMDGPUMacroFusion.cpp
(2.28 KB)
📄
AMDGPUMacroFusion.h
(679 B)
📄
AMDGPUOpenCLEnqueuedBlockLowering.cpp
(5.31 KB)
📄
AMDGPUPTNote.h
(1.29 KB)
📄
AMDGPUPerfHintAnalysis.cpp
(12.17 KB)
📄
AMDGPUPerfHintAnalysis.h
(1.67 KB)
📄
AMDGPUPostLegalizerCombiner.cpp
(12.02 KB)
📄
AMDGPUPreLegalizerCombiner.cpp
(5.45 KB)
📄
AMDGPUPrintfRuntimeBinding.cpp
(21.7 KB)
📄
AMDGPUPromoteAlloca.cpp
(35.24 KB)
📄
AMDGPUPropagateAttributes.cpp
(11.76 KB)
📄
AMDGPURegBankCombiner.cpp
(5.36 KB)
📄
AMDGPURegisterBankInfo.cpp
(161.67 KB)
📄
AMDGPURegisterBankInfo.h
(7.41 KB)
📄
AMDGPURegisterBanks.td
(921 B)
📄
AMDGPURewriteOutArguments.cpp
(15.82 KB)
📄
AMDGPUSearchableTables.td
(21.04 KB)
📄
AMDGPUSubtarget.cpp
(29.62 KB)
📄
AMDGPUSubtarget.h
(35.82 KB)
📄
AMDGPUTargetMachine.cpp
(42.67 KB)
📄
AMDGPUTargetMachine.h
(4.52 KB)
📄
AMDGPUTargetObjectFile.cpp
(1.54 KB)
📄
AMDGPUTargetObjectFile.h
(1.14 KB)
📄
AMDGPUTargetTransformInfo.cpp
(39.07 KB)
📄
AMDGPUTargetTransformInfo.h
(11.11 KB)
📄
AMDGPUUnifyDivergentExitNodes.cpp
(13.84 KB)
📄
AMDGPUUnifyMetadata.cpp
(4.46 KB)
📄
AMDILCFGStructurizer.cpp
(56.32 KB)
📄
AMDKernelCodeT.h
(32.84 KB)
📁
AsmParser
📄
BUFInstructions.td
(110.75 KB)
📄
CaymanInstructions.td
(7.93 KB)
📄
DSInstructions.td
(52.37 KB)
📁
Disassembler
📄
EvergreenInstructions.td
(28.24 KB)
📄
FLATInstructions.td
(66.93 KB)
📄
GCNDPPCombine.cpp
(19.92 KB)
📄
GCNHazardRecognizer.cpp
(45.3 KB)
📄
GCNHazardRecognizer.h
(3.96 KB)
📄
GCNILPSched.cpp
(11.3 KB)
📄
GCNIterativeScheduler.cpp
(20.62 KB)
📄
GCNIterativeScheduler.h
(4.16 KB)
📄
GCNMinRegStrategy.cpp
(8.47 KB)
📄
GCNNSAReassign.cpp
(10.92 KB)
📄
GCNProcessors.td
(4.84 KB)
📄
GCNRegBankReassign.cpp
(26.68 KB)
📄
GCNRegPressure.cpp
(16.27 KB)
📄
GCNRegPressure.h
(9.15 KB)
📄
GCNSchedStrategy.cpp
(21.67 KB)
📄
GCNSchedStrategy.h
(3.77 KB)
📁
MCTargetDesc
📄
MIMGInstructions.td
(39.85 KB)
📄
R600.td
(1.51 KB)
📄
R600AsmPrinter.cpp
(4.46 KB)
📄
R600AsmPrinter.h
(1.5 KB)
📄
R600ClauseMergePass.cpp
(7.38 KB)
📄
R600ControlFlowFinalizer.cpp
(23.4 KB)
📄
R600Defines.h
(4.25 KB)
📄
R600EmitClauseMarkers.cpp
(12.1 KB)
📄
R600ExpandSpecialInstrs.cpp
(10.11 KB)
📄
R600FrameLowering.cpp
(1.83 KB)
📄
R600FrameLowering.h
(1.25 KB)
📄
R600ISelLowering.cpp
(81.88 KB)
📄
R600ISelLowering.h
(4.8 KB)
📄
R600InstrFormats.td
(11.58 KB)
📄
R600InstrInfo.cpp
(49.47 KB)
📄
R600InstrInfo.h
(13.7 KB)
📄
R600Instructions.td
(55.13 KB)
📄
R600MachineFunctionInfo.cpp
(551 B)
📄
R600MachineFunctionInfo.h
(824 B)
📄
R600MachineScheduler.cpp
(13.57 KB)
📄
R600MachineScheduler.h
(2.53 KB)
📄
R600OpenCLImageTypeLoweringPass.cpp
(11.75 KB)
📄
R600OptimizeVectorRegisters.cpp
(13.4 KB)
📄
R600Packetizer.cpp
(13.4 KB)
📄
R600Processors.td
(4.42 KB)
📄
R600RegisterInfo.cpp
(3.95 KB)
📄
R600RegisterInfo.h
(2 KB)
📄
R600RegisterInfo.td
(9.75 KB)
📄
R600Schedule.td
(1.62 KB)
📄
R700Instructions.td
(783 B)
📄
SIAddIMGInit.cpp
(6.24 KB)
📄
SIAnnotateControlFlow.cpp
(11.18 KB)
📄
SIDefines.h
(20.86 KB)
📄
SIFixSGPRCopies.cpp
(29.46 KB)
📄
SIFixVGPRCopies.cpp
(2 KB)
📄
SIFixupVectorISel.cpp
(8.75 KB)
📄
SIFoldOperands.cpp
(54.56 KB)
📄
SIFormMemoryClauses.cpp
(12.76 KB)
📄
SIFrameLowering.cpp
(48.08 KB)
📄
SIFrameLowering.h
(2.98 KB)
📄
SIISelLowering.cpp
(423.43 KB)
📄
SIISelLowering.h
(22.13 KB)
📄
SIInsertHardClauses.cpp
(7.01 KB)
📄
SIInsertSkips.cpp
(15.29 KB)
📄
SIInsertWaitcnts.cpp
(58.33 KB)
📄
SIInstrFormats.td
(9.44 KB)
📄
SIInstrInfo.cpp
(247.15 KB)
📄
SIInstrInfo.h
(41.24 KB)
📄
SIInstrInfo.td
(90.7 KB)
📄
SIInstructions.td
(77.7 KB)
📄
SILoadStoreOptimizer.cpp
(76.21 KB)
📄
SILowerControlFlow.cpp
(22.66 KB)
📄
SILowerI1Copies.cpp
(27.83 KB)
📄
SILowerSGPRSpills.cpp
(12.68 KB)
📄
SIMachineFunctionInfo.cpp
(20.01 KB)
📄
SIMachineFunctionInfo.h
(26.91 KB)
📄
SIMachineScheduler.cpp
(69.44 KB)
📄
SIMachineScheduler.h
(15.65 KB)
📄
SIMemoryLegalizer.cpp
(45.84 KB)
📄
SIModeRegister.cpp
(17.43 KB)
📄
SIOptimizeExecMasking.cpp
(12.81 KB)
📄
SIOptimizeExecMaskingPreRA.cpp
(11.13 KB)
📄
SIPeepholeSDWA.cpp
(42.84 KB)
📄
SIPostRABundler.cpp
(3.6 KB)
📄
SIPreAllocateWWMRegs.cpp
(6.09 KB)
📄
SIPreEmitPeephole.cpp
(10.51 KB)
📄
SIProgramInfo.h
(2.04 KB)
📄
SIRegisterInfo.cpp
(71.51 KB)
📄
SIRegisterInfo.h
(13.04 KB)
📄
SIRegisterInfo.td
(37.28 KB)
📄
SIRemoveShortExecBranches.cpp
(4.96 KB)
📄
SISchedule.td
(7.58 KB)
📄
SIShrinkInstructions.cpp
(26.86 KB)
📄
SIWholeQuadMode.cpp
(30.22 KB)
📄
SMInstructions.td
(48.14 KB)
📄
SOPInstructions.td
(60.51 KB)
📁
TargetInfo
📁
Utils
📄
VIInstrFormats.td
(645 B)
📄
VOP1Instructions.td
(35.53 KB)
📄
VOP2Instructions.td
(65.04 KB)
📄
VOP3Instructions.td
(53.14 KB)
📄
VOP3PInstructions.td
(26.47 KB)
📄
VOPCInstructions.td
(63.31 KB)
📄
VOPInstructions.td
(23.76 KB)
Editing: GCNNSAReassign.cpp
//===-- GCNNSAReassign.cpp - Reassign registers in NSA unstructions -------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // /// \file /// \brief Try to reassign registers on GFX10+ from non-sequential to sequential /// in NSA image instructions. Later SIShrinkInstructions pass will relace NSA /// with sequential versions where possible. /// //===----------------------------------------------------------------------===// #include "AMDGPU.h" #include "AMDGPUSubtarget.h" #include "SIInstrInfo.h" #include "SIMachineFunctionInfo.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveRegMatrix.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/VirtRegMap.h" #include "llvm/InitializePasses.h" #include "llvm/Support/MathExtras.h" #include <algorithm> using namespace llvm; #define DEBUG_TYPE "amdgpu-nsa-reassign" STATISTIC(NumNSAInstructions, "Number of NSA instructions with non-sequential address found"); STATISTIC(NumNSAConverted, "Number of NSA instructions changed to sequential"); namespace { class GCNNSAReassign : public MachineFunctionPass { public: static char ID; GCNNSAReassign() : MachineFunctionPass(ID) { initializeGCNNSAReassignPass(*PassRegistry::getPassRegistry()); } bool runOnMachineFunction(MachineFunction &MF) override; StringRef getPassName() const override { return "GCN NSA Reassign"; } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired<LiveIntervals>(); AU.addRequired<VirtRegMap>(); AU.addRequired<LiveRegMatrix>(); AU.setPreservesAll(); MachineFunctionPass::getAnalysisUsage(AU); } private: typedef enum { NOT_NSA, // Not an NSA instruction FIXED, // NSA which we cannot modify NON_CONTIGUOUS, // NSA with non-sequential address which we can try // to optimize. CONTIGUOUS // NSA with all sequential address registers } NSA_Status; const GCNSubtarget *ST; const MachineRegisterInfo *MRI; const SIRegisterInfo *TRI; VirtRegMap *VRM; LiveRegMatrix *LRM; LiveIntervals *LIS; unsigned MaxNumVGPRs; const MCPhysReg *CSRegs; NSA_Status CheckNSA(const MachineInstr &MI, bool Fast = false) const; bool tryAssignRegisters(SmallVectorImpl<LiveInterval *> &Intervals, unsigned StartReg) const; bool canAssign(unsigned StartReg, unsigned NumRegs) const; bool scavengeRegs(SmallVectorImpl<LiveInterval *> &Intervals) const; }; } // End anonymous namespace. INITIALIZE_PASS_BEGIN(GCNNSAReassign, DEBUG_TYPE, "GCN NSA Reassign", false, false) INITIALIZE_PASS_DEPENDENCY(LiveIntervals) INITIALIZE_PASS_DEPENDENCY(VirtRegMap) INITIALIZE_PASS_DEPENDENCY(LiveRegMatrix) INITIALIZE_PASS_END(GCNNSAReassign, DEBUG_TYPE, "GCN NSA Reassign", false, false) char GCNNSAReassign::ID = 0; char &llvm::GCNNSAReassignID = GCNNSAReassign::ID; bool GCNNSAReassign::tryAssignRegisters(SmallVectorImpl<LiveInterval *> &Intervals, unsigned StartReg) const { unsigned NumRegs = Intervals.size(); for (unsigned N = 0; N < NumRegs; ++N) if (VRM->hasPhys(Intervals[N]->reg)) LRM->unassign(*Intervals[N]); for (unsigned N = 0; N < NumRegs; ++N) if (LRM->checkInterference(*Intervals[N], StartReg + N)) return false; for (unsigned N = 0; N < NumRegs; ++N) LRM->assign(*Intervals[N], StartReg + N); return true; } bool GCNNSAReassign::canAssign(unsigned StartReg, unsigned NumRegs) const { for (unsigned N = 0; N < NumRegs; ++N) { unsigned Reg = StartReg + N; if (!MRI->isAllocatable(Reg)) return false; for (unsigned I = 0; CSRegs[I]; ++I) if (TRI->isSubRegisterEq(Reg, CSRegs[I]) && !LRM->isPhysRegUsed(CSRegs[I])) return false; } return true; } bool GCNNSAReassign::scavengeRegs(SmallVectorImpl<LiveInterval *> &Intervals) const { unsigned NumRegs = Intervals.size(); if (NumRegs > MaxNumVGPRs) return false; unsigned MaxReg = MaxNumVGPRs - NumRegs + AMDGPU::VGPR0; for (unsigned Reg = AMDGPU::VGPR0; Reg <= MaxReg; ++Reg) { if (!canAssign(Reg, NumRegs)) continue; if (tryAssignRegisters(Intervals, Reg)) return true; } return false; } GCNNSAReassign::NSA_Status GCNNSAReassign::CheckNSA(const MachineInstr &MI, bool Fast) const { const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI.getOpcode()); if (!Info || Info->MIMGEncoding != AMDGPU::MIMGEncGfx10NSA) return NSA_Status::NOT_NSA; int VAddr0Idx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::vaddr0); unsigned VgprBase = 0; bool NSA = false; for (unsigned I = 0; I < Info->VAddrDwords; ++I) { const MachineOperand &Op = MI.getOperand(VAddr0Idx + I); Register Reg = Op.getReg(); if (Register::isPhysicalRegister(Reg) || !VRM->isAssignedReg(Reg)) return NSA_Status::FIXED; Register PhysReg = VRM->getPhys(Reg); if (!Fast) { if (!PhysReg) return NSA_Status::FIXED; // Bail if address is not a VGPR32. That should be possible to extend the // optimization to work with subregs of a wider register tuples, but the // logic to find free registers will be much more complicated with much // less chances for success. That seems reasonable to assume that in most // cases a tuple is used because a vector variable contains different // parts of an address and it is either already consequitive or cannot // be reassigned if not. If needed it is better to rely on register // coalescer to process such address tuples. if (MRI->getRegClass(Reg) != &AMDGPU::VGPR_32RegClass || Op.getSubReg()) return NSA_Status::FIXED; const MachineInstr *Def = MRI->getUniqueVRegDef(Reg); if (Def && Def->isCopy() && Def->getOperand(1).getReg() == PhysReg) return NSA_Status::FIXED; for (auto U : MRI->use_nodbg_operands(Reg)) { if (U.isImplicit()) return NSA_Status::FIXED; const MachineInstr *UseInst = U.getParent(); if (UseInst->isCopy() && UseInst->getOperand(0).getReg() == PhysReg) return NSA_Status::FIXED; } if (!LIS->hasInterval(Reg)) return NSA_Status::FIXED; } if (I == 0) VgprBase = PhysReg; else if (VgprBase + I != PhysReg) NSA = true; } return NSA ? NSA_Status::NON_CONTIGUOUS : NSA_Status::CONTIGUOUS; } bool GCNNSAReassign::runOnMachineFunction(MachineFunction &MF) { ST = &MF.getSubtarget<GCNSubtarget>(); if (ST->getGeneration() < GCNSubtarget::GFX10) return false; MRI = &MF.getRegInfo(); TRI = ST->getRegisterInfo(); VRM = &getAnalysis<VirtRegMap>(); LRM = &getAnalysis<LiveRegMatrix>(); LIS = &getAnalysis<LiveIntervals>(); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); MaxNumVGPRs = ST->getMaxNumVGPRs(MF); MaxNumVGPRs = std::min(ST->getMaxNumVGPRs(MFI->getOccupancy()), MaxNumVGPRs); CSRegs = MRI->getCalleeSavedRegs(); using Candidate = std::pair<const MachineInstr*, bool>; SmallVector<Candidate, 32> Candidates; for (const MachineBasicBlock &MBB : MF) { for (const MachineInstr &MI : MBB) { switch (CheckNSA(MI)) { default: continue; case NSA_Status::CONTIGUOUS: Candidates.push_back(std::make_pair(&MI, true)); break; case NSA_Status::NON_CONTIGUOUS: Candidates.push_back(std::make_pair(&MI, false)); ++NumNSAInstructions; break; } } } bool Changed = false; for (auto &C : Candidates) { if (C.second) continue; const MachineInstr *MI = C.first; if (CheckNSA(*MI, true) == NSA_Status::CONTIGUOUS) { // Already happen to be fixed. C.second = true; ++NumNSAConverted; continue; } const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(MI->getOpcode()); int VAddr0Idx = AMDGPU::getNamedOperandIdx(MI->getOpcode(), AMDGPU::OpName::vaddr0); SmallVector<LiveInterval *, 16> Intervals; SmallVector<unsigned, 16> OrigRegs; SlotIndex MinInd, MaxInd; for (unsigned I = 0; I < Info->VAddrDwords; ++I) { const MachineOperand &Op = MI->getOperand(VAddr0Idx + I); Register Reg = Op.getReg(); LiveInterval *LI = &LIS->getInterval(Reg); if (llvm::find(Intervals, LI) != Intervals.end()) { // Same register used, unable to make sequential Intervals.clear(); break; } Intervals.push_back(LI); OrigRegs.push_back(VRM->getPhys(Reg)); if (LI->empty()) { // The address input is undef, so it doesn't contribute to the relevant // range. Seed a reasonable index range if required. if (I == 0) MinInd = MaxInd = LIS->getInstructionIndex(*MI); continue; } MinInd = I != 0 ? std::min(MinInd, LI->beginIndex()) : LI->beginIndex(); MaxInd = I != 0 ? std::max(MaxInd, LI->endIndex()) : LI->endIndex(); } if (Intervals.empty()) continue; LLVM_DEBUG(dbgs() << "Attempting to reassign NSA: " << *MI << "\tOriginal allocation:\t"; for(auto *LI : Intervals) dbgs() << " " << llvm::printReg((VRM->getPhys(LI->reg)), TRI); dbgs() << '\n'); bool Success = scavengeRegs(Intervals); if (!Success) { LLVM_DEBUG(dbgs() << "\tCannot reallocate.\n"); if (VRM->hasPhys(Intervals.back()->reg)) // Did not change allocation. continue; } else { // Check we did not make it worse for other instructions. auto I = std::lower_bound(Candidates.begin(), &C, MinInd, [this](const Candidate &C, SlotIndex I) { return LIS->getInstructionIndex(*C.first) < I; }); for (auto E = Candidates.end(); Success && I != E && LIS->getInstructionIndex(*I->first) < MaxInd; ++I) { if (I->second && CheckNSA(*I->first, true) < NSA_Status::CONTIGUOUS) { Success = false; LLVM_DEBUG(dbgs() << "\tNSA conversion conflict with " << *I->first); } } } if (!Success) { for (unsigned I = 0; I < Info->VAddrDwords; ++I) if (VRM->hasPhys(Intervals[I]->reg)) LRM->unassign(*Intervals[I]); for (unsigned I = 0; I < Info->VAddrDwords; ++I) LRM->assign(*Intervals[I], OrigRegs[I]); continue; } C.second = true; ++NumNSAConverted; LLVM_DEBUG(dbgs() << "\tNew allocation:\t\t [" << llvm::printReg((VRM->getPhys(Intervals.front()->reg)), TRI) << " : " << llvm::printReg((VRM->getPhys(Intervals.back()->reg)), TRI) << "]\n"); Changed = true; } return Changed; }
Upload File
Create Folder