File Manager

003 File Manager

Current Path: /usr/src/contrib/llvm-project/llvm/lib/Target/AMDGPU

usr / src / contrib / llvm-project / llvm / lib / Target / AMDGPU /

📁 ..
📄 AMDGPU.h(11.46 KB)
📄 AMDGPU.td(36.97 KB)
📄 AMDGPUAliasAnalysis.cpp(5.58 KB)
📄 AMDGPUAliasAnalysis.h(3.32 KB)
📄 AMDGPUAlwaysInlinePass.cpp(4.83 KB)
📄 AMDGPUAnnotateKernelFeatures.cpp(11.94 KB)
📄 AMDGPUAnnotateUniformValues.cpp(6.13 KB)
📄 AMDGPUArgumentUsageInfo.cpp(7.66 KB)
📄 AMDGPUArgumentUsageInfo.h(4.81 KB)
📄 AMDGPUAsmPrinter.cpp(50.42 KB)
📄 AMDGPUAsmPrinter.h(5.13 KB)
📄 AMDGPUAtomicOptimizer.cpp(23.79 KB)
📄 AMDGPUCallLowering.cpp(28.66 KB)
📄 AMDGPUCallLowering.h(2.37 KB)
📄 AMDGPUCallingConv.td(7.33 KB)
📄 AMDGPUCodeGenPrepare.cpp(46.42 KB)
📄 AMDGPUCombine.td(2.79 KB)
📄 AMDGPUExportClustering.cpp(4.52 KB)
📄 AMDGPUExportClustering.h(533 B)
📄 AMDGPUFeatures.td(1.81 KB)
📄 AMDGPUFixFunctionBitcasts.cpp(1.87 KB)
📄 AMDGPUFrameLowering.cpp(1.98 KB)
📄 AMDGPUFrameLowering.h(1.39 KB)
📄 AMDGPUGISel.td(11.57 KB)
📄 AMDGPUGenRegisterBankInfo.def(5.83 KB)
📄 AMDGPUGlobalISelUtils.cpp(1.77 KB)
📄 AMDGPUGlobalISelUtils.h(2.07 KB)
📄 AMDGPUHSAMetadataStreamer.cpp(31.21 KB)
📄 AMDGPUHSAMetadataStreamer.h(5.46 KB)
📄 AMDGPUISelDAGToDAG.cpp(101.59 KB)
📄 AMDGPUISelLowering.cpp(168.65 KB)
📄 AMDGPUISelLowering.h(19.23 KB)
📄 AMDGPUInline.cpp(7.97 KB)
📄 AMDGPUInstrInfo.cpp(1.71 KB)
📄 AMDGPUInstrInfo.h(1.66 KB)
📄 AMDGPUInstrInfo.td(17.18 KB)
📄 AMDGPUInstructionSelector.cpp(128.53 KB)
📄 AMDGPUInstructionSelector.h(11.04 KB)
📄 AMDGPUInstructions.td(25.36 KB)
📄 AMDGPULegalizerInfo.cpp(149.32 KB)
📄 AMDGPULegalizerInfo.h(8.49 KB)
📄 AMDGPULibCalls.cpp(53.89 KB)
📄 AMDGPULibFunc.cpp(37.85 KB)
📄 AMDGPULibFunc.h(10.99 KB)
📄 AMDGPULowerIntrinsics.cpp(4.55 KB)
📄 AMDGPULowerKernelArguments.cpp(8.89 KB)
📄 AMDGPULowerKernelAttributes.cpp(7.78 KB)
📄 AMDGPUMCInstLower.cpp(14.27 KB)
📄 AMDGPUMachineCFGStructurizer.cpp(101.97 KB)
📄 AMDGPUMachineFunction.cpp(2.24 KB)
📄 AMDGPUMachineFunction.h(2.13 KB)
📄 AMDGPUMachineModuleInfo.cpp(1.34 KB)
📄 AMDGPUMachineModuleInfo.h(5.46 KB)
📄 AMDGPUMacroFusion.cpp(2.28 KB)
📄 AMDGPUMacroFusion.h(679 B)
📄 AMDGPUOpenCLEnqueuedBlockLowering.cpp(5.31 KB)
📄 AMDGPUPTNote.h(1.29 KB)
📄 AMDGPUPerfHintAnalysis.cpp(12.17 KB)
📄 AMDGPUPerfHintAnalysis.h(1.67 KB)
📄 AMDGPUPostLegalizerCombiner.cpp(12.02 KB)
📄 AMDGPUPreLegalizerCombiner.cpp(5.45 KB)
📄 AMDGPUPrintfRuntimeBinding.cpp(21.7 KB)
📄 AMDGPUPromoteAlloca.cpp(35.24 KB)
📄 AMDGPUPropagateAttributes.cpp(11.76 KB)
📄 AMDGPURegBankCombiner.cpp(5.36 KB)
📄 AMDGPURegisterBankInfo.cpp(161.67 KB)
📄 AMDGPURegisterBankInfo.h(7.41 KB)
📄 AMDGPURegisterBanks.td(921 B)
📄 AMDGPURewriteOutArguments.cpp(15.82 KB)
📄 AMDGPUSearchableTables.td(21.04 KB)
📄 AMDGPUSubtarget.cpp(29.62 KB)
📄 AMDGPUSubtarget.h(35.82 KB)
📄 AMDGPUTargetMachine.cpp(42.67 KB)
📄 AMDGPUTargetMachine.h(4.52 KB)
📄 AMDGPUTargetObjectFile.cpp(1.54 KB)
📄 AMDGPUTargetObjectFile.h(1.14 KB)
📄 AMDGPUTargetTransformInfo.cpp(39.07 KB)
📄 AMDGPUTargetTransformInfo.h(11.11 KB)
📄 AMDGPUUnifyDivergentExitNodes.cpp(13.84 KB)
📄 AMDGPUUnifyMetadata.cpp(4.46 KB)
📄 AMDILCFGStructurizer.cpp(56.32 KB)
📄 AMDKernelCodeT.h(32.84 KB)
📁 AsmParser
📄 BUFInstructions.td(110.75 KB)
📄 CaymanInstructions.td(7.93 KB)
📄 DSInstructions.td(52.37 KB)
📁 Disassembler
📄 EvergreenInstructions.td(28.24 KB)
📄 FLATInstructions.td(66.93 KB)
📄 GCNDPPCombine.cpp(19.92 KB)
📄 GCNHazardRecognizer.cpp(45.3 KB)
📄 GCNHazardRecognizer.h(3.96 KB)
📄 GCNILPSched.cpp(11.3 KB)
📄 GCNIterativeScheduler.cpp(20.62 KB)
📄 GCNIterativeScheduler.h(4.16 KB)
📄 GCNMinRegStrategy.cpp(8.47 KB)
📄 GCNNSAReassign.cpp(10.92 KB)
📄 GCNProcessors.td(4.84 KB)
📄 GCNRegBankReassign.cpp(26.68 KB)
📄 GCNRegPressure.cpp(16.27 KB)
📄 GCNRegPressure.h(9.15 KB)
📄 GCNSchedStrategy.cpp(21.67 KB)
📄 GCNSchedStrategy.h(3.77 KB)
📁 MCTargetDesc
📄 MIMGInstructions.td(39.85 KB)
📄 R600.td(1.51 KB)
📄 R600AsmPrinter.cpp(4.46 KB)
📄 R600AsmPrinter.h(1.5 KB)
📄 R600ClauseMergePass.cpp(7.38 KB)
📄 R600ControlFlowFinalizer.cpp(23.4 KB)
📄 R600Defines.h(4.25 KB)
📄 R600EmitClauseMarkers.cpp(12.1 KB)
📄 R600ExpandSpecialInstrs.cpp(10.11 KB)
📄 R600FrameLowering.cpp(1.83 KB)
📄 R600FrameLowering.h(1.25 KB)
📄 R600ISelLowering.cpp(81.88 KB)
📄 R600ISelLowering.h(4.8 KB)
📄 R600InstrFormats.td(11.58 KB)
📄 R600InstrInfo.cpp(49.47 KB)
📄 R600InstrInfo.h(13.7 KB)
📄 R600Instructions.td(55.13 KB)
📄 R600MachineFunctionInfo.cpp(551 B)
📄 R600MachineFunctionInfo.h(824 B)
📄 R600MachineScheduler.cpp(13.57 KB)
📄 R600MachineScheduler.h(2.53 KB)
📄 R600OpenCLImageTypeLoweringPass.cpp(11.75 KB)
📄 R600OptimizeVectorRegisters.cpp(13.4 KB)
📄 R600Packetizer.cpp(13.4 KB)
📄 R600Processors.td(4.42 KB)
📄 R600RegisterInfo.cpp(3.95 KB)
📄 R600RegisterInfo.h(2 KB)
📄 R600RegisterInfo.td(9.75 KB)
📄 R600Schedule.td(1.62 KB)
📄 R700Instructions.td(783 B)
📄 SIAddIMGInit.cpp(6.24 KB)
📄 SIAnnotateControlFlow.cpp(11.18 KB)
📄 SIDefines.h(20.86 KB)
📄 SIFixSGPRCopies.cpp(29.46 KB)
📄 SIFixVGPRCopies.cpp(2 KB)
📄 SIFixupVectorISel.cpp(8.75 KB)
📄 SIFoldOperands.cpp(54.56 KB)
📄 SIFormMemoryClauses.cpp(12.76 KB)
📄 SIFrameLowering.cpp(48.08 KB)
📄 SIFrameLowering.h(2.98 KB)
📄 SIISelLowering.cpp(423.43 KB)
📄 SIISelLowering.h(22.13 KB)
📄 SIInsertHardClauses.cpp(7.01 KB)
📄 SIInsertSkips.cpp(15.29 KB)
📄 SIInsertWaitcnts.cpp(58.33 KB)
📄 SIInstrFormats.td(9.44 KB)
📄 SIInstrInfo.cpp(247.15 KB)
📄 SIInstrInfo.h(41.24 KB)
📄 SIInstrInfo.td(90.7 KB)
📄 SIInstructions.td(77.7 KB)
📄 SILoadStoreOptimizer.cpp(76.21 KB)
📄 SILowerControlFlow.cpp(22.66 KB)
📄 SILowerI1Copies.cpp(27.83 KB)
📄 SILowerSGPRSpills.cpp(12.68 KB)
📄 SIMachineFunctionInfo.cpp(20.01 KB)
📄 SIMachineFunctionInfo.h(26.91 KB)
📄 SIMachineScheduler.cpp(69.44 KB)
📄 SIMachineScheduler.h(15.65 KB)
📄 SIMemoryLegalizer.cpp(45.84 KB)
📄 SIModeRegister.cpp(17.43 KB)
📄 SIOptimizeExecMasking.cpp(12.81 KB)
📄 SIOptimizeExecMaskingPreRA.cpp(11.13 KB)
📄 SIPeepholeSDWA.cpp(42.84 KB)
📄 SIPostRABundler.cpp(3.6 KB)
📄 SIPreAllocateWWMRegs.cpp(6.09 KB)
📄 SIPreEmitPeephole.cpp(10.51 KB)
📄 SIProgramInfo.h(2.04 KB)
📄 SIRegisterInfo.cpp(71.51 KB)
📄 SIRegisterInfo.h(13.04 KB)
📄 SIRegisterInfo.td(37.28 KB)
📄 SIRemoveShortExecBranches.cpp(4.96 KB)
📄 SISchedule.td(7.58 KB)
📄 SIShrinkInstructions.cpp(26.86 KB)
📄 SIWholeQuadMode.cpp(30.22 KB)
📄 SMInstructions.td(48.14 KB)
📄 SOPInstructions.td(60.51 KB)
📁 TargetInfo
📁 Utils
📄 VIInstrFormats.td(645 B)
📄 VOP1Instructions.td(35.53 KB)
📄 VOP2Instructions.td(65.04 KB)
📄 VOP3Instructions.td(53.14 KB)
📄 VOP3PInstructions.td(26.47 KB)
📄 VOPCInstructions.td(63.31 KB)
📄 VOPInstructions.td(23.76 KB)

Editing: SIOptimizeExecMasking.cpp

//===-- SIOptimizeExecMasking.cpp -----------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "AMDGPU.h"
#include "AMDGPUSubtarget.h"
#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
#include "SIInstrInfo.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/InitializePasses.h"
#include "llvm/Support/Debug.h"

using namespace llvm;

#define DEBUG_TYPE "si-optimize-exec-masking"

namespace {

class SIOptimizeExecMasking : public MachineFunctionPass {
public:
  static char ID;

public:
  SIOptimizeExecMasking() : MachineFunctionPass(ID) {
    initializeSIOptimizeExecMaskingPass(*PassRegistry::getPassRegistry());
  }

bool runOnMachineFunction(MachineFunction &MF) override;

StringRef getPassName() const override {
    return "SI optimize exec mask operations";
  }

void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.setPreservesCFG();
    MachineFunctionPass::getAnalysisUsage(AU);
  }
};

} // End anonymous namespace.

INITIALIZE_PASS_BEGIN(SIOptimizeExecMasking, DEBUG_TYPE,
                      "SI optimize exec mask operations", false, false)
INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
INITIALIZE_PASS_END(SIOptimizeExecMasking, DEBUG_TYPE,
                    "SI optimize exec mask operations", false, false)

char SIOptimizeExecMasking::ID = 0;

char &llvm::SIOptimizeExecMaskingID = SIOptimizeExecMasking::ID;

/// If \p MI is a copy from exec, return the register copied to.
static Register isCopyFromExec(const MachineInstr &MI, const GCNSubtarget &ST) {
  switch (MI.getOpcode()) {
  case AMDGPU::COPY:
  case AMDGPU::S_MOV_B64:
  case AMDGPU::S_MOV_B64_term:
  case AMDGPU::S_MOV_B32:
  case AMDGPU::S_MOV_B32_term: {
    const MachineOperand &Src = MI.getOperand(1);
    if (Src.isReg() &&
        Src.getReg() == (ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC))
      return MI.getOperand(0).getReg();
  }
  }

return AMDGPU::NoRegister;
}

/// If \p MI is a copy to exec, return the register copied from.
static Register isCopyToExec(const MachineInstr &MI, const GCNSubtarget &ST) {
  switch (MI.getOpcode()) {
  case AMDGPU::COPY:
  case AMDGPU::S_MOV_B64:
  case AMDGPU::S_MOV_B32: {
    const MachineOperand &Dst = MI.getOperand(0);
    if (Dst.isReg() &&
        Dst.getReg() == (ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC) &&
        MI.getOperand(1).isReg())
      return MI.getOperand(1).getReg();
    break;
  }
  case AMDGPU::S_MOV_B64_term:
  case AMDGPU::S_MOV_B32_term:
    llvm_unreachable("should have been replaced");
  }

return Register();
}

/// If \p MI is a logical operation on an exec value,
/// return the register copied to.
static Register isLogicalOpOnExec(const MachineInstr &MI) {
  switch (MI.getOpcode()) {
  case AMDGPU::S_AND_B64:
  case AMDGPU::S_OR_B64:
  case AMDGPU::S_XOR_B64:
  case AMDGPU::S_ANDN2_B64:
  case AMDGPU::S_ORN2_B64:
  case AMDGPU::S_NAND_B64:
  case AMDGPU::S_NOR_B64:
  case AMDGPU::S_XNOR_B64: {
    const MachineOperand &Src1 = MI.getOperand(1);
    if (Src1.isReg() && Src1.getReg() == AMDGPU::EXEC)
      return MI.getOperand(0).getReg();
    const MachineOperand &Src2 = MI.getOperand(2);
    if (Src2.isReg() && Src2.getReg() == AMDGPU::EXEC)
      return MI.getOperand(0).getReg();
    break;
  }
  case AMDGPU::S_AND_B32:
  case AMDGPU::S_OR_B32:
  case AMDGPU::S_XOR_B32:
  case AMDGPU::S_ANDN2_B32:
  case AMDGPU::S_ORN2_B32:
  case AMDGPU::S_NAND_B32:
  case AMDGPU::S_NOR_B32:
  case AMDGPU::S_XNOR_B32: {
    const MachineOperand &Src1 = MI.getOperand(1);
    if (Src1.isReg() && Src1.getReg() == AMDGPU::EXEC_LO)
      return MI.getOperand(0).getReg();
    const MachineOperand &Src2 = MI.getOperand(2);
    if (Src2.isReg() && Src2.getReg() == AMDGPU::EXEC_LO)
      return MI.getOperand(0).getReg();
    break;
  }
  }

return AMDGPU::NoRegister;
}

static unsigned getSaveExecOp(unsigned Opc) {
  switch (Opc) {
  case AMDGPU::S_AND_B64:
    return AMDGPU::S_AND_SAVEEXEC_B64;
  case AMDGPU::S_OR_B64:
    return AMDGPU::S_OR_SAVEEXEC_B64;
  case AMDGPU::S_XOR_B64:
    return AMDGPU::S_XOR_SAVEEXEC_B64;
  case AMDGPU::S_ANDN2_B64:
    return AMDGPU::S_ANDN2_SAVEEXEC_B64;
  case AMDGPU::S_ORN2_B64:
    return AMDGPU::S_ORN2_SAVEEXEC_B64;
  case AMDGPU::S_NAND_B64:
    return AMDGPU::S_NAND_SAVEEXEC_B64;
  case AMDGPU::S_NOR_B64:
    return AMDGPU::S_NOR_SAVEEXEC_B64;
  case AMDGPU::S_XNOR_B64:
    return AMDGPU::S_XNOR_SAVEEXEC_B64;
  case AMDGPU::S_AND_B32:
    return AMDGPU::S_AND_SAVEEXEC_B32;
  case AMDGPU::S_OR_B32:
    return AMDGPU::S_OR_SAVEEXEC_B32;
  case AMDGPU::S_XOR_B32:
    return AMDGPU::S_XOR_SAVEEXEC_B32;
  case AMDGPU::S_ANDN2_B32:
    return AMDGPU::S_ANDN2_SAVEEXEC_B32;
  case AMDGPU::S_ORN2_B32:
    return AMDGPU::S_ORN2_SAVEEXEC_B32;
  case AMDGPU::S_NAND_B32:
    return AMDGPU::S_NAND_SAVEEXEC_B32;
  case AMDGPU::S_NOR_B32:
    return AMDGPU::S_NOR_SAVEEXEC_B32;
  case AMDGPU::S_XNOR_B32:
    return AMDGPU::S_XNOR_SAVEEXEC_B32;
  default:
    return AMDGPU::INSTRUCTION_LIST_END;
  }
}

// These are only terminators to get correct spill code placement during
// register allocation, so turn them back into normal instructions. Only one of
// these is expected per block.
static bool removeTerminatorBit(const SIInstrInfo &TII, MachineInstr &MI) {
  switch (MI.getOpcode()) {
  case AMDGPU::S_MOV_B64_term:
  case AMDGPU::S_MOV_B32_term: {
    MI.setDesc(TII.get(AMDGPU::COPY));
    return true;
  }
  case AMDGPU::S_XOR_B64_term: {
    // This is only a terminator to get the correct spill code placement during
    // register allocation.
    MI.setDesc(TII.get(AMDGPU::S_XOR_B64));
    return true;
  }
  case AMDGPU::S_XOR_B32_term: {
    // This is only a terminator to get the correct spill code placement during
    // register allocation.
    MI.setDesc(TII.get(AMDGPU::S_XOR_B32));
    return true;
  }
  case AMDGPU::S_OR_B32_term: {
    // This is only a terminator to get the correct spill code placement during
    // register allocation.
    MI.setDesc(TII.get(AMDGPU::S_OR_B32));
    return true;
  }
  case AMDGPU::S_ANDN2_B64_term: {
    // This is only a terminator to get the correct spill code placement during
    // register allocation.
    MI.setDesc(TII.get(AMDGPU::S_ANDN2_B64));
    return true;
  }
  case AMDGPU::S_ANDN2_B32_term: {
    // This is only a terminator to get the correct spill code placement during
    // register allocation.
    MI.setDesc(TII.get(AMDGPU::S_ANDN2_B32));
    return true;
  }
  default:
    return false;
  }
}

static MachineBasicBlock::reverse_iterator fixTerminators(
  const SIInstrInfo &TII,
  MachineBasicBlock &MBB) {
  MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend();
  for (; I != E; ++I) {
    if (!I->isTerminator())
      return I;

if (removeTerminatorBit(TII, *I))
      return I;
  }

return E;
}

static MachineBasicBlock::reverse_iterator findExecCopy(
  const SIInstrInfo &TII,
  const GCNSubtarget &ST,
  MachineBasicBlock &MBB,
  MachineBasicBlock::reverse_iterator I,
  unsigned CopyToExec) {
  const unsigned InstLimit = 25;

auto E = MBB.rend();
  for (unsigned N = 0; N <= InstLimit && I != E; ++I, ++N) {
    Register CopyFromExec = isCopyFromExec(*I, ST);
    if (CopyFromExec.isValid())
      return I;
  }

return E;
}

// XXX - Seems LivePhysRegs doesn't work correctly since it will incorrectly
// report the register as unavailable because a super-register with a lane mask
// is unavailable.
static bool isLiveOut(const MachineBasicBlock &MBB, unsigned Reg) {
  for (MachineBasicBlock *Succ : MBB.successors()) {
    if (Succ->isLiveIn(Reg))
      return true;
  }

return false;
}

bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) {
  if (skipFunction(MF.getFunction()))
    return false;

const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
  const SIRegisterInfo *TRI = ST.getRegisterInfo();
  const SIInstrInfo *TII = ST.getInstrInfo();
  MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;

// Optimize sequences emitted for control flow lowering. They are originally
  // emitted as the separate operations because spill code may need to be
  // inserted for the saved copy of exec.
  //
  //     x = copy exec
  //     z = s_<op>_b64 x, y
  //     exec = copy z
  // =>
  //     x = s_<op>_saveexec_b64 y
  //

for (MachineBasicBlock &MBB : MF) {
    MachineBasicBlock::reverse_iterator I = fixTerminators(*TII, MBB);
    MachineBasicBlock::reverse_iterator E = MBB.rend();
    if (I == E)
      continue;

// Scan backwards to find the def.
    auto CopyToExecInst = &*I;
    auto CopyFromExecInst = findExecCopy(*TII, ST, MBB, I, CopyToExec);
    if (CopyFromExecInst == E) {
      auto PrepareExecInst = std::next(I);
      if (PrepareExecInst == E)
        continue;
      // Fold exec = COPY (S_AND_B64 reg, exec) -> exec = S_AND_B64 reg, exec
      if (CopyToExecInst->getOperand(1).isKill() &&
          isLogicalOpOnExec(*PrepareExecInst) == CopyToExec) {
        LLVM_DEBUG(dbgs() << "Fold exec copy: " << *PrepareExecInst);

PrepareExecInst->getOperand(0).setReg(Exec);

LLVM_DEBUG(dbgs() << "into: " << *PrepareExecInst << '\n');

CopyToExecInst->eraseFromParent();
      }

continue;
    }

if (isLiveOut(MBB, CopyToExec)) {
      // The copied register is live out and has a second use in another block.
      LLVM_DEBUG(dbgs() << "Exec copy source register is live out\n");
      continue;
    }

Register CopyFromExec = CopyFromExecInst->getOperand(0).getReg();
    MachineInstr *SaveExecInst = nullptr;
    SmallVector<MachineInstr *, 4> OtherUseInsts;

for (MachineBasicBlock::iterator J
           = std::next(CopyFromExecInst->getIterator()), JE = I->getIterator();
         J != JE; ++J) {
      if (SaveExecInst && J->readsRegister(Exec, TRI)) {
        LLVM_DEBUG(dbgs() << "exec read prevents saveexec: " << *J << '\n');
        // Make sure this is inserted after any VALU ops that may have been
        // scheduled in between.
        SaveExecInst = nullptr;
        break;
      }

bool ReadsCopyFromExec = J->readsRegister(CopyFromExec, TRI);

if (J->modifiesRegister(CopyToExec, TRI)) {
        if (SaveExecInst) {
          LLVM_DEBUG(dbgs() << "Multiple instructions modify "
                            << printReg(CopyToExec, TRI) << '\n');
          SaveExecInst = nullptr;
          break;
        }

unsigned SaveExecOp = getSaveExecOp(J->getOpcode());
        if (SaveExecOp == AMDGPU::INSTRUCTION_LIST_END)
          break;

if (ReadsCopyFromExec) {
          SaveExecInst = &*J;
          LLVM_DEBUG(dbgs() << "Found save exec op: " << *SaveExecInst << '\n');
          continue;
        } else {
          LLVM_DEBUG(dbgs()
                     << "Instruction does not read exec copy: " << *J << '\n');
          break;
        }
      } else if (ReadsCopyFromExec && !SaveExecInst) {
        // Make sure no other instruction is trying to use this copy, before it
        // will be rewritten by the saveexec, i.e. hasOneUse. There may have
        // been another use, such as an inserted spill. For example:
        //
        // %sgpr0_sgpr1 = COPY %exec
        // spill %sgpr0_sgpr1
        // %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1
        //
        LLVM_DEBUG(dbgs() << "Found second use of save inst candidate: " << *J
                          << '\n');
        break;
      }

if (SaveExecInst && J->readsRegister(CopyToExec, TRI)) {
        assert(SaveExecInst != &*J);
        OtherUseInsts.push_back(&*J);
      }
    }

if (!SaveExecInst)
      continue;

LLVM_DEBUG(dbgs() << "Insert save exec op: " << *SaveExecInst << '\n');

MachineOperand &Src0 = SaveExecInst->getOperand(1);
    MachineOperand &Src1 = SaveExecInst->getOperand(2);

MachineOperand *OtherOp = nullptr;

if (Src0.isReg() && Src0.getReg() == CopyFromExec) {
      OtherOp = &Src1;
    } else if (Src1.isReg() && Src1.getReg() == CopyFromExec) {
      if (!SaveExecInst->isCommutable())
        break;

OtherOp = &Src0;
    } else
      llvm_unreachable("unexpected");

CopyFromExecInst->eraseFromParent();

auto InsPt = SaveExecInst->getIterator();
    const DebugLoc &DL = SaveExecInst->getDebugLoc();

BuildMI(MBB, InsPt, DL, TII->get(getSaveExecOp(SaveExecInst->getOpcode())),
            CopyFromExec)
      .addReg(OtherOp->getReg());
    SaveExecInst->eraseFromParent();

CopyToExecInst->eraseFromParent();

for (MachineInstr *OtherInst : OtherUseInsts) {
      OtherInst->substituteRegister(CopyToExec, Exec,
                                    AMDGPU::NoSubRegister, *TRI);
    }
  }

return true;

}

003 File Manager

Editing: SIOptimizeExecMasking.cpp

Upload File

Create Folder