File Manager

003 File Manager

Current Path: /usr/src/contrib/llvm-project/llvm/lib/Target/X86

usr / src / contrib / llvm-project / llvm / lib / Target / X86 /

📁 ..
📁 AsmParser
📁 Disassembler
📄 ImmutableGraph.h(15.15 KB)
📁 MCTargetDesc
📁 TargetInfo
📄 X86.h(7.41 KB)
📄 X86.td(68.44 KB)
📄 X86AsmPrinter.cpp(27.18 KB)
📄 X86AsmPrinter.h(5.96 KB)
📄 X86AvoidStoreForwardingBlocks.cpp(27.94 KB)
📄 X86AvoidTrailingCall.cpp(4.91 KB)
📄 X86CallFrameOptimization.cpp(23.07 KB)
📄 X86CallLowering.cpp(17.62 KB)
📄 X86CallLowering.h(1.74 KB)
📄 X86CallingConv.cpp(13.34 KB)
📄 X86CallingConv.h(1.09 KB)
📄 X86CallingConv.td(46.15 KB)
📄 X86CmovConversion.cpp(34.07 KB)
📄 X86CondBrFolding.cpp(18.4 KB)
📄 X86DiscriminateMemOps.cpp(7.11 KB)
📄 X86DomainReassignment.cpp(25.87 KB)
📄 X86EvexToVex.cpp(8.8 KB)
📄 X86ExpandPseudo.cpp(16.95 KB)
📄 X86FastISel.cpp(139.28 KB)
📄 X86FixupBWInsts.cpp(18.09 KB)
📄 X86FixupLEAs.cpp(24.44 KB)
📄 X86FixupSetCC.cpp(4.44 KB)
📄 X86FlagsCopyLowering.cpp(40.36 KB)
📄 X86FloatingPoint.cpp(62.66 KB)
📄 X86FrameLowering.cpp(138.71 KB)
📄 X86FrameLowering.h(11.64 KB)
📄 X86GenRegisterBankInfo.def(3.32 KB)
📄 X86ISelDAGToDAG.cpp(208.37 KB)
📄 X86ISelLowering.cpp(1.94 MB)
📄 X86ISelLowering.h(60.88 KB)
📄 X86IndirectBranchTracking.cpp(6.17 KB)
📄 X86IndirectThunks.cpp(9.78 KB)
📄 X86InsertPrefetch.cpp(9.64 KB)
📄 X86InsertWait.cpp(4.47 KB)
📄 X86Instr3DNow.td(5.24 KB)
📄 X86InstrAMX.td(5.6 KB)
📄 X86InstrAVX512.td(653.76 KB)
📄 X86InstrArithmetic.td(75.61 KB)
📄 X86InstrBuilder.h(8.45 KB)
📄 X86InstrCMovSetCC.td(5.76 KB)
📄 X86InstrCompiler.td(95.78 KB)
📄 X86InstrControl.td(20.53 KB)
📄 X86InstrExtension.td(11.64 KB)
📄 X86InstrFMA.td(33.23 KB)
📄 X86InstrFMA3Info.cpp(6.21 KB)
📄 X86InstrFMA3Info.h(3.25 KB)
📄 X86InstrFPStack.td(39.52 KB)
📄 X86InstrFoldTables.cpp(393.01 KB)
📄 X86InstrFoldTables.h(3.03 KB)
📄 X86InstrFormats.td(41.05 KB)
📄 X86InstrFragmentsSIMD.td(61.14 KB)
📄 X86InstrInfo.cpp(322.72 KB)
📄 X86InstrInfo.h(29.34 KB)
📄 X86InstrInfo.td(169.76 KB)
📄 X86InstrMMX.td(29.55 KB)
📄 X86InstrMPX.td(3.63 KB)
📄 X86InstrSGX.td(1.12 KB)
📄 X86InstrSSE.td(385.01 KB)
📄 X86InstrSVM.td(2.16 KB)
📄 X86InstrShiftRotate.td(49.56 KB)
📄 X86InstrSystem.td(34.03 KB)
📄 X86InstrTSX.td(2.1 KB)
📄 X86InstrVMX.td(3.53 KB)
📄 X86InstrVecCompiler.td(21.09 KB)
📄 X86InstrXOP.td(23.81 KB)
📄 X86InstructionSelector.cpp(61.11 KB)
📄 X86InterleavedAccess.cpp(32.7 KB)
📄 X86IntrinsicsInfo.h(73.96 KB)
📄 X86LegalizerInfo.cpp(15.6 KB)
📄 X86LegalizerInfo.h(1.65 KB)
📄 X86LoadValueInjectionLoadHardening.cpp(32.4 KB)
📄 X86LoadValueInjectionRetHardening.cpp(4.93 KB)
📄 X86MCInstLower.cpp(96.53 KB)
📄 X86MachineFunctionInfo.cpp(1.1 KB)
📄 X86MachineFunctionInfo.h(8.87 KB)
📄 X86MacroFusion.cpp(2.62 KB)
📄 X86MacroFusion.h(992 B)
📄 X86OptimizeLEAs.cpp(27.47 KB)
📄 X86PadShortFunction.cpp(7.33 KB)
📄 X86PartialReduction.cpp(15.46 KB)
📄 X86PfmCounters.td(10.18 KB)
📄 X86RegisterBankInfo.cpp(10.55 KB)
📄 X86RegisterBankInfo.h(2.87 KB)
📄 X86RegisterBanks.td(629 B)
📄 X86RegisterInfo.cpp(29 KB)
📄 X86RegisterInfo.h(5.61 KB)
📄 X86RegisterInfo.td(26.07 KB)
📄 X86SchedBroadwell.td(69.45 KB)
📄 X86SchedHaswell.td(73.96 KB)
📄 X86SchedPredicates.td(4.23 KB)
📄 X86SchedSandyBridge.td(50 KB)
📄 X86SchedSkylakeClient.td(74.65 KB)
📄 X86SchedSkylakeServer.td(113.85 KB)
📄 X86Schedule.td(36.9 KB)
📄 X86ScheduleAtom.td(38.26 KB)
📄 X86ScheduleBdVer2.td(56.78 KB)
📄 X86ScheduleBtVer2.td(46.98 KB)
📄 X86ScheduleSLM.td(22.91 KB)
📄 X86ScheduleZnver1.td(48.97 KB)
📄 X86ScheduleZnver2.td(48.12 KB)
📄 X86SelectionDAGInfo.cpp(12.02 KB)
📄 X86SelectionDAGInfo.h(1.8 KB)
📄 X86ShuffleDecodeConstantPool.cpp(11.22 KB)
📄 X86ShuffleDecodeConstantPool.h(2.13 KB)
📄 X86SpeculativeExecutionSideEffectSuppression.cpp(6.97 KB)
📄 X86SpeculativeLoadHardening.cpp(93.16 KB)
📄 X86Subtarget.cpp(13.25 KB)
📄 X86Subtarget.h(32.08 KB)
📄 X86TargetMachine.cpp(18.88 KB)
📄 X86TargetMachine.h(2.04 KB)
📄 X86TargetObjectFile.cpp(2.61 KB)
📄 X86TargetObjectFile.h(2.13 KB)
📄 X86TargetTransformInfo.cpp(189.14 KB)
📄 X86TargetTransformInfo.h(9.63 KB)
📄 X86VZeroUpper.cpp(12.59 KB)
📄 X86WinAllocaExpander.cpp(9.54 KB)
📄 X86WinEHState.cpp(28.97 KB)

Editing: X86VZeroUpper.cpp

//===- X86VZeroUpper.cpp - AVX vzeroupper instruction inserter ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines the pass which inserts x86 AVX vzeroupper instructions
// before calls to SSE encoded functions. This avoids transition latency
// penalty when transferring control between AVX encoded instructions and old
// SSE encoding mode.
//
//===----------------------------------------------------------------------===//

#include "X86.h"
#include "X86InstrInfo.h"
#include "X86Subtarget.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Function.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>

using namespace llvm;

#define DEBUG_TYPE "x86-vzeroupper"

static cl::opt<bool>
UseVZeroUpper("x86-use-vzeroupper", cl::Hidden,
  cl::desc("Minimize AVX to SSE transition penalty"),
  cl::init(true));

STATISTIC(NumVZU, "Number of vzeroupper instructions inserted");

namespace {

class VZeroUpperInserter : public MachineFunctionPass {
  public:
    VZeroUpperInserter() : MachineFunctionPass(ID) {}

bool runOnMachineFunction(MachineFunction &MF) override;

MachineFunctionProperties getRequiredProperties() const override {
      return MachineFunctionProperties().set(
          MachineFunctionProperties::Property::NoVRegs);
    }

StringRef getPassName() const override { return "X86 vzeroupper inserter"; }

private:
    void processBasicBlock(MachineBasicBlock &MBB);
    void insertVZeroUpper(MachineBasicBlock::iterator I,
                          MachineBasicBlock &MBB);
    void addDirtySuccessor(MachineBasicBlock &MBB);

using BlockExitState = enum { PASS_THROUGH, EXITS_CLEAN, EXITS_DIRTY };

static const char* getBlockExitStateName(BlockExitState ST);

// Core algorithm state:
    // BlockState - Each block is either:
    //   - PASS_THROUGH: There are neither YMM/ZMM dirtying instructions nor
    //                   vzeroupper instructions in this block.
    //   - EXITS_CLEAN: There is (or will be) a vzeroupper instruction in this
    //                  block that will ensure that YMM/ZMM is clean on exit.
    //   - EXITS_DIRTY: An instruction in the block dirties YMM/ZMM and no
    //                  subsequent vzeroupper in the block clears it.
    //
    // AddedToDirtySuccessors - This flag is raised when a block is added to the
    //                          DirtySuccessors list to ensure that it's not
    //                          added multiple times.
    //
    // FirstUnguardedCall - Records the location of the first unguarded call in
    //                      each basic block that may need to be guarded by a
    //                      vzeroupper. We won't know whether it actually needs
    //                      to be guarded until we discover a predecessor that
    //                      is DIRTY_OUT.
    struct BlockState {
      BlockExitState ExitState = PASS_THROUGH;
      bool AddedToDirtySuccessors = false;
      MachineBasicBlock::iterator FirstUnguardedCall;

BlockState() = default;
    };

using BlockStateMap = SmallVector<BlockState, 8>;
    using DirtySuccessorsWorkList = SmallVector<MachineBasicBlock *, 8>;

BlockStateMap BlockStates;
    DirtySuccessorsWorkList DirtySuccessors;
    bool EverMadeChange;
    bool IsX86INTR;
    const TargetInstrInfo *TII;

static char ID;
  };

} // end anonymous namespace

char VZeroUpperInserter::ID = 0;

FunctionPass *llvm::createX86IssueVZeroUpperPass() {
  return new VZeroUpperInserter();
}

#ifndef NDEBUG
const char* VZeroUpperInserter::getBlockExitStateName(BlockExitState ST) {
  switch (ST) {
    case PASS_THROUGH: return "Pass-through";
    case EXITS_DIRTY: return "Exits-dirty";
    case EXITS_CLEAN: return "Exits-clean";
  }
  llvm_unreachable("Invalid block exit state.");
}
#endif

/// VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
/// Thus, there is no need to check for Y/ZMM16 and above.
static bool isYmmOrZmmReg(unsigned Reg) {
  return (Reg >= X86::YMM0 && Reg <= X86::YMM15) ||
         (Reg >= X86::ZMM0 && Reg <= X86::ZMM15);
}

static bool checkFnHasLiveInYmmOrZmm(MachineRegisterInfo &MRI) {
  for (std::pair<unsigned, unsigned> LI : MRI.liveins())
    if (isYmmOrZmmReg(LI.first))
      return true;

return false;
}

static bool clobbersAllYmmAndZmmRegs(const MachineOperand &MO) {
  for (unsigned reg = X86::YMM0; reg <= X86::YMM15; ++reg) {
    if (!MO.clobbersPhysReg(reg))
      return false;
  }
  for (unsigned reg = X86::ZMM0; reg <= X86::ZMM15; ++reg) {
    if (!MO.clobbersPhysReg(reg))
      return false;
  }
  return true;
}

static bool hasYmmOrZmmReg(MachineInstr &MI) {
  for (const MachineOperand &MO : MI.operands()) {
    if (MI.isCall() && MO.isRegMask() && !clobbersAllYmmAndZmmRegs(MO))
      return true;
    if (!MO.isReg())
      continue;
    if (MO.isDebug())
      continue;
    if (isYmmOrZmmReg(MO.getReg()))
      return true;
  }
  return false;
}

/// Check if given call instruction has a RegMask operand.
static bool callHasRegMask(MachineInstr &MI) {
  assert(MI.isCall() && "Can only be called on call instructions.");
  for (const MachineOperand &MO : MI.operands()) {
    if (MO.isRegMask())
      return true;
  }
  return false;
}

/// Insert a vzeroupper instruction before I.
void VZeroUpperInserter::insertVZeroUpper(MachineBasicBlock::iterator I,
                                          MachineBasicBlock &MBB) {
  DebugLoc dl = I->getDebugLoc();
  BuildMI(MBB, I, dl, TII->get(X86::VZEROUPPER));
  ++NumVZU;
  EverMadeChange = true;
}

/// Add MBB to the DirtySuccessors list if it hasn't already been added.
void VZeroUpperInserter::addDirtySuccessor(MachineBasicBlock &MBB) {
  if (!BlockStates[MBB.getNumber()].AddedToDirtySuccessors) {
    DirtySuccessors.push_back(&MBB);
    BlockStates[MBB.getNumber()].AddedToDirtySuccessors = true;
  }
}

/// Loop over all of the instructions in the basic block, inserting vzeroupper
/// instructions before function calls.
void VZeroUpperInserter::processBasicBlock(MachineBasicBlock &MBB) {
  // Start by assuming that the block is PASS_THROUGH which implies no unguarded
  // calls.
  BlockExitState CurState = PASS_THROUGH;
  BlockStates[MBB.getNumber()].FirstUnguardedCall = MBB.end();

for (MachineInstr &MI : MBB) {
    bool IsCall = MI.isCall();
    bool IsReturn = MI.isReturn();
    bool IsControlFlow = IsCall || IsReturn;

// No need for vzeroupper before iret in interrupt handler function,
    // epilogue will restore YMM/ZMM registers if needed.
    if (IsX86INTR && IsReturn)
      continue;

// An existing VZERO* instruction resets the state.
    if (MI.getOpcode() == X86::VZEROALL || MI.getOpcode() == X86::VZEROUPPER) {
      CurState = EXITS_CLEAN;
      continue;
    }

// Shortcut: don't need to check regular instructions in dirty state.
    if (!IsControlFlow && CurState == EXITS_DIRTY)
      continue;

if (hasYmmOrZmmReg(MI)) {
      // We found a ymm/zmm-using instruction; this could be an AVX/AVX512
      // instruction, or it could be control flow.
      CurState = EXITS_DIRTY;
      continue;
    }

// Check for control-flow out of the current function (which might
    // indirectly execute SSE instructions).
    if (!IsControlFlow)
      continue;

// If the call has no RegMask, skip it as well. It usually happens on
    // helper function calls (such as '_chkstk', '_ftol2') where standard
    // calling convention is not used (RegMask is not used to mark register
    // clobbered and register usage (def/implicit-def/use) is well-defined and
    // explicitly specified.
    if (IsCall && !callHasRegMask(MI))
      continue;

// The VZEROUPPER instruction resets the upper 128 bits of YMM0-YMM15
    // registers. In addition, the processor changes back to Clean state, after
    // which execution of SSE instructions or AVX instructions has no transition
    // penalty. Add the VZEROUPPER instruction before any function call/return
    // that might execute SSE code.
    // FIXME: In some cases, we may want to move the VZEROUPPER into a
    // predecessor block.
    if (CurState == EXITS_DIRTY) {
      // After the inserted VZEROUPPER the state becomes clean again, but
      // other YMM/ZMM may appear before other subsequent calls or even before
      // the end of the BB.
      insertVZeroUpper(MI, MBB);
      CurState = EXITS_CLEAN;
    } else if (CurState == PASS_THROUGH) {
      // If this block is currently in pass-through state and we encounter a
      // call then whether we need a vzeroupper or not depends on whether this
      // block has successors that exit dirty. Record the location of the call,
      // and set the state to EXITS_CLEAN, but do not insert the vzeroupper yet.
      // It will be inserted later if necessary.
      BlockStates[MBB.getNumber()].FirstUnguardedCall = MI;
      CurState = EXITS_CLEAN;
    }
  }

LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber() << " exit state: "
                    << getBlockExitStateName(CurState) << '\n');

if (CurState == EXITS_DIRTY)
    for (MachineBasicBlock::succ_iterator SI = MBB.succ_begin(),
                                          SE = MBB.succ_end();
         SI != SE; ++SI)
      addDirtySuccessor(**SI);

BlockStates[MBB.getNumber()].ExitState = CurState;
}

/// Loop over all of the basic blocks, inserting vzeroupper instructions before
/// function calls.
bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
  if (!UseVZeroUpper)
    return false;

const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
  if (!ST.hasAVX() || !ST.insertVZEROUPPER())
    return false;
  TII = ST.getInstrInfo();
  MachineRegisterInfo &MRI = MF.getRegInfo();
  EverMadeChange = false;
  IsX86INTR = MF.getFunction().getCallingConv() == CallingConv::X86_INTR;

bool FnHasLiveInYmmOrZmm = checkFnHasLiveInYmmOrZmm(MRI);

// Fast check: if the function doesn't use any ymm/zmm registers, we don't
  // need to insert any VZEROUPPER instructions.  This is constant-time, so it
  // is cheap in the common case of no ymm/zmm use.
  bool YmmOrZmmUsed = FnHasLiveInYmmOrZmm;
  for (auto *RC : {&X86::VR256RegClass, &X86::VR512_0_15RegClass}) {
    if (!YmmOrZmmUsed) {
      for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end(); i != e;
           i++) {
        if (!MRI.reg_nodbg_empty(*i)) {
          YmmOrZmmUsed = true;
          break;
        }
      }
    }
  }
  if (!YmmOrZmmUsed)
    return false;

assert(BlockStates.empty() && DirtySuccessors.empty() &&
         "X86VZeroUpper state should be clear");
  BlockStates.resize(MF.getNumBlockIDs());

// Process all blocks. This will compute block exit states, record the first
  // unguarded call in each block, and add successors of dirty blocks to the
  // DirtySuccessors list.
  for (MachineBasicBlock &MBB : MF)
    processBasicBlock(MBB);

// If any YMM/ZMM regs are live-in to this function, add the entry block to
  // the DirtySuccessors list
  if (FnHasLiveInYmmOrZmm)
    addDirtySuccessor(MF.front());

// Re-visit all blocks that are successors of EXITS_DIRTY blocks. Add
  // vzeroupper instructions to unguarded calls, and propagate EXITS_DIRTY
  // through PASS_THROUGH blocks.
  while (!DirtySuccessors.empty()) {
    MachineBasicBlock &MBB = *DirtySuccessors.back();
    DirtySuccessors.pop_back();
    BlockState &BBState = BlockStates[MBB.getNumber()];

// MBB is a successor of a dirty block, so its first call needs to be
    // guarded.
    if (BBState.FirstUnguardedCall != MBB.end())
      insertVZeroUpper(BBState.FirstUnguardedCall, MBB);

// If this successor was a pass-through block, then it is now dirty. Its
    // successors need to be added to the worklist (if they haven't been
    // already).
    if (BBState.ExitState == PASS_THROUGH) {
      LLVM_DEBUG(dbgs() << "MBB #" << MBB.getNumber()
                        << " was Pass-through, is now Dirty-out.\n");
      for (MachineBasicBlock *Succ : MBB.successors())
        addDirtySuccessor(*Succ);
    }
  }

BlockStates.clear();
  return EverMadeChange;
}

003 File Manager

Editing: X86VZeroUpper.cpp

Upload File

Create Folder