File Manager

003 File Manager

Current Path: /usr/src/contrib/llvm-project/llvm/lib/Target/AArch64

usr / src / contrib / llvm-project / llvm / lib / Target / AArch64 /

📁 ..
📄 AArch64.h(4.09 KB)
📄 AArch64.td(49.85 KB)
📄 AArch64A53Fix835769.cpp(8.3 KB)
📄 AArch64A57FPLoadBalancing.cpp(25.73 KB)
📄 AArch64AdvSIMDScalarPass.cpp(16.1 KB)
📄 AArch64AsmPrinter.cpp(49.26 KB)
📄 AArch64BranchTargets.cpp(4.91 KB)
📄 AArch64CallingConvention.cpp(6.63 KB)
📄 AArch64CallingConvention.h(2.62 KB)
📄 AArch64CallingConvention.td(23.84 KB)
📄 AArch64CleanupLocalDynamicTLSPass.cpp(5.53 KB)
📄 AArch64CollectLOH.cpp(20.07 KB)
📄 AArch64Combine.td(3.27 KB)
📄 AArch64CompressJumpTables.cpp(5.06 KB)
📄 AArch64CondBrTuning.cpp(10.19 KB)
📄 AArch64ConditionOptimizer.cpp(15.26 KB)
📄 AArch64ConditionalCompares.cpp(33.26 KB)
📄 AArch64DeadRegisterDefinitionsPass.cpp(7.76 KB)
📄 AArch64ExpandImm.cpp(14.25 KB)
📄 AArch64ExpandImm.h(959 B)
📄 AArch64ExpandPseudoInsts.cpp(38.08 KB)
📄 AArch64FalkorHWPFFix.cpp(23.3 KB)
📄 AArch64FastISel.cpp(171.76 KB)
📄 AArch64FrameLowering.cpp(124.13 KB)
📄 AArch64FrameLowering.h(5.54 KB)
📄 AArch64GenRegisterBankInfo.def(11 KB)
📄 AArch64ISelDAGToDAG.cpp(180.26 KB)
📄 AArch64ISelLowering.cpp(578.95 KB)
📄 AArch64ISelLowering.h(33.88 KB)
📄 AArch64InstrAtomics.td(20.33 KB)
📄 AArch64InstrFormats.td(430.93 KB)
📄 AArch64InstrGISel.td(4.29 KB)
📄 AArch64InstrInfo.cpp(243.23 KB)
📄 AArch64InstrInfo.h(19.92 KB)
📄 AArch64InstrInfo.td(374.84 KB)
📄 AArch64LoadStoreOptimizer.cpp(77.04 KB)
📄 AArch64MCInstLower.cpp(11.72 KB)
📄 AArch64MCInstLower.h(1.69 KB)
📄 AArch64MachineFunctionInfo.cpp(1.02 KB)
📄 AArch64MachineFunctionInfo.h(12.9 KB)
📄 AArch64MacroFusion.cpp(11.47 KB)
📄 AArch64MacroFusion.h(891 B)
📄 AArch64PBQPRegAlloc.cpp(11.35 KB)
📄 AArch64PBQPRegAlloc.h(1.3 KB)
📄 AArch64PerfectShuffle.h(382.04 KB)
📄 AArch64PfmCounters.td(713 B)
📄 AArch64PromoteConstant.cpp(22.43 KB)
📄 AArch64RedundantCopyElimination.cpp(17.09 KB)
📄 AArch64RegisterBanks.td(719 B)
📄 AArch64RegisterInfo.cpp(29.6 KB)
📄 AArch64RegisterInfo.h(5.5 KB)
📄 AArch64RegisterInfo.td(51 KB)
📄 AArch64SIMDInstrOpt.cpp(26.07 KB)
📄 AArch64SLSHardening.cpp(15.92 KB)
📄 AArch64SVEInstrInfo.td(169.43 KB)
📄 AArch64SchedA53.td(15.28 KB)
📄 AArch64SchedA57.td(34.69 KB)
📄 AArch64SchedA57WriteRes.td(19.87 KB)
📄 AArch64SchedCyclone.td(29.82 KB)
📄 AArch64SchedExynosM3.td(42.57 KB)
📄 AArch64SchedExynosM4.td(49.81 KB)
📄 AArch64SchedExynosM5.td(50.74 KB)
📄 AArch64SchedFalkor.td(5.3 KB)
📄 AArch64SchedFalkorDetails.td(67.66 KB)
📄 AArch64SchedKryo.td(6.21 KB)
📄 AArch64SchedKryoDetails.td(82.63 KB)
📄 AArch64SchedPredExynos.td(7.5 KB)
📄 AArch64SchedPredicates.td(27.86 KB)
📄 AArch64SchedThunderX.td(14.99 KB)
📄 AArch64SchedThunderX2T99.td(68.58 KB)
📄 AArch64SchedThunderX3T110.td(68.77 KB)
📄 AArch64Schedule.td(3.86 KB)
📄 AArch64SelectionDAGInfo.cpp(5.54 KB)
📄 AArch64SelectionDAGInfo.h(1.46 KB)
📄 AArch64SpeculationHardening.cpp(29.6 KB)
📄 AArch64StackOffset.h(5.01 KB)
📄 AArch64StackTagging.cpp(24.31 KB)
📄 AArch64StackTaggingPreRA.cpp(7.25 KB)
📄 AArch64StorePairSuppress.cpp(6.26 KB)
📄 AArch64Subtarget.cpp(12.38 KB)
📄 AArch64Subtarget.h(18.49 KB)
📄 AArch64SystemOperands.td(81.75 KB)
📄 AArch64TargetMachine.cpp(26.65 KB)
📄 AArch64TargetMachine.h(3.25 KB)
📄 AArch64TargetObjectFile.cpp(3.35 KB)
📄 AArch64TargetObjectFile.h(2.31 KB)
📄 AArch64TargetTransformInfo.cpp(42.6 KB)
📄 AArch64TargetTransformInfo.h(8.68 KB)
📁 AsmParser
📁 Disassembler
📁 GISel
📁 MCTargetDesc
📄 SVEInstrFormats.td(304.03 KB)
📄 SVEIntrinsicOpts.cpp(8.13 KB)
📁 TargetInfo
📁 Utils

Editing: AArch64MacroFusion.cpp

//===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
/// \file This file contains the AArch64 implementation of the DAG scheduling
///  mutation to pair instructions back to back.
//
//===----------------------------------------------------------------------===//

#include "AArch64Subtarget.h"
#include "llvm/CodeGen/MacroFusion.h"
#include "llvm/CodeGen/TargetInstrInfo.h"

using namespace llvm;

namespace {

/// CMN, CMP, TST followed by Bcc
static bool isArithmeticBccPair(const MachineInstr *FirstMI,
                                const MachineInstr &SecondMI) {
  if (SecondMI.getOpcode() != AArch64::Bcc)
    return false;

// Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr)
    return true;

switch (FirstMI->getOpcode()) {
  case AArch64::ADDSWri:
  case AArch64::ADDSWrr:
  case AArch64::ADDSXri:
  case AArch64::ADDSXrr:
  case AArch64::ANDSWri:
  case AArch64::ANDSWrr:
  case AArch64::ANDSXri:
  case AArch64::ANDSXrr:
  case AArch64::SUBSWri:
  case AArch64::SUBSWrr:
  case AArch64::SUBSXri:
  case AArch64::SUBSXrr:
  case AArch64::BICSWrr:
  case AArch64::BICSXrr:
    return true;
  case AArch64::ADDSWrs:
  case AArch64::ADDSXrs:
  case AArch64::ANDSWrs:
  case AArch64::ANDSXrs:
  case AArch64::SUBSWrs:
  case AArch64::SUBSXrs:
  case AArch64::BICSWrs:
  case AArch64::BICSXrs:
    // Shift value can be 0 making these behave like the "rr" variant...
    return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
  }

return false;
}

/// ALU operations followed by CBZ/CBNZ.
static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
                                const MachineInstr &SecondMI) {
  if (SecondMI.getOpcode() != AArch64::CBZW &&
      SecondMI.getOpcode() != AArch64::CBZX &&
      SecondMI.getOpcode() != AArch64::CBNZW &&
      SecondMI.getOpcode() != AArch64::CBNZX)
    return false;

// Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr)
    return true;

switch (FirstMI->getOpcode()) {
  case AArch64::ADDWri:
  case AArch64::ADDWrr:
  case AArch64::ADDXri:
  case AArch64::ADDXrr:
  case AArch64::ANDWri:
  case AArch64::ANDWrr:
  case AArch64::ANDXri:
  case AArch64::ANDXrr:
  case AArch64::EORWri:
  case AArch64::EORWrr:
  case AArch64::EORXri:
  case AArch64::EORXrr:
  case AArch64::ORRWri:
  case AArch64::ORRWrr:
  case AArch64::ORRXri:
  case AArch64::ORRXrr:
  case AArch64::SUBWri:
  case AArch64::SUBWrr:
  case AArch64::SUBXri:
  case AArch64::SUBXrr:
    return true;
  case AArch64::ADDWrs:
  case AArch64::ADDXrs:
  case AArch64::ANDWrs:
  case AArch64::ANDXrs:
  case AArch64::SUBWrs:
  case AArch64::SUBXrs:
  case AArch64::BICWrs:
  case AArch64::BICXrs:
    // Shift value can be 0 making these behave like the "rr" variant...
    return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
  }

return false;
}

/// AES crypto encoding or decoding.
static bool isAESPair(const MachineInstr *FirstMI,
                      const MachineInstr &SecondMI) {
  // Assume the 1st instr to be a wildcard if it is unspecified.
  switch (SecondMI.getOpcode()) {
  // AES encode.
  case AArch64::AESMCrr:
  case AArch64::AESMCrrTied:
    return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
  // AES decode.
  case AArch64::AESIMCrr:
  case AArch64::AESIMCrrTied:
    return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
  }

return false;
}

/// AESE/AESD/PMULL + EOR.
static bool isCryptoEORPair(const MachineInstr *FirstMI,
                            const MachineInstr &SecondMI) {
  if (SecondMI.getOpcode() != AArch64::EORv16i8)
    return false;

// Assume the 1st instr to be a wildcard if it is unspecified.
  if (FirstMI == nullptr)
    return true;

switch (FirstMI->getOpcode()) {
  case AArch64::AESErr:
  case AArch64::AESDrr:
  case AArch64::PMULLv16i8:
  case AArch64::PMULLv8i8:
  case AArch64::PMULLv1i64:
  case AArch64::PMULLv2i64:
    return true;
  }

return false;
}

/// Literal generation.
static bool isLiteralsPair(const MachineInstr *FirstMI,
                           const MachineInstr &SecondMI) {
  // Assume the 1st instr to be a wildcard if it is unspecified.

// PC relative address.
  if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
      SecondMI.getOpcode() == AArch64::ADDXri)
    return true;

// 32 bit immediate.
  if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
      (SecondMI.getOpcode() == AArch64::MOVKWi &&
       SecondMI.getOperand(3).getImm() == 16))
    return true;

// Lower half of 64 bit immediate.
  if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
     (SecondMI.getOpcode() == AArch64::MOVKXi &&
      SecondMI.getOperand(3).getImm() == 16))
    return true;

// Upper half of 64 bit immediate.
  if ((FirstMI == nullptr ||
       (FirstMI->getOpcode() == AArch64::MOVKXi &&
        FirstMI->getOperand(3).getImm() == 32)) &&
      (SecondMI.getOpcode() == AArch64::MOVKXi &&
       SecondMI.getOperand(3).getImm() == 48))
    return true;

return false;
}

/// Fuse address generation and loads or stores.
static bool isAddressLdStPair(const MachineInstr *FirstMI,
                              const MachineInstr &SecondMI) {
  switch (SecondMI.getOpcode()) {
  case AArch64::STRBBui:
  case AArch64::STRBui:
  case AArch64::STRDui:
  case AArch64::STRHHui:
  case AArch64::STRHui:
  case AArch64::STRQui:
  case AArch64::STRSui:
  case AArch64::STRWui:
  case AArch64::STRXui:
  case AArch64::LDRBBui:
  case AArch64::LDRBui:
  case AArch64::LDRDui:
  case AArch64::LDRHHui:
  case AArch64::LDRHui:
  case AArch64::LDRQui:
  case AArch64::LDRSui:
  case AArch64::LDRWui:
  case AArch64::LDRXui:
  case AArch64::LDRSBWui:
  case AArch64::LDRSBXui:
  case AArch64::LDRSHWui:
  case AArch64::LDRSHXui:
  case AArch64::LDRSWui:
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;

switch (FirstMI->getOpcode()) {
    case AArch64::ADR:
      return SecondMI.getOperand(2).getImm() == 0;
    case AArch64::ADRP:
      return true;
    }
  }

return false;
}

/// Compare and conditional select.
static bool isCCSelectPair(const MachineInstr *FirstMI,
                           const MachineInstr &SecondMI) {
  // 32 bits
  if (SecondMI.getOpcode() == AArch64::CSELWr) {
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;

if (FirstMI->definesRegister(AArch64::WZR))
      switch (FirstMI->getOpcode()) {
      case AArch64::SUBSWrs:
        return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
      case AArch64::SUBSWrx:
        return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
      case AArch64::SUBSWrr:
      case AArch64::SUBSWri:
        return true;
      }
  }

// 64 bits
  if (SecondMI.getOpcode() == AArch64::CSELXr) {
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;

if (FirstMI->definesRegister(AArch64::XZR))
      switch (FirstMI->getOpcode()) {
      case AArch64::SUBSXrs:
        return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
      case AArch64::SUBSXrx:
      case AArch64::SUBSXrx64:
        return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
      case AArch64::SUBSXrr:
      case AArch64::SUBSXri:
        return true;
      }
  }

return false;
}

// Arithmetic and logic.
static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
                                  const MachineInstr &SecondMI) {
  if (AArch64InstrInfo::hasShiftedReg(SecondMI))
    return false;

switch (SecondMI.getOpcode()) {
  // Arithmetic
  case AArch64::ADDWrr:
  case AArch64::ADDXrr:
  case AArch64::SUBWrr:
  case AArch64::SUBXrr:
  case AArch64::ADDWrs:
  case AArch64::ADDXrs:
  case AArch64::SUBWrs:
  case AArch64::SUBXrs:
  // Logic
  case AArch64::ANDWrr:
  case AArch64::ANDXrr:
  case AArch64::BICWrr:
  case AArch64::BICXrr:
  case AArch64::EONWrr:
  case AArch64::EONXrr:
  case AArch64::EORWrr:
  case AArch64::EORXrr:
  case AArch64::ORNWrr:
  case AArch64::ORNXrr:
  case AArch64::ORRWrr:
  case AArch64::ORRXrr:
  case AArch64::ANDWrs:
  case AArch64::ANDXrs:
  case AArch64::BICWrs:
  case AArch64::BICXrs:
  case AArch64::EONWrs:
  case AArch64::EONXrs:
  case AArch64::EORWrs:
  case AArch64::EORXrs:
  case AArch64::ORNWrs:
  case AArch64::ORNXrs:
  case AArch64::ORRWrs:
  case AArch64::ORRXrs:
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;

// Arithmetic
    switch (FirstMI->getOpcode()) {
    case AArch64::ADDWrr:
    case AArch64::ADDXrr:
    case AArch64::ADDSWrr:
    case AArch64::ADDSXrr:
    case AArch64::SUBWrr:
    case AArch64::SUBXrr:
    case AArch64::SUBSWrr:
    case AArch64::SUBSXrr:
      return true;
    case AArch64::ADDWrs:
    case AArch64::ADDXrs:
    case AArch64::ADDSWrs:
    case AArch64::ADDSXrs:
    case AArch64::SUBWrs:
    case AArch64::SUBXrs:
    case AArch64::SUBSWrs:
    case AArch64::SUBSXrs:
      return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
    }
    break;

// Arithmetic, setting flags.
  case AArch64::ADDSWrr:
  case AArch64::ADDSXrr:
  case AArch64::SUBSWrr:
  case AArch64::SUBSXrr:
  case AArch64::ADDSWrs:
  case AArch64::ADDSXrs:
  case AArch64::SUBSWrs:
  case AArch64::SUBSXrs:
    // Assume the 1st instr to be a wildcard if it is unspecified.
    if (FirstMI == nullptr)
      return true;

// Arithmetic, not setting flags.
    switch (FirstMI->getOpcode()) {
    case AArch64::ADDWrr:
    case AArch64::ADDXrr:
    case AArch64::SUBWrr:
    case AArch64::SUBXrr:
      return true;
    case AArch64::ADDWrs:
    case AArch64::ADDXrs:
    case AArch64::SUBWrs:
    case AArch64::SUBXrs:
      return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
    }
    break;
  }

return false;
}

/// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
/// together. Given SecondMI, when FirstMI is unspecified, then check if
/// SecondMI may be part of a fused pair at all.
static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
                                   const TargetSubtargetInfo &TSI,
                                   const MachineInstr *FirstMI,
                                   const MachineInstr &SecondMI) {
  const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);

// All checking functions assume that the 1st instr is a wildcard if it is
  // unspecified.
  if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI))
    return true;
  if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
    return true;
  if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
    return true;
  if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
    return true;
  if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
    return true;
  if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
    return true;
  if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
    return true;
  if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
    return true;

return false;
}

} // end namespace

namespace llvm {

std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
  return createMacroFusionDAGMutation(shouldScheduleAdjacent);
}

} // end namespace llvm

003 File Manager

Editing: AArch64MacroFusion.cpp

Upload File

Create Folder