File Manager

003 File Manager

Current Path: /usr/src/contrib/llvm-project/llvm/lib/Target/X86

usr / src / contrib / llvm-project / llvm / lib / Target / X86 /

📁 ..
📁 AsmParser
📁 Disassembler
📄 ImmutableGraph.h(15.15 KB)
📁 MCTargetDesc
📁 TargetInfo
📄 X86.h(7.41 KB)
📄 X86.td(68.44 KB)
📄 X86AsmPrinter.cpp(27.18 KB)
📄 X86AsmPrinter.h(5.96 KB)
📄 X86AvoidStoreForwardingBlocks.cpp(27.94 KB)
📄 X86AvoidTrailingCall.cpp(4.91 KB)
📄 X86CallFrameOptimization.cpp(23.07 KB)
📄 X86CallLowering.cpp(17.62 KB)
📄 X86CallLowering.h(1.74 KB)
📄 X86CallingConv.cpp(13.34 KB)
📄 X86CallingConv.h(1.09 KB)
📄 X86CallingConv.td(46.15 KB)
📄 X86CmovConversion.cpp(34.07 KB)
📄 X86CondBrFolding.cpp(18.4 KB)
📄 X86DiscriminateMemOps.cpp(7.11 KB)
📄 X86DomainReassignment.cpp(25.87 KB)
📄 X86EvexToVex.cpp(8.8 KB)
📄 X86ExpandPseudo.cpp(16.95 KB)
📄 X86FastISel.cpp(139.28 KB)
📄 X86FixupBWInsts.cpp(18.09 KB)
📄 X86FixupLEAs.cpp(24.44 KB)
📄 X86FixupSetCC.cpp(4.44 KB)
📄 X86FlagsCopyLowering.cpp(40.36 KB)
📄 X86FloatingPoint.cpp(62.66 KB)
📄 X86FrameLowering.cpp(138.71 KB)
📄 X86FrameLowering.h(11.64 KB)
📄 X86GenRegisterBankInfo.def(3.32 KB)
📄 X86ISelDAGToDAG.cpp(208.37 KB)
📄 X86ISelLowering.cpp(1.94 MB)
📄 X86ISelLowering.h(60.88 KB)
📄 X86IndirectBranchTracking.cpp(6.17 KB)
📄 X86IndirectThunks.cpp(9.78 KB)
📄 X86InsertPrefetch.cpp(9.64 KB)
📄 X86InsertWait.cpp(4.47 KB)
📄 X86Instr3DNow.td(5.24 KB)
📄 X86InstrAMX.td(5.6 KB)
📄 X86InstrAVX512.td(653.76 KB)
📄 X86InstrArithmetic.td(75.61 KB)
📄 X86InstrBuilder.h(8.45 KB)
📄 X86InstrCMovSetCC.td(5.76 KB)
📄 X86InstrCompiler.td(95.78 KB)
📄 X86InstrControl.td(20.53 KB)
📄 X86InstrExtension.td(11.64 KB)
📄 X86InstrFMA.td(33.23 KB)
📄 X86InstrFMA3Info.cpp(6.21 KB)
📄 X86InstrFMA3Info.h(3.25 KB)
📄 X86InstrFPStack.td(39.52 KB)
📄 X86InstrFoldTables.cpp(393.01 KB)
📄 X86InstrFoldTables.h(3.03 KB)
📄 X86InstrFormats.td(41.05 KB)
📄 X86InstrFragmentsSIMD.td(61.14 KB)
📄 X86InstrInfo.cpp(322.72 KB)
📄 X86InstrInfo.h(29.34 KB)
📄 X86InstrInfo.td(169.76 KB)
📄 X86InstrMMX.td(29.55 KB)
📄 X86InstrMPX.td(3.63 KB)
📄 X86InstrSGX.td(1.12 KB)
📄 X86InstrSSE.td(385.01 KB)
📄 X86InstrSVM.td(2.16 KB)
📄 X86InstrShiftRotate.td(49.56 KB)
📄 X86InstrSystem.td(34.03 KB)
📄 X86InstrTSX.td(2.1 KB)
📄 X86InstrVMX.td(3.53 KB)
📄 X86InstrVecCompiler.td(21.09 KB)
📄 X86InstrXOP.td(23.81 KB)
📄 X86InstructionSelector.cpp(61.11 KB)
📄 X86InterleavedAccess.cpp(32.7 KB)
📄 X86IntrinsicsInfo.h(73.96 KB)
📄 X86LegalizerInfo.cpp(15.6 KB)
📄 X86LegalizerInfo.h(1.65 KB)
📄 X86LoadValueInjectionLoadHardening.cpp(32.4 KB)
📄 X86LoadValueInjectionRetHardening.cpp(4.93 KB)
📄 X86MCInstLower.cpp(96.53 KB)
📄 X86MachineFunctionInfo.cpp(1.1 KB)
📄 X86MachineFunctionInfo.h(8.87 KB)
📄 X86MacroFusion.cpp(2.62 KB)
📄 X86MacroFusion.h(992 B)
📄 X86OptimizeLEAs.cpp(27.47 KB)
📄 X86PadShortFunction.cpp(7.33 KB)
📄 X86PartialReduction.cpp(15.46 KB)
📄 X86PfmCounters.td(10.18 KB)
📄 X86RegisterBankInfo.cpp(10.55 KB)
📄 X86RegisterBankInfo.h(2.87 KB)
📄 X86RegisterBanks.td(629 B)
📄 X86RegisterInfo.cpp(29 KB)
📄 X86RegisterInfo.h(5.61 KB)
📄 X86RegisterInfo.td(26.07 KB)
📄 X86SchedBroadwell.td(69.45 KB)
📄 X86SchedHaswell.td(73.96 KB)
📄 X86SchedPredicates.td(4.23 KB)
📄 X86SchedSandyBridge.td(50 KB)
📄 X86SchedSkylakeClient.td(74.65 KB)
📄 X86SchedSkylakeServer.td(113.85 KB)
📄 X86Schedule.td(36.9 KB)
📄 X86ScheduleAtom.td(38.26 KB)
📄 X86ScheduleBdVer2.td(56.78 KB)
📄 X86ScheduleBtVer2.td(46.98 KB)
📄 X86ScheduleSLM.td(22.91 KB)
📄 X86ScheduleZnver1.td(48.97 KB)
📄 X86ScheduleZnver2.td(48.12 KB)
📄 X86SelectionDAGInfo.cpp(12.02 KB)
📄 X86SelectionDAGInfo.h(1.8 KB)
📄 X86ShuffleDecodeConstantPool.cpp(11.22 KB)
📄 X86ShuffleDecodeConstantPool.h(2.13 KB)
📄 X86SpeculativeExecutionSideEffectSuppression.cpp(6.97 KB)
📄 X86SpeculativeLoadHardening.cpp(93.16 KB)
📄 X86Subtarget.cpp(13.25 KB)
📄 X86Subtarget.h(32.08 KB)
📄 X86TargetMachine.cpp(18.88 KB)
📄 X86TargetMachine.h(2.04 KB)
📄 X86TargetObjectFile.cpp(2.61 KB)
📄 X86TargetObjectFile.h(2.13 KB)
📄 X86TargetTransformInfo.cpp(189.14 KB)
📄 X86TargetTransformInfo.h(9.63 KB)
📄 X86VZeroUpper.cpp(12.59 KB)
📄 X86WinAllocaExpander.cpp(9.54 KB)
📄 X86WinEHState.cpp(28.97 KB)

Editing: X86ShuffleDecodeConstantPool.cpp

//===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// Define several functions to decode x86 specific shuffle semantics using
// constants from the constant pool.
//
//===----------------------------------------------------------------------===//

#include "X86ShuffleDecodeConstantPool.h"
#include "MCTargetDesc/X86ShuffleDecode.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/IR/Constants.h"

//===----------------------------------------------------------------------===//
//  Vector Mask Decoding
//===----------------------------------------------------------------------===//

namespace llvm {

static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits,
                                APInt &UndefElts,
                                SmallVectorImpl<uint64_t> &RawMask) {
  // It is not an error for shuffle masks to not be a vector of
  // MaskEltSizeInBits because the constant pool uniques constants by their
  // bit representation.
  // e.g. the following take up the same space in the constant pool:
  //   i128 -170141183420855150465331762880109871104
  //
  //   <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160>
  //
  //   <4 x i32> <i32 -2147483648, i32 -2147483648,
  //              i32 -2147483648, i32 -2147483648>
  auto *CstTy = dyn_cast<FixedVectorType>(C->getType());
  if (!CstTy)
    return false;

Type *CstEltTy = CstTy->getElementType();
  if (!CstEltTy->isIntegerTy())
    return false;

unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
  unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits();
  unsigned NumCstElts = CstTy->getNumElements();

assert((CstSizeInBits % MaskEltSizeInBits) == 0 &&
         "Unaligned shuffle mask size");

unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits;
  UndefElts = APInt(NumMaskElts, 0);
  RawMask.resize(NumMaskElts, 0);

// Fast path - if the constants match the mask size then copy direct.
  if (MaskEltSizeInBits == CstEltSizeInBits) {
    assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size");
    for (unsigned i = 0; i != NumMaskElts; ++i) {
      Constant *COp = C->getAggregateElement(i);
      if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
        return false;

if (isa<UndefValue>(COp)) {
        UndefElts.setBit(i);
        RawMask[i] = 0;
        continue;
      }

auto *Elt = cast<ConstantInt>(COp);
      RawMask[i] = Elt->getValue().getZExtValue();
    }
    return true;
  }

// Extract all the undef/constant element data and pack into single bitsets.
  APInt UndefBits(CstSizeInBits, 0);
  APInt MaskBits(CstSizeInBits, 0);
  for (unsigned i = 0; i != NumCstElts; ++i) {
    Constant *COp = C->getAggregateElement(i);
    if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp)))
      return false;

unsigned BitOffset = i * CstEltSizeInBits;

if (isa<UndefValue>(COp)) {
      UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits);
      continue;
    }

MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset);
  }

// Now extract the undef/constant bit data into the raw shuffle masks.
  for (unsigned i = 0; i != NumMaskElts; ++i) {
    unsigned BitOffset = i * MaskEltSizeInBits;
    APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset);

// Only treat the element as UNDEF if all bits are UNDEF, otherwise
    // treat it as zero.
    if (EltUndef.isAllOnesValue()) {
      UndefElts.setBit(i);
      RawMask[i] = 0;
      continue;
    }

APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset);
    RawMask[i] = EltBits.getZExtValue();
  }

return true;
}

void DecodePSHUFBMask(const Constant *C, unsigned Width,
                      SmallVectorImpl<int> &ShuffleMask) {
  assert((Width == 128 || Width == 256 || Width == 512) &&
         C->getType()->getPrimitiveSizeInBits() >= Width &&
         "Unexpected vector size.");

// The shuffle mask requires a byte vector.
  APInt UndefElts;
  SmallVector<uint64_t, 64> RawMask;
  if (!extractConstantMask(C, 8, UndefElts, RawMask))
    return;

unsigned NumElts = Width / 8;
  assert((NumElts == 16 || NumElts == 32 || NumElts == 64) &&
         "Unexpected number of vector elements.");

for (unsigned i = 0; i != NumElts; ++i) {
    if (UndefElts[i]) {
      ShuffleMask.push_back(SM_SentinelUndef);
      continue;
    }

uint64_t Element = RawMask[i];
    // If the high bit (7) of the byte is set, the element is zeroed.
    if (Element & (1 << 7))
      ShuffleMask.push_back(SM_SentinelZero);
    else {
      // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte
      // lane of the vector we're inside.
      unsigned Base = i & ~0xf;

// Only the least significant 4 bits of the byte are used.
      int Index = Base + (Element & 0xf);
      ShuffleMask.push_back(Index);
    }
  }
}

void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width,
                        SmallVectorImpl<int> &ShuffleMask) {
  assert((Width == 128 || Width == 256 || Width == 512) &&
         C->getType()->getPrimitiveSizeInBits() >= Width &&
         "Unexpected vector size.");
  assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size.");

// The shuffle mask requires elements the same size as the target.
  APInt UndefElts;
  SmallVector<uint64_t, 16> RawMask;
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
    return;

unsigned NumElts = Width / ElSize;
  unsigned NumEltsPerLane = 128 / ElSize;
  assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) &&
         "Unexpected number of vector elements.");

for (unsigned i = 0; i != NumElts; ++i) {
    if (UndefElts[i]) {
      ShuffleMask.push_back(SM_SentinelUndef);
      continue;
    }

int Index = i & ~(NumEltsPerLane - 1);
    uint64_t Element = RawMask[i];
    if (ElSize == 64)
      Index += (Element >> 1) & 0x1;
    else
      Index += Element & 0x3;

ShuffleMask.push_back(Index);
  }
}

void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize,
                         unsigned Width, SmallVectorImpl<int> &ShuffleMask) {
  Type *MaskTy = C->getType();
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
  (void)MaskTySize;
  assert((MaskTySize == 128 || MaskTySize == 256) && Width >= MaskTySize &&
         "Unexpected vector size.");

// The shuffle mask requires elements the same size as the target.
  APInt UndefElts;
  SmallVector<uint64_t, 8> RawMask;
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
    return;

unsigned NumElts = Width / ElSize;
  unsigned NumEltsPerLane = 128 / ElSize;
  assert((NumElts == 2 || NumElts == 4 || NumElts == 8) &&
         "Unexpected number of vector elements.");

for (unsigned i = 0; i != NumElts; ++i) {
    if (UndefElts[i]) {
      ShuffleMask.push_back(SM_SentinelUndef);
      continue;
    }

// VPERMIL2 Operation.
    // Bits[3] - Match Bit.
    // Bits[2:1] - (Per Lane) PD Shuffle Mask.
    // Bits[2:0] - (Per Lane) PS Shuffle Mask.
    uint64_t Selector = RawMask[i];
    unsigned MatchBit = (Selector >> 3) & 0x1;

// M2Z[0:1]     MatchBit
    //   0Xb           X        Source selected by Selector index.
    //   10b           0        Source selected by Selector index.
    //   10b           1        Zero.
    //   11b           0        Zero.
    //   11b           1        Source selected by Selector index.
    if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) {
      ShuffleMask.push_back(SM_SentinelZero);
      continue;
    }

int Index = i & ~(NumEltsPerLane - 1);
    if (ElSize == 64)
      Index += (Selector >> 1) & 0x1;
    else
      Index += Selector & 0x3;

int Src = (Selector >> 2) & 0x1;
    Index += Src * NumElts;
    ShuffleMask.push_back(Index);
  }
}

void DecodeVPPERMMask(const Constant *C, unsigned Width,
                      SmallVectorImpl<int> &ShuffleMask) {
  Type *MaskTy = C->getType();
  unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits();
  (void)MaskTySize;
  assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size.");

// The shuffle mask requires a byte vector.
  APInt UndefElts;
  SmallVector<uint64_t, 16> RawMask;
  if (!extractConstantMask(C, 8, UndefElts, RawMask))
    return;

unsigned NumElts = Width / 8;
  assert(NumElts == 16 && "Unexpected number of vector elements.");

for (unsigned i = 0; i != NumElts; ++i) {
    if (UndefElts[i]) {
      ShuffleMask.push_back(SM_SentinelUndef);
      continue;
    }

// VPPERM Operation
    // Bits[4:0] - Byte Index (0 - 31)
    // Bits[7:5] - Permute Operation
    //
    // Permute Operation:
    // 0 - Source byte (no logical operation).
    // 1 - Invert source byte.
    // 2 - Bit reverse of source byte.
    // 3 - Bit reverse of inverted source byte.
    // 4 - 00h (zero - fill).
    // 5 - FFh (ones - fill).
    // 6 - Most significant bit of source byte replicated in all bit positions.
    // 7 - Invert most significant bit of source byte and replicate in all bit
    // positions.
    uint64_t Element = RawMask[i];
    uint64_t Index = Element & 0x1F;
    uint64_t PermuteOp = (Element >> 5) & 0x7;

if (PermuteOp == 4) {
      ShuffleMask.push_back(SM_SentinelZero);
      continue;
    }
    if (PermuteOp != 0) {
      ShuffleMask.clear();
      return;
    }
    ShuffleMask.push_back((int)Index);
  }
}

void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width,
                      SmallVectorImpl<int> &ShuffleMask) {
  assert((Width == 128 || Width == 256 || Width == 512) &&
         C->getType()->getPrimitiveSizeInBits() >= Width &&
         "Unexpected vector size.");
  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
         "Unexpected vector element size.");

// The shuffle mask requires elements the same size as the target.
  APInt UndefElts;
  SmallVector<uint64_t, 64> RawMask;
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
    return;

unsigned NumElts = Width / ElSize;

for (unsigned i = 0; i != NumElts; ++i) {
    if (UndefElts[i]) {
      ShuffleMask.push_back(SM_SentinelUndef);
      continue;
    }
    int Index = RawMask[i] & (NumElts - 1);
    ShuffleMask.push_back(Index);
  }
}

void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width,
                       SmallVectorImpl<int> &ShuffleMask) {
  assert((Width == 128 || Width == 256 || Width == 512) &&
         C->getType()->getPrimitiveSizeInBits() >= Width &&
         "Unexpected vector size.");
  assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) &&
         "Unexpected vector element size.");

// The shuffle mask requires elements the same size as the target.
  APInt UndefElts;
  SmallVector<uint64_t, 64> RawMask;
  if (!extractConstantMask(C, ElSize, UndefElts, RawMask))
    return;

unsigned NumElts = Width / ElSize;

for (unsigned i = 0; i != NumElts; ++i) {
    if (UndefElts[i]) {
      ShuffleMask.push_back(SM_SentinelUndef);
      continue;
    }
    int Index = RawMask[i] & (NumElts*2 - 1);
    ShuffleMask.push_back(Index);
  }
}
} // llvm namespace

003 File Manager

Editing: X86ShuffleDecodeConstantPool.cpp

Upload File

Create Folder