File Manager

003 File Manager

Current Path: /usr/src/contrib/llvm-project/compiler-rt/lib/xray

usr / src / contrib / llvm-project / compiler-rt / lib / xray /

📁 ..
📄 weak_symbols.txt(88 B)
📄 xray_AArch64.cpp(4.62 KB)
📄 xray_allocator.h(9.33 KB)
📄 xray_always_instrument.txt(255 B)
📄 xray_arm.cpp(5.87 KB)
📄 xray_basic_flags.cpp(1.52 KB)
📄 xray_basic_flags.h(1.18 KB)
📄 xray_basic_flags.inc(1011 B)
📄 xray_basic_logging.cpp(17.59 KB)
📄 xray_basic_logging.h(1.68 KB)
📄 xray_buffer_queue.cpp(7.35 KB)
📄 xray_buffer_queue.h(8.79 KB)
📄 xray_defs.h(967 B)
📄 xray_fdr_controller.h(11.79 KB)
📄 xray_fdr_flags.cpp(1.47 KB)
📄 xray_fdr_flags.h(1.2 KB)
📄 xray_fdr_flags.inc(1.28 KB)
📄 xray_fdr_log_records.h(2.5 KB)
📄 xray_fdr_log_writer.h(8.72 KB)
📄 xray_fdr_logging.cpp(27.62 KB)
📄 xray_fdr_logging.h(1.53 KB)
📄 xray_flags.cpp(2.76 KB)
📄 xray_flags.h(1.12 KB)
📄 xray_flags.inc(2.29 KB)
📄 xray_function_call_trie.h(22.36 KB)
📄 xray_init.cpp(4.6 KB)
📄 xray_interface.cpp(16.43 KB)
📄 xray_interface_internal.h(3.2 KB)
📄 xray_log_interface.cpp(7.53 KB)
📄 xray_mips.cpp(7.16 KB)
📄 xray_mips64.cpp(7.66 KB)
📄 xray_never_instrument.txt(282 B)
📄 xray_powerpc64.cpp(3.63 KB)
📄 xray_powerpc64.inc(1019 B)
📄 xray_profile_collector.cpp(14.06 KB)
📄 xray_profile_collector.h(2.73 KB)
📄 xray_profiling.cpp(17.55 KB)
📄 xray_profiling_flags.cpp(1.34 KB)
📄 xray_profiling_flags.h(1.17 KB)
📄 xray_profiling_flags.inc(1.4 KB)
📄 xray_recursion_guard.h(1.8 KB)
📄 xray_segmented_array.h(21.22 KB)
📄 xray_trampoline_AArch64.S(6.1 KB)
📄 xray_trampoline_arm.S(3.85 KB)
📄 xray_trampoline_mips.S(2.67 KB)
📄 xray_trampoline_mips64.S(3.47 KB)
📄 xray_trampoline_powerpc64.cpp(420 B)
📄 xray_trampoline_powerpc64_asm.S(3.91 KB)
📄 xray_trampoline_x86_64.S(7.06 KB)
📄 xray_tsc.h(2.75 KB)
📄 xray_utils.cpp(6.04 KB)
📄 xray_utils.h(2.18 KB)
📄 xray_x86_64.cpp(11.73 KB)
📄 xray_x86_64.inc(997 B)

Editing: xray_profiling.cpp

//===-- xray_profiling.cpp --------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file is a part of XRay, a dynamic runtime instrumentation system.
//
// This is the implementation of a profiling handler.
//
//===----------------------------------------------------------------------===//
#include <memory>
#include <time.h>

#include "sanitizer_common/sanitizer_atomic.h"
#include "sanitizer_common/sanitizer_flags.h"
#include "xray/xray_interface.h"
#include "xray/xray_log_interface.h"
#include "xray_buffer_queue.h"
#include "xray_flags.h"
#include "xray_profile_collector.h"
#include "xray_profiling_flags.h"
#include "xray_recursion_guard.h"
#include "xray_tsc.h"
#include "xray_utils.h"
#include <pthread.h>

namespace __xray {

namespace {

static atomic_sint32_t ProfilerLogFlushStatus = {
    XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING};

static atomic_sint32_t ProfilerLogStatus = {
    XRayLogInitStatus::XRAY_LOG_UNINITIALIZED};

static SpinMutex ProfilerOptionsMutex;

struct ProfilingData {
  atomic_uintptr_t Allocators;
  atomic_uintptr_t FCT;
};

static pthread_key_t ProfilingKey;

// We use a global buffer queue, which gets initialized once at initialisation
// time, and gets reset when profiling is "done".
static std::aligned_storage<sizeof(BufferQueue), alignof(BufferQueue)>::type
    BufferQueueStorage;
static BufferQueue *BQ = nullptr;

thread_local FunctionCallTrie::Allocators::Buffers ThreadBuffers;
thread_local std::aligned_storage<sizeof(FunctionCallTrie::Allocators),
                                  alignof(FunctionCallTrie::Allocators)>::type
    AllocatorsStorage;
thread_local std::aligned_storage<sizeof(FunctionCallTrie),
                                  alignof(FunctionCallTrie)>::type
    FunctionCallTrieStorage;
thread_local ProfilingData TLD{{0}, {0}};
thread_local atomic_uint8_t ReentranceGuard{0};

// We use a separate guard for ensuring that for this thread, if we're already
// cleaning up, that any signal handlers don't attempt to cleanup nor
// initialise.
thread_local atomic_uint8_t TLDInitGuard{0};

// We also use a separate latch to signal that the thread is exiting, and
// non-essential work should be ignored (things like recording events, etc.).
thread_local atomic_uint8_t ThreadExitingLatch{0};

static ProfilingData *getThreadLocalData() XRAY_NEVER_INSTRUMENT {
  thread_local auto ThreadOnce = []() XRAY_NEVER_INSTRUMENT {
    pthread_setspecific(ProfilingKey, &TLD);
    return false;
  }();
  (void)ThreadOnce;

RecursionGuard TLDInit(TLDInitGuard);
  if (!TLDInit)
    return nullptr;

if (atomic_load_relaxed(&ThreadExitingLatch))
    return nullptr;

uptr Allocators = 0;
  if (atomic_compare_exchange_strong(&TLD.Allocators, &Allocators, 1,
                                     memory_order_acq_rel)) {
    bool Success = false;
    auto AllocatorsUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT {
      if (!Success)
        atomic_store(&TLD.Allocators, 0, memory_order_release);
    });

// Acquire a set of buffers for this thread.
    if (BQ == nullptr)
      return nullptr;

if (BQ->getBuffer(ThreadBuffers.NodeBuffer) != BufferQueue::ErrorCode::Ok)
      return nullptr;
    auto NodeBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT {
      if (!Success)
        BQ->releaseBuffer(ThreadBuffers.NodeBuffer);
    });

if (BQ->getBuffer(ThreadBuffers.RootsBuffer) != BufferQueue::ErrorCode::Ok)
      return nullptr;
    auto RootsBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT {
      if (!Success)
        BQ->releaseBuffer(ThreadBuffers.RootsBuffer);
    });

if (BQ->getBuffer(ThreadBuffers.ShadowStackBuffer) !=
        BufferQueue::ErrorCode::Ok)
      return nullptr;
    auto ShadowStackBufferUndo = at_scope_exit([&]() XRAY_NEVER_INSTRUMENT {
      if (!Success)
        BQ->releaseBuffer(ThreadBuffers.ShadowStackBuffer);
    });

if (BQ->getBuffer(ThreadBuffers.NodeIdPairBuffer) !=
        BufferQueue::ErrorCode::Ok)
      return nullptr;

Success = true;
    new (&AllocatorsStorage) FunctionCallTrie::Allocators(
        FunctionCallTrie::InitAllocatorsFromBuffers(ThreadBuffers));
    Allocators = reinterpret_cast<uptr>(
        reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage));
    atomic_store(&TLD.Allocators, Allocators, memory_order_release);
  }

if (Allocators == 1)
    return nullptr;

uptr FCT = 0;
  if (atomic_compare_exchange_strong(&TLD.FCT, &FCT, 1, memory_order_acq_rel)) {
    new (&FunctionCallTrieStorage)
        FunctionCallTrie(*reinterpret_cast<FunctionCallTrie::Allocators *>(
            atomic_load_relaxed(&TLD.Allocators)));
    FCT = reinterpret_cast<uptr>(
        reinterpret_cast<FunctionCallTrie *>(&FunctionCallTrieStorage));
    atomic_store(&TLD.FCT, FCT, memory_order_release);
  }

if (FCT == 1)
    return nullptr;

return &TLD;
}

static void cleanupTLD() XRAY_NEVER_INSTRUMENT {
  auto FCT = atomic_exchange(&TLD.FCT, 0, memory_order_acq_rel);
  if (FCT == reinterpret_cast<uptr>(reinterpret_cast<FunctionCallTrie *>(
                 &FunctionCallTrieStorage)))
    reinterpret_cast<FunctionCallTrie *>(FCT)->~FunctionCallTrie();

auto Allocators = atomic_exchange(&TLD.Allocators, 0, memory_order_acq_rel);
  if (Allocators ==
      reinterpret_cast<uptr>(
          reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage)))
    reinterpret_cast<FunctionCallTrie::Allocators *>(Allocators)->~Allocators();
}

static void postCurrentThreadFCT(ProfilingData &T) XRAY_NEVER_INSTRUMENT {
  RecursionGuard TLDInit(TLDInitGuard);
  if (!TLDInit)
    return;

uptr P = atomic_exchange(&T.FCT, 0, memory_order_acq_rel);
  if (P != reinterpret_cast<uptr>(
               reinterpret_cast<FunctionCallTrie *>(&FunctionCallTrieStorage)))
    return;

auto FCT = reinterpret_cast<FunctionCallTrie *>(P);
  DCHECK_NE(FCT, nullptr);

uptr A = atomic_exchange(&T.Allocators, 0, memory_order_acq_rel);
  if (A !=
      reinterpret_cast<uptr>(
          reinterpret_cast<FunctionCallTrie::Allocators *>(&AllocatorsStorage)))
    return;

auto Allocators = reinterpret_cast<FunctionCallTrie::Allocators *>(A);
  DCHECK_NE(Allocators, nullptr);

// Always move the data into the profile collector.
  profileCollectorService::post(BQ, std::move(*FCT), std::move(*Allocators),
                                std::move(ThreadBuffers), GetTid());

// Re-initialize the ThreadBuffers object to a known "default" state.
  ThreadBuffers = FunctionCallTrie::Allocators::Buffers{};
}

} // namespace

const char *profilingCompilerDefinedFlags() XRAY_NEVER_INSTRUMENT {
#ifdef XRAY_PROFILER_DEFAULT_OPTIONS
  return SANITIZER_STRINGIFY(XRAY_PROFILER_DEFAULT_OPTIONS);
#else
  return "";
#endif
}

XRayLogFlushStatus profilingFlush() XRAY_NEVER_INSTRUMENT {
  if (atomic_load(&ProfilerLogStatus, memory_order_acquire) !=
      XRayLogInitStatus::XRAY_LOG_FINALIZED) {
    if (Verbosity())
      Report("Not flushing profiles, profiling not been finalized.\n");
    return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
  }

RecursionGuard SignalGuard(ReentranceGuard);
  if (!SignalGuard) {
    if (Verbosity())
      Report("Cannot finalize properly inside a signal handler!\n");
    atomic_store(&ProfilerLogFlushStatus,
                 XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING,
                 memory_order_release);
    return XRayLogFlushStatus::XRAY_LOG_NOT_FLUSHING;
  }

s32 Previous = atomic_exchange(&ProfilerLogFlushStatus,
                                 XRayLogFlushStatus::XRAY_LOG_FLUSHING,
                                 memory_order_acq_rel);
  if (Previous == XRayLogFlushStatus::XRAY_LOG_FLUSHING) {
    if (Verbosity())
      Report("Not flushing profiles, implementation still flushing.\n");
    return XRayLogFlushStatus::XRAY_LOG_FLUSHING;
  }

// At this point, we'll create the file that will contain the profile, but
  // only if the options say so.
  if (!profilingFlags()->no_flush) {
    // First check whether we have data in the profile collector service
    // before we try and write anything down.
    XRayBuffer B = profileCollectorService::nextBuffer({nullptr, 0});
    if (B.Data == nullptr) {
      if (Verbosity())
        Report("profiling: No data to flush.\n");
    } else {
      LogWriter *LW = LogWriter::Open();
      if (LW == nullptr) {
        if (Verbosity())
          Report("profiling: Failed to flush to file, dropping data.\n");
      } else {
        // Now for each of the buffers, write out the profile data as we would
        // see it in memory, verbatim.
        while (B.Data != nullptr && B.Size != 0) {
          LW->WriteAll(reinterpret_cast<const char *>(B.Data),
                       reinterpret_cast<const char *>(B.Data) + B.Size);
          B = profileCollectorService::nextBuffer(B);
        }
      }
      LogWriter::Close(LW);
    }
  }

profileCollectorService::reset();

atomic_store(&ProfilerLogFlushStatus, XRayLogFlushStatus::XRAY_LOG_FLUSHED,
               memory_order_release);
  atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_UNINITIALIZED,
               memory_order_release);

return XRayLogFlushStatus::XRAY_LOG_FLUSHED;
}

void profilingHandleArg0(int32_t FuncId,
                         XRayEntryType Entry) XRAY_NEVER_INSTRUMENT {
  unsigned char CPU;
  auto TSC = readTSC(CPU);
  RecursionGuard G(ReentranceGuard);
  if (!G)
    return;

auto Status = atomic_load(&ProfilerLogStatus, memory_order_acquire);
  if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_UNINITIALIZED ||
               Status == XRayLogInitStatus::XRAY_LOG_INITIALIZING))
    return;

if (UNLIKELY(Status == XRayLogInitStatus::XRAY_LOG_FINALIZED ||
               Status == XRayLogInitStatus::XRAY_LOG_FINALIZING)) {
    postCurrentThreadFCT(TLD);
    return;
  }

auto T = getThreadLocalData();
  if (T == nullptr)
    return;

auto FCT = reinterpret_cast<FunctionCallTrie *>(atomic_load_relaxed(&T->FCT));
  switch (Entry) {
  case XRayEntryType::ENTRY:
  case XRayEntryType::LOG_ARGS_ENTRY:
    FCT->enterFunction(FuncId, TSC, CPU);
    break;
  case XRayEntryType::EXIT:
  case XRayEntryType::TAIL:
    FCT->exitFunction(FuncId, TSC, CPU);
    break;
  default:
    // FIXME: Handle bugs.
    break;
  }
}

void profilingHandleArg1(int32_t FuncId, XRayEntryType Entry,
                         uint64_t) XRAY_NEVER_INSTRUMENT {
  return profilingHandleArg0(FuncId, Entry);
}

XRayLogInitStatus profilingFinalize() XRAY_NEVER_INSTRUMENT {
  s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_INITIALIZED;
  if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus,
                                      XRayLogInitStatus::XRAY_LOG_FINALIZING,
                                      memory_order_release)) {
    if (Verbosity())
      Report("Cannot finalize profile, the profiling is not initialized.\n");
    return static_cast<XRayLogInitStatus>(CurrentStatus);
  }

// Mark then finalize the current generation of buffers. This allows us to let
  // the threads currently holding onto new buffers still use them, but let the
  // last reference do the memory cleanup.
  DCHECK_NE(BQ, nullptr);
  BQ->finalize();

// Wait a grace period to allow threads to see that we're finalizing.
  SleepForMillis(profilingFlags()->grace_period_ms);

// If we for some reason are entering this function from an instrumented
  // handler, we bail out.
  RecursionGuard G(ReentranceGuard);
  if (!G)
    return static_cast<XRayLogInitStatus>(CurrentStatus);

// Post the current thread's data if we have any.
  postCurrentThreadFCT(TLD);

// Then we force serialize the log data.
  profileCollectorService::serialize();

atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_FINALIZED,
               memory_order_release);
  return XRayLogInitStatus::XRAY_LOG_FINALIZED;
}

XRayLogInitStatus
profilingLoggingInit(size_t, size_t, void *Options,
                     size_t OptionsSize) XRAY_NEVER_INSTRUMENT {
  RecursionGuard G(ReentranceGuard);
  if (!G)
    return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;

s32 CurrentStatus = XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
  if (!atomic_compare_exchange_strong(&ProfilerLogStatus, &CurrentStatus,
                                      XRayLogInitStatus::XRAY_LOG_INITIALIZING,
                                      memory_order_acq_rel)) {
    if (Verbosity())
      Report("Cannot initialize already initialised profiling "
             "implementation.\n");
    return static_cast<XRayLogInitStatus>(CurrentStatus);
  }

{
    SpinMutexLock Lock(&ProfilerOptionsMutex);
    FlagParser ConfigParser;
    ProfilerFlags Flags;
    Flags.setDefaults();
    registerProfilerFlags(&ConfigParser, &Flags);
    ConfigParser.ParseString(profilingCompilerDefinedFlags());
    const char *Env = GetEnv("XRAY_PROFILING_OPTIONS");
    if (Env == nullptr)
      Env = "";
    ConfigParser.ParseString(Env);

// Then parse the configuration string provided.
    ConfigParser.ParseString(static_cast<const char *>(Options));
    if (Verbosity())
      ReportUnrecognizedFlags();
    *profilingFlags() = Flags;
  }

// We need to reset the profile data collection implementation now.
  profileCollectorService::reset();

// Then also reset the buffer queue implementation.
  if (BQ == nullptr) {
    bool Success = false;
    new (&BufferQueueStorage)
        BufferQueue(profilingFlags()->per_thread_allocator_max,
                    profilingFlags()->buffers_max, Success);
    if (!Success) {
      if (Verbosity())
        Report("Failed to initialize preallocated memory buffers!");
      atomic_store(&ProfilerLogStatus,
                   XRayLogInitStatus::XRAY_LOG_UNINITIALIZED,
                   memory_order_release);
      return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
    }

// If we've succeded, set the global pointer to the initialised storage.
    BQ = reinterpret_cast<BufferQueue *>(&BufferQueueStorage);
  } else {
    BQ->finalize();
    auto InitStatus = BQ->init(profilingFlags()->per_thread_allocator_max,
                               profilingFlags()->buffers_max);

if (InitStatus != BufferQueue::ErrorCode::Ok) {
      if (Verbosity())
        Report("Failed to initialize preallocated memory buffers; error: %s",
               BufferQueue::getErrorString(InitStatus));
      atomic_store(&ProfilerLogStatus,
                   XRayLogInitStatus::XRAY_LOG_UNINITIALIZED,
                   memory_order_release);
      return XRayLogInitStatus::XRAY_LOG_UNINITIALIZED;
    }

DCHECK(!BQ->finalizing());
  }

// We need to set up the exit handlers.
  static pthread_once_t Once = PTHREAD_ONCE_INIT;
  pthread_once(
      &Once, +[] {
        pthread_key_create(
            &ProfilingKey, +[](void *P) XRAY_NEVER_INSTRUMENT {
              if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel))
                return;

if (P == nullptr)
                return;

auto T = reinterpret_cast<ProfilingData *>(P);
              if (atomic_load_relaxed(&T->Allocators) == 0)
                return;

{
                // If we're somehow executing this while inside a
                // non-reentrant-friendly context, we skip attempting to post
                // the current thread's data.
                RecursionGuard G(ReentranceGuard);
                if (!G)
                  return;

postCurrentThreadFCT(*T);
              }
            });

// We also need to set up an exit handler, so that we can get the
        // profile information at exit time. We use the C API to do this, to not
        // rely on C++ ABI functions for registering exit handlers.
        Atexit(+[]() XRAY_NEVER_INSTRUMENT {
          if (atomic_exchange(&ThreadExitingLatch, 1, memory_order_acq_rel))
            return;

auto Cleanup =
              at_scope_exit([]() XRAY_NEVER_INSTRUMENT { cleanupTLD(); });

// Finalize and flush.
          if (profilingFinalize() != XRAY_LOG_FINALIZED ||
              profilingFlush() != XRAY_LOG_FLUSHED)
            return;

if (Verbosity())
            Report("XRay Profile flushed at exit.");
        });
      });

__xray_log_set_buffer_iterator(profileCollectorService::nextBuffer);
  __xray_set_handler(profilingHandleArg0);
  __xray_set_handler_arg1(profilingHandleArg1);

atomic_store(&ProfilerLogStatus, XRayLogInitStatus::XRAY_LOG_INITIALIZED,
               memory_order_release);
  if (Verbosity())
    Report("XRay Profiling init successful.\n");

return XRayLogInitStatus::XRAY_LOG_INITIALIZED;
}

bool profilingDynamicInitializer() XRAY_NEVER_INSTRUMENT {
  // Set up the flag defaults from the static defaults and the
  // compiler-provided defaults.
  {
    SpinMutexLock Lock(&ProfilerOptionsMutex);
    auto *F = profilingFlags();
    F->setDefaults();
    FlagParser ProfilingParser;
    registerProfilerFlags(&ProfilingParser, F);
    ProfilingParser.ParseString(profilingCompilerDefinedFlags());
  }

XRayLogImpl Impl{
      profilingLoggingInit,
      profilingFinalize,
      profilingHandleArg0,
      profilingFlush,
  };
  auto RegistrationResult = __xray_log_register_mode("xray-profiling", Impl);
  if (RegistrationResult != XRayLogRegisterStatus::XRAY_REGISTRATION_OK) {
    if (Verbosity())
      Report("Cannot register XRay Profiling mode to 'xray-profiling'; error = "
             "%d\n",
             RegistrationResult);
    return false;
  }

if (!internal_strcmp(flags()->xray_mode, "xray-profiling"))
    __xray_log_select_mode("xray_profiling");
  return true;
}

} // namespace __xray

static auto UNUSED Unused = __xray::profilingDynamicInitializer();

003 File Manager

Editing: xray_profiling.cpp

Upload File

Create Folder