//===-- combined_test.cpp ---------------------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//

#include "memtag.h"
#include "stack_depot.h"
#include "tests/scudo_unit_test.h"

#include "allocator_config.h"
#include "chunk.h"
#include "combined.h"
#include "condition_variable.h"
#include "mem_map.h"
#include "size_class_map.h"

#include <algorithm>
#include <atomic>
#include <condition_variable>
#include <cstdio>
#include <memory>
#include <mutex>
#include <set>
#include <stdlib.h>
#include <thread>
#include <unordered_map>
#include <vector>

static constexpr scudo::Chunk::Origin Origin = scudo::Chunk::Origin::Malloc;
static constexpr scudo::uptr MinAlignLog = FIRST_32_SECOND_64(3U, 4U);

// Fuchsia complains that the function is not used.
UNUSED static void disableDebuggerdMaybe() {
#if SCUDO_ANDROID
  // Disable the debuggerd signal handler on Android, without this we can end
  // up spending a significant amount of time creating tombstones.
  signal(SIGSEGV, SIG_DFL);
#endif
}

template <class AllocatorT>
bool isPrimaryAllocation(scudo::uptr Size, scudo::uptr Alignment) {
  const scudo::uptr MinAlignment = 1UL << SCUDO_MIN_ALIGNMENT_LOG;
  if (Alignment < MinAlignment)
    Alignment = MinAlignment;
  const scudo::uptr NeededSize =
      scudo::roundUp(Size, MinAlignment) +
      ((Alignment > MinAlignment) ? Alignment : scudo::Chunk::getHeaderSize());
  return AllocatorT::PrimaryT::canAllocate(NeededSize);
}

template <class AllocatorT>
void checkMemoryTaggingMaybe(AllocatorT *Allocator, void *P, scudo::uptr Size,
                             scudo::uptr Alignment) {
  const scudo::uptr MinAlignment = 1UL << SCUDO_MIN_ALIGNMENT_LOG;
  Size = scudo::roundUp(Size, MinAlignment);
  if (Allocator->useMemoryTaggingTestOnly()) {
    EXPECT_DEATH(
        {
          disableDebuggerdMaybe();
          reinterpret_cast<char *>(P)[-1] = 'A';
        },
        "");
  }
  if (isPrimaryAllocation<AllocatorT>(Size, Alignment)
          ? Allocator->useMemoryTaggingTestOnly()
          : Alignment == MinAlignment &&
                AllocatorT::SecondaryT::getGuardPageSize() > 0) {
    EXPECT_DEATH(
        {
          disableDebuggerdMaybe();
          reinterpret_cast<char *>(P)[Size] = 'A';
        },
        "");
  }
}

template <typename Config> struct TestAllocator : scudo::Allocator<Config> {
  TestAllocator() {
    this->initThreadMaybe();
    if (scudo::archSupportsMemoryTagging() &&
        !scudo::systemDetectsMemoryTagFaultsTestOnly())
      this->disableMemoryTagging();
  }
  ~TestAllocator() { this->unmapTestOnly(); }

  void *operator new(size_t size);
  void operator delete(void *ptr);
};

constexpr size_t kMaxAlign = std::max({
  alignof(scudo::Allocator<scudo::DefaultConfig>),
#if SCUDO_CAN_USE_PRIMARY64
      alignof(scudo::Allocator<scudo::FuchsiaConfig>),
#endif
      alignof(scudo::Allocator<scudo::AndroidConfig>)
});

#if SCUDO_RISCV64
// The allocator is over 4MB large. Rather than creating an instance of this on
// the heap, keep it in a global storage to reduce fragmentation from having to
// mmap this at the start of every test.
struct TestAllocatorStorage {
  static constexpr size_t kMaxSize = std::max({
    sizeof(scudo::Allocator<scudo::DefaultConfig>),
#if SCUDO_CAN_USE_PRIMARY64
        sizeof(scudo::Allocator<scudo::FuchsiaConfig>),
#endif
        sizeof(scudo::Allocator<scudo::AndroidConfig>)
  });

  // To alleviate some problem, let's skip the thread safety analysis here.
  static void *get(size_t size) NO_THREAD_SAFETY_ANALYSIS {
    CHECK(size <= kMaxSize &&
          "Allocation size doesn't fit in the allocator storage");
    M.lock();
    return AllocatorStorage;
  }

  static void release(void *ptr) NO_THREAD_SAFETY_ANALYSIS {
    M.assertHeld();
    M.unlock();
    ASSERT_EQ(ptr, AllocatorStorage);
  }

  static scudo::HybridMutex M;
  static uint8_t AllocatorStorage[kMaxSize];
};
scudo::HybridMutex TestAllocatorStorage::M;
alignas(kMaxAlign) uint8_t TestAllocatorStorage::AllocatorStorage[kMaxSize];
#else
struct TestAllocatorStorage {
  static void *get(size_t size) NO_THREAD_SAFETY_ANALYSIS {
    void *p = nullptr;
    EXPECT_EQ(0, posix_memalign(&p, kMaxAlign, size));
    return p;
  }
  static void release(void *ptr) NO_THREAD_SAFETY_ANALYSIS { free(ptr); }
};
#endif

template <typename Config>
void *TestAllocator<Config>::operator new(size_t size) {
  return TestAllocatorStorage::get(size);
}

template <typename Config>
void TestAllocator<Config>::operator delete(void *ptr) {
  TestAllocatorStorage::release(ptr);
}

class ScopedScudoOptions {
public:
  explicit ScopedScudoOptions(const char *Options) {
    setenv("SCUDO_OPTIONS", Options, 1);
  }

  ~ScopedScudoOptions() { unsetenv("SCUDO_OPTIONS"); }
};

template <class TypeParam> struct ScudoCombinedTest : public Test {
  ScudoCombinedTest() { Allocator = std::make_unique<AllocatorT>(); }
  ~ScudoCombinedTest() { Allocator->releaseToOS(scudo::ReleaseToOS::Force); }

  void RunTest();

  void BasicTest(scudo::uptr SizeLog);

  using AllocatorT = TestAllocator<TypeParam>;
  std::unique_ptr<AllocatorT> Allocator;
};

template <typename T> using ScudoCombinedDeathTest = ScudoCombinedTest<T>;

namespace scudo {
struct TestConditionVariableConfig {
  static const bool MaySupportMemoryTagging = true;
  template <class A>
  using TSDRegistryT =
      scudo::TSDRegistrySharedT<A, 8U, 4U>; // Shared, max 8 TSDs.

  struct Primary {
    using SizeClassMap = scudo::AndroidSizeClassMap;
#if SCUDO_CAN_USE_PRIMARY64
    static const scudo::uptr RegionSizeLog = 28U;
    typedef scudo::u32 CompactPtrT;
    static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
    static const scudo::uptr GroupSizeLog = 20U;
    static const bool EnableRandomOffset = true;
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
#else
    static const scudo::uptr RegionSizeLog = 18U;
    static const scudo::uptr GroupSizeLog = 18U;
    typedef scudo::uptr CompactPtrT;
#endif
    static const scudo::s32 MinReleaseToOsIntervalMs = 1000;
    static const scudo::s32 MaxReleaseToOsIntervalMs = 1000;
#if SCUDO_LINUX
    using ConditionVariableT = scudo::ConditionVariableLinux;
#else
    using ConditionVariableT = scudo::ConditionVariableDummy;
#endif
  };
#if SCUDO_CAN_USE_PRIMARY64
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;
#else
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator32<Config>;
#endif

  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };
  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

struct TestNoCacheConfig {
  static const bool MaySupportMemoryTagging = true;
  template <class A>
  using TSDRegistryT =
      scudo::TSDRegistrySharedT<A, 8U, 4U>; // Shared, max 8 TSDs.

  struct Primary {
    using SizeClassMap = scudo::AndroidSizeClassMap;
#if SCUDO_CAN_USE_PRIMARY64
    static const scudo::uptr RegionSizeLog = 28U;
    typedef scudo::u32 CompactPtrT;
    static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
    static const scudo::uptr GroupSizeLog = 20U;
    static const bool EnableRandomOffset = true;
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
#else
    static const scudo::uptr RegionSizeLog = 18U;
    static const scudo::uptr GroupSizeLog = 18U;
    typedef scudo::uptr CompactPtrT;
#endif
    static const bool EnableBlockCache = false;
    static const scudo::s32 MinReleaseToOsIntervalMs = 1000;
    static const scudo::s32 MaxReleaseToOsIntervalMs = 1000;
  };

#if SCUDO_CAN_USE_PRIMARY64
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;
#else
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator32<Config>;
#endif

  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };
  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

} // namespace scudo

#if SCUDO_FUCHSIA
#define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME)                              \
  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, FuchsiaConfig)
#else
#define SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME)                              \
  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, DefaultConfig)                          \
  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, AndroidConfig)                          \
  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestConditionVariableConfig)            \
  SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TestNoCacheConfig)
#endif

#define SCUDO_TYPED_TEST_TYPE(FIXTURE, NAME, TYPE)                             \
  using FIXTURE##NAME##_##TYPE = FIXTURE##NAME<scudo::TYPE>;                   \
  TEST_F(FIXTURE##NAME##_##TYPE, NAME) { FIXTURE##NAME<scudo::TYPE>::Run(); }

#define SCUDO_TYPED_TEST(FIXTURE, NAME)                                        \
  template <class TypeParam>                                                   \
  struct FIXTURE##NAME : public FIXTURE<TypeParam> {                           \
    using BaseT = FIXTURE<TypeParam>;                                          \
    void Run();                                                                \
  };                                                                           \
  SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME)                                    \
  template <class TypeParam> void FIXTURE##NAME<TypeParam>::Run()

// Accessing `TSD->getCache()` requires `TSD::Mutex` which isn't easy to test
// using thread-safety analysis. Alternatively, we verify the thread safety
// through a runtime check in ScopedTSD and mark the test body with
// NO_THREAD_SAFETY_ANALYSIS.
#define SCUDO_TYPED_TEST_SKIP_THREAD_SAFETY(FIXTURE, NAME)                     \
  template <class TypeParam>                                                   \
  struct FIXTURE##NAME : public FIXTURE<TypeParam> {                           \
    using BaseT = FIXTURE<TypeParam>;                                          \
    void Run() NO_THREAD_SAFETY_ANALYSIS;                                      \
  };                                                                           \
  SCUDO_TYPED_TEST_ALL_TYPES(FIXTURE, NAME)                                    \
  template <class TypeParam> void FIXTURE##NAME<TypeParam>::Run()

SCUDO_TYPED_TEST(ScudoCombinedTest, IsOwned) {
  auto *Allocator = this->Allocator.get();
  static scudo::u8 StaticBuffer[scudo::Chunk::getHeaderSize() + 1];
  EXPECT_FALSE(
      Allocator->isOwned(&StaticBuffer[scudo::Chunk::getHeaderSize()]));

  scudo::u8 StackBuffer[scudo::Chunk::getHeaderSize() + 1];
  for (scudo::uptr I = 0; I < sizeof(StackBuffer); I++)
    StackBuffer[I] = 0x42U;
  EXPECT_FALSE(Allocator->isOwned(&StackBuffer[scudo::Chunk::getHeaderSize()]));
  for (scudo::uptr I = 0; I < sizeof(StackBuffer); I++)
    EXPECT_EQ(StackBuffer[I], 0x42U);
}

template <class Config>
void ScudoCombinedTest<Config>::BasicTest(scudo::uptr SizeLog) {
  auto *Allocator = this->Allocator.get();

  // This allocates and deallocates a bunch of chunks, with a wide range of
  // sizes and alignments, with a focus on sizes that could trigger weird
  // behaviors (plus or minus a small delta of a power of two for example).
  for (scudo::uptr AlignLog = MinAlignLog; AlignLog <= 16U; AlignLog++) {
    const scudo::uptr Align = 1U << AlignLog;
    for (scudo::sptr Delta = -32; Delta <= 32; Delta++) {
      if ((1LL << SizeLog) + Delta < 0)
        continue;
      const scudo::uptr Size =
          static_cast<scudo::uptr>((1LL << SizeLog) + Delta);
      void *P = Allocator->allocate(Size, Origin, Align);
      EXPECT_NE(P, nullptr);
      EXPECT_TRUE(Allocator->isOwned(P));
      EXPECT_TRUE(scudo::isAligned(reinterpret_cast<scudo::uptr>(P), Align));
      EXPECT_LE(Size, Allocator->getUsableSize(P));
      memset(P, 0xaa, Size);
      checkMemoryTaggingMaybe(Allocator, P, Size, Align);
      Allocator->deallocateSized(P, Origin, Size);
    }
  }

  if (TEST_HAS_FAILURE) {
    Allocator->printStats();
    Allocator->printFragmentationInfo();
  }
}

#define SCUDO_MAKE_BASIC_TEST(SizeLog)                                         \
  SCUDO_TYPED_TEST(ScudoCombinedDeathTest, BasicCombined##SizeLog) {           \
    this->BasicTest(SizeLog);                                                  \
  }

SCUDO_MAKE_BASIC_TEST(0)
SCUDO_MAKE_BASIC_TEST(1)
SCUDO_MAKE_BASIC_TEST(2)
SCUDO_MAKE_BASIC_TEST(3)
SCUDO_MAKE_BASIC_TEST(4)
SCUDO_MAKE_BASIC_TEST(5)
SCUDO_MAKE_BASIC_TEST(6)
SCUDO_MAKE_BASIC_TEST(7)
SCUDO_MAKE_BASIC_TEST(8)
SCUDO_MAKE_BASIC_TEST(9)
SCUDO_MAKE_BASIC_TEST(10)
SCUDO_MAKE_BASIC_TEST(11)
SCUDO_MAKE_BASIC_TEST(12)
SCUDO_MAKE_BASIC_TEST(13)
SCUDO_MAKE_BASIC_TEST(14)
SCUDO_MAKE_BASIC_TEST(15)
SCUDO_MAKE_BASIC_TEST(16)
SCUDO_MAKE_BASIC_TEST(17)
SCUDO_MAKE_BASIC_TEST(18)
SCUDO_MAKE_BASIC_TEST(19)
SCUDO_MAKE_BASIC_TEST(20)

SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroContents) {
  auto *Allocator = this->Allocator.get();

  // Ensure that specifying ZeroContents returns a zero'd out block.
  for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
    for (scudo::uptr Delta = 0U; Delta <= 4U; Delta++) {
      const scudo::uptr Size = (1U << SizeLog) + Delta * 128U;
      void *P = Allocator->allocate(Size, Origin, 1U << MinAlignLog, true);
      EXPECT_NE(P, nullptr);
      for (scudo::uptr I = 0; I < Size; I++)
        ASSERT_EQ((reinterpret_cast<char *>(P))[I], '\0');
      memset(P, 0xaa, Size);
      Allocator->deallocateSized(P, Origin, Size);
    }
  }
}

SCUDO_TYPED_TEST(ScudoCombinedTest, ZeroFill) {
  auto *Allocator = this->Allocator.get();

  // Ensure that specifying ZeroFill returns a zero'd out block.
  Allocator->setFillContents(scudo::ZeroFill);
  for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
    for (scudo::uptr Delta = 0U; Delta <= 4U; Delta++) {
      const scudo::uptr Size = (1U << SizeLog) + Delta * 128U;
      void *P = Allocator->allocate(Size, Origin, 1U << MinAlignLog, false);
      EXPECT_NE(P, nullptr);
      for (scudo::uptr I = 0; I < Size; I++)
        ASSERT_EQ((reinterpret_cast<char *>(P))[I], '\0');
      memset(P, 0xaa, Size);
      Allocator->deallocateSized(P, Origin, Size);
    }
  }
}

SCUDO_TYPED_TEST(ScudoCombinedTest, PatternOrZeroFill) {
  auto *Allocator = this->Allocator.get();

  // Ensure that specifying PatternOrZeroFill returns a pattern or zero filled
  // block. The primary allocator only produces pattern filled blocks if MTE
  // is disabled, so we only require pattern filled blocks in that case.
  Allocator->setFillContents(scudo::PatternOrZeroFill);
  for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
    for (scudo::uptr Delta = 0U; Delta <= 4U; Delta++) {
      const scudo::uptr Size = (1U << SizeLog) + Delta * 128U;
      void *P = Allocator->allocate(Size, Origin, 1U << MinAlignLog, false);
      EXPECT_NE(P, nullptr);
      for (scudo::uptr I = 0; I < Size; I++) {
        unsigned char V = (reinterpret_cast<unsigned char *>(P))[I];
        if (isPrimaryAllocation<TestAllocator<TypeParam>>(Size,
                                                          1U << MinAlignLog) &&
            !Allocator->useMemoryTaggingTestOnly())
          ASSERT_EQ(V, scudo::PatternFillByte);
        else
          ASSERT_TRUE(V == scudo::PatternFillByte || V == 0);
      }
      memset(P, 0xaa, Size);
      Allocator->deallocateSized(P, Origin, Size);
    }
  }
}

SCUDO_TYPED_TEST(ScudoCombinedTest, BlockReuse) {
  auto *Allocator = this->Allocator.get();

  // Verify that a chunk will end up being reused, at some point.
  const scudo::uptr NeedleSize = 1024U;
  void *NeedleP = Allocator->allocate(NeedleSize, Origin);
  Allocator->deallocate(NeedleP, Origin);
  bool Found = false;
  for (scudo::uptr I = 0; I < 1024U && !Found; I++) {
    void *P = Allocator->allocate(NeedleSize, Origin);
    if (Allocator->getHeaderTaggedPointer(P) ==
        Allocator->getHeaderTaggedPointer(NeedleP))
      Found = true;
    Allocator->deallocate(P, Origin);
  }
  EXPECT_TRUE(Found);
}

SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeIncreasing) {
  auto *Allocator = this->Allocator.get();

  // Reallocate a chunk all the way up to a secondary allocation, verifying that
  // we preserve the data in the process.
  scudo::uptr Size = 16;
  void *P = Allocator->allocate(Size, Origin);
  const char Marker = 'A';
  memset(P, Marker, Size);
  while (Size < TypeParam::Primary::SizeClassMap::MaxSize * 4) {
    void *NewP = Allocator->reallocate(P, Size * 2);
    EXPECT_NE(NewP, nullptr);
    for (scudo::uptr J = 0; J < Size; J++)
      EXPECT_EQ((reinterpret_cast<char *>(NewP))[J], Marker);
    memset(reinterpret_cast<char *>(NewP) + Size, Marker, Size);
    Size *= 2U;
    P = NewP;
  }
  Allocator->deallocate(P, Origin);
}

SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateLargeDecreasing) {
  auto *Allocator = this->Allocator.get();

  // Reallocate a large chunk all the way down to a byte, verifying that we
  // preserve the data in the process.
  scudo::uptr Size = TypeParam::Primary::SizeClassMap::MaxSize * 2;
  const scudo::uptr DataSize = 2048U;
  void *P = Allocator->allocate(Size, Origin);
  const char Marker = 'A';
  memset(P, Marker, scudo::Min(Size, DataSize));
  while (Size > 1U) {
    Size /= 2U;
    void *NewP = Allocator->reallocate(P, Size);
    EXPECT_NE(NewP, nullptr);
    for (scudo::uptr J = 0; J < scudo::Min(Size, DataSize); J++)
      EXPECT_EQ((reinterpret_cast<char *>(NewP))[J], Marker);
    P = NewP;
  }
  Allocator->deallocate(P, Origin);
}

SCUDO_TYPED_TEST(ScudoCombinedDeathTest, ReallocateSame) {
  auto *Allocator = this->Allocator.get();

  // Check that reallocating a chunk to a slightly smaller or larger size
  // returns the same chunk. This requires that all the sizes we iterate on use
  // the same block size, but that should be the case for MaxSize - 64 with our
  // default class size maps.
  constexpr scudo::uptr InitialSize =
      TypeParam::Primary::SizeClassMap::MaxSize - 64;
  const char Marker = 'A';
  Allocator->setFillContents(scudo::PatternOrZeroFill);

  void *P = Allocator->allocate(InitialSize, Origin);
  scudo::uptr CurrentSize = InitialSize;
  for (scudo::sptr Delta = -32; Delta < 32; Delta += 8) {
    memset(P, Marker, CurrentSize);
    const scudo::uptr NewSize =
        static_cast<scudo::uptr>(static_cast<scudo::sptr>(InitialSize) + Delta);
    void *NewP = Allocator->reallocate(P, NewSize);
    EXPECT_EQ(NewP, P);

    // Verify that existing contents have been preserved.
    for (scudo::uptr I = 0; I < scudo::Min(CurrentSize, NewSize); I++)
      EXPECT_EQ((reinterpret_cast<char *>(NewP))[I], Marker);

    checkMemoryTaggingMaybe(Allocator, NewP, NewSize, 0);
    CurrentSize = NewSize;
  }
  Allocator->deallocate(P, Origin);
}

SCUDO_TYPED_TEST(ScudoCombinedTest, IterateOverChunks) {
  auto *Allocator = this->Allocator.get();
  // Allocates a bunch of chunks, then iterate over all the chunks, ensuring
  // they are the ones we allocated. This requires the allocator to not have any
  // other allocated chunk at this point.
  std::vector<void *> V;
  for (scudo::uptr I = 0; I < 64U; I++)
    V.push_back(Allocator->allocate(
        static_cast<scudo::uptr>(std::rand()) %
            (TypeParam::Primary::SizeClassMap::MaxSize / 2U),
        Origin));
  Allocator->disable();
  Allocator->iterateOverChunks(
      0U, static_cast<scudo::uptr>(SCUDO_MMAP_RANGE_SIZE - 1),
      [](uintptr_t Base, UNUSED size_t Size, void *Arg) {
        std::vector<void *> *V = reinterpret_cast<std::vector<void *> *>(Arg);
        void *P = reinterpret_cast<void *>(Base);
        EXPECT_NE(std::find(V->begin(), V->end(), P), V->end());
      },
      reinterpret_cast<void *>(&V));
  Allocator->enable();
  for (auto P : V)
    Allocator->deallocate(P, Origin);
}

SCUDO_TYPED_TEST(ScudoCombinedDeathTest, UseAfterFree) {
  auto *Allocator = this->Allocator.get();

  // Check that use-after-free is detected.
  for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
    const scudo::uptr Size = 1U << SizeLog;
    if (!Allocator->useMemoryTaggingTestOnly())
      continue;
    EXPECT_DEATH(
        {
          disableDebuggerdMaybe();
          void *P = Allocator->allocate(Size, Origin);
          Allocator->deallocate(P, Origin);
          reinterpret_cast<char *>(P)[0] = 'A';
        },
        "");
    EXPECT_DEATH(
        {
          disableDebuggerdMaybe();
          void *P = Allocator->allocate(Size, Origin);
          Allocator->deallocate(P, Origin);
          reinterpret_cast<char *>(P)[Size - 1] = 'A';
        },
        "");
  }
}

SCUDO_TYPED_TEST(ScudoCombinedDeathTest, DoubleFreeFromPrimary) {
  auto *Allocator = this->Allocator.get();

  for (scudo::uptr SizeLog = 0U; SizeLog <= 20U; SizeLog++) {
    const scudo::uptr Size = 1U << SizeLog;
    if (!isPrimaryAllocation<TestAllocator<TypeParam>>(Size, 0))
      break;

    // Verify that a double free results in a chunk state error.
    EXPECT_DEATH(
        {
          // Allocate from primary
          void *P = Allocator->allocate(Size, Origin);
          ASSERT_TRUE(P != nullptr);
          Allocator->deallocate(P, Origin);
          Allocator->deallocate(P, Origin);
        },
        "invalid chunk state");
  }
}

SCUDO_TYPED_TEST(ScudoCombinedDeathTest, DisableMemoryTagging) {
  auto *Allocator = this->Allocator.get();

  if (Allocator->useMemoryTaggingTestOnly()) {
    // Check that disabling memory tagging works correctly.
    void *P = Allocator->allocate(2048, Origin);
    EXPECT_DEATH(reinterpret_cast<char *>(P)[2048] = 'A', "");
    scudo::ScopedDisableMemoryTagChecks NoTagChecks;
    Allocator->disableMemoryTagging();
    reinterpret_cast<char *>(P)[2048] = 'A';
    Allocator->deallocate(P, Origin);

    P = Allocator->allocate(2048, Origin);
    EXPECT_EQ(scudo::untagPointer(P), P);
    reinterpret_cast<char *>(P)[2048] = 'A';
    Allocator->deallocate(P, Origin);

    Allocator->releaseToOS(scudo::ReleaseToOS::Force);
  }
}

SCUDO_TYPED_TEST(ScudoCombinedTest, Stats) {
  auto *Allocator = this->Allocator.get();

  std::string Stats(10000, '\0');
  scudo::uptr ActualSize = Allocator->getStats(Stats.data(), Stats.size());
  if (ActualSize > Stats.size()) {
    Stats.resize(ActualSize);
    ActualSize = Allocator->getStats(Stats.data(), Stats.size());
  }
  EXPECT_GE(Stats.size(), ActualSize);

  // Basic checks on the contents of the statistics output, which also allows us
  // to verify that we got it all.
  EXPECT_NE(Stats.find("Stats: SizeClassAllocator"), std::string::npos);
  EXPECT_NE(Stats.find("Stats: MapAllocator"), std::string::npos);
  // Do not explicitly check for quarantine stats since a config can disable
  // them. Other tests verify this (QuarantineEnabled/QuarantineDisabled).
}

SCUDO_TYPED_TEST_SKIP_THREAD_SAFETY(ScudoCombinedTest, Drain) {
  using AllocatorT = typename BaseT::AllocatorT;
  auto *Allocator = this->Allocator.get();

  std::vector<void *> V;
  for (scudo::uptr I = 0; I < 64U; I++)
    V.push_back(Allocator->allocate(
        static_cast<scudo::uptr>(std::rand()) %
            (TypeParam::Primary::SizeClassMap::MaxSize / 2U),
        Origin));
  for (auto P : V)
    Allocator->deallocate(P, Origin);

  typename AllocatorT::TSDRegistryT::ScopedTSD TSD(
      *Allocator->getTSDRegistry());
  EXPECT_TRUE(!TSD->getSizeClassAllocator().isEmpty());
  TSD->getSizeClassAllocator().drain();
  EXPECT_TRUE(TSD->getSizeClassAllocator().isEmpty());
}

SCUDO_TYPED_TEST_SKIP_THREAD_SAFETY(ScudoCombinedTest, ForceCacheDrain) {
  using AllocatorT = typename BaseT::AllocatorT;
  auto *Allocator = this->Allocator.get();

  std::vector<void *> V;
  for (scudo::uptr I = 0; I < 64U; I++)
    V.push_back(Allocator->allocate(
        static_cast<scudo::uptr>(std::rand()) %
            (TypeParam::Primary::SizeClassMap::MaxSize / 2U),
        Origin));
  for (auto P : V)
    Allocator->deallocate(P, Origin);

  // `ForceAll` will also drain the caches.
  Allocator->releaseToOS(scudo::ReleaseToOS::ForceAll);

  typename AllocatorT::TSDRegistryT::ScopedTSD TSD(
      *Allocator->getTSDRegistry());
  EXPECT_TRUE(TSD->getSizeClassAllocator().isEmpty());
  EXPECT_EQ(TSD->getQuarantineCache().getSize(), 0U);
  EXPECT_TRUE(Allocator->getQuarantine()->isEmpty());
}

SCUDO_TYPED_TEST(ScudoCombinedTest, ThreadedCombined) {
  std::mutex Mutex;
  std::condition_variable Cv;
  bool Ready = false;
  auto *Allocator = this->Allocator.get();
  std::thread Threads[32];
  for (scudo::uptr I = 0; I < ARRAY_SIZE(Threads); I++)
    Threads[I] = std::thread([&]() {
      {
        std::unique_lock<std::mutex> Lock(Mutex);
        while (!Ready)
          Cv.wait(Lock);
      }
      std::vector<std::pair<void *, scudo::uptr>> V;
      for (scudo::uptr I = 0; I < 256U; I++) {
        const scudo::uptr Size = static_cast<scudo::uptr>(std::rand()) % 4096U;
        void *P = Allocator->allocate(Size, Origin);
        // A region could have ran out of memory, resulting in a null P.
        if (P)
          V.push_back(std::make_pair(P, Size));
      }

      // Try to interleave pushBlocks(), popBatch() and releaseToOS().
      Allocator->releaseToOS(scudo::ReleaseToOS::Force);

      while (!V.empty()) {
        auto Pair = V.back();
        Allocator->deallocateSized(Pair.first, Origin, Pair.second);
        V.pop_back();
      }
    });
  {
    std::unique_lock<std::mutex> Lock(Mutex);
    Ready = true;
    Cv.notify_all();
  }
  for (auto &T : Threads)
    T.join();
  Allocator->releaseToOS(scudo::ReleaseToOS::Force);
}

SCUDO_TYPED_TEST(ScudoCombinedTest, ForceFast) {
  auto *Allocator = this->Allocator.get();

  // Simple smoke test to verify that ForceFast does crash.
  void *P = Allocator->allocate(2048, Origin);
  memset(P, 0xff, 2048);
  Allocator->deallocate(P, Origin);

  Allocator->releaseToOS(scudo::ReleaseToOS::ForceFast);
}

// Test that multiple instantiations of the allocator have not messed up the
// process's signal handlers (GWP-ASan used to do this).
TEST(ScudoCombinedDeathTest, SKIP_ON_FUCHSIA(testSEGV)) {
  const scudo::uptr Size = 4 * scudo::getPageSizeCached();
  scudo::ReservedMemoryT ReservedMemory;
  ASSERT_TRUE(ReservedMemory.create(/*Addr=*/0U, Size, "testSEGV"));
  void *P = reinterpret_cast<void *>(ReservedMemory.getBase());
  ASSERT_NE(P, nullptr);
  EXPECT_DEATH(memset(P, 0xaa, Size), "");
  ReservedMemory.release();
}

struct DeathSizeClassConfig {
  static const scudo::uptr NumBits = 1;
  static const scudo::uptr MinSizeLog = 10;
  static const scudo::uptr MidSizeLog = 10;
  static const scudo::uptr MaxSizeLog = 13;
  static const scudo::u16 MaxNumCachedHint = 8;
  static const scudo::uptr MaxBytesCachedLog = 12;
  static const scudo::uptr SizeDelta = 0;
};

static const scudo::uptr DeathRegionSizeLog = 21U;
struct DeathConfig {
  static const bool MaySupportMemoryTagging = false;
  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>;

  struct Primary {
    // Tiny allocator, its Primary only serves chunks of four sizes.
    using SizeClassMap = scudo::FixedSizeClassMap<DeathSizeClassConfig>;
    static const scudo::uptr RegionSizeLog = DeathRegionSizeLog;
    static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN;
    static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX;
    typedef scudo::uptr CompactPtrT;
    static const scudo::uptr CompactPtrScale = 0;
    static const bool EnableRandomOffset = true;
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
    static const scudo::uptr GroupSizeLog = 18;
  };
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;

  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };

  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

TEST(ScudoCombinedDeathTest, DeathCombined) {
  using AllocatorT = TestAllocator<DeathConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  const scudo::uptr Size = 1000U;
  void *P = Allocator->allocate(Size, Origin);
  EXPECT_NE(P, nullptr);

  // Invalid sized deallocation.
  EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size + 8U), "");

  // Misaligned pointer. Potentially unused if EXPECT_DEATH isn't available.
  UNUSED void *MisalignedP =
      reinterpret_cast<void *>(reinterpret_cast<scudo::uptr>(P) | 1U);
  EXPECT_DEATH(Allocator->deallocateSized(MisalignedP, Origin, Size), "");
  EXPECT_DEATH(Allocator->reallocate(MisalignedP, Size * 2U), "");

  // Header corruption.
  scudo::u64 *H =
      reinterpret_cast<scudo::u64 *>(scudo::Chunk::getAtomicHeader(P));
  *H ^= 0x42U;
  EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size), "");
  *H ^= 0x420042U;
  EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size), "");
  *H ^= 0x420000U;

  // Invalid chunk state.
  Allocator->deallocateSized(P, Origin, Size);
  EXPECT_DEATH(Allocator->deallocateSized(P, Origin, Size), "");
  EXPECT_DEATH(Allocator->reallocate(P, Size * 2U), "");
  EXPECT_DEATH(Allocator->getUsableSize(P), "");
}

// Verify that when a region gets full, the allocator will still manage to
// fulfill the allocation through a larger size class.
TEST(ScudoCombinedTest, FullRegion) {
  using AllocatorT = TestAllocator<DeathConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  std::vector<void *> V;
  scudo::uptr FailedAllocationsCount = 0;
  for (scudo::uptr ClassId = 1U;
       ClassId <= DeathConfig::Primary::SizeClassMap::LargestClassId;
       ClassId++) {
    const scudo::uptr Size =
        DeathConfig::Primary::SizeClassMap::getSizeByClassId(ClassId);
    // Allocate enough to fill all of the regions above this one.
    const scudo::uptr MaxNumberOfChunks =
        ((1U << DeathRegionSizeLog) / Size) *
        (DeathConfig::Primary::SizeClassMap::LargestClassId - ClassId + 1);
    void *P;
    for (scudo::uptr I = 0; I <= MaxNumberOfChunks; I++) {
      P = Allocator->allocate(Size - 64U, Origin);
      if (!P)
        FailedAllocationsCount++;
      else
        V.push_back(P);
    }
    while (!V.empty()) {
      Allocator->deallocate(V.back(), Origin);
      V.pop_back();
    }
  }
  EXPECT_EQ(FailedAllocationsCount, 0U);
}

// Ensure that releaseToOS can be called prior to any other allocator
// operation without issue.
SCUDO_TYPED_TEST(ScudoCombinedTest, ReleaseToOS) {
  auto *Allocator = this->Allocator.get();
  Allocator->releaseToOS(scudo::ReleaseToOS::Force);
}

SCUDO_TYPED_TEST(ScudoCombinedTest, OddEven) {
  auto *Allocator = this->Allocator.get();
  Allocator->setOption(scudo::Option::MemtagTuning, M_MEMTAG_TUNING_BUFFER_OVERFLOW);

  if (!Allocator->useMemoryTaggingTestOnly())
    return;

  auto CheckOddEven = [](scudo::uptr P1, scudo::uptr P2) {
    scudo::uptr Tag1 = scudo::extractTag(scudo::loadTag(P1));
    scudo::uptr Tag2 = scudo::extractTag(scudo::loadTag(P2));
    EXPECT_NE(Tag1 % 2, Tag2 % 2);
  };

  using SizeClassMap = typename TypeParam::Primary::SizeClassMap;
  for (scudo::uptr ClassId = 1U; ClassId <= SizeClassMap::LargestClassId;
       ClassId++) {
    const scudo::uptr Size = SizeClassMap::getSizeByClassId(ClassId);

    std::set<scudo::uptr> Ptrs;
    bool Found = false;
    for (unsigned I = 0; I != 65536; ++I) {
      scudo::uptr P = scudo::untagPointer(reinterpret_cast<scudo::uptr>(
          Allocator->allocate(Size - scudo::Chunk::getHeaderSize(), Origin)));
      if (Ptrs.count(P - Size)) {
        Found = true;
        CheckOddEven(P, P - Size);
        break;
      }
      if (Ptrs.count(P + Size)) {
        Found = true;
        CheckOddEven(P, P + Size);
        break;
      }
      Ptrs.insert(P);
    }
    EXPECT_TRUE(Found);
  }
}

SCUDO_TYPED_TEST(ScudoCombinedTest, DisableMemInit) {
  auto *Allocator = this->Allocator.get();

  std::vector<void *> Ptrs(65536);

  Allocator->setOption(scudo::Option::ThreadDisableMemInit, 1);

  constexpr scudo::uptr MinAlignLog = FIRST_32_SECOND_64(3U, 4U);

  // Test that if mem-init is disabled on a thread, calloc should still work as
  // expected. This is tricky to ensure when MTE is enabled, so this test tries
  // to exercise the relevant code on our MTE path.
  for (scudo::uptr ClassId = 1U; ClassId <= 8; ClassId++) {
    using SizeClassMap = typename TypeParam::Primary::SizeClassMap;
    const scudo::uptr Size =
        SizeClassMap::getSizeByClassId(ClassId) - scudo::Chunk::getHeaderSize();
    if (Size < 8)
      continue;
    for (unsigned I = 0; I != Ptrs.size(); ++I) {
      Ptrs[I] = Allocator->allocate(Size, Origin);
      memset(Ptrs[I], 0xaa, Size);
    }
    for (unsigned I = 0; I != Ptrs.size(); ++I)
      Allocator->deallocateSized(Ptrs[I], Origin, Size);
    for (unsigned I = 0; I != Ptrs.size(); ++I) {
      Ptrs[I] = Allocator->allocate(Size - 8, Origin);
      memset(Ptrs[I], 0xbb, Size - 8);
    }
    for (unsigned I = 0; I != Ptrs.size(); ++I)
      Allocator->deallocateSized(Ptrs[I], Origin, Size - 8);
    for (unsigned I = 0; I != Ptrs.size(); ++I) {
      Ptrs[I] = Allocator->allocate(Size, Origin, 1U << MinAlignLog, true);
      for (scudo::uptr J = 0; J < Size; ++J)
        ASSERT_EQ((reinterpret_cast<char *>(Ptrs[I]))[J], '\0');
    }
  }

  Allocator->setOption(scudo::Option::ThreadDisableMemInit, 0);
}

SCUDO_TYPED_TEST(ScudoCombinedTest, ReallocateInPlaceStress) {
  auto *Allocator = this->Allocator.get();

  // Regression test: make realloc-in-place happen at the very right end of a
  // mapped region.
  constexpr size_t nPtrs = 10000;
  for (scudo::uptr i = 1; i < 32; ++i) {
    scudo::uptr Size = 16 * i - 1;
    std::vector<void *> Ptrs;
    for (size_t i = 0; i < nPtrs; ++i) {
      void *P = Allocator->allocate(Size, Origin);
      P = Allocator->reallocate(P, Size + 1);
      Ptrs.push_back(P);
    }

    for (size_t i = 0; i < nPtrs; ++i)
      Allocator->deallocate(Ptrs[i], Origin);
  }
}

SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferDefaultDisabled) {
  // The RingBuffer is not initialized until tracking is enabled for the
  // first time.
  auto *Allocator = this->Allocator.get();
  EXPECT_EQ(0u, Allocator->getRingBufferSize());
  EXPECT_EQ(nullptr, Allocator->getRingBufferAddress());
}

SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferInitOnce) {
  auto *Allocator = this->Allocator.get();
  Allocator->setTrackAllocationStacks(true);

  auto RingBufferSize = Allocator->getRingBufferSize();
  ASSERT_GT(RingBufferSize, 0u);
  auto *RingBufferAddress = Allocator->getRingBufferAddress();
  EXPECT_NE(nullptr, RingBufferAddress);

  // Enable tracking again to verify that the initialization only happens once.
  Allocator->setTrackAllocationStacks(true);
  ASSERT_EQ(RingBufferSize, Allocator->getRingBufferSize());
  EXPECT_EQ(RingBufferAddress, Allocator->getRingBufferAddress());
}

SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferSize) {
  auto *Allocator = this->Allocator.get();
  Allocator->setTrackAllocationStacks(true);

  auto RingBufferSize = Allocator->getRingBufferSize();
  ASSERT_GT(RingBufferSize, 0u);
  EXPECT_EQ(Allocator->getRingBufferAddress()[RingBufferSize - 1], '\0');
}

SCUDO_TYPED_TEST(ScudoCombinedTest, RingBufferAddress) {
  auto *Allocator = this->Allocator.get();
  Allocator->setTrackAllocationStacks(true);

  auto *RingBufferAddress = Allocator->getRingBufferAddress();
  EXPECT_NE(RingBufferAddress, nullptr);
  EXPECT_EQ(RingBufferAddress, Allocator->getRingBufferAddress());
}

SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepotDefaultDisabled) {
  // The StackDepot is not initialized until tracking is enabled for the
  // first time.
  auto *Allocator = this->Allocator.get();
  EXPECT_EQ(0u, Allocator->getStackDepotSize());
  EXPECT_EQ(nullptr, Allocator->getStackDepotAddress());
}

SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepotInitOnce) {
  auto *Allocator = this->Allocator.get();
  Allocator->setTrackAllocationStacks(true);

  auto StackDepotSize = Allocator->getStackDepotSize();
  EXPECT_GT(StackDepotSize, 0u);
  auto *StackDepotAddress = Allocator->getStackDepotAddress();
  EXPECT_NE(nullptr, StackDepotAddress);

  // Enable tracking again to verify that the initialization only happens once.
  Allocator->setTrackAllocationStacks(true);
  EXPECT_EQ(StackDepotSize, Allocator->getStackDepotSize());
  EXPECT_EQ(StackDepotAddress, Allocator->getStackDepotAddress());
}

SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepotSize) {
  auto *Allocator = this->Allocator.get();
  Allocator->setTrackAllocationStacks(true);

  auto StackDepotSize = Allocator->getStackDepotSize();
  EXPECT_GT(StackDepotSize, 0u);
  EXPECT_EQ(Allocator->getStackDepotAddress()[StackDepotSize - 1], '\0');
}

SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepotAddress) {
  auto *Allocator = this->Allocator.get();
  Allocator->setTrackAllocationStacks(true);

  auto *StackDepotAddress = Allocator->getStackDepotAddress();
  EXPECT_NE(StackDepotAddress, nullptr);
  EXPECT_EQ(StackDepotAddress, Allocator->getStackDepotAddress());
}

SCUDO_TYPED_TEST(ScudoCombinedTest, StackDepot) {
  alignas(scudo::StackDepot) char Buf[sizeof(scudo::StackDepot) +
                                      1024 * sizeof(scudo::atomic_u64) +
                                      1024 * sizeof(scudo::atomic_u32)] = {};
  auto *Depot = reinterpret_cast<scudo::StackDepot *>(Buf);
  Depot->init(1024, 1024);
  ASSERT_TRUE(Depot->isValid(sizeof(Buf)));
  ASSERT_FALSE(Depot->isValid(sizeof(Buf) - 1));
  scudo::uptr Stack[] = {1, 2, 3};
  scudo::u32 Elem = Depot->insert(&Stack[0], &Stack[3]);
  scudo::uptr RingPosPtr = 0;
  scudo::uptr SizePtr = 0;
  ASSERT_TRUE(Depot->find(Elem, &RingPosPtr, &SizePtr));
  ASSERT_EQ(SizePtr, 3u);
  EXPECT_EQ(Depot->at(RingPosPtr), 1u);
  EXPECT_EQ(Depot->at(RingPosPtr + 1), 2u);
  EXPECT_EQ(Depot->at(RingPosPtr + 2), 3u);
}

#if SCUDO_CAN_USE_PRIMARY64
#if SCUDO_TRUSTY

// TrustyConfig is designed for a domain-specific allocator. Add a basic test
// which covers only simple operations and ensure the configuration is able to
// compile.
TEST(ScudoCombinedTest, BasicTrustyConfig) {
  using AllocatorT = TestAllocator<scudo::TrustyConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  for (scudo::uptr ClassId = 1U;
       ClassId <= scudo::TrustyConfig::SizeClassMap::LargestClassId;
       ClassId++) {
    const scudo::uptr Size =
        scudo::TrustyConfig::SizeClassMap::getSizeByClassId(ClassId);
    void *p = Allocator->allocate(Size - scudo::Chunk::getHeaderSize(), Origin);
    ASSERT_NE(p, nullptr);
    free(p);
  }

  bool UnlockRequired;
  typename AllocatorT::TSDRegistryT::ScopedTSD TSD(
      *Allocator->getTSDRegistry());
  TSD->getSizeClassAllocator().drain();

  Allocator->releaseToOS(scudo::ReleaseToOS::Force);
}

#endif
#endif

struct TestQuarantineSizeClassConfig {
  static const scudo::uptr NumBits = 1;
  static const scudo::uptr MinSizeLog = 10;
  static const scudo::uptr MidSizeLog = 10;
  static const scudo::uptr MaxSizeLog = 13;
  static const scudo::u16 MaxNumCachedHint = 8;
  static const scudo::uptr MaxBytesCachedLog = 12;
  static const scudo::uptr SizeDelta = 0;
};

#if SCUDO_FUCHSIA
struct TestZeroOnDeallocConfig {
  static const bool MaySupportMemoryTagging = false;
  static const bool EnableZeroOnDealloc = true;
  static const bool QuarantineDisabled = true;

  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>;
  struct Primary {
    // Tiny allocator, its Primary only serves chunks of four sizes.
    using SizeClassMap =
        scudo::FixedSizeClassMap<TestQuarantineSizeClassConfig>;
    static const scudo::uptr RegionSizeLog = DeathRegionSizeLog;
    static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN;
    static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX;
    typedef scudo::uptr CompactPtrT;
    static const scudo::uptr CompactPtrScale = 0;
    static const bool EnableRandomOffset = true;
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
    static const scudo::uptr GroupSizeLog = 18;
  };

  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;

  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };

  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

struct TestNoZeroOnDeallocConfig : public TestZeroOnDeallocConfig {
  static const bool EnableZeroOnDealloc = false;
};

TEST(ScudoCombinedTest, ZeroOnDeallocEnabledAndFlag) {
  for (scudo::uptr FlagValue = 128; FlagValue <= 2048; FlagValue *= 2) {
    // Set the size limit flag via the environment variable.
    char OptionsStr[256];
    snprintf(OptionsStr, sizeof(OptionsStr), "zero_on_dealloc_max_size=%ld",
             static_cast<long>(FlagValue));
    ScopedScudoOptions Options(OptionsStr);

    // Creates an allocator, configured from the environment.
    using AllocatorT = TestAllocator<TestZeroOnDeallocConfig>;
    auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

    for (scudo::uptr AllocatedSize : {FlagValue / 2, FlagValue}) {
      // Allocates and sets the memory.
      void *P = Allocator->allocate(AllocatedSize, Origin);
      ASSERT_NE(P, nullptr);
      memset(P, 'B', AllocatedSize);

      char *Begin =
          reinterpret_cast<char *>(Allocator->getBlockBeginTestOnly(P));
      char *End = reinterpret_cast<char *>(P) + AllocatedSize;
      // Deallocates and eventually clears the memory.
      Allocator->deallocate(P, Origin);

      if (End - Begin <= static_cast<long>(FlagValue)) {
        // Verifies the memory was cleared, including the header.
        for (char *T = Begin; T < End; ++T) {
          ASSERT_EQ(*T, 0);
        }
      }
    }
  }
}
#endif

struct TestQuarantineConfig {
  static const bool MaySupportMemoryTagging = false;

  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>;

  struct Primary {
    // Tiny allocator, its Primary only serves chunks of four sizes.
    using SizeClassMap = scudo::FixedSizeClassMap<DeathSizeClassConfig>;
    static const scudo::uptr RegionSizeLog = DeathRegionSizeLog;
    static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN;
    static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX;
    typedef scudo::uptr CompactPtrT;
    static const scudo::uptr CompactPtrScale = 0;
    static const bool EnableRandomOffset = true;
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
    static const scudo::uptr GroupSizeLog = 18;
  };
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;

  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };

  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

// Verify that the quarantine exists by default.
TEST(ScudoCombinedTest, QuarantineEnabled) {
  using AllocatorT = TestAllocator<TestQuarantineConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  const scudo::uptr Size = 1000U;
  void *P = Allocator->allocate(Size, Origin);
  EXPECT_NE(P, nullptr);
  Allocator->deallocate(P, Origin);

  std::string Stats(10000, '\0');
  scudo::uptr ActualSize = Allocator->getStats(Stats.data(), Stats.size());
  if (ActualSize > Stats.size()) {
    Stats.resize(ActualSize);
    ActualSize = Allocator->getStats(Stats.data(), Stats.size());
  }
  EXPECT_GE(Stats.size(), ActualSize);

  // Quarantine stats should be present.
  EXPECT_NE(Stats.find("Stats: Quarantine"), std::string::npos);
}

struct TestQuarantineDisabledConfig : TestQuarantineConfig {
  static const bool QuarantineDisabled = true;
};

TEST(ScudoCombinedTest, QuarantineDisabled) {
  using AllocatorT = TestAllocator<TestQuarantineDisabledConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  const scudo::uptr Size = 1000U;
  void *P = Allocator->allocate(Size, Origin);
  EXPECT_NE(P, nullptr);
  Allocator->deallocate(P, Origin);

  std::string Stats(10000, '\0');
  scudo::uptr ActualSize = Allocator->getStats(Stats.data(), Stats.size());
  if (ActualSize > Stats.size()) {
    Stats.resize(ActualSize);
    ActualSize = Allocator->getStats(Stats.data(), Stats.size());
  }
  EXPECT_GE(Stats.size(), ActualSize);

  // No quarantine stats should not be present.
  EXPECT_EQ(Stats.find("Stats: Quarantine"), std::string::npos);
}

struct UsableSizeClassConfig {
  static const scudo::uptr NumBits = 1;
  static const scudo::uptr MinSizeLog = 10;
  static const scudo::uptr MidSizeLog = 10;
  static const scudo::uptr MaxSizeLog = 13;
  static const scudo::u16 MaxNumCachedHint = 8;
  static const scudo::uptr MaxBytesCachedLog = 12;
  static const scudo::uptr SizeDelta = 0;
};

struct TestExactUsableSizeConfig {
  static const bool MaySupportMemoryTagging = false;
  static const bool QuarantineDisabled = true;

  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>;

  struct Primary {
    // In order to properly test the usable size, this Primary config has
    // four real size classes: 1024, 2048, 4096, 8192.
    using SizeClassMap = scudo::FixedSizeClassMap<UsableSizeClassConfig>;
    static const scudo::uptr RegionSizeLog = 21U;
    static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN;
    static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX;
    typedef scudo::uptr CompactPtrT;
    static const scudo::uptr CompactPtrScale = 0;
    static const bool EnableRandomOffset = true;
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
    static const scudo::uptr GroupSizeLog = 18;
  };
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;

  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };

  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

template <class AllocatorT> void VerifyExactUsableSize(AllocatorT &Allocator) {
  // Scan through all sizes up to 10000 then some larger sizes.
  for (scudo::uptr Size = 1; Size < 10000; Size++) {
    void *P = Allocator.allocate(Size, Origin);
    EXPECT_EQ(Size, Allocator.getUsableSize(P))
        << "Failed usable size at allocation size " << Size;
    Allocator.deallocate(P, Origin);
  }

  // Verify that aligned allocations also return the exact size allocated.
  const scudo::uptr AllocSize = 313;
  for (scudo::uptr Align = 1; Align <= 8; Align++) {
    void *P = Allocator.allocate(AllocSize, Origin, 1U << Align);
    EXPECT_EQ(AllocSize, Allocator.getUsableSize(P))
        << "Failed usable size at allocation size " << AllocSize << " at align "
        << 1 << Align;
    Allocator.deallocate(P, Origin);
  }

  // Verify an explicitly large allocations.
  const scudo::uptr LargeAllocSize = 1000000;
  void *P = Allocator.allocate(LargeAllocSize, Origin);
  EXPECT_EQ(LargeAllocSize, Allocator.getUsableSize(P));
  Allocator.deallocate(P, Origin);

  // Now do it for aligned allocations for large allocations.
  for (scudo::uptr Align = 1; Align <= 8; Align++) {
    void *P = Allocator.allocate(LargeAllocSize, Origin, 1U << Align);
    EXPECT_EQ(LargeAllocSize, Allocator.getUsableSize(P))
        << "Failed usable size at allocation size " << AllocSize << " at align "
        << 1 << Align;
    Allocator.deallocate(P, Origin);
  }
}

template <class AllocatorT>
void VerifyIterateOverUsableSize(AllocatorT &Allocator) {
  // This will not verify if the size is the exact size or the size of the
  // size class. Instead verify that the size matches the usable size and
  // assume the other tests have verified getUsableSize.
  std::unordered_map<void *, size_t> Pointers;
  Pointers.insert({Allocator.allocate(128, Origin), 0U});
  Pointers.insert({Allocator.allocate(128, Origin, 32), 0U});
  Pointers.insert({Allocator.allocate(2000, Origin), 0U});
  Pointers.insert({Allocator.allocate(2000, Origin, 64), 0U});
  Pointers.insert({Allocator.allocate(8000, Origin), 0U});
  Pointers.insert({Allocator.allocate(8000, Origin, 128), 0U});
  Pointers.insert({Allocator.allocate(2000205, Origin), 0U});
  Pointers.insert({Allocator.allocate(2000205, Origin, 128), 0U});
  Pointers.insert({Allocator.allocate(2000205, Origin, 256), 0U});

  Allocator.disable();
  Allocator.iterateOverChunks(
      0, static_cast<scudo::uptr>(SCUDO_MMAP_RANGE_SIZE - 1),
      [](uintptr_t Base, size_t Size, void *Arg) {
        std::unordered_map<void *, size_t> *Pointers =
            reinterpret_cast<std::unordered_map<void *, size_t> *>(Arg);
        (*Pointers)[reinterpret_cast<void *>(Base)] = Size;
      },
      reinterpret_cast<void *>(&Pointers));
  Allocator.enable();

  for (auto [Ptr, IterateSize] : Pointers) {
    EXPECT_NE(0U, IterateSize)
        << "Pointer " << Ptr << " not found in iterateOverChunks call.";
    EXPECT_EQ(IterateSize, Allocator.getUsableSize(Ptr))
        << "Pointer " << Ptr
        << " mismatch between iterate size and usable size.";
    Allocator.deallocate(Ptr, Origin);
  }
}

TEST(ScudoCombinedTest, ExactUsableSize) {
  using AllocatorT = scudo::Allocator<TestExactUsableSizeConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  VerifyExactUsableSize<AllocatorT>(*Allocator);
  VerifyIterateOverUsableSize<AllocatorT>(*Allocator);
}

struct TestExactUsableSizeMTEConfig : TestExactUsableSizeConfig {
  static const bool MaySupportMemoryTagging = true;
};

TEST(ScudoCombinedTest, ExactUsableSizeMTE) {
  if (!scudo::archSupportsMemoryTagging() ||
      !scudo::systemDetectsMemoryTagFaultsTestOnly())
    TEST_SKIP("Only supported on systems that can enable MTE.");

  scudo::enableSystemMemoryTaggingTestOnly();

  using AllocatorT = scudo::Allocator<TestExactUsableSizeMTEConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  VerifyExactUsableSize<AllocatorT>(*Allocator);
  VerifyIterateOverUsableSize<AllocatorT>(*Allocator);
}

template <class AllocatorT>
void VerifyUsableSizePrimary(AllocatorT &Allocator) {
  std::vector<scudo::uptr> SizeClasses = {1024U, 2048U, 4096U, 8192U};
  for (size_t I = 0; I < SizeClasses.size(); I++) {
    scudo::uptr SizeClass = SizeClasses[I];
    scudo::uptr StartSize;
    if (I == 0)
      StartSize = 1;
    else
      StartSize = SizeClasses[I - 1];
    scudo::uptr UsableSize = SizeClass - scudo::Chunk::getHeaderSize();
    for (scudo::uptr Size = StartSize; Size < UsableSize; Size++) {
      void *P = Allocator.allocate(Size, Origin);
      EXPECT_EQ(UsableSize, Allocator.getUsableSize(P))
          << "Failed usable size at allocation size " << Size
          << " for size class " << SizeClass;
      memset(P, 0xff, UsableSize);
      EXPECT_EQ(Allocator.getBlockBeginTestOnly(P) + SizeClass,
                reinterpret_cast<scudo::uptr>(P) + UsableSize);
      Allocator.deallocate(P, Origin);
    }

    StartSize = UsableSize + 1;
  }

  std::vector<scudo::uptr> Alignments = {32U, 128U};
  for (size_t I = 0; I < SizeClasses.size(); I++) {
    scudo::uptr SizeClass = SizeClasses[I];
    scudo::uptr AllocSize;
    if (I == 0)
      AllocSize = 1;
    else
      AllocSize = SizeClasses[I - 1] + 1;

    for (auto Alignment : Alignments) {
      void *P = Allocator.allocate(AllocSize, Origin, Alignment);
      scudo::uptr UsableSize = Allocator.getUsableSize(P);
      memset(P, 0xff, UsableSize);
      EXPECT_EQ(Allocator.getBlockBeginTestOnly(P) + SizeClass,
                reinterpret_cast<scudo::uptr>(P) + UsableSize)
          << "Failed usable size at allocation size " << AllocSize
          << " for size class " << SizeClass << " at alignment " << Alignment;
      Allocator.deallocate(P, Origin);
    }
  }
}

template <class AllocatorT>
void VerifyUsableSizeSecondary(AllocatorT &Allocator) {
  const scudo::uptr LargeAllocSize = 996780;
  const scudo::uptr PageSize = scudo::getPageSizeCached();
  void *P = Allocator.allocate(LargeAllocSize, Origin);
  scudo::uptr UsableSize = Allocator.getUsableSize(P);
  memset(P, 0xff, UsableSize);
  // Assumes that the secondary always rounds up allocations to a page boundary.
  EXPECT_EQ(scudo::roundUp(reinterpret_cast<scudo::uptr>(P) + LargeAllocSize,
                           PageSize),
            reinterpret_cast<scudo::uptr>(P) + UsableSize);
  Allocator.deallocate(P, Origin);

  // Check aligned allocations now.
  for (scudo::uptr Alignment = 1; Alignment <= 8; Alignment++) {
    void *P = Allocator.allocate(LargeAllocSize, Origin, 1U << Alignment);
    scudo::uptr UsableSize = Allocator.getUsableSize(P);
    EXPECT_EQ(scudo::roundUp(reinterpret_cast<scudo::uptr>(P) + LargeAllocSize,
                             PageSize),
              reinterpret_cast<scudo::uptr>(P) + UsableSize)
        << "Failed usable size at allocation size " << LargeAllocSize
        << " at alignment " << Alignment;
    Allocator.deallocate(P, Origin);
  }
}

struct TestFullUsableSizeConfig : TestExactUsableSizeConfig {
  static const bool ExactUsableSize = false;
};

TEST(ScudoCombinedTest, FullUsableSize) {
  using AllocatorT = scudo::Allocator<TestFullUsableSizeConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  VerifyUsableSizePrimary<AllocatorT>(*Allocator);
  VerifyUsableSizeSecondary<AllocatorT>(*Allocator);
  VerifyIterateOverUsableSize<AllocatorT>(*Allocator);
}

TEST(ScudoCombinedTest, ReallocUsableSize) {
  using AllocatorT = TestAllocator<TestFullUsableSizeConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  scudo::uptr Size = 1000;
  void *P = Allocator->allocate(Size, Origin);
  scudo::uptr UsableSize = Allocator->getUsableSize(P);
  std::vector<unsigned char> Buffer(UsableSize);
  for (size_t I = 0; I < UsableSize; I++) {
    Buffer[I] = I & 0xff;
  }
  memcpy(P, Buffer.data(), UsableSize);
  EXPECT_LE(Size, UsableSize);

  scudo::uptr NewSize = 2 * UsableSize;
  void *NewP = Allocator->reallocate(P, NewSize);
  EXPECT_NE(NewP, P);
  for (size_t I = 0; I < UsableSize; I++) {
    EXPECT_EQ((reinterpret_cast<unsigned char *>(NewP))[I], I & 0xff)
        << "Failed at index " << I;
  }
  Allocator->deallocate(NewP, Origin);
}

struct TestFullUsableSizeMTEConfig : TestFullUsableSizeConfig {
  static const bool MaySupportMemoryTagging = true;
};

TEST(ScudoCombinedTest, FullUsableSizeMTE) {
  if (!scudo::archSupportsMemoryTagging() ||
      !scudo::systemDetectsMemoryTagFaultsTestOnly())
    TEST_SKIP("Only supported on systems that can enable MTE.");

  scudo::enableSystemMemoryTaggingTestOnly();

  using AllocatorT = scudo::Allocator<TestFullUsableSizeMTEConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  // When MTE is enabled, you get exact sizes.
  VerifyExactUsableSize<AllocatorT>(*Allocator);
  VerifyIterateOverUsableSize<AllocatorT>(*Allocator);
}

// Verify that no special quarantine blocks appear in iterateOverChunks.
TEST(ScudoCombinedTest, QuarantineIterateOverChunks) {
  using AllocatorT = TestAllocator<TestQuarantineConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  // Do a bunch of allocations and deallocations. At the end there should
  // be no special quarantine blocks in our callbacks, and no blocks at all.
  std::vector<scudo::uptr> Sizes = {128, 128, 256, 256};
  for (auto const Size : Sizes) {
    void *Ptr = Allocator->allocate(Size, Origin);
    EXPECT_NE(Ptr, nullptr);
    Allocator->deallocate(Ptr, Origin);
  }
  std::unordered_map<uintptr_t, size_t> Pointers;
  Allocator->disable();
  Allocator->iterateOverChunks(
      0, static_cast<scudo::uptr>(SCUDO_MMAP_RANGE_SIZE - 1),
      [](uintptr_t Base, size_t Size, void *Arg) {
        std::unordered_map<uintptr_t, size_t> *Pointers =
            reinterpret_cast<std::unordered_map<uintptr_t, size_t> *>(Arg);
        (*Pointers)[Base] = Size;
      },
      reinterpret_cast<void *>(&Pointers));
  Allocator->enable();

  for (const auto [Base, Size] : Pointers) {
    EXPECT_TRUE(false) << "Unexpected pointer found in iterateOverChunks "
                       << std::hex << Base << " Size " << std::dec << Size;
  }
}

struct InitSizeClassConfig {
  static const scudo::uptr NumBits = 1;
  static const scudo::uptr MinSizeLog = 10;
  static const scudo::uptr MidSizeLog = 10;
  static const scudo::uptr MaxSizeLog = 13;
  static const scudo::u16 MaxNumCachedHint = 8;
  static const scudo::uptr MaxBytesCachedLog = 12;
  static const scudo::uptr SizeDelta = 0;
};

struct TestInitSizeConfig {
  static const bool MaySupportMemoryTagging = false;
  static const bool QuarantineDisabled = true;

  struct Primary {
    using SizeClassMap = scudo::FixedSizeClassMap<InitSizeClassConfig>;
    static const scudo::uptr RegionSizeLog = 21U;
    static const scudo::s32 MinReleaseToOsIntervalMs = INT32_MIN;
    static const scudo::s32 MaxReleaseToOsIntervalMs = INT32_MAX;
    typedef scudo::uptr CompactPtrT;
    static const scudo::uptr CompactPtrScale = 0;
    static const bool EnableRandomOffset = true;
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
    static const scudo::uptr GroupSizeLog = 18;
  };
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;

  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };

  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

struct TestInitSizeTSDSharedConfig : public TestInitSizeConfig {
  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 4U, 4U>;
};

struct TestInitSizeTSDExclusiveConfig : public TestInitSizeConfig {
  template <class A> using TSDRegistryT = scudo::TSDRegistryExT<A>;
};

template <class AllocatorT> void RunStress() {
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  // This test is designed to try and have many threads trying to initialize
  // the TSD at the same time. Make sure this doesn't crash.
  std::atomic_bool StartRunning = false;
  std::vector<std::thread *> threads;
  for (size_t I = 0; I < 16; I++) {
    threads.emplace_back(new std::thread([&Allocator, &StartRunning]() {
      while (!StartRunning.load())
        ;

      void *Ptr = Allocator->allocate(10, Origin);
      EXPECT_TRUE(Ptr != nullptr);
      // Make sure this value is not optimized away.
      asm volatile("" : : "r,m"(Ptr) : "memory");
      Allocator->deallocate(Ptr, Origin);
    }));
  }

  StartRunning = true;

  for (auto *thread : threads) {
    thread->join();
    delete thread;
  }
}

TEST(ScudoCombinedTest, StressThreadInitTSDShared) {
  using AllocatorT = scudo::Allocator<TestInitSizeTSDSharedConfig>;
  // Run the stress test a few times.
  for (size_t I = 0; I < 10; I++)
    RunStress<AllocatorT>();
}

TEST(ScudoCombinedTest, StressThreadInitTSDExclusive) {
  using AllocatorT = scudo::Allocator<TestInitSizeTSDExclusiveConfig>;
  // Run the stress test a few times.
  for (size_t I = 0; I < 10; I++)
    RunStress<AllocatorT>();
}

struct TestMatchConfig {
  static const bool MaySupportMemoryTagging = false;
  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>;

  struct Primary {
    using SizeClassMap = scudo::AndroidSizeClassMap;
#if SCUDO_CAN_USE_PRIMARY64
    static const scudo::uptr RegionSizeLog = 28U;
    typedef scudo::u32 CompactPtrT;
    static const scudo::uptr CompactPtrScale = SCUDO_MIN_ALIGNMENT_LOG;
    static const scudo::uptr GroupSizeLog = 20U;
    static const bool EnableRandomOffset = true;
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
#else
    static const scudo::uptr RegionSizeLog = 18U;
    static const scudo::uptr GroupSizeLog = 18U;
    typedef scudo::uptr CompactPtrT;
#endif
    static const bool EnableBlockCache = false;
    static const scudo::s32 MinReleaseToOsIntervalMs = 1000;
    static const scudo::s32 MaxReleaseToOsIntervalMs = 1000;
  };

#if SCUDO_CAN_USE_PRIMARY64
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;
#else
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator32<Config>;
#endif

  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };
};

TEST(ScudoCombinedDeathTest, DeallocateAlignNotPowerOfTwo) {
  using AllocatorT = scudo::Allocator<TestMatchConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
  EXPECT_DEATH(
      Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Memalign, 9),
      "alignment must be a power of two");
  Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Memalign, 32);
}

TEST(ScudoCombinedDeathTest, TypeMismatch) {
  ScopedScudoOptions Options("dealloc_type_mismatch=true");

  using AllocatorT = scudo::Allocator<TestMatchConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Malloc);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::New),
               "deallocating.*\\(malloc vs new\\)");
  EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray),
               "deallocating.*\\(malloc vs new\\[\\]\\)");
  Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc);

  Ptr = Allocator->allocate(10, scudo::Chunk::Origin::New);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc),
               "deallocating.*\\(new vs malloc\\)");
  EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray),
               "deallocating.*\\(new vs new\\[\\]\\)");
  Allocator->deallocate(Ptr, scudo::Chunk::Origin::New);

  Ptr = Allocator->allocate(10, scudo::Chunk::Origin::NewArray);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc),
               "deallocating.*\\(new\\[\\] vs malloc\\)");
  EXPECT_DEATH(Allocator->deallocate(Ptr, scudo::Chunk::Origin::New),
               "deallocating.*\\(new\\[\\] vs new\\)");
  Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray);
}

TEST(ScudoCombinedDeathTest, ReallocTypeMismatch) {
  ScopedScudoOptions Options("dealloc_type_mismatch=true");

  using AllocatorT = scudo::Allocator<TestMatchConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::New);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(Allocator->reallocate(Ptr, 1000000),
               "reallocating.*\\(new vs malloc\\)");
  Allocator->deallocate(Ptr, scudo::Chunk::Origin::New);

  Ptr = Allocator->allocate(10, scudo::Chunk::Origin::NewArray);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(Allocator->reallocate(Ptr, 1000000),
               "reallocating.*\\(new\\[\\] vs malloc\\)");
  Allocator->deallocate(Ptr, scudo::Chunk::Origin::NewArray);

  Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(Allocator->reallocate(Ptr, 1000000),
               "reallocating.*\\(aligned malloc vs malloc\\)");
  Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Memalign, 32);
}

TEST(ScudoCombinedDeathTest, AlignTypeMismatch) {
  ScopedScudoOptions Options("dealloc_type_mismatch=true");

  using AllocatorT = scudo::Allocator<TestMatchConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(Allocator->reallocate(Ptr, 1000),
               "reallocating.*\\(aligned malloc vs malloc\\)");
  // Aligned allocate with non-aligned deallocate is okay.
  Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc);
}

// Scudo currently cannot verify that a pointer allocated with an aligned
// new/new [] is deallocated with an aligned delete/delete [].
TEST(ScudoCombinedTest, DISABLED_NewType) {
  ScopedScudoOptions Options("dealloc_type_mismatch=true");

  using AllocatorT = scudo::Allocator<TestMatchConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::New, 1024);
  EXPECT_TRUE(Ptr != nullptr);
  Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::New, 1024);

  Ptr = Allocator->allocate(10, scudo::Chunk::Origin::NewArray, 1024);
  EXPECT_TRUE(Ptr != nullptr);
  Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::NewArray, 1024);
}

TEST(ScudoCombinedDeathTest, SizeMismatch) {
  ScopedScudoOptions Options("delete_size_mismatch=true");

  using AllocatorT = scudo::Allocator<TestMatchConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Malloc);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(
      Allocator->deallocateSized(Ptr, scudo::Chunk::Origin::Malloc, 1000000),
      "invalid sized delete when deallocating.*\\(1000000 vs 10\\)");
  Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc);

  Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(Allocator->deallocateSizedAligned(
                   Ptr, scudo::Chunk::Origin::Malloc, 1000000, 32),
               "invalid sized delete when deallocating.*\\(1000000 vs 10\\)");
  Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Malloc, 32);
}

struct TestMatchNotExactUsableSizeConfig : public TestMatchConfig {
  static const bool ExactUsableSize = false;
};

TEST(ScudoCombinedDeathTest, SizeNotExactUsableMismatch) {
  ScopedScudoOptions Options("delete_size_mismatch=true");

  using AllocatorT = scudo::Allocator<TestMatchNotExactUsableSizeConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Malloc);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(
      Allocator->deallocateSized(Ptr, scudo::Chunk::Origin::Malloc, 1000000),
      "invalid sized delete when deallocating.*\\(1000000 vs 10 or .*\\)");
  Allocator->deallocateSized(Ptr, scudo::Chunk::Origin::Malloc, 10);

  Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
  EXPECT_TRUE(Ptr != nullptr);
  EXPECT_DEATH(
      Allocator->deallocateSizedAligned(Ptr, scudo::Chunk::Origin::Malloc,
                                        1000000, 32),
      "invalid sized delete when deallocating.*\\(1000000 vs 10 or .*\\)");
  EXPECT_NE(10U, Allocator->getUsableSize(Ptr));
  Allocator->deallocateSizedAligned(Ptr, scudo::Chunk::Origin::Malloc,
                                    Allocator->getUsableSize(Ptr), 32);
}

template <class AllocatorT>
static void *getMinAlignedPointer(AllocatorT *Allocator) {
  // The actual alignment is not stored in the header, so the only check
  // that can be made is to verify how the pointer is aligned.
  // Therefore, try and allocate a pointer which has a low alignment but
  // not a high alignment so we can check for death.
  std::vector<void *> Ptrs;
  void *AlignedPtr = nullptr;
  scudo::uptr AlignmentMask = 2 * scudo::getPageSizeCached() - 1;
  for (size_t I = 0; I < 1000; ++I) {
    void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Memalign, 32);
    EXPECT_TRUE(Ptr != nullptr);
    scudo::uptr FlatPtr = reinterpret_cast<scudo::uptr>(Ptr);
    if ((FlatPtr & AlignmentMask) != 0) {
      AlignedPtr = Ptr;
      break;
    }
    Ptrs.push_back(Ptr);
  }

  // Free all of the other pointers.
  for (auto Ptr : Ptrs) {
    Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Memalign, 32);
  }

  return AlignedPtr;
}

TEST(ScudoCombinedDeathTest, AlignMismatch) {
  ScopedScudoOptions Options("dealloc_align_mismatch=true");

  using AllocatorT = scudo::Allocator<TestMatchConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  // Pointer is guaranteed to not be aligned to 2 * page size.
  void *AlignedPtr = getMinAlignedPointer<AllocatorT>(Allocator.get());
  if (AlignedPtr == nullptr) {
    TEST_SKIP("Cannot allocate aligned pointer for test.");
  }

  scudo::uptr Alignment = 2 * scudo::getPageSizeCached();
  EXPECT_DEATH(Allocator->deallocateAligned(
                   AlignedPtr, scudo::Chunk::Origin::Malloc, Alignment),
               "invalid aligned delete when deallocating");
  EXPECT_DEATH(Allocator->deallocateSizedAligned(
                   AlignedPtr, scudo::Chunk::Origin::Malloc, 10, Alignment),
               "invalid aligned delete when deallocating");

  Allocator->deallocateAligned(AlignedPtr, scudo::Chunk::Origin::Malloc, 32);
}

struct TestMatchOverrideConfig : public TestMatchConfig {
  // Disable all type/size/alignment checks.
  static const bool AbortOnDeallocTypeMismatch = false;
  static const bool AbortOnDeallocSizeMismatch = false;
  static const bool AbortOnDeallocAlignmentMismatch = false;
};

TEST(ScudoCombinedTest, VerifyConfigOverrideMatchChecks) {
  ScopedScudoOptions Options(
      "dealloc_type_mismatch=true:delete_size_mismatch=true:dealloc_align_"
      "mismatch=true");

  using AllocatorT = scudo::Allocator<TestMatchOverrideConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  // Verify type mismatch is ignored.
  void *Ptr = Allocator->allocate(10, scudo::Chunk::Origin::New);
  EXPECT_TRUE(Ptr != nullptr);
  Allocator->deallocate(Ptr, scudo::Chunk::Origin::Malloc);

  // Verify size mismatch is ignored.
  Ptr = Allocator->allocate(10, scudo::Chunk::Origin::Malloc);
  EXPECT_TRUE(Ptr != nullptr);
  Allocator->deallocateSized(Ptr, scudo::Chunk::Origin::Malloc, 1000000);

  // Pointer is guaranteed to not be aligned to 2 * page size.
  scudo::uptr Alignment = 2 * scudo::getPageSizeCached();
  Ptr = getMinAlignedPointer<AllocatorT>(Allocator.get());
  if (Ptr != nullptr) {
    Allocator->deallocateAligned(Ptr, scudo::Chunk::Origin::Malloc, Alignment);
  }
}

struct TestNumericValuesConfig {
  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 8U, 4U>;

  struct Primary {
    using SizeClassMap = scudo::AndroidSizeClassMap;
    static const scudo::uptr RegionSizeLog = 18U;
    static const scudo::uptr GroupSizeLog = 18U;
    static const scudo::uptr CompactPtrScale = 0;
    static const scudo::s32 DefaultReleaseToOsIntervalMs = INT32_MIN;
#if SCUDO_CAN_USE_PRIMARY64
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
#endif
  };

#if SCUDO_CAN_USE_PRIMARY64
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;
#else
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator32<Config>;
#endif

  struct Secondary {
    template <typename Config> using CacheT = scudo::MapAllocatorCache<Config>;
    struct Cache {
      static const scudo::u32 EntriesArraySize = 128;
      static const scudo::u32 QuarantineSize = 0;
      static const scudo::u32 DefaultMaxEntriesCount = 64;
      static const scudo::uptr DefaultMaxEntrySize = 1UL << 20;
      static const scudo::s32 MinReleaseToOsIntervalMs = 1000;
      static const scudo::s32 MaxReleaseToOsIntervalMs = 2000;
      static const scudo::s32 DefaultReleaseToOsIntervalMs = 1500;
    };
  };

  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

TEST(ScudoCombinedTest, VerifyNumericValuesConfig) {
  using AllocatorT = scudo::Allocator<TestNumericValuesConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  std::string Stats(10000, '\0');
  Allocator->getStats(Stats.data(), Stats.size());
  EXPECT_NE(Stats.find("CompactPtrScale: 0"), std::string::npos);
  EXPECT_NE(Stats.find("EntriesArraySize: 128"), std::string::npos);
  EXPECT_NE(Stats.find("QuarantineSize: 0"), std::string::npos);
  EXPECT_NE(Stats.find("DefaultMaxEntriesCount: 64"), std::string::npos);
  EXPECT_NE(Stats.find("DefaultMaxEntrySize: 1048576"), std::string::npos);
  EXPECT_NE(Stats.find("MinReleaseToOsIntervalMs: 1000"), std::string::npos);
  EXPECT_NE(Stats.find("MaxReleaseToOsIntervalMs: 2000"), std::string::npos);
  EXPECT_NE(Stats.find("DefaultReleaseToOsIntervalMs: 1500"),
            std::string::npos);
}

struct TestAllBoolFalseConfig {
  static const bool MaySupportMemoryTagging = false;
  static const bool QuarantineDisabled = false;
  static const bool ExactUsableSize = false;
  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 8U, 4U>;

  struct Primary {
    using SizeClassMap = scudo::AndroidSizeClassMap;
    static const scudo::uptr RegionSizeLog = 18U;
    static const scudo::uptr GroupSizeLog = 18U;
#if SCUDO_CAN_USE_PRIMARY64
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
#endif
    static const bool EnableBlockCache = false;
    static const scudo::uptr CompactPtrScale = 0;
    static const bool EnableRandomOffset = false;
    static const bool EnableContiguousRegions = false;
  };

#if SCUDO_CAN_USE_PRIMARY64
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;
#else
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator32<Config>;
#endif

  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };
  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

TEST(ScudoCombinedTest, VerifyAllBoolFalseConfig) {
  using AllocatorT = scudo::Allocator<TestAllBoolFalseConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  std::string Stats(10000, '\0');
  Allocator->getStats(Stats.data(), Stats.size());
  EXPECT_NE(Stats.find("MaySupportMemoryTagging: false"), std::string::npos);
  EXPECT_NE(Stats.find("QuarantineDisabled: false"), std::string::npos);
  EXPECT_NE(Stats.find("ExactUsableSize: false"), std::string::npos);
  EXPECT_NE(Stats.find("EnableBlockCache: false"), std::string::npos);
  EXPECT_NE(Stats.find("CompactPtrScale: 0"), std::string::npos);
  EXPECT_NE(Stats.find("EnableRandomOffset: false"), std::string::npos);
  EXPECT_NE(Stats.find("EnableContiguousRegions: false"), std::string::npos);
}

struct TestAllBoolTrueConfig {
  static const bool MaySupportMemoryTagging = true;
  static const bool QuarantineDisabled = true;
  static const bool ExactUsableSize = true;
  template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 8U, 4U>;

  struct Primary {
    using SizeClassMap = scudo::AndroidSizeClassMap;
    static const scudo::uptr RegionSizeLog = 18U;
    static const scudo::uptr GroupSizeLog = 18U;
#if SCUDO_CAN_USE_PRIMARY64
    static const scudo::uptr MapSizeIncrement = 1UL << 18;
#endif
    static const bool EnableBlockCache = true;
    static const scudo::uptr CompactPtrScale = 0;
    static const bool EnableRandomOffset = true;
    static const bool EnableContiguousRegions = true;
  };

#if SCUDO_CAN_USE_PRIMARY64
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator64<Config>;
#else
  template <typename Config>
  using PrimaryT = scudo::SizeClassAllocator32<Config>;
#endif

  struct Secondary {
    template <typename Config>
    using CacheT = scudo::MapAllocatorNoCache<Config>;
  };
  template <typename Config> using SecondaryT = scudo::MapAllocator<Config>;
};

TEST(ScudoCombinedTest, VerifyAllBoolTrueConfig) {
  using AllocatorT = scudo::Allocator<TestAllBoolTrueConfig>;
  auto Allocator = std::unique_ptr<AllocatorT>(new AllocatorT());

  std::string Stats(10000, '\0');
  Allocator->getStats(Stats.data(), Stats.size());
  EXPECT_NE(Stats.find("MaySupportMemoryTagging: true"), std::string::npos);
  EXPECT_NE(Stats.find("QuarantineDisabled: true"), std::string::npos);
  EXPECT_NE(Stats.find("ExactUsableSize: true"), std::string::npos);
  EXPECT_NE(Stats.find("EnableBlockCache: true"), std::string::npos);
  EXPECT_NE(Stats.find("CompactPtrScale: 0"), std::string::npos);
  EXPECT_NE(Stats.find("EnableRandomOffset: true"), std::string::npos);
  EXPECT_NE(Stats.find("EnableContiguousRegions: true"), std::string::npos);
}
