Author: Peter Collingbourne Date: 2020-12-22T14:54:40-08:00 New Revision: 6dfe5801e01d259d00198147b27438ffea39e59f
URL: https://github.com/llvm/llvm-project/commit/6dfe5801e01d259d00198147b27438ffea39e59f DIFF: https://github.com/llvm/llvm-project/commit/6dfe5801e01d259d00198147b27438ffea39e59f.diff LOG: scudo: Move the configuration for the primary allocator to Config. NFCI. This will allow the primary and secondary allocators to share the MaySupportMemoryTagging bool. Differential Revision: https://reviews.llvm.org/D93728 Added: Modified: compiler-rt/lib/scudo/standalone/allocator_config.h compiler-rt/lib/scudo/standalone/memtag.h compiler-rt/lib/scudo/standalone/primary32.h compiler-rt/lib/scudo/standalone/primary64.h compiler-rt/lib/scudo/standalone/tests/combined_test.cpp compiler-rt/lib/scudo/standalone/tests/primary_test.cpp Removed: ################################################################################ diff --git a/compiler-rt/lib/scudo/standalone/allocator_config.h b/compiler-rt/lib/scudo/standalone/allocator_config.h index 8f1757dab322..12daaa2f6b44 100644 --- a/compiler-rt/lib/scudo/standalone/allocator_config.h +++ b/compiler-rt/lib/scudo/standalone/allocator_config.h @@ -25,13 +25,18 @@ namespace scudo { struct DefaultConfig { using SizeClassMap = DefaultSizeClassMap; + static const bool MaySupportMemoryTagging = false; + #if SCUDO_CAN_USE_PRIMARY64 - // 1GB Regions - typedef SizeClassAllocator64<SizeClassMap, 30U> Primary; + typedef SizeClassAllocator64<DefaultConfig> Primary; + static const uptr PrimaryRegionSizeLog = 30U; #else - // 512KB regions - typedef SizeClassAllocator32<SizeClassMap, 19U> Primary; + typedef SizeClassAllocator32<DefaultConfig> Primary; + static const uptr PrimaryRegionSizeLog = 19U; #endif + static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + typedef MapAllocatorCache<DefaultConfig> SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 32U; static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; @@ -44,15 +49,18 @@ struct DefaultConfig { struct AndroidConfig { using SizeClassMap = AndroidSizeClassMap; + static const bool MaySupportMemoryTagging = true; + #if SCUDO_CAN_USE_PRIMARY64 - // 256MB regions - typedef SizeClassAllocator64<SizeClassMap, 28U, 1000, 1000, - /*MaySupportMemoryTagging=*/true> - Primary; + typedef SizeClassAllocator64<AndroidConfig> Primary; + static const uptr PrimaryRegionSizeLog = 28U; #else - // 256KB regions - typedef SizeClassAllocator32<SizeClassMap, 18U, 1000, 1000> Primary; + typedef SizeClassAllocator32<AndroidConfig> Primary; + static const uptr PrimaryRegionSizeLog = 18U; #endif + static const s32 PrimaryMinReleaseToOsIntervalMs = 1000; + static const s32 PrimaryMaxReleaseToOsIntervalMs = 1000; + typedef MapAllocatorCache<AndroidConfig> SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 256U; static const u32 SecondaryCacheDefaultMaxEntriesCount = 32U; @@ -66,13 +74,18 @@ struct AndroidConfig { struct AndroidSvelteConfig { using SizeClassMap = SvelteSizeClassMap; + static const bool MaySupportMemoryTagging = false; + #if SCUDO_CAN_USE_PRIMARY64 - // 128MB regions - typedef SizeClassAllocator64<SizeClassMap, 27U, 1000, 1000> Primary; + typedef SizeClassAllocator64<AndroidSvelteConfig> Primary; + static const uptr PrimaryRegionSizeLog = 27U; #else - // 64KB regions - typedef SizeClassAllocator32<SizeClassMap, 16U, 1000, 1000> Primary; + typedef SizeClassAllocator32<AndroidSvelteConfig> Primary; + static const uptr PrimaryRegionSizeLog = 16U; #endif + static const s32 PrimaryMinReleaseToOsIntervalMs = 1000; + static const s32 PrimaryMaxReleaseToOsIntervalMs = 1000; + typedef MapAllocatorCache<AndroidSvelteConfig> SecondaryCache; static const u32 SecondaryCacheEntriesArraySize = 16U; static const u32 SecondaryCacheDefaultMaxEntriesCount = 4U; @@ -86,8 +99,14 @@ struct AndroidSvelteConfig { #if SCUDO_CAN_USE_PRIMARY64 struct FuchsiaConfig { - // 1GB Regions - typedef SizeClassAllocator64<DefaultSizeClassMap, 30U> Primary; + using SizeClassMap = DefaultSizeClassMap; + static const bool MaySupportMemoryTagging = false; + + typedef SizeClassAllocator64<FuchsiaConfig> Primary; + static const uptr PrimaryRegionSizeLog = 30U; + static const s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + typedef MapAllocatorNoCache SecondaryCache; template <class A> using TSDRegistryT = TSDRegistrySharedT<A, 8U, 4U>; // Shared, max 8 TSDs. diff --git a/compiler-rt/lib/scudo/standalone/memtag.h b/compiler-rt/lib/scudo/standalone/memtag.h index c3c4f574b4fc..d848e3a0b6c9 100644 --- a/compiler-rt/lib/scudo/standalone/memtag.h +++ b/compiler-rt/lib/scudo/standalone/memtag.h @@ -268,6 +268,11 @@ inline void setRandomTag(void *Ptr, uptr Size, uptr ExcludeMask, *TaggedEnd = storeTags(*TaggedBegin, *TaggedBegin + Size); } +template <typename Config> +inline constexpr bool allocatorSupportsMemoryTagging() { + return archSupportsMemoryTagging() && Config::MaySupportMemoryTagging; +} + } // namespace scudo #endif diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h index 0db95d2e1f11..c744670b4392 100644 --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -39,20 +39,15 @@ namespace scudo { // Memory used by this allocator is never unmapped but can be partially // reclaimed if the platform allows for it. -template <class SizeClassMapT, uptr RegionSizeLog, - s32 MinReleaseToOsIntervalMs = INT32_MIN, - s32 MaxReleaseToOsIntervalMs = INT32_MAX> -class SizeClassAllocator32 { +template <typename Config> class SizeClassAllocator32 { public: - typedef SizeClassMapT SizeClassMap; + typedef typename Config::SizeClassMap SizeClassMap; // The bytemap can only track UINT8_MAX - 1 classes. static_assert(SizeClassMap::LargestClassId <= (UINT8_MAX - 1), ""); // Regions should be large enough to hold the largest Block. - static_assert((1UL << RegionSizeLog) >= SizeClassMap::MaxSize, ""); - typedef SizeClassAllocator32<SizeClassMapT, RegionSizeLog, - MinReleaseToOsIntervalMs, - MaxReleaseToOsIntervalMs> - ThisT; + static_assert((1UL << Config::PrimaryRegionSizeLog) >= SizeClassMap::MaxSize, + ""); + typedef SizeClassAllocator32<Config> ThisT; typedef SizeClassAllocatorLocalCache<ThisT> CacheT; typedef typename CacheT::TransferBatch TransferBatch; static const bool SupportsMemoryTagging = false; @@ -199,9 +194,9 @@ class SizeClassAllocator32 { bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { - const s32 Interval = - Max(Min(static_cast<s32>(Value), MaxReleaseToOsIntervalMs), - MinReleaseToOsIntervalMs); + const s32 Interval = Max( + Min(static_cast<s32>(Value), Config::PrimaryMaxReleaseToOsIntervalMs), + Config::PrimaryMinReleaseToOsIntervalMs); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } @@ -236,8 +231,9 @@ class SizeClassAllocator32 { private: static const uptr NumClasses = SizeClassMap::NumClasses; - static const uptr RegionSize = 1UL << RegionSizeLog; - static const uptr NumRegions = SCUDO_MMAP_RANGE_SIZE >> RegionSizeLog; + static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog; + static const uptr NumRegions = + SCUDO_MMAP_RANGE_SIZE >> Config::PrimaryRegionSizeLog; static const u32 MaxNumBatches = SCUDO_ANDROID ? 4U : 8U; typedef FlatByteMap<NumRegions> ByteMap; @@ -270,7 +266,7 @@ class SizeClassAllocator32 { static_assert(sizeof(SizeClassInfo) % SCUDO_CACHE_LINE_SIZE == 0, ""); uptr computeRegionId(uptr Mem) { - const uptr Id = Mem >> RegionSizeLog; + const uptr Id = Mem >> Config::PrimaryRegionSizeLog; CHECK_LT(Id, NumRegions); return Id; } diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h index f9854cbfd4d6..df1310aa8e95 100644 --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -40,21 +40,14 @@ namespace scudo { // The memory used by this allocator is never unmapped, but can be partially // released if the platform allows for it. -template <class SizeClassMapT, uptr RegionSizeLog, - s32 MinReleaseToOsIntervalMs = INT32_MIN, - s32 MaxReleaseToOsIntervalMs = INT32_MAX, - bool MaySupportMemoryTagging = false> -class SizeClassAllocator64 { +template <typename Config> class SizeClassAllocator64 { public: - typedef SizeClassMapT SizeClassMap; - typedef SizeClassAllocator64< - SizeClassMap, RegionSizeLog, MinReleaseToOsIntervalMs, - MaxReleaseToOsIntervalMs, MaySupportMemoryTagging> - ThisT; + typedef typename Config::SizeClassMap SizeClassMap; + typedef SizeClassAllocator64<Config> ThisT; typedef SizeClassAllocatorLocalCache<ThisT> CacheT; typedef typename CacheT::TransferBatch TransferBatch; static const bool SupportsMemoryTagging = - MaySupportMemoryTagging && archSupportsMemoryTagging(); + allocatorSupportsMemoryTagging<Config>(); static uptr getSizeByClassId(uptr ClassId) { return (ClassId == SizeClassMap::BatchClassId) @@ -178,9 +171,9 @@ class SizeClassAllocator64 { bool setOption(Option O, sptr Value) { if (O == Option::ReleaseInterval) { - const s32 Interval = - Max(Min(static_cast<s32>(Value), MaxReleaseToOsIntervalMs), - MinReleaseToOsIntervalMs); + const s32 Interval = Max( + Min(static_cast<s32>(Value), Config::PrimaryMaxReleaseToOsIntervalMs), + Config::PrimaryMinReleaseToOsIntervalMs); atomic_store_relaxed(&ReleaseToOsIntervalMs, Interval); return true; } @@ -258,7 +251,7 @@ class SizeClassAllocator64 { AtomicOptions Options; private: - static const uptr RegionSize = 1UL << RegionSizeLog; + static const uptr RegionSize = 1UL << Config::PrimaryRegionSizeLog; static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr PrimarySize = RegionSize * NumClasses; @@ -308,7 +301,7 @@ class SizeClassAllocator64 { } uptr getRegionBaseByClassId(uptr ClassId) const { - return PrimaryBase + (ClassId << RegionSizeLog); + return PrimaryBase + (ClassId << Config::PrimaryRegionSizeLog); } NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId, diff --git a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp index b0ab0244e877..53874933e7ac 100644 --- a/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/combined_test.cpp @@ -401,10 +401,15 @@ struct DeathSizeClassConfig { static const scudo::uptr DeathRegionSizeLog = 20U; struct DeathConfig { + static const bool MaySupportMemoryTagging = false; + // Tiny allocator, its Primary only serves chunks of four sizes. - using DeathSizeClassMap = scudo::FixedSizeClassMap<DeathSizeClassConfig>; - typedef scudo::SizeClassAllocator64<DeathSizeClassMap, DeathRegionSizeLog> - Primary; + using SizeClassMap = scudo::FixedSizeClassMap<DeathSizeClassConfig>; + typedef scudo::SizeClassAllocator64<DeathConfig> Primary; + static const scudo::uptr PrimaryRegionSizeLog = DeathRegionSizeLog; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + typedef scudo::MapAllocatorNoCache SecondaryCache; template <class A> using TSDRegistryT = scudo::TSDRegistrySharedT<A, 1U, 1U>; }; @@ -460,13 +465,13 @@ TEST(ScudoCombinedTest, FullRegion) { std::vector<void *> V; scudo::uptr FailedAllocationsCount = 0; for (scudo::uptr ClassId = 1U; - ClassId <= DeathConfig::DeathSizeClassMap::LargestClassId; ClassId++) { + ClassId <= DeathConfig::SizeClassMap::LargestClassId; ClassId++) { const scudo::uptr Size = - DeathConfig::DeathSizeClassMap::getSizeByClassId(ClassId); + DeathConfig::SizeClassMap::getSizeByClassId(ClassId); // Allocate enough to fill all of the regions above this one. const scudo::uptr MaxNumberOfChunks = ((1U << DeathRegionSizeLog) / Size) * - (DeathConfig::DeathSizeClassMap::LargestClassId - ClassId + 1); + (DeathConfig::SizeClassMap::LargestClassId - ClassId + 1); void *P; for (scudo::uptr I = 0; I <= MaxNumberOfChunks; I++) { P = Allocator->allocate(Size - 64U, Origin); diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp index 67d1fe52acef..eed64314cc3d 100644 --- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp @@ -52,20 +52,51 @@ template <typename Primary> static void testPrimary() { Str.output(); } +template <typename SizeClassMapT> struct TestConfig1 { + using SizeClassMap = SizeClassMapT; + static const scudo::uptr PrimaryRegionSizeLog = 18U; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + static const bool MaySupportMemoryTagging = false; +}; + +template <typename SizeClassMapT> struct TestConfig2 { + using SizeClassMap = SizeClassMapT; + static const scudo::uptr PrimaryRegionSizeLog = 24U; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + static const bool MaySupportMemoryTagging = false; +}; + +template <typename SizeClassMapT> struct TestConfig3 { + using SizeClassMap = SizeClassMapT; + static const scudo::uptr PrimaryRegionSizeLog = 24U; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + static const bool MaySupportMemoryTagging = true; +}; + TEST(ScudoPrimaryTest, BasicPrimary) { using SizeClassMap = scudo::DefaultSizeClassMap; #if !SCUDO_FUCHSIA - testPrimary<scudo::SizeClassAllocator32<SizeClassMap, 18U>>(); + testPrimary<scudo::SizeClassAllocator32<TestConfig1<SizeClassMap>>>(); #endif - testPrimary<scudo::SizeClassAllocator64<SizeClassMap, 24U>>(); - testPrimary<scudo::SizeClassAllocator64<SizeClassMap, 24U, INT32_MIN, - INT32_MAX, true>>(); + testPrimary<scudo::SizeClassAllocator64<TestConfig2<SizeClassMap>>>(); + testPrimary<scudo::SizeClassAllocator64<TestConfig3<SizeClassMap>>>(); } +struct SmallRegionsConfig { + using SizeClassMap = scudo::DefaultSizeClassMap; + static const scudo::uptr PrimaryRegionSizeLog = 20U; + static const scudo::s32 PrimaryMinReleaseToOsIntervalMs = INT32_MIN; + static const scudo::s32 PrimaryMaxReleaseToOsIntervalMs = INT32_MAX; + static const bool MaySupportMemoryTagging = false; +}; + // The 64-bit SizeClassAllocator can be easily OOM'd with small region sizes. // For the 32-bit one, it requires actually exhausting memory, so we skip it. TEST(ScudoPrimaryTest, Primary64OOM) { - using Primary = scudo::SizeClassAllocator64<scudo::DefaultSizeClassMap, 20U>; + using Primary = scudo::SizeClassAllocator64<SmallRegionsConfig>; using TransferBatch = Primary::CacheT::TransferBatch; Primary Allocator; Allocator.init(/*ReleaseToOsInterval=*/-1); @@ -142,11 +173,10 @@ template <typename Primary> static void testIteratePrimary() { TEST(ScudoPrimaryTest, PrimaryIterate) { using SizeClassMap = scudo::DefaultSizeClassMap; #if !SCUDO_FUCHSIA - testIteratePrimary<scudo::SizeClassAllocator32<SizeClassMap, 18U>>(); + testIteratePrimary<scudo::SizeClassAllocator32<TestConfig1<SizeClassMap>>>(); #endif - testIteratePrimary<scudo::SizeClassAllocator64<SizeClassMap, 24U>>(); - testIteratePrimary<scudo::SizeClassAllocator64<SizeClassMap, 24U, INT32_MIN, - INT32_MAX, true>>(); + testIteratePrimary<scudo::SizeClassAllocator64<TestConfig2<SizeClassMap>>>(); + testIteratePrimary<scudo::SizeClassAllocator64<TestConfig3<SizeClassMap>>>(); } static std::mutex Mutex; @@ -204,11 +234,10 @@ template <typename Primary> static void testPrimaryThreaded() { TEST(ScudoPrimaryTest, PrimaryThreaded) { using SizeClassMap = scudo::SvelteSizeClassMap; #if !SCUDO_FUCHSIA - testPrimaryThreaded<scudo::SizeClassAllocator32<SizeClassMap, 18U>>(); + testPrimaryThreaded<scudo::SizeClassAllocator32<TestConfig1<SizeClassMap>>>(); #endif - testPrimaryThreaded<scudo::SizeClassAllocator64<SizeClassMap, 24U>>(); - testPrimaryThreaded<scudo::SizeClassAllocator64<SizeClassMap, 24U, INT32_MIN, - INT32_MAX, true>>(); + testPrimaryThreaded<scudo::SizeClassAllocator64<TestConfig2<SizeClassMap>>>(); + testPrimaryThreaded<scudo::SizeClassAllocator64<TestConfig3<SizeClassMap>>>(); } // Through a simple allocation that spans two pages, verify that releaseToOS @@ -236,9 +265,8 @@ template <typename Primary> static void testReleaseToOS() { TEST(ScudoPrimaryTest, ReleaseToOS) { using SizeClassMap = scudo::DefaultSizeClassMap; #if !SCUDO_FUCHSIA - testReleaseToOS<scudo::SizeClassAllocator32<SizeClassMap, 18U>>(); + testReleaseToOS<scudo::SizeClassAllocator32<TestConfig1<SizeClassMap>>>(); #endif - testReleaseToOS<scudo::SizeClassAllocator64<SizeClassMap, 24U>>(); - testReleaseToOS<scudo::SizeClassAllocator64<SizeClassMap, 24U, INT32_MIN, - INT32_MAX, true>>(); + testReleaseToOS<scudo::SizeClassAllocator64<TestConfig2<SizeClassMap>>>(); + testReleaseToOS<scudo::SizeClassAllocator64<TestConfig3<SizeClassMap>>>(); } _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
