[scudo] Drain caches when release with M_PURGE_ALL
This will drain both quarantine and local caches.
Reviewed By: cferris
Differential Revision: https://reviews.llvm.org/D150242
GitOrigin-RevId: 6a057e7b51beaa54ebaece825ba1c87db0a7322d
diff --git a/combined.h b/combined.h
index d5365b6..0066056 100644
--- a/combined.h
+++ b/combined.h
@@ -239,6 +239,7 @@
}
TSDRegistryT *getTSDRegistry() { return &TSDRegistry; }
+ QuarantineT *getQuarantine() { return &Quarantine; }
// The Cache must be provided zero-initialized.
void initCache(CacheT *Cache) { Cache->init(&Stats, &Primary); }
@@ -254,6 +255,13 @@
TSD->getCache().destroy(&Stats);
}
+ void drainCache(TSD<ThisT> *TSD) {
+ Quarantine.drainAndRecycle(&TSD->getQuarantineCache(),
+ QuarantineCallback(*this, TSD->getCache()));
+ TSD->getCache().drain();
+ }
+ void drainCaches() { TSDRegistry.drainCaches(this); }
+
ALWAYS_INLINE void *getHeaderTaggedPointer(void *Ptr) {
if (!allocatorSupportsMemoryTagging<Params>())
return Ptr;
@@ -747,6 +755,8 @@
void releaseToOS(ReleaseToOS ReleaseType) {
initThreadMaybe();
+ if (ReleaseType == ReleaseToOS::ForceAll)
+ drainCaches();
Primary.releaseToOS(ReleaseType);
Secondary.releaseToOS();
}
diff --git a/quarantine.h b/quarantine.h
index e65a733..b5f8db0 100644
--- a/quarantine.h
+++ b/quarantine.h
@@ -192,6 +192,12 @@
uptr getMaxSize() const { return atomic_load_relaxed(&MaxSize); }
uptr getCacheSize() const { return atomic_load_relaxed(&MaxCacheSize); }
+ // This is supposed to be used in test only.
+ bool isEmpty() {
+ ScopedLock L(CacheMutex);
+ return Cache.getSize() == 0U;
+ }
+
void put(CacheT *C, Callback Cb, Node *Ptr, uptr Size) {
C->enqueue(Cb, Ptr, Size);
if (C->getSize() > getCacheSize())
diff --git a/tests/combined_test.cpp b/tests/combined_test.cpp
index 33a309e..44ba639 100644
--- a/tests/combined_test.cpp
+++ b/tests/combined_test.cpp
@@ -457,6 +457,28 @@
TSD->unlock();
}
+SCUDO_TYPED_TEST(ScudoCombinedTest, ForceCacheDrain) NO_THREAD_SAFETY_ANALYSIS {
+ auto *Allocator = this->Allocator.get();
+
+ std::vector<void *> V;
+ for (scudo::uptr I = 0; I < 64U; I++)
+ V.push_back(Allocator->allocate(
+ rand() % (TypeParam::Primary::SizeClassMap::MaxSize / 2U), Origin));
+ for (auto P : V)
+ Allocator->deallocate(P, Origin);
+
+ // `ForceAll` will also drain the caches.
+ Allocator->releaseToOS(scudo::ReleaseToOS::ForceAll);
+
+ bool UnlockRequired;
+ auto *TSD = Allocator->getTSDRegistry()->getTSDAndLock(&UnlockRequired);
+ EXPECT_TRUE(TSD->getCache().isEmpty());
+ EXPECT_EQ(TSD->getQuarantineCache().getSize(), 0U);
+ EXPECT_TRUE(Allocator->getQuarantine()->isEmpty());
+ if (UnlockRequired)
+ TSD->unlock();
+}
+
SCUDO_TYPED_TEST(ScudoCombinedTest, ThreadedCombined) {
std::mutex Mutex;
std::condition_variable Cv;
diff --git a/tsd_exclusive.h b/tsd_exclusive.h
index aca9fc9..9d03773 100644
--- a/tsd_exclusive.h
+++ b/tsd_exclusive.h
@@ -59,6 +59,13 @@
Initialized = false;
}
+ void drainCaches(Allocator *Instance) {
+ // We don't have a way to iterate all thread local `ThreadTSD`s. Simply
+ // drain the `ThreadTSD` of current thread and `FallbackTSD`.
+ Instance->drainCache(&ThreadTSD);
+ Instance->drainCache(&FallbackTSD);
+ }
+
ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) {
if (LIKELY(State.InitState != ThreadState::NotInitialized))
return;
diff --git a/tsd_shared.h b/tsd_shared.h
index e193281..dcb0948 100644
--- a/tsd_shared.h
+++ b/tsd_shared.h
@@ -54,6 +54,15 @@
Initialized = false;
}
+ void drainCaches(Allocator *Instance) {
+ ScopedLock L(MutexTSDs);
+ for (uptr I = 0; I < NumberOfTSDs; ++I) {
+ TSDs[I].lock();
+ Instance->drainCache(&TSDs[I]);
+ TSDs[I].unlock();
+ }
+ }
+
ALWAYS_INLINE void initThreadMaybe(Allocator *Instance,
UNUSED bool MinimalInit) {
if (LIKELY(getCurrentTSD()))
