|
150
|
1 //===-- guarded_pool_allocator.h --------------------------------*- C++ -*-===//
|
|
|
2 //
|
|
|
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
4 // See https://llvm.org/LICENSE.txt for license information.
|
|
|
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
6 //
|
|
|
7 //===----------------------------------------------------------------------===//
|
|
|
8
|
|
|
9 #ifndef GWP_ASAN_GUARDED_POOL_ALLOCATOR_H_
|
|
|
10 #define GWP_ASAN_GUARDED_POOL_ALLOCATOR_H_
|
|
|
11
|
|
|
12 #include "gwp_asan/common.h"
|
|
|
13 #include "gwp_asan/definitions.h"
|
|
|
14 #include "gwp_asan/mutex.h"
|
|
|
15 #include "gwp_asan/options.h"
|
|
221
|
16 #include "gwp_asan/platform_specific/guarded_pool_allocator_fuchsia.h" // IWYU pragma: keep
|
|
|
17 #include "gwp_asan/platform_specific/guarded_pool_allocator_posix.h" // IWYU pragma: keep
|
|
|
18 #include "gwp_asan/platform_specific/guarded_pool_allocator_tls.h"
|
|
150
|
19
|
|
|
20 #include <stddef.h>
|
|
|
21 #include <stdint.h>
|
|
221
|
22 // IWYU pragma: no_include <__stddef_max_align_t.h>
|
|
150
|
23
|
|
|
24 namespace gwp_asan {
|
|
|
25 // This class is the primary implementation of the allocator portion of GWP-
|
|
|
26 // ASan. It is the sole owner of the pool of sequentially allocated guarded
|
|
|
27 // slots. It should always be treated as a singleton.
|
|
|
28
|
|
|
29 // Functions in the public interface of this class are thread-compatible until
|
|
|
30 // init() is called, at which point they become thread-safe (unless specified
|
|
|
31 // otherwise).
|
|
|
32 class GuardedPoolAllocator {
|
|
|
33 public:
|
|
|
34 // Name of the GWP-ASan mapping that for `Metadata`.
|
|
|
35 static constexpr const char *kGwpAsanMetadataName = "GWP-ASan Metadata";
|
|
|
36
|
|
|
37 // During program startup, we must ensure that memory allocations do not land
|
|
|
38 // in this allocation pool if the allocator decides to runtime-disable
|
|
|
39 // GWP-ASan. The constructor value-initialises the class such that if no
|
|
|
40 // further initialisation takes place, calls to shouldSample() and
|
|
|
41 // pointerIsMine() will return false.
|
|
221
|
42 constexpr GuardedPoolAllocator() {}
|
|
150
|
43 GuardedPoolAllocator(const GuardedPoolAllocator &) = delete;
|
|
|
44 GuardedPoolAllocator &operator=(const GuardedPoolAllocator &) = delete;
|
|
|
45
|
|
|
46 // Note: This class is expected to be a singleton for the lifetime of the
|
|
|
47 // program. If this object is initialised, it will leak the guarded page pool
|
|
|
48 // and metadata allocations during destruction. We can't clean up these areas
|
|
|
49 // as this may cause a use-after-free on shutdown.
|
|
|
50 ~GuardedPoolAllocator() = default;
|
|
|
51
|
|
|
52 // Initialise the rest of the members of this class. Create the allocation
|
|
|
53 // pool using the provided options. See options.inc for runtime configuration
|
|
|
54 // options.
|
|
|
55 void init(const options::Options &Opts);
|
|
|
56 void uninitTestOnly();
|
|
|
57
|
|
|
58 // Functions exported for libmemunreachable's use on Android. disable()
|
|
|
59 // installs a lock in the allocator that prevents any thread from being able
|
|
|
60 // to allocate memory, until enable() is called.
|
|
|
61 void disable();
|
|
|
62 void enable();
|
|
|
63
|
|
|
64 typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg);
|
|
|
65 // Execute the callback Cb for every allocation the lies in [Base, Base +
|
|
|
66 // Size). Must be called while the allocator is disabled. The callback can not
|
|
|
67 // allocate.
|
|
|
68 void iterate(void *Base, size_t Size, iterate_callback Cb, void *Arg);
|
|
|
69
|
|
|
70 // Return whether the allocation should be randomly chosen for sampling.
|
|
|
71 GWP_ASAN_ALWAYS_INLINE bool shouldSample() {
|
|
|
72 // NextSampleCounter == 0 means we "should regenerate the counter".
|
|
|
73 // == 1 means we "should sample this allocation".
|
|
|
74 // AdjustedSampleRatePlusOne is designed to intentionally underflow. This
|
|
|
75 // class must be valid when zero-initialised, and we wish to sample as
|
|
|
76 // infrequently as possible when this is the case, hence we underflow to
|
|
|
77 // UINT32_MAX.
|
|
221
|
78 if (GWP_ASAN_UNLIKELY(getThreadLocals()->NextSampleCounter == 0))
|
|
|
79 getThreadLocals()->NextSampleCounter =
|
|
|
80 ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) &
|
|
|
81 ThreadLocalPackedVariables::NextSampleCounterMask;
|
|
150
|
82
|
|
221
|
83 return GWP_ASAN_UNLIKELY(--getThreadLocals()->NextSampleCounter == 0);
|
|
150
|
84 }
|
|
|
85
|
|
|
86 // Returns whether the provided pointer is a current sampled allocation that
|
|
|
87 // is owned by this pool.
|
|
|
88 GWP_ASAN_ALWAYS_INLINE bool pointerIsMine(const void *Ptr) const {
|
|
|
89 return State.pointerIsMine(Ptr);
|
|
|
90 }
|
|
|
91
|
|
221
|
92 // Allocate memory in a guarded slot, with the specified `Alignment`. Returns
|
|
|
93 // nullptr if the pool is empty, if the alignnment is not a power of two, or
|
|
|
94 // if the size/alignment makes the allocation too large for this pool to
|
|
|
95 // handle. By default, uses strong alignment (i.e. `max_align_t`), see
|
|
|
96 // http://www.open-std.org/jtc1/sc22/wg14/www/docs/n2293.htm for discussion of
|
|
|
97 // alignment issues in the standard.
|
|
|
98 void *allocate(size_t Size, size_t Alignment = alignof(max_align_t));
|
|
150
|
99
|
|
|
100 // Deallocate memory in a guarded slot. The provided pointer must have been
|
|
|
101 // allocated using this pool. This will set the guarded slot as inaccessible.
|
|
|
102 void deallocate(void *Ptr);
|
|
|
103
|
|
|
104 // Returns the size of the allocation at Ptr.
|
|
|
105 size_t getSize(const void *Ptr);
|
|
|
106
|
|
|
107 // Returns a pointer to the Metadata region, or nullptr if it doesn't exist.
|
|
|
108 const AllocationMetadata *getMetadataRegion() const { return Metadata; }
|
|
|
109
|
|
|
110 // Returns a pointer to the AllocatorState region.
|
|
|
111 const AllocatorState *getAllocatorState() const { return &State; }
|
|
|
112
|
|
252
|
113 // Functions that the signal handler is responsible for calling, while
|
|
|
114 // providing the SEGV pointer, prior to dumping the crash, and after dumping
|
|
|
115 // the crash (in recoverable mode only).
|
|
|
116 void preCrashReport(void *Ptr);
|
|
|
117 void postCrashReportRecoverableOnly(void *Ptr);
|
|
|
118
|
|
221
|
119 // Exposed as protected for testing.
|
|
|
120 protected:
|
|
|
121 // Returns the actual allocation size required to service an allocation with
|
|
|
122 // the provided Size and Alignment.
|
|
|
123 static size_t getRequiredBackingSize(size_t Size, size_t Alignment,
|
|
|
124 size_t PageSize);
|
|
|
125
|
|
|
126 // Returns the provided pointer that meets the specified alignment, depending
|
|
|
127 // on whether it's left or right aligned.
|
|
|
128 static uintptr_t alignUp(uintptr_t Ptr, size_t Alignment);
|
|
|
129 static uintptr_t alignDown(uintptr_t Ptr, size_t Alignment);
|
|
|
130
|
|
150
|
131 private:
|
|
|
132 // Name of actively-occupied slot mappings.
|
|
|
133 static constexpr const char *kGwpAsanAliveSlotName = "GWP-ASan Alive Slot";
|
|
|
134 // Name of the guard pages. This includes all slots that are not actively in
|
|
|
135 // use (i.e. were never used, or have been free()'d).)
|
|
|
136 static constexpr const char *kGwpAsanGuardPageName = "GWP-ASan Guard Page";
|
|
|
137 // Name of the mapping for `FreeSlots`.
|
|
|
138 static constexpr const char *kGwpAsanFreeSlotsName = "GWP-ASan Metadata";
|
|
|
139
|
|
|
140 static constexpr size_t kInvalidSlotID = SIZE_MAX;
|
|
|
141
|
|
|
142 // These functions anonymously map memory or change the permissions of mapped
|
|
|
143 // memory into this process in a platform-specific way. Pointer and size
|
|
|
144 // arguments are expected to be page-aligned. These functions will never
|
|
|
145 // return on error, instead electing to kill the calling process on failure.
|
|
221
|
146 // The pool memory is initially reserved and inaccessible, and RW mappings are
|
|
|
147 // subsequently created and destroyed via allocateInGuardedPool() and
|
|
|
148 // deallocateInGuardedPool(). Each mapping is named on platforms that support
|
|
|
149 // it, primarily Android. This name must be a statically allocated string, as
|
|
|
150 // the Android kernel uses the string pointer directly.
|
|
|
151 void *map(size_t Size, const char *Name) const;
|
|
|
152 void unmap(void *Ptr, size_t Size) const;
|
|
|
153
|
|
|
154 // The pool is managed separately, as some platforms (particularly Fuchsia)
|
|
|
155 // manage virtual memory regions as a chunk where individual pages can still
|
|
|
156 // have separate permissions. These platforms maintain metadata about the
|
|
|
157 // region in order to perform operations. The pool is unique as it's the only
|
|
|
158 // thing in GWP-ASan that treats pages in a single VM region on an individual
|
|
|
159 // basis for page protection.
|
|
|
160 // The pointer returned by reserveGuardedPool() is the reserved address range
|
|
|
161 // of (at least) Size bytes.
|
|
|
162 void *reserveGuardedPool(size_t Size);
|
|
|
163 // allocateInGuardedPool() Ptr and Size must be a subrange of the previously
|
|
|
164 // reserved pool range.
|
|
|
165 void allocateInGuardedPool(void *Ptr, size_t Size) const;
|
|
|
166 // deallocateInGuardedPool() Ptr and Size must be an exact pair previously
|
|
|
167 // passed to allocateInGuardedPool().
|
|
|
168 void deallocateInGuardedPool(void *Ptr, size_t Size) const;
|
|
|
169 void unreserveGuardedPool();
|
|
150
|
170
|
|
|
171 // Get the page size from the platform-specific implementation. Only needs to
|
|
|
172 // be called once, and the result should be cached in PageSize in this class.
|
|
|
173 static size_t getPlatformPageSize();
|
|
|
174
|
|
|
175 // Returns a pointer to the metadata for the owned pointer. If the pointer is
|
|
|
176 // not owned by this pool, the result is undefined.
|
|
|
177 AllocationMetadata *addrToMetadata(uintptr_t Ptr) const;
|
|
|
178
|
|
|
179 // Reserve a slot for a new guarded allocation. Returns kInvalidSlotID if no
|
|
|
180 // slot is available to be reserved.
|
|
|
181 size_t reserveSlot();
|
|
|
182
|
|
|
183 // Unreserve the guarded slot.
|
|
|
184 void freeSlot(size_t SlotIndex);
|
|
|
185
|
|
|
186 // Raise a SEGV and set the corresponding fields in the Allocator's State in
|
|
|
187 // order to tell the crash handler what happened. Used when errors are
|
|
|
188 // detected internally (Double Free, Invalid Free).
|
|
252
|
189 void raiseInternallyDetectedError(uintptr_t Address, Error E);
|
|
150
|
190
|
|
|
191 static GuardedPoolAllocator *getSingleton();
|
|
|
192
|
|
|
193 // Install a pthread_atfork handler.
|
|
|
194 void installAtFork();
|
|
|
195
|
|
|
196 gwp_asan::AllocatorState State;
|
|
|
197
|
|
|
198 // A mutex to protect the guarded slot and metadata pool for this class.
|
|
|
199 Mutex PoolMutex;
|
|
221
|
200 // Some unwinders can grab the libdl lock. In order to provide atfork
|
|
|
201 // protection, we need to ensure that we allow an unwinding thread to release
|
|
|
202 // the libdl lock before forking.
|
|
|
203 Mutex BacktraceMutex;
|
|
150
|
204 // Record the number allocations that we've sampled. We store this amount so
|
|
|
205 // that we don't randomly choose to recycle a slot that previously had an
|
|
|
206 // allocation before all the slots have been utilised.
|
|
|
207 size_t NumSampledAllocations = 0;
|
|
|
208 // Pointer to the allocation metadata (allocation/deallocation stack traces),
|
|
|
209 // if any.
|
|
|
210 AllocationMetadata *Metadata = nullptr;
|
|
|
211
|
|
|
212 // Pointer to an array of free slot indexes.
|
|
|
213 size_t *FreeSlots = nullptr;
|
|
|
214 // The current length of the list of free slots.
|
|
|
215 size_t FreeSlotsLength = 0;
|
|
|
216
|
|
|
217 // See options.{h, inc} for more information.
|
|
|
218 bool PerfectlyRightAlign = false;
|
|
|
219
|
|
|
220 // Backtrace function provided by the supporting allocator. See `options.h`
|
|
|
221 // for more information.
|
|
|
222 options::Backtrace_t Backtrace = nullptr;
|
|
|
223
|
|
|
224 // The adjusted sample rate for allocation sampling. Default *must* be
|
|
|
225 // nonzero, as dynamic initialisation may call malloc (e.g. from libstdc++)
|
|
|
226 // before GPA::init() is called. This would cause an error in shouldSample(),
|
|
|
227 // where we would calculate modulo zero. This value is set UINT32_MAX, as when
|
|
|
228 // GWP-ASan is disabled, we wish to never spend wasted cycles recalculating
|
|
|
229 // the sample rate.
|
|
|
230 uint32_t AdjustedSampleRatePlusOne = 0;
|
|
|
231
|
|
221
|
232 // Additional platform specific data structure for the guarded pool mapping.
|
|
|
233 PlatformSpecificMapData GuardedPagePoolPlatformData = {};
|
|
|
234
|
|
|
235 class ScopedRecursiveGuard {
|
|
|
236 public:
|
|
|
237 ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = true; }
|
|
|
238 ~ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = false; }
|
|
150
|
239 };
|
|
221
|
240
|
|
|
241 // Initialise the PRNG, platform-specific.
|
|
|
242 void initPRNG();
|
|
|
243
|
|
|
244 // xorshift (32-bit output), extremely fast PRNG that uses arithmetic
|
|
|
245 // operations only. Seeded using platform-specific mechanisms by initPRNG().
|
|
|
246 uint32_t getRandomUnsigned32();
|
|
150
|
247 };
|
|
|
248 } // namespace gwp_asan
|
|
|
249
|
|
|
250 #endif // GWP_ASAN_GUARDED_POOL_ALLOCATOR_H_
|
