|
150
|
1 //===-- guarded_pool_allocator.h --------------------------------*- C++ -*-===//
|
|
|
2 //
|
|
|
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
4 // See https://llvm.org/LICENSE.txt for license information.
|
|
|
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
6 //
|
|
|
7 //===----------------------------------------------------------------------===//
|
|
|
8
|
|
|
9 #ifndef GWP_ASAN_GUARDED_POOL_ALLOCATOR_H_
|
|
|
10 #define GWP_ASAN_GUARDED_POOL_ALLOCATOR_H_
|
|
|
11
|
|
|
12 #include "gwp_asan/common.h"
|
|
|
13 #include "gwp_asan/definitions.h"
|
|
|
14 #include "gwp_asan/mutex.h"
|
|
|
15 #include "gwp_asan/options.h"
|
|
207
|
16 #include "gwp_asan/platform_specific/guarded_pool_allocator_fuchsia.h" // IWYU pragma: keep
|
|
|
17 #include "gwp_asan/platform_specific/guarded_pool_allocator_posix.h" // IWYU pragma: keep
|
|
|
18 #include "gwp_asan/platform_specific/guarded_pool_allocator_tls.h"
|
|
150
|
19
|
|
|
20 #include <stddef.h>
|
|
|
21 #include <stdint.h>
|
|
207
|
22 // IWYU pragma: no_include <__stddef_max_align_t.h>
|
|
150
|
23
|
|
|
24 namespace gwp_asan {
|
|
|
25 // This class is the primary implementation of the allocator portion of GWP-
|
|
|
26 // ASan. It is the sole owner of the pool of sequentially allocated guarded
|
|
|
27 // slots. It should always be treated as a singleton.
|
|
|
28
|
|
|
29 // Functions in the public interface of this class are thread-compatible until
|
|
|
30 // init() is called, at which point they become thread-safe (unless specified
|
|
|
31 // otherwise).
|
|
|
32 class GuardedPoolAllocator {
|
|
|
33 public:
|
|
|
34 // Name of the GWP-ASan mapping that for `Metadata`.
|
|
|
35 static constexpr const char *kGwpAsanMetadataName = "GWP-ASan Metadata";
|
|
|
36
|
|
|
37 // During program startup, we must ensure that memory allocations do not land
|
|
|
38 // in this allocation pool if the allocator decides to runtime-disable
|
|
|
39 // GWP-ASan. The constructor value-initialises the class such that if no
|
|
|
40 // further initialisation takes place, calls to shouldSample() and
|
|
|
41 // pointerIsMine() will return false.
|
|
207
|
42 constexpr GuardedPoolAllocator() {}
|
|
150
|
43 GuardedPoolAllocator(const GuardedPoolAllocator &) = delete;
|
|
|
44 GuardedPoolAllocator &operator=(const GuardedPoolAllocator &) = delete;
|
|
|
45
|
|
|
46 // Note: This class is expected to be a singleton for the lifetime of the
|
|
|
47 // program. If this object is initialised, it will leak the guarded page pool
|
|
|
48 // and metadata allocations during destruction. We can't clean up these areas
|
|
|
49 // as this may cause a use-after-free on shutdown.
|
|
|
50 ~GuardedPoolAllocator() = default;
|
|
|
51
|
|
|
52 // Initialise the rest of the members of this class. Create the allocation
|
|
|
53 // pool using the provided options. See options.inc for runtime configuration
|
|
|
54 // options.
|
|
|
55 void init(const options::Options &Opts);
|
|
|
56 void uninitTestOnly();
|
|
|
57
|
|
|
58 // Functions exported for libmemunreachable's use on Android. disable()
|
|
|
59 // installs a lock in the allocator that prevents any thread from being able
|
|
|
60 // to allocate memory, until enable() is called.
|
|
|
61 void disable();
|
|
|
62 void enable();
|
|
|
63
|
|
|
64 typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg);
|
|
|
65 // Execute the callback Cb for every allocation the lies in [Base, Base +
|
|
|
66 // Size). Must be called while the allocator is disabled. The callback can not
|
|
|
67 // allocate.
|
|
|
68 void iterate(void *Base, size_t Size, iterate_callback Cb, void *Arg);
|
|
|
69
|
|
|
70 // This function is used to signal the allocator to indefinitely stop
|
|
|
71 // functioning, as a crash has occurred. This stops the allocator from
|
|
|
72 // servicing any further allocations permanently.
|
|
|
73 void stop();
|
|
|
74
|
|
|
75 // Return whether the allocation should be randomly chosen for sampling.
|
|
|
76 GWP_ASAN_ALWAYS_INLINE bool shouldSample() {
|
|
|
77 // NextSampleCounter == 0 means we "should regenerate the counter".
|
|
|
78 // == 1 means we "should sample this allocation".
|
|
|
79 // AdjustedSampleRatePlusOne is designed to intentionally underflow. This
|
|
|
80 // class must be valid when zero-initialised, and we wish to sample as
|
|
|
81 // infrequently as possible when this is the case, hence we underflow to
|
|
|
82 // UINT32_MAX.
|
|
207
|
83 if (GWP_ASAN_UNLIKELY(getThreadLocals()->NextSampleCounter == 0))
|
|
|
84 getThreadLocals()->NextSampleCounter =
|
|
|
85 ((getRandomUnsigned32() % (AdjustedSampleRatePlusOne - 1)) + 1) &
|
|
|
86 ThreadLocalPackedVariables::NextSampleCounterMask;
|
|
150
|
87
|
|
207
|
88 return GWP_ASAN_UNLIKELY(--getThreadLocals()->NextSampleCounter == 0);
|
|
150
|
89 }
|
|
|
90
|
|
|
91 // Returns whether the provided pointer is a current sampled allocation that
|
|
|
92 // is owned by this pool.
|
|
|
93 GWP_ASAN_ALWAYS_INLINE bool pointerIsMine(const void *Ptr) const {
|
|
|
94 return State.pointerIsMine(Ptr);
|
|
|
95 }
|
|
|
96
|
|
207
|
97 // Allocate memory in a guarded slot, with the specified `Alignment`. Returns
|
|
|
98 // nullptr if the pool is empty, if the alignnment is not a power of two, or
|
|
|
99 // if the size/alignment makes the allocation too large for this pool to
|
|
|
100 // handle. By default, uses strong alignment (i.e. `max_align_t`), see
|
|
|
101 // http://www.open-std.org/jtc1/sc22/wg14/www/docs/n2293.htm for discussion of
|
|
|
102 // alignment issues in the standard.
|
|
|
103 void *allocate(size_t Size, size_t Alignment = alignof(max_align_t));
|
|
150
|
104
|
|
|
105 // Deallocate memory in a guarded slot. The provided pointer must have been
|
|
|
106 // allocated using this pool. This will set the guarded slot as inaccessible.
|
|
|
107 void deallocate(void *Ptr);
|
|
|
108
|
|
|
109 // Returns the size of the allocation at Ptr.
|
|
|
110 size_t getSize(const void *Ptr);
|
|
|
111
|
|
|
112 // Returns a pointer to the Metadata region, or nullptr if it doesn't exist.
|
|
|
113 const AllocationMetadata *getMetadataRegion() const { return Metadata; }
|
|
|
114
|
|
|
115 // Returns a pointer to the AllocatorState region.
|
|
|
116 const AllocatorState *getAllocatorState() const { return &State; }
|
|
|
117
|
|
207
|
118 // Exposed as protected for testing.
|
|
|
119 protected:
|
|
|
120 // Returns the actual allocation size required to service an allocation with
|
|
|
121 // the provided Size and Alignment.
|
|
|
122 static size_t getRequiredBackingSize(size_t Size, size_t Alignment,
|
|
|
123 size_t PageSize);
|
|
|
124
|
|
|
125 // Returns the provided pointer that meets the specified alignment, depending
|
|
|
126 // on whether it's left or right aligned.
|
|
|
127 static uintptr_t alignUp(uintptr_t Ptr, size_t Alignment);
|
|
|
128 static uintptr_t alignDown(uintptr_t Ptr, size_t Alignment);
|
|
|
129
|
|
150
|
130 private:
|
|
|
131 // Name of actively-occupied slot mappings.
|
|
|
132 static constexpr const char *kGwpAsanAliveSlotName = "GWP-ASan Alive Slot";
|
|
|
133 // Name of the guard pages. This includes all slots that are not actively in
|
|
|
134 // use (i.e. were never used, or have been free()'d).)
|
|
|
135 static constexpr const char *kGwpAsanGuardPageName = "GWP-ASan Guard Page";
|
|
|
136 // Name of the mapping for `FreeSlots`.
|
|
|
137 static constexpr const char *kGwpAsanFreeSlotsName = "GWP-ASan Metadata";
|
|
|
138
|
|
|
139 static constexpr size_t kInvalidSlotID = SIZE_MAX;
|
|
|
140
|
|
|
141 // These functions anonymously map memory or change the permissions of mapped
|
|
|
142 // memory into this process in a platform-specific way. Pointer and size
|
|
|
143 // arguments are expected to be page-aligned. These functions will never
|
|
|
144 // return on error, instead electing to kill the calling process on failure.
|
|
207
|
145 // The pool memory is initially reserved and inaccessible, and RW mappings are
|
|
|
146 // subsequently created and destroyed via allocateInGuardedPool() and
|
|
|
147 // deallocateInGuardedPool(). Each mapping is named on platforms that support
|
|
|
148 // it, primarily Android. This name must be a statically allocated string, as
|
|
|
149 // the Android kernel uses the string pointer directly.
|
|
|
150 void *map(size_t Size, const char *Name) const;
|
|
|
151 void unmap(void *Ptr, size_t Size) const;
|
|
|
152
|
|
|
153 // The pool is managed separately, as some platforms (particularly Fuchsia)
|
|
|
154 // manage virtual memory regions as a chunk where individual pages can still
|
|
|
155 // have separate permissions. These platforms maintain metadata about the
|
|
|
156 // region in order to perform operations. The pool is unique as it's the only
|
|
|
157 // thing in GWP-ASan that treats pages in a single VM region on an individual
|
|
|
158 // basis for page protection.
|
|
|
159 // The pointer returned by reserveGuardedPool() is the reserved address range
|
|
|
160 // of (at least) Size bytes.
|
|
|
161 void *reserveGuardedPool(size_t Size);
|
|
|
162 // allocateInGuardedPool() Ptr and Size must be a subrange of the previously
|
|
|
163 // reserved pool range.
|
|
|
164 void allocateInGuardedPool(void *Ptr, size_t Size) const;
|
|
|
165 // deallocateInGuardedPool() Ptr and Size must be an exact pair previously
|
|
|
166 // passed to allocateInGuardedPool().
|
|
|
167 void deallocateInGuardedPool(void *Ptr, size_t Size) const;
|
|
|
168 void unreserveGuardedPool();
|
|
150
|
169
|
|
|
170 // Get the page size from the platform-specific implementation. Only needs to
|
|
|
171 // be called once, and the result should be cached in PageSize in this class.
|
|
|
172 static size_t getPlatformPageSize();
|
|
|
173
|
|
|
174 // Returns a pointer to the metadata for the owned pointer. If the pointer is
|
|
|
175 // not owned by this pool, the result is undefined.
|
|
|
176 AllocationMetadata *addrToMetadata(uintptr_t Ptr) const;
|
|
|
177
|
|
|
178 // Reserve a slot for a new guarded allocation. Returns kInvalidSlotID if no
|
|
|
179 // slot is available to be reserved.
|
|
|
180 size_t reserveSlot();
|
|
|
181
|
|
|
182 // Unreserve the guarded slot.
|
|
|
183 void freeSlot(size_t SlotIndex);
|
|
|
184
|
|
|
185 // Raise a SEGV and set the corresponding fields in the Allocator's State in
|
|
|
186 // order to tell the crash handler what happened. Used when errors are
|
|
|
187 // detected internally (Double Free, Invalid Free).
|
|
|
188 void trapOnAddress(uintptr_t Address, Error E);
|
|
|
189
|
|
|
190 static GuardedPoolAllocator *getSingleton();
|
|
|
191
|
|
|
192 // Install a pthread_atfork handler.
|
|
|
193 void installAtFork();
|
|
|
194
|
|
|
195 gwp_asan::AllocatorState State;
|
|
|
196
|
|
|
197 // A mutex to protect the guarded slot and metadata pool for this class.
|
|
|
198 Mutex PoolMutex;
|
|
207
|
199 // Some unwinders can grab the libdl lock. In order to provide atfork
|
|
|
200 // protection, we need to ensure that we allow an unwinding thread to release
|
|
|
201 // the libdl lock before forking.
|
|
|
202 Mutex BacktraceMutex;
|
|
150
|
203 // Record the number allocations that we've sampled. We store this amount so
|
|
|
204 // that we don't randomly choose to recycle a slot that previously had an
|
|
|
205 // allocation before all the slots have been utilised.
|
|
|
206 size_t NumSampledAllocations = 0;
|
|
|
207 // Pointer to the allocation metadata (allocation/deallocation stack traces),
|
|
|
208 // if any.
|
|
|
209 AllocationMetadata *Metadata = nullptr;
|
|
|
210
|
|
|
211 // Pointer to an array of free slot indexes.
|
|
|
212 size_t *FreeSlots = nullptr;
|
|
|
213 // The current length of the list of free slots.
|
|
|
214 size_t FreeSlotsLength = 0;
|
|
|
215
|
|
|
216 // See options.{h, inc} for more information.
|
|
|
217 bool PerfectlyRightAlign = false;
|
|
|
218
|
|
|
219 // Backtrace function provided by the supporting allocator. See `options.h`
|
|
|
220 // for more information.
|
|
|
221 options::Backtrace_t Backtrace = nullptr;
|
|
|
222
|
|
|
223 // The adjusted sample rate for allocation sampling. Default *must* be
|
|
|
224 // nonzero, as dynamic initialisation may call malloc (e.g. from libstdc++)
|
|
|
225 // before GPA::init() is called. This would cause an error in shouldSample(),
|
|
|
226 // where we would calculate modulo zero. This value is set UINT32_MAX, as when
|
|
|
227 // GWP-ASan is disabled, we wish to never spend wasted cycles recalculating
|
|
|
228 // the sample rate.
|
|
|
229 uint32_t AdjustedSampleRatePlusOne = 0;
|
|
|
230
|
|
207
|
231 // Additional platform specific data structure for the guarded pool mapping.
|
|
|
232 PlatformSpecificMapData GuardedPagePoolPlatformData = {};
|
|
|
233
|
|
|
234 class ScopedRecursiveGuard {
|
|
|
235 public:
|
|
|
236 ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = true; }
|
|
|
237 ~ScopedRecursiveGuard() { getThreadLocals()->RecursiveGuard = false; }
|
|
150
|
238 };
|
|
207
|
239
|
|
|
240 // Initialise the PRNG, platform-specific.
|
|
|
241 void initPRNG();
|
|
|
242
|
|
|
243 // xorshift (32-bit output), extremely fast PRNG that uses arithmetic
|
|
|
244 // operations only. Seeded using platform-specific mechanisms by initPRNG().
|
|
|
245 uint32_t getRandomUnsigned32();
|
|
150
|
246 };
|
|
|
247 } // namespace gwp_asan
|
|
|
248
|
|
|
249 #endif // GWP_ASAN_GUARDED_POOL_ALLOCATOR_H_
|
