tvl-depot/absl/base/internal/low_level_alloc.cc
Abseil Team 53c239d1fc Changes imported from Abseil "staging" branch:
- 06c8c67f5a564d00696e023060f05a5c34e7e164 IWYU | absl/base by Juemin Yang <jueminyang@google.com>
  - 2b1a054a09bda55843b449843b2a125741e936e7 Internal refactoring by Greg Miller <jgm@google.com>
  - f43f7f1f91bee26b5ddcd0c5bbbc47cb977aef77 Make std::hash<absl::optional<T>> to be standard compliant: by Xiaoyi Zhang <zhangxy@google.com>
  - 539bad2ebc22e610e1f292285a30a87945bc663e Update utility.h to Abseil standards by Tom Manshreck <shreck@google.com>
  - d05ec10a5f16a5d6640e0db91ecc7ab3ea971fd4 Add a test for absl::Barrier. by Derek Mauro <dmauro@google.com>
  - d707e27acb3c06f0d74c5f7ad7861e3841a5471f Run leak-checking tool over all outbound code. by Daniel Katz <katzdm@google.com>
  - 55f07f482a50422b8f99f7176374a19d0d473c5f Add alignas(16) to uint128. by Alex Strelnikov <strel@google.com>
  - 94999b7edde82308f736fb939501537ee9edbca6 Update attributes.h to Abseil standards by Tom Manshreck <shreck@google.com>
  - 321bed0061c41b53d0206ad4865528c00dd6d825 Test git merge + piper cl process by Juemin Yang <jueminyang@google.com>
  - 69920e7351a1053a7f4940bbde1768e839ce84bc Adds support for "/etc/localtime" as an embedded time zon... by Greg Miller <jgm@google.com>
  - 6839c06bf232903d3a9cbffa6eb2c960db78e67b Add copyright notices to inlined_vector code. by Greg Falcon <gfalcon@google.com>
  - 4e2714f6266263515cdfd31675c30c6ed6f98e1a Adding Apache 2.0 License by Gennadiy Civil <misterg@google.com>
  - 7402e7594016a4cd0a8b823fe6bc1bad1874bb85 IWYU | absl/utility by Juemin Yang <jueminyang@google.com>
  - 271a3812337eed97c412042738482688a80e19bd IWYU | absl/memory by Juemin Yang <jueminyang@google.com>
  - 32bda13a8098c2b06e25a5cf7bb782d6b79eb006 IWYU | absl/numeric by Juemin Yang <jueminyang@google.com>
  - 62d375cedc133108904bc06e340e303091a565df Remove "no_test"-annotations on span_test_noexceptions. by Daniel Katz <katzdm@google.com>
  - ebcbae9a55a93a7f1bb6862edc2715a6d9877206 Move CI-testing support files out of public-facing reposi... by Daniel Katz <katzdm@google.com>
  - d3f05eff4daa6030bfacb31cca0f9213fb702247 Fixes ToInt64Minutes() and ToInt64Hours() to properly sat... by Greg Miller <jgm@google.com>
  - b8dfae3facb6bb002622f083a10d14448f19e6e0 Fix typo. by Abseil Team <absl-team@google.com>
  - 150f03baa0afa231c2fc01597ea2321da586caba Update README.md by Yilei Yang <yileiyang@google.com>
  - 05276aa837dd081686518fd27bda4bd206ac4443 Adding Apache 2.0 License by Gennadiy Civil <misterg@google.com>
  - 37bf8e223e79ad06a195e28db9499e0c3d140f73 IWYU | absl/container by Juemin Yang <jueminyang@google.com>
  - 49164928f220978a32f88d16a55549bdf871daef IWYU | absl/algorithm by Juemin Yang <jueminyang@google.com>
  - a6804734e129039f9580a4fcd0f66425d0d0ac30 Move throw delegate wrappers to an internal namespace. by Greg Falcon <gfalcon@google.com>
  - ac83e73f67f593e2aff957b2be0b28e59c552a71 Fix error in comment stripping directives. by Greg Falcon <gfalcon@google.com>
  - e018a24185a984e787fb81a75fc35b74ad3a4d3d Update copyright headers all BUILD files by Gennadiy Civil <misterg@google.com>
  - a3be0990bfd76b0dec76bd85cecfa4dcec68b3ea Fix closing namespace comment typo. by Abseil Team <absl-team@google.com>
  - be3e3c4327e4f83949e0f29fd7a190d7eaa8b50b Update TODO by Abseil Team <absl-team@google.com>
  - f56a5d6f72685d92bb9c2905841b950d8177210c Add test-coverage for leak-sanitizer. by Daniel Katz <katzdm@google.com>
  - 7694bf161c7e00fdd08bfadc2aaf8e0fb09335f8 span.h: further touch up wording around the std::initiali... by Abseil Team <absl-team@google.com>
  - 3a12e081c0f8b359973e020d1e91f65356548ebc Update time.h to Abseil standards by Tom Manshreck <shreck@google.com>
  - 48d28f6468129420f4b20d451dca8e08012a7a77 Remove references to google from comments in Abseil. by Greg Falcon <gfalcon@google.com>
  - 773e34402d15fcad6370d5ed2430482d17db910d Rename the ExpectTime macro to match Abseil naming conven... by Greg Falcon <gfalcon@google.com>
  - 774d2ff1fe26c7313b301ff203e83e1aaac86627 Internal change. by Daniel Katz <katzdm@google.com>
  - 2e8a5830e95c8a1b839721bb2f1d4f5c85b9fb60 Fix typo in comment (missing '*' on a pointer). by Abseil Team <absl-team@google.com>
  - 458106feb707cf9609dd243713bde44aa9679e2a Correct capitalization: github -> GitHub by Yilei Yang <yileiyang@google.com>
  - af440725f02c2a83ca5cbaf176e1142f9e9d9b2f Update copyright headers by Gennadiy Civil <misterg@google.com>
  - 05b1118cce4ab87d23c33d48e64a96bcfec08761 Update copyright headers by Gennadiy Civil <misterg@google.com>
  - d5c6669a62d047156bb77055c5da03ee1b3c61b9 Update Abseil README to include descriptions of the inclu... by Tom Manshreck <shreck@google.com>
  - 3cd7e4663dddc840087469a6495f6cf433bfad8d Update copyright headers in //base by Tom Manshreck <shreck@google.com>
  - 7a876da657cd6698c5da2008a582d52eedc85dd1 Update strings overview with robust string library docume... by Tom Manshreck <shreck@google.com>
  - d9e3d0768d6f1c77d30992bdbef7b47ec92994bb Update copyright headers all BUILD files by Abseil Team <absl-team@google.com>
  - 6fe942728bceb0625f7c79b2840c4a6154d076b3 Make InlinedVector, FixedArray, and Span's at() throw on ... by Jon Cohen <cohenjon@google.com>
  - 5b52d5ec6cb9fbb07fc2e2fa020bd3eeb48c4953 Update clock.h to Abseil standards by Tom Manshreck <shreck@google.com>
  - c03c1ca3aee8bb7e40aa0315f6c432d31a72c30c Update //algorithm copyright headers by Tom Manshreck <shreck@google.com>
  - d46f40ddc596aaacb0459351d0e4aa6871289fa2 Temporarily prevent running mutex_test on crosstool17, wh... by Jon Cohen <cohenjon@google.com>
  - 61f11476189df68edfb9908308d677a91f03ff67 Update copyright headers in //container by Tom Manshreck <shreck@google.com>
  - 91832c00948954edf0b3dda12219c9a0202421ac Update copyright headers for //synchronization by Tom Manshreck <shreck@google.com>
  - 4e09100264b4585af6b4508ff35b9c627ac1f1ce Update copyright headers all BUILD files by Gennadiy Civil <misterg@google.com>
  - 13a0e8aebedec0f95b33750cbcd6b5548619b2a5 Update copyright headers in //memory by Tom Manshreck <shreck@google.com>
  - 63e1b9d4fdbcdf097e5276050ad1f76f0053e553 Update copyright headers for //strings (+ one from //nume... by Tom Manshreck <shreck@google.com>
  - 0108e7cfc50777a94c56d00e9c305161364df341 Convert ASSERT and EXPECT to ABSL_RAW_CHECK in helper fun... by Derek Mauro <dmauro@google.com>
  - 0122306fe47a3093248254a1b475c3a1d82abec5 Internal change by Abseil Team <absl-team@google.com>
  - 89c0c2698c98a12cd63172eeb02063b2f67e7c81 #absl Fix comment. by Abseil Team <absl-team@google.com>
  - 6621cc1ff54800e0aadb5e3071dbaa84b2077ceb Publishing contributing guidelines. by Gennadiy Civil <misterg@google.com>
  - e48c5be3c75e794b3e3827d40915b01fe1a1afc5 Avoid PRIdPTR.  Cast to long long and use %lld insted. by Abseil Team <absl-team@google.com>
  - 2640ea4a260d89b94b07a3142660327e47db33fd No algorithmic changes. by Abseil Team <absl-team@google.com>
  - 1bc6c1bad17754f5d84963bf1d0db279402a0a1d Internal change. by Derek Mauro <dmauro@google.com>
  - 6845d24733e8c95bebde825ba78a2abfd9e35bdb span.h: fix up incorrect wording around lvalues. by Abseil Team <absl-team@google.com>
  - d8f5caee721e252e5f9b1080fb996363f498ac28 Add more exaustive Mutex testing. by Derek Mauro <dmauro@google.com>
  - e8b4cb053eb98858eef10cc53280b6ed5d6815a7 Change Span::subspan to not call into a deprecated constr... by Jon Cohen <cohenjon@google.com>
  - 49c36a82b3114926390557670aaaf0ea25b5760c s/std::size_t/size_t/g by Jon Cohen <cohenjon@google.com>
  - e17487c3c4d4a99f2fd8bc3e42176fc3171614d1 Account for the case of timeval::tv_sec being smaller than by Abseil Team <absl-team@google.com>
  - ecbb89d5fb98483e777c03d97ac02d7b7b54985e Alias absl::string_view to std::string_view when C++17 st... by Xiaoyi Zhang <zhangxy@google.com>
  - 6820e5a51459cdbb6a423cbae25a0cc839c85d44 Internal cleanup. by Xiaoyi Zhang <zhangxy@google.com>
  - 6976469b76a6faaf4111a24ddb37f40211ffadae More Span constexpr by Jon Cohen <cohenjon@google.com>
  - 8521c8956eee1125b7759eb272ec4a5a86fcefc5 #absl Fix comment. by Abseil Team <absl-team@google.com>
  - 20eae7a67fde5dd809aa47e5f7de8a493701645e Embed enough zoneinfo data to make time:time_test (under ... by Abseil Team <absl-team@google.com>
  - 841f5d98ceef4a423839ea73ee06c2f47a9b9680 Clean up macros in attributes.h | ATTRIBUTE_INITIAL_EXEC by Juemin Yang <jueminyang@google.com>
  - 83d8b36656e47919b5d0bac82eece897e195697e Update any.h comments to Abseil standards by Tom Manshreck <shreck@google.com>
  - bb3fae11d3459eeae2f63bfd22e65d3193187cc8 Update type_traits.h comments to Abseil standards by Tom Manshreck <shreck@google.com>
  - 992e1b07c0dec64271f8c44f22fd8df3734d0c47 Renamespace CycleClock code. by Greg Falcon <gfalcon@google.com>
  - 08d6fb0594098493ffbc0e737405182638122e7e Eliminate more existing lint warnings by Gennadiy Rozental <rogeeff@google.com>
  - 1cc6fdc71eb777497239f8c3e9168e6c9d40ea53 Update optional.h to Abseil standards by Tom Manshreck <shreck@google.com>
  - baa91747aa55009a9eb31b6072e33db06cfce2d2 Enforce internal namespace symbol reference policy by Gennadiy Rozental <rogeeff@google.com>
  - 939251e39342ce559e5d23fe43799671581f7cf5 Add CycleClock scaling shift to mitigate a possible overf... by Derek Mauro <dmauro@google.com>
  - c6dfdeecea0c7470938bed47c99ea2b2a95889d8 Add constexpr tests for absl::make_optional(). by Xiaoyi Zhang <zhangxy@google.com>
  - 509e949b992db33041d840746fbd05cc01cb206e Alias absl::optional to std::optional when C++17 std::opt... by Xiaoyi Zhang <zhangxy@google.com>
  - a1ae6d96a8826ba75281cac8632a766b5856acaf Remove no_test_* tags from span_test to increase test cov... by Xiaoyi Zhang <zhangxy@google.com>
  - 3c2a43cc09791723c8a324836629644ac44cb9c8 Remove accidental bits of Google-internal code and short ... by Greg Falcon <gfalcon@google.com>
  - 4874d49d496ac0b6ec36f4280a14b2159e7af930 Replaces the macro-generated Duration factory functions e... by Greg Miller <jgm@google.com>
  - 90e62695e03cb4a57e137ca0c3e116b1d802db57 Fix namespacing for a couple files in base/internal, and ... by Greg Falcon <gfalcon@google.com>
  - b0e6e00e34f967924849aaf8c123bba068f093e3 Publishing contributing guidelines. by Gennadiy Civil <misterg@google.com>
  - d74eafbccc3dffa6c25f9b6a2219425a24b5a959 Internal change by Abseil Team <absl-team@google.com>
  - 27477badbbf720265f5b9509b6c0e01913dc0a9f Update escaping.h comments to Abseil standards by Tom Manshreck <shreck@google.com>
  - 67002f55738319c2875197c3b6282de215ec250d adds absl namespace to debugging/stacktrace.h by Behzad Nouri <bnouri@google.com>
  - e608018f7faa384d5b202ac0a4c7a0d5166f4d9c Update string_view.h comments to Abseil standards by Tom Manshreck <shreck@google.com>
  - e884f04d4c648e01ed7dcde2fda80c24e8452047 Exclude strings/ files we are not releasing from OSS univ... by Gennadiy Rozental <rogeeff@google.com>
  - 3f4c4032ed520f2dd10a81d58ef4f399c001c5cc Strip out eventmanager reference from release. by Gennadiy Rozental <rogeeff@google.com>
  - fb0f1c204793c3792bad101dbaa734e7c2a35887 Fix copybara strip comments by Gennadiy Rozental <rogeeff@google.com>
  - 3eaaac942f77c3d41d63d414630403bfd0f6b70c Strip out style guide waivers. by Gennadiy Rozental <rogeeff@google.com>
  - 020e045058173178b51266b99a2a5dc9ed921960 substitute_test portability | MUST_USE_RESULT cast-to-voi... by Juemin Yang <jueminyang@google.com>
  - 86c093bf81d80ff537ed5e8b89225ce75a636220 Internal change. by Derek Mauro <dmauro@google.com>
  - 330375eb952fe78276e75631a28e750d5bfdb198 Prefer absl::FixedTimeZone() over loading "Etc/GMT[-+]<N>". by Abseil Team <absl-team@google.com>
  - 2e07ebee46a8201adc0dfd2c4ddb3df76e524357 Internal change. by Derek Mauro <dmauro@google.com>
  - 1f0c8b78c8ebd66f14cdf39fcba9f4c9986dcdca ::absl -> absl by Gennadiy Rozental <rogeeff@google.com>
  - ad163566d12ea08f1da2c23931eeacfffc564139 Avoid old style loops where possible. by Gennadiy Rozental <rogeeff@google.com>
  - bce2108818fe57b5617ce0090ddd4f753808f0a1 Update comments in str_cat.h in line with recent changes. by Abseil Team <absl-team@google.com>
  - cfd593a80f4897256f2ce1ea0be55dc14e3fcad4 Copybara-out gtl aliases. by Gennadiy Rozental <rogeeff@google.com>
  - 584f1524d717993c1a16093caccd9ed2b1e5409e Fix a warning for Windows/Kokoro time_test.cc. by Daniel Katz <katzdm@google.com>
  (And 562 more changes)

GitOrigin-RevId: 06c8c67f5a564d00696e023060f05a5c34e7e164
Change-Id: I89907a6188fe7de05da400bf49ddfeba242aff8e
2017-09-20 10:37:35 -04:00

600 lines
23 KiB
C++

// Copyright 2017 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// A low-level allocator that can be used by other low-level
// modules without introducing dependency cycles.
// This allocator is slow and wasteful of memory;
// it should not be used when performance is key.
#include "absl/base/internal/low_level_alloc.h"
#include "absl/base/config.h"
#include "absl/base/internal/scheduling_mode.h"
#include "absl/base/macros.h"
#include "absl/base/thread_annotations.h"
// LowLevelAlloc requires that the platform support low-level
// allocation of virtual memory. Platforms lacking this cannot use
// LowLevelAlloc.
#ifndef ABSL_LOW_LEVEL_ALLOC_MISSING
#ifndef _WIN32
#include <signal.h>
#include <sys/mman.h>
#include <unistd.h>
#else
#include <windows.h>
#endif
#include <string.h>
#include <algorithm>
#include <atomic>
#include <cerrno>
#include <cstddef>
#include <new> // for placement-new
#include "absl/base/dynamic_annotations.h"
#include "absl/base/internal/malloc_hook.h"
#include "absl/base/internal/malloc_hook_invoke.h"
#include "absl/base/internal/raw_logging.h"
#include "absl/base/internal/spinlock.h"
// MAP_ANONYMOUS
#if defined(__APPLE__)
// For mmap, Linux defines both MAP_ANONYMOUS and MAP_ANON and says MAP_ANON is
// deprecated. In Darwin, MAP_ANON is all there is.
#if !defined MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif // !MAP_ANONYMOUS
#endif // __APPLE__
namespace absl {
namespace base_internal {
// A first-fit allocator with amortized logarithmic free() time.
// ---------------------------------------------------------------------------
static const int kMaxLevel = 30;
namespace {
// This struct describes one allocated block, or one free block.
struct AllocList {
struct Header {
// Size of entire region, including this field. Must be
// first. Valid in both allocated and unallocated blocks.
uintptr_t size;
// kMagicAllocated or kMagicUnallocated xor this.
uintptr_t magic;
// Pointer to parent arena.
LowLevelAlloc::Arena *arena;
// Aligns regions to 0 mod 2*sizeof(void*).
void *dummy_for_alignment;
} header;
// Next two fields: in unallocated blocks: freelist skiplist data
// in allocated blocks: overlaps with client data
// Levels in skiplist used.
int levels;
// Actually has levels elements. The AllocList node may not have room
// for all kMaxLevel entries. See max_fit in LLA_SkiplistLevels().
AllocList *next[kMaxLevel];
};
} // namespace
// ---------------------------------------------------------------------------
// A trivial skiplist implementation. This is used to keep the freelist
// in address order while taking only logarithmic time per insert and delete.
// An integer approximation of log2(size/base)
// Requires size >= base.
static int IntLog2(size_t size, size_t base) {
int result = 0;
for (size_t i = size; i > base; i >>= 1) { // i == floor(size/2**result)
result++;
}
// floor(size / 2**result) <= base < floor(size / 2**(result-1))
// => log2(size/(base+1)) <= result < 1+log2(size/base)
// => result ~= log2(size/base)
return result;
}
// Return a random integer n: p(n)=1/(2**n) if 1 <= n; p(n)=0 if n < 1.
static int Random(uint32_t *state) {
uint32_t r = *state;
int result = 1;
while ((((r = r*1103515245 + 12345) >> 30) & 1) == 0) {
result++;
}
*state = r;
return result;
}
// Return a number of skiplist levels for a node of size bytes, where
// base is the minimum node size. Compute level=log2(size / base)+n
// where n is 1 if random is false and otherwise a random number generated with
// the standard distribution for a skiplist: See Random() above.
// Bigger nodes tend to have more skiplist levels due to the log2(size / base)
// term, so first-fit searches touch fewer nodes. "level" is clipped so
// level<kMaxLevel and next[level-1] will fit in the node.
// 0 < LLA_SkiplistLevels(x,y,false) <= LLA_SkiplistLevels(x,y,true) < kMaxLevel
static int LLA_SkiplistLevels(size_t size, size_t base, uint32_t *random) {
// max_fit is the maximum number of levels that will fit in a node for the
// given size. We can't return more than max_fit, no matter what the
// random number generator says.
size_t max_fit = (size - offsetof(AllocList, next)) / sizeof(AllocList *);
int level = IntLog2(size, base) + (random != nullptr ? Random(random) : 1);
if (static_cast<size_t>(level) > max_fit) level = static_cast<int>(max_fit);
if (level > kMaxLevel-1) level = kMaxLevel - 1;
ABSL_RAW_CHECK(level >= 1, "block not big enough for even one level");
return level;
}
// Return "atleast", the first element of AllocList *head s.t. *atleast >= *e.
// For 0 <= i < head->levels, set prev[i] to "no_greater", where no_greater
// points to the last element at level i in the AllocList less than *e, or is
// head if no such element exists.
static AllocList *LLA_SkiplistSearch(AllocList *head,
AllocList *e, AllocList **prev) {
AllocList *p = head;
for (int level = head->levels - 1; level >= 0; level--) {
for (AllocList *n; (n = p->next[level]) != nullptr && n < e; p = n) {
}
prev[level] = p;
}
return (head->levels == 0) ? nullptr : prev[0]->next[0];
}
// Insert element *e into AllocList *head. Set prev[] as LLA_SkiplistSearch.
// Requires that e->levels be previously set by the caller (using
// LLA_SkiplistLevels())
static void LLA_SkiplistInsert(AllocList *head, AllocList *e,
AllocList **prev) {
LLA_SkiplistSearch(head, e, prev);
for (; head->levels < e->levels; head->levels++) { // extend prev pointers
prev[head->levels] = head; // to all *e's levels
}
for (int i = 0; i != e->levels; i++) { // add element to list
e->next[i] = prev[i]->next[i];
prev[i]->next[i] = e;
}
}
// Remove element *e from AllocList *head. Set prev[] as LLA_SkiplistSearch().
// Requires that e->levels be previous set by the caller (using
// LLA_SkiplistLevels())
static void LLA_SkiplistDelete(AllocList *head, AllocList *e,
AllocList **prev) {
AllocList *found = LLA_SkiplistSearch(head, e, prev);
ABSL_RAW_CHECK(e == found, "element not in freelist");
for (int i = 0; i != e->levels && prev[i]->next[i] == e; i++) {
prev[i]->next[i] = e->next[i];
}
while (head->levels > 0 && head->next[head->levels - 1] == nullptr) {
head->levels--; // reduce head->levels if level unused
}
}
// ---------------------------------------------------------------------------
// Arena implementation
struct LowLevelAlloc::Arena {
// This constructor does nothing, and relies on zero-initialization to get
// the proper initial state.
Arena() : mu(base_internal::kLinkerInitialized) {} // NOLINT
explicit Arena(int) // NOLINT(readability/casting)
: // Avoid recursive cooperative scheduling w/ kernel scheduling.
mu(base_internal::SCHEDULE_KERNEL_ONLY),
// Set pagesize to zero explicitly for non-static init.
pagesize(0),
random(0) {}
base_internal::SpinLock mu; // protects freelist, allocation_count,
// pagesize, roundup, min_size
AllocList freelist; // head of free list; sorted by addr (under mu)
int32_t allocation_count; // count of allocated blocks (under mu)
std::atomic<uint32_t> flags; // flags passed to NewArena (ro after init)
size_t pagesize; // ==getpagesize() (init under mu, then ro)
size_t roundup; // lowest 2^n >= max(16,sizeof (AllocList))
// (init under mu, then ro)
size_t min_size; // smallest allocation block size
// (init under mu, then ro)
uint32_t random; // PRNG state
};
// The default arena, which is used when 0 is passed instead of an Arena
// pointer.
static struct LowLevelAlloc::Arena default_arena; // NOLINT
// Non-malloc-hooked arenas: used only to allocate metadata for arenas that
// do not want malloc hook reporting, so that for them there's no malloc hook
// reporting even during arena creation.
static struct LowLevelAlloc::Arena unhooked_arena; // NOLINT
#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
static struct LowLevelAlloc::Arena unhooked_async_sig_safe_arena; // NOLINT
#endif
// magic numbers to identify allocated and unallocated blocks
static const uintptr_t kMagicAllocated = 0x4c833e95U;
static const uintptr_t kMagicUnallocated = ~kMagicAllocated;
namespace {
class SCOPED_LOCKABLE ArenaLock {
public:
explicit ArenaLock(LowLevelAlloc::Arena *arena)
EXCLUSIVE_LOCK_FUNCTION(arena->mu)
: arena_(arena) {
#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
if (arena == &unhooked_async_sig_safe_arena ||
(arena->flags.load(std::memory_order_relaxed) &
LowLevelAlloc::kAsyncSignalSafe) != 0) {
sigset_t all;
sigfillset(&all);
mask_valid_ = pthread_sigmask(SIG_BLOCK, &all, &mask_) == 0;
}
#endif
arena_->mu.Lock();
}
~ArenaLock() { ABSL_RAW_CHECK(left_, "haven't left Arena region"); }
void Leave() UNLOCK_FUNCTION() {
arena_->mu.Unlock();
#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
if (mask_valid_) {
pthread_sigmask(SIG_SETMASK, &mask_, nullptr);
}
#endif
left_ = true;
}
private:
bool left_ = false; // whether left region
#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
bool mask_valid_ = false;
sigset_t mask_; // old mask of blocked signals
#endif
LowLevelAlloc::Arena *arena_;
ArenaLock(const ArenaLock &) = delete;
ArenaLock &operator=(const ArenaLock &) = delete;
};
} // namespace
// create an appropriate magic number for an object at "ptr"
// "magic" should be kMagicAllocated or kMagicUnallocated
inline static uintptr_t Magic(uintptr_t magic, AllocList::Header *ptr) {
return magic ^ reinterpret_cast<uintptr_t>(ptr);
}
// Initialize the fields of an Arena
static void ArenaInit(LowLevelAlloc::Arena *arena) {
if (arena->pagesize == 0) {
#ifdef _WIN32
SYSTEM_INFO system_info;
GetSystemInfo(&system_info);
arena->pagesize = std::max(system_info.dwPageSize,
system_info.dwAllocationGranularity);
#else
arena->pagesize = getpagesize();
#endif
// Round up block sizes to a power of two close to the header size.
arena->roundup = 16;
while (arena->roundup < sizeof (arena->freelist.header)) {
arena->roundup += arena->roundup;
}
// Don't allocate blocks less than twice the roundup size to avoid tiny
// free blocks.
arena->min_size = 2 * arena->roundup;
arena->freelist.header.size = 0;
arena->freelist.header.magic =
Magic(kMagicUnallocated, &arena->freelist.header);
arena->freelist.header.arena = arena;
arena->freelist.levels = 0;
memset(arena->freelist.next, 0, sizeof (arena->freelist.next));
arena->allocation_count = 0;
if (arena == &default_arena) {
// Default arena should be hooked, e.g. for heap-checker to trace
// pointer chains through objects in the default arena.
arena->flags.store(LowLevelAlloc::kCallMallocHook,
std::memory_order_relaxed);
}
#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
else if (arena == // NOLINT(readability/braces)
&unhooked_async_sig_safe_arena) {
arena->flags.store(LowLevelAlloc::kAsyncSignalSafe,
std::memory_order_relaxed);
}
#endif
else { // NOLINT(readability/braces)
// other arenas' flags may be overridden by client,
// but unhooked_arena will have 0 in 'flags'.
arena->flags.store(0, std::memory_order_relaxed);
}
}
}
// L < meta_data_arena->mu
LowLevelAlloc::Arena *LowLevelAlloc::NewArena(int32_t flags,
Arena *meta_data_arena) {
ABSL_RAW_CHECK(meta_data_arena != nullptr, "must pass a valid arena");
if (meta_data_arena == &default_arena) {
#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING
if ((flags & LowLevelAlloc::kAsyncSignalSafe) != 0) {
meta_data_arena = &unhooked_async_sig_safe_arena;
} else // NOLINT(readability/braces)
#endif
if ((flags & LowLevelAlloc::kCallMallocHook) == 0) {
meta_data_arena = &unhooked_arena;
}
}
// Arena(0) uses the constructor for non-static contexts
Arena *result =
new (AllocWithArena(sizeof (*result), meta_data_arena)) Arena(0);
ArenaInit(result);
result->flags.store(flags, std::memory_order_relaxed);
return result;
}
// L < arena->mu, L < arena->arena->mu
bool LowLevelAlloc::DeleteArena(Arena *arena) {
ABSL_RAW_CHECK(
arena != nullptr && arena != &default_arena && arena != &unhooked_arena,
"may not delete default arena");
ArenaLock section(arena);
bool empty = (arena->allocation_count == 0);
section.Leave();
if (empty) {
while (arena->freelist.next[0] != nullptr) {
AllocList *region = arena->freelist.next[0];
size_t size = region->header.size;
arena->freelist.next[0] = region->next[0];
ABSL_RAW_CHECK(
region->header.magic == Magic(kMagicUnallocated, &region->header),
"bad magic number in DeleteArena()");
ABSL_RAW_CHECK(region->header.arena == arena,
"bad arena pointer in DeleteArena()");
ABSL_RAW_CHECK(size % arena->pagesize == 0,
"empty arena has non-page-aligned block size");
ABSL_RAW_CHECK(reinterpret_cast<uintptr_t>(region) % arena->pagesize == 0,
"empty arena has non-page-aligned block");
int munmap_result;
#ifdef _WIN32
munmap_result = VirtualFree(region, 0, MEM_RELEASE);
ABSL_RAW_CHECK(munmap_result != 0,
"LowLevelAlloc::DeleteArena: VitualFree failed");
#else
if ((arena->flags.load(std::memory_order_relaxed) &
LowLevelAlloc::kAsyncSignalSafe) == 0) {
munmap_result = munmap(region, size);
} else {
munmap_result = MallocHook::UnhookedMUnmap(region, size);
}
if (munmap_result != 0) {
ABSL_RAW_LOG(FATAL, "LowLevelAlloc::DeleteArena: munmap failed: %d",
errno);
}
#endif
}
Free(arena);
}
return empty;
}
// ---------------------------------------------------------------------------
// Addition, checking for overflow. The intent is to die if an external client
// manages to push through a request that would cause arithmetic to fail.
static inline uintptr_t CheckedAdd(uintptr_t a, uintptr_t b) {
uintptr_t sum = a + b;
ABSL_RAW_CHECK(sum >= a, "LowLevelAlloc arithmetic overflow");
return sum;
}
// Return value rounded up to next multiple of align.
// align must be a power of two.
static inline uintptr_t RoundUp(uintptr_t addr, uintptr_t align) {
return CheckedAdd(addr, align - 1) & ~(align - 1);
}
// Equivalent to "return prev->next[i]" but with sanity checking
// that the freelist is in the correct order, that it
// consists of regions marked "unallocated", and that no two regions
// are adjacent in memory (they should have been coalesced).
// L < arena->mu
static AllocList *Next(int i, AllocList *prev, LowLevelAlloc::Arena *arena) {
ABSL_RAW_CHECK(i < prev->levels, "too few levels in Next()");
AllocList *next = prev->next[i];
if (next != nullptr) {
ABSL_RAW_CHECK(
next->header.magic == Magic(kMagicUnallocated, &next->header),
"bad magic number in Next()");
ABSL_RAW_CHECK(next->header.arena == arena, "bad arena pointer in Next()");
if (prev != &arena->freelist) {
ABSL_RAW_CHECK(prev < next, "unordered freelist");
ABSL_RAW_CHECK(reinterpret_cast<char *>(prev) + prev->header.size <
reinterpret_cast<char *>(next),
"malformed freelist");
}
}
return next;
}
// Coalesce list item "a" with its successor if they are adjacent.
static void Coalesce(AllocList *a) {
AllocList *n = a->next[0];
if (n != nullptr && reinterpret_cast<char *>(a) + a->header.size ==
reinterpret_cast<char *>(n)) {
LowLevelAlloc::Arena *arena = a->header.arena;
a->header.size += n->header.size;
n->header.magic = 0;
n->header.arena = nullptr;
AllocList *prev[kMaxLevel];
LLA_SkiplistDelete(&arena->freelist, n, prev);
LLA_SkiplistDelete(&arena->freelist, a, prev);
a->levels = LLA_SkiplistLevels(a->header.size, arena->min_size,
&arena->random);
LLA_SkiplistInsert(&arena->freelist, a, prev);
}
}
// Adds block at location "v" to the free list
// L >= arena->mu
static void AddToFreelist(void *v, LowLevelAlloc::Arena *arena) {
AllocList *f = reinterpret_cast<AllocList *>(
reinterpret_cast<char *>(v) - sizeof (f->header));
ABSL_RAW_CHECK(f->header.magic == Magic(kMagicAllocated, &f->header),
"bad magic number in AddToFreelist()");
ABSL_RAW_CHECK(f->header.arena == arena,
"bad arena pointer in AddToFreelist()");
f->levels = LLA_SkiplistLevels(f->header.size, arena->min_size,
&arena->random);
AllocList *prev[kMaxLevel];
LLA_SkiplistInsert(&arena->freelist, f, prev);
f->header.magic = Magic(kMagicUnallocated, &f->header);
Coalesce(f); // maybe coalesce with successor
Coalesce(prev[0]); // maybe coalesce with predecessor
}
// Frees storage allocated by LowLevelAlloc::Alloc().
// L < arena->mu
void LowLevelAlloc::Free(void *v) {
if (v != nullptr) {
AllocList *f = reinterpret_cast<AllocList *>(
reinterpret_cast<char *>(v) - sizeof (f->header));
ABSL_RAW_CHECK(f->header.magic == Magic(kMagicAllocated, &f->header),
"bad magic number in Free()");
LowLevelAlloc::Arena *arena = f->header.arena;
if ((arena->flags.load(std::memory_order_relaxed) & kCallMallocHook) != 0) {
MallocHook::InvokeDeleteHook(v);
}
ArenaLock section(arena);
AddToFreelist(v, arena);
ABSL_RAW_CHECK(arena->allocation_count > 0, "nothing in arena to free");
arena->allocation_count--;
section.Leave();
}
}
// allocates and returns a block of size bytes, to be freed with Free()
// L < arena->mu
static void *DoAllocWithArena(size_t request, LowLevelAlloc::Arena *arena) {
void *result = nullptr;
if (request != 0) {
AllocList *s; // will point to region that satisfies request
ArenaLock section(arena);
ArenaInit(arena);
// round up with header
size_t req_rnd = RoundUp(CheckedAdd(request, sizeof (s->header)),
arena->roundup);
for (;;) { // loop until we find a suitable region
// find the minimum levels that a block of this size must have
int i = LLA_SkiplistLevels(req_rnd, arena->min_size, nullptr) - 1;
if (i < arena->freelist.levels) { // potential blocks exist
AllocList *before = &arena->freelist; // predecessor of s
while ((s = Next(i, before, arena)) != nullptr &&
s->header.size < req_rnd) {
before = s;
}
if (s != nullptr) { // we found a region
break;
}
}
// we unlock before mmap() both because mmap() may call a callback hook,
// and because it may be slow.
arena->mu.Unlock();
// mmap generous 64K chunks to decrease
// the chances/impact of fragmentation:
size_t new_pages_size = RoundUp(req_rnd, arena->pagesize * 16);
void *new_pages;
#ifdef _WIN32
new_pages = VirtualAlloc(0, new_pages_size,
MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
ABSL_RAW_CHECK(new_pages != nullptr, "VirtualAlloc failed");
#else
if ((arena->flags.load(std::memory_order_relaxed) &
LowLevelAlloc::kAsyncSignalSafe) != 0) {
new_pages = MallocHook::UnhookedMMap(nullptr, new_pages_size,
PROT_WRITE|PROT_READ, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
} else {
new_pages = mmap(nullptr, new_pages_size, PROT_WRITE | PROT_READ,
MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
}
if (new_pages == MAP_FAILED) {
ABSL_RAW_LOG(FATAL, "mmap error: %d", errno);
}
#endif
arena->mu.Lock();
s = reinterpret_cast<AllocList *>(new_pages);
s->header.size = new_pages_size;
// Pretend the block is allocated; call AddToFreelist() to free it.
s->header.magic = Magic(kMagicAllocated, &s->header);
s->header.arena = arena;
AddToFreelist(&s->levels, arena); // insert new region into free list
}
AllocList *prev[kMaxLevel];
LLA_SkiplistDelete(&arena->freelist, s, prev); // remove from free list
// s points to the first free region that's big enough
if (CheckedAdd(req_rnd, arena->min_size) <= s->header.size) {
// big enough to split
AllocList *n = reinterpret_cast<AllocList *>
(req_rnd + reinterpret_cast<char *>(s));
n->header.size = s->header.size - req_rnd;
n->header.magic = Magic(kMagicAllocated, &n->header);
n->header.arena = arena;
s->header.size = req_rnd;
AddToFreelist(&n->levels, arena);
}
s->header.magic = Magic(kMagicAllocated, &s->header);
ABSL_RAW_CHECK(s->header.arena == arena, "");
arena->allocation_count++;
section.Leave();
result = &s->levels;
}
ANNOTATE_MEMORY_IS_UNINITIALIZED(result, request);
return result;
}
void *LowLevelAlloc::Alloc(size_t request) {
void *result = DoAllocWithArena(request, &default_arena);
if ((default_arena.flags.load(std::memory_order_relaxed) &
kCallMallocHook) != 0) {
// this call must be directly in the user-called allocator function
// for MallocHook::GetCallerStackTrace to work properly
MallocHook::InvokeNewHook(result, request);
}
return result;
}
void *LowLevelAlloc::AllocWithArena(size_t request, Arena *arena) {
ABSL_RAW_CHECK(arena != nullptr, "must pass a valid arena");
void *result = DoAllocWithArena(request, arena);
if ((arena->flags.load(std::memory_order_relaxed) & kCallMallocHook) != 0) {
// this call must be directly in the user-called allocator function
// for MallocHook::GetCallerStackTrace to work properly
MallocHook::InvokeNewHook(result, request);
}
return result;
}
LowLevelAlloc::Arena *LowLevelAlloc::DefaultArena() {
return &default_arena;
}
} // namespace base_internal
} // namespace absl
#endif // ABSL_LOW_LEVEL_ALLOC_MISSING