tvl-depot/absl/base/internal/spinlock.h
Abseil Team 518f17501e Export of internal Abseil changes
--
79913a12f0cad4baf948430315aabf53f03b6475 by Abseil Team <absl-team@google.com>:

Don't inline (Un)LockSlow.

PiperOrigin-RevId: 302502344

--
6b340e80f0690655f24799c8de6707b3a95b8579 by Derek Mauro <dmauro@google.com>:

Add hardening assertions to absl::optional's dereference operators

PiperOrigin-RevId: 302492862

--
a9951bf4852d8c1aec472cb4b539830411270e4c by Derek Mauro <dmauro@google.com>:

Correctly add hardware AES compiler flags under Linux X86-64
Fixes #643

PiperOrigin-RevId: 302490673

--
314c3621ee4d57b6bc8d64338a1f1d48a69741d1 by Derek Mauro <dmauro@google.com>:

Upgrade to hardening assertions in absl::Span::remove_prefix and absl::Span::remove_suffix

PiperOrigin-RevId: 302481191

--
a142b8c6c62705c5f0d4fe3113150f0c0b7822b9 by Derek Mauro <dmauro@google.com>:

Update docker containers to Bazel 2.2.0, GCC 9.3, and new Clang snapshot

PiperOrigin-RevId: 302454042

--
afedeb70a2adc87010030c9ba6f06fe35ec26407 by Derek Mauro <dmauro@google.com>:

Add hardening assertions for the preconditions of absl::FixedArray

PiperOrigin-RevId: 302441767

--
44442bfbc0a9a742df32f07cee86a47712efb8b4 by Derek Mauro <dmauro@google.com>:

Fix new Clang warning about SpinLock doing operations on enums of different types

PiperOrigin-RevId: 302430387

--
69eaff7f97231779f696321c2ba8b88debf6dd9e by Derek Mauro <dmauro@google.com>:

Convert precondition assertions to ABSL_HARDENING_ASSERT for
absl::InlinedVector

PiperOrigin-RevId: 302427894

--
26b6db906a0942fd18583dc2cdd1bab32919d964 by Gennadiy Rozental <rogeeff@google.com>:

Internal change

PiperOrigin-RevId: 302425283

--
e62e81422979e922505d2cd9000e1de58123c088 by Derek Mauro <dmauro@google.com>:

Add an option to build Abseil in hardened mode

In hardened mode, the ABSL_HARDENING_ASSERT() macro is active even
when NDEBUG is defined. This allows Abseil to perform runtime checks
even in release mode. This should be used to implement things like
bounds checks that could otherwise lead to security vulnerabilities.

Use the new assertion in absl::string_view and absl::Span to test it.

PiperOrigin-RevId: 302119187
GitOrigin-RevId: 79913a12f0cad4baf948430315aabf53f03b6475
Change-Id: I0cc3341fd333a1df313167bab72dc5a759c4a048
2020-03-23 16:24:45 -04:00

244 lines
8.9 KiB
C++

//
// Copyright 2017 The Abseil Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Most users requiring mutual exclusion should use Mutex.
// SpinLock is provided for use in three situations:
// - for use in code that Mutex itself depends on
// - to get a faster fast-path release under low contention (without an
// atomic read-modify-write) In return, SpinLock has worse behaviour under
// contention, which is why Mutex is preferred in most situations.
// - for async signal safety (see below)
// SpinLock is async signal safe. If a spinlock is used within a signal
// handler, all code that acquires the lock must ensure that the signal cannot
// arrive while they are holding the lock. Typically, this is done by blocking
// the signal.
#ifndef ABSL_BASE_INTERNAL_SPINLOCK_H_
#define ABSL_BASE_INTERNAL_SPINLOCK_H_
#include <stdint.h>
#include <sys/types.h>
#include <atomic>
#include "absl/base/attributes.h"
#include "absl/base/dynamic_annotations.h"
#include "absl/base/internal/low_level_scheduling.h"
#include "absl/base/internal/raw_logging.h"
#include "absl/base/internal/scheduling_mode.h"
#include "absl/base/internal/tsan_mutex_interface.h"
#include "absl/base/macros.h"
#include "absl/base/port.h"
#include "absl/base/thread_annotations.h"
namespace absl {
ABSL_NAMESPACE_BEGIN
namespace base_internal {
class ABSL_LOCKABLE SpinLock {
public:
SpinLock() : lockword_(kSpinLockCooperative) {
ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static);
}
// Special constructor for use with static SpinLock objects. E.g.,
//
// static SpinLock lock(base_internal::kLinkerInitialized);
//
// When initialized using this constructor, we depend on the fact
// that the linker has already initialized the memory appropriately. The lock
// is initialized in non-cooperative mode.
//
// A SpinLock constructed like this can be freely used from global
// initializers without worrying about the order in which global
// initializers run.
explicit SpinLock(base_internal::LinkerInitialized) {
// Does nothing; lockword_ is already initialized
ABSL_TSAN_MUTEX_CREATE(this, 0);
}
// Constructors that allow non-cooperative spinlocks to be created for use
// inside thread schedulers. Normal clients should not use these.
explicit SpinLock(base_internal::SchedulingMode mode);
SpinLock(base_internal::LinkerInitialized,
base_internal::SchedulingMode mode);
~SpinLock() { ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); }
// Acquire this SpinLock.
inline void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() {
ABSL_TSAN_MUTEX_PRE_LOCK(this, 0);
if (!TryLockImpl()) {
SlowLock();
}
ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0);
}
// Try to acquire this SpinLock without blocking and return true if the
// acquisition was successful. If the lock was not acquired, false is
// returned. If this SpinLock is free at the time of the call, TryLock
// will return true with high probability.
inline bool TryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) {
ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_try_lock);
bool res = TryLockImpl();
ABSL_TSAN_MUTEX_POST_LOCK(
this, __tsan_mutex_try_lock | (res ? 0 : __tsan_mutex_try_lock_failed),
0);
return res;
}
// Release this SpinLock, which must be held by the calling thread.
inline void Unlock() ABSL_UNLOCK_FUNCTION() {
ABSL_TSAN_MUTEX_PRE_UNLOCK(this, 0);
uint32_t lock_value = lockword_.load(std::memory_order_relaxed);
lock_value = lockword_.exchange(lock_value & kSpinLockCooperative,
std::memory_order_release);
if ((lock_value & kSpinLockDisabledScheduling) != 0) {
base_internal::SchedulingGuard::EnableRescheduling(true);
}
if ((lock_value & kWaitTimeMask) != 0) {
// Collect contentionz profile info, and speed the wakeup of any waiter.
// The wait_cycles value indicates how long this thread spent waiting
// for the lock.
SlowUnlock(lock_value);
}
ABSL_TSAN_MUTEX_POST_UNLOCK(this, 0);
}
// Determine if the lock is held. When the lock is held by the invoking
// thread, true will always be returned. Intended to be used as
// CHECK(lock.IsHeld()).
inline bool IsHeld() const {
return (lockword_.load(std::memory_order_relaxed) & kSpinLockHeld) != 0;
}
protected:
// These should not be exported except for testing.
// Store number of cycles between wait_start_time and wait_end_time in a
// lock value.
static uint32_t EncodeWaitCycles(int64_t wait_start_time,
int64_t wait_end_time);
// Extract number of wait cycles in a lock value.
static uint64_t DecodeWaitCycles(uint32_t lock_value);
// Provide access to protected method above. Use for testing only.
friend struct SpinLockTest;
private:
// lockword_ is used to store the following:
//
// bit[0] encodes whether a lock is being held.
// bit[1] encodes whether a lock uses cooperative scheduling.
// bit[2] encodes whether a lock disables scheduling.
// bit[3:31] encodes time a lock spent on waiting as a 29-bit unsigned int.
static constexpr uint32_t kSpinLockHeld = 1;
static constexpr uint32_t kSpinLockCooperative = 2;
static constexpr uint32_t kSpinLockDisabledScheduling = 4;
static constexpr uint32_t kSpinLockSleeper = 8;
// Includes kSpinLockSleeper.
static constexpr uint32_t kWaitTimeMask =
~(kSpinLockHeld | kSpinLockCooperative | kSpinLockDisabledScheduling);
// Returns true if the provided scheduling mode is cooperative.
static constexpr bool IsCooperative(
base_internal::SchedulingMode scheduling_mode) {
return scheduling_mode == base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL;
}
uint32_t TryLockInternal(uint32_t lock_value, uint32_t wait_cycles);
void InitLinkerInitializedAndCooperative();
void SlowLock() ABSL_ATTRIBUTE_COLD;
void SlowUnlock(uint32_t lock_value) ABSL_ATTRIBUTE_COLD;
uint32_t SpinLoop();
inline bool TryLockImpl() {
uint32_t lock_value = lockword_.load(std::memory_order_relaxed);
return (TryLockInternal(lock_value, 0) & kSpinLockHeld) == 0;
}
std::atomic<uint32_t> lockword_;
SpinLock(const SpinLock&) = delete;
SpinLock& operator=(const SpinLock&) = delete;
};
// Corresponding locker object that arranges to acquire a spinlock for
// the duration of a C++ scope.
class ABSL_SCOPED_LOCKABLE SpinLockHolder {
public:
inline explicit SpinLockHolder(SpinLock* l) ABSL_EXCLUSIVE_LOCK_FUNCTION(l)
: lock_(l) {
l->Lock();
}
inline ~SpinLockHolder() ABSL_UNLOCK_FUNCTION() { lock_->Unlock(); }
SpinLockHolder(const SpinLockHolder&) = delete;
SpinLockHolder& operator=(const SpinLockHolder&) = delete;
private:
SpinLock* lock_;
};
// Register a hook for profiling support.
//
// The function pointer registered here will be called whenever a spinlock is
// contended. The callback is given an opaque handle to the contended spinlock
// and the number of wait cycles. This is thread-safe, but only a single
// profiler can be registered. It is an error to call this function multiple
// times with different arguments.
void RegisterSpinLockProfiler(void (*fn)(const void* lock,
int64_t wait_cycles));
//------------------------------------------------------------------------------
// Public interface ends here.
//------------------------------------------------------------------------------
// If (result & kSpinLockHeld) == 0, then *this was successfully locked.
// Otherwise, returns last observed value for lockword_.
inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value,
uint32_t wait_cycles) {
if ((lock_value & kSpinLockHeld) != 0) {
return lock_value;
}
uint32_t sched_disabled_bit = 0;
if ((lock_value & kSpinLockCooperative) == 0) {
// For non-cooperative locks we must make sure we mark ourselves as
// non-reschedulable before we attempt to CompareAndSwap.
if (base_internal::SchedulingGuard::DisableRescheduling()) {
sched_disabled_bit = kSpinLockDisabledScheduling;
}
}
if (!lockword_.compare_exchange_strong(
lock_value,
kSpinLockHeld | lock_value | wait_cycles | sched_disabled_bit,
std::memory_order_acquire, std::memory_order_relaxed)) {
base_internal::SchedulingGuard::EnableRescheduling(sched_disabled_bit != 0);
}
return lock_value;
}
} // namespace base_internal
ABSL_NAMESPACE_END
} // namespace absl
#endif // ABSL_BASE_INTERNAL_SPINLOCK_H_