study_objc4/objc4-750/include/os/lock_private.h

/*
 * Copyright (c) 2013-2016 Apple Inc. All rights reserved.
 *
 * @APPLE_APACHE_LICENSE_HEADER_START@
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @APPLE_APACHE_LICENSE_HEADER_END@
 */

#ifndef __OS_LOCK_PRIVATE__
#define __OS_LOCK_PRIVATE__

#include <Availability.h>
#include <TargetConditionals.h>
#include <sys/cdefs.h>
#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
#include <os/base_private.h>
#include <os/lock.h>

OS_ASSUME_NONNULL_BEGIN

/*! @header
 * Low-level lock SPI
 */

#define OS_LOCK_SPI_VERSION 20171006

/*!
 * @typedef os_lock_t
 *
 * @abstract
 * Pointer to one of the os_lock variants.
 */

#define OS_LOCK_TYPE_STRUCT(type) const struct _os_lock_type_##type##_s
#define OS_LOCK_TYPE_REF(type) _os_lock_type_##type
#define OS_LOCK_TYPE_DECL(type) OS_LOCK_TYPE_STRUCT(type) OS_LOCK_TYPE_REF(type)

#define OS_LOCK(type) os_lock_##type##_s
#define OS_LOCK_STRUCT(type) struct OS_LOCK(type)

#if defined(__cplusplus) && __cplusplus >= 201103L

#define OS_LOCK_DECL(type, size) \
		typedef OS_LOCK_STRUCT(type) : public OS_LOCK(base) { \
			private: \
			OS_LOCK_TYPE_STRUCT(type) * osl_type OS_UNUSED; \
			uintptr_t _osl_##type##_opaque[size-1] OS_UNUSED; \
			public: \
            constexpr OS_LOCK(type)() : \
				osl_type(&OS_LOCK_TYPE_REF(type)), _osl_##type##_opaque() {} \
		} OS_LOCK(type)
#define OS_LOCK_INIT(type) {}

typedef OS_LOCK_STRUCT(base) {
	protected:
	constexpr OS_LOCK(base)() {}
} *os_lock_t;

#else

#define OS_LOCK_DECL(type, size) \
		typedef OS_LOCK_STRUCT(type) { \
			OS_LOCK_TYPE_STRUCT(type) * osl_type; \
			uintptr_t _osl_##type##_opaque[size-1]; \
		} OS_LOCK(type)

#define OS_LOCK_INIT(type) { .osl_type = &OS_LOCK_TYPE_REF(type), }

#ifndef OS_LOCK_T_MEMBER
#define OS_LOCK_T_MEMBER(type) OS_LOCK_STRUCT(type) *_osl_##type
#endif

typedef OS_TRANSPARENT_UNION union {
	OS_LOCK_T_MEMBER(base);
	OS_LOCK_T_MEMBER(unfair);
	OS_LOCK_T_MEMBER(nospin);
	OS_LOCK_T_MEMBER(spin);
	OS_LOCK_T_MEMBER(handoff);
	OS_LOCK_T_MEMBER(eliding);
	OS_LOCK_T_MEMBER(transactional);
} os_lock_t;

#endif

/*!
 * @typedef os_lock_unfair_s
 *
 * @abstract
 * os_lock variant equivalent to os_unfair_lock. Does not spin on contention but
 * waits in the kernel to be woken up by an unlock. The lock value contains
 * ownership information that the system may use to attempt to resolve priority
 * inversions.
 *
 * @discussion
 * Intended as a replacement for os_lock_spin_s or OSSpinLock. Like with
 * OSSpinLock there is no attempt at fairness or lock ordering, e.g. an unlocker
 * can potentially immediately reacquire the lock before a woken up waiter gets
 * an opportunity to attempt to acquire the lock, so starvation is possibile.
 *
 * Must be initialized with OS_LOCK_UNFAIR_INIT
 */
__OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0)
__TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0)
OS_EXPORT OS_LOCK_TYPE_DECL(unfair);
OS_LOCK_DECL(unfair, 2);
#define OS_LOCK_UNFAIR_INIT OS_LOCK_INIT(unfair)

/*!
 * @typedef os_lock_nospin_s
 *
 * @abstract
 * os_lock variant that does not spin on contention but waits in the kernel to
 * be woken up by an unlock. No attempt to resolve priority inversions is made
 * so os_unfair_lock or os_lock_unfair_s should generally be preferred.
 *
 * @discussion
 * Intended as a replacement for os_lock_spin_s or OSSpinLock. Like with
 * OSSpinLock there is no attempt at fairness or lock ordering, e.g. an unlocker
 * can potentially immediately reacquire the lock before a woken up waiter gets
 * an opportunity to attempt to acquire the lock, so starvation is possibile.
 *
 * Must be initialized with OS_LOCK_NOSPIN_INIT
 */
__OSX_AVAILABLE(10.12) __IOS_AVAILABLE(10.0)
__TVOS_AVAILABLE(10.0) __WATCHOS_AVAILABLE(3.0)
OS_EXPORT OS_LOCK_TYPE_DECL(nospin);
OS_LOCK_DECL(nospin, 2);
#define OS_LOCK_NOSPIN_INIT OS_LOCK_INIT(nospin)

/*!
 * @typedef os_lock_spin_s
 *
 * @abstract
 * Deprecated os_lock variant that on contention starts by spinning trying to
 * acquire the lock, then depressing the priority of the current thread and
 * finally blocking the thread waiting for the lock to become available.
 * Equivalent to OSSpinLock and equally not recommended, see discussion in
 * libkern/OSAtomic.h headerdoc.
 *
 * @discussion
 * Spinlocks are intended to be held only for very brief periods of time. The
 * critical section must not make syscalls and should avoid touching areas of
 * memory that may trigger a page fault, in particular if the critical section
 * may be executing on threads of widely differing priorities or on a mix of
 * IO-throttled and unthrottled threads.
 *
 * Must be initialized with OS_LOCK_SPIN_INIT
 */
__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0)
OS_EXPORT OS_LOCK_TYPE_DECL(spin);
OS_LOCK_DECL(spin, 2);
#define OS_LOCK_SPIN_INIT OS_LOCK_INIT(spin)

/*!
 * @typedef os_lock_handoff_s
 *
 * @abstract
 * os_lock variant that on contention hands off the current kernel thread to the
 * lock-owning userspace thread (if it is not running), temporarily overriding
 * its priority and IO throttle if necessary.
 *
 * @discussion
 * Intended for use in limited circumstances where the critical section might
 * be executing on threads of widely differing priorities or on a mix of
 * IO-throttled and unthrottled threads where the ordinary os_lock_spin_s would
 * be likely to encounter a priority inversion.
 *
 * IMPORTANT: This lock variant is NOT intended as a general replacement for all
 * uses of os_lock_spin_s or OSSpinLock.
 *
 * Must be initialized with OS_LOCK_HANDOFF_INIT
 */
__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0)
OS_EXPORT OS_LOCK_TYPE_DECL(handoff);
OS_LOCK_DECL(handoff, 2);
#define OS_LOCK_HANDOFF_INIT OS_LOCK_INIT(handoff)


#if !TARGET_OS_IPHONE
/*!
 * @typedef os_lock_eliding_s
 *
 * @abstract
 * os_lock variant that uses hardware lock elision support if available to allow
 * multiple processors to concurrently execute a critical section as long as
 * they don't perform conflicting operations on each other's data. In case of
 * conflict, the lock reverts to exclusive operation and os_lock_spin_s behavior
 * on contention (at potential extra cost for the aborted attempt at lock-elided
 * concurrent execution). If hardware HLE support is not present, this lock
 * variant behaves like os_lock_spin_s.
 *
 * @discussion
 * IMPORTANT: Use of this lock variant MUST be extensively tested on hardware
 * with HLE support to ensure the data access pattern and length of the critical
 * section allows lock-elided execution to succeed frequently enough to offset
 * the cost of any aborted concurrent execution.
 *
 * Must be initialized with OS_LOCK_ELIDING_INIT
 */
__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_NA)
OS_EXPORT OS_LOCK_TYPE_DECL(eliding);
OS_LOCK_DECL(eliding, 8) OS_ALIGNED(64);
#define OS_LOCK_ELIDING_INIT OS_LOCK_INIT(eliding)

/*!
 * @typedef os_lock_transactional_s
 *
 * @abstract
 * os_lock variant that uses hardware restricted transactional memory support if
 * available to allow multiple processors to concurrently execute the critical
 * section as a transactional region. If transactional execution aborts, the
 * lock reverts to exclusive operation and os_lock_spin_s behavior on contention
 * (at potential extra cost for the aborted attempt at transactional concurrent
 * execution). If hardware RTM support is not present, this lock variant behaves
 * like os_lock_eliding_s.
 *
 * @discussion
 * IMPORTANT: Use of this lock variant MUST be extensively tested on hardware
 * with RTM support to ensure the data access pattern and length of the critical
 * section allows transactional execution to succeed frequently enough to offset
 * the cost of any aborted transactions.
 *
 * Must be initialized with OS_LOCK_TRANSACTIONAL_INIT
 */
__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_NA)
OS_EXPORT OS_LOCK_TYPE_DECL(transactional);
OS_LOCK_DECL(transactional, 8) OS_ALIGNED(64);
#define OS_LOCK_TRANSACTIONAL_INIT OS_LOCK_INIT(transactional)
#endif

__BEGIN_DECLS

/*!
 * @function os_lock_lock
 *
 * @abstract
 * Locks an os_lock variant.
 *
 * @param lock
 * Pointer to one of the os_lock variants.
 */
__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0)
OS_EXPORT OS_NOTHROW OS_NONNULL_ALL
void os_lock_lock(os_lock_t lock);

/*!
 * @function os_lock_trylock
 *
 * @abstract
 * Locks an os_lock variant if it is not already locked.
 *
 * @param lock
 * Pointer to one of the os_lock variants.
 *
 * @result
 * Returns true if the lock was succesfully locked and false if the lock was
 * already locked.
 */
__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0)
OS_EXPORT OS_NOTHROW OS_NONNULL_ALL
bool os_lock_trylock(os_lock_t lock);

/*!
 * @function os_lock_unlock
 *
 * @abstract
 * Unlocks an os_lock variant.
 *
 * @param lock
 * Pointer to one of the os_lock variants.
 */
__OSX_AVAILABLE_STARTING(__MAC_10_9,__IPHONE_7_0)
OS_EXPORT OS_NOTHROW OS_NONNULL_ALL
void os_lock_unlock(os_lock_t lock);

/*! @group os_unfair_lock SPI
 *
 * @abstract
 * Replacement for the deprecated OSSpinLock. Does not spin on contention but
 * waits in the kernel to be woken up by an unlock. The opaque lock value
 * contains thread ownership information that the system may use to attempt to
 * resolve priority inversions.
 *
 * This lock must be unlocked from the same thread that locked it, attemps to
 * unlock from a different thread will cause an assertion aborting the process.
 *
 * This lock must not be accessed from multiple processes or threads via shared
 * or multiply-mapped memory, the lock implementation relies on the address of
 * the lock value and owning process.
 *
 * @discussion
 * As with OSSpinLock there is no attempt at fairness or lock ordering, e.g. an
 * unlocker can potentially immediately reacquire the lock before a woken up
 * waiter gets an opportunity to attempt to acquire the lock. This may be
 * advantageous for performance reasons, but also makes starvation of waiters a
 * possibility.
 *
 * Must be initialized with OS_UNFAIR_LOCK_INIT
 */

/*!
 * @typedef os_unfair_lock_options_t
 *
 * @const OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION
 * This flag informs the runtime that the specified lock is used for data
 * synchronization and that the lock owner is always able to make progress
 * toward releasing the lock without the help of another thread in the same
 * process. This hint will cause the workqueue subsystem to not create new
 * threads to offset for threads waiting for the lock.
 *
 * When this flag is used, the code running under the critical section should
 * be well known and under your control  (Generally it should not call into
 * framework code).
 */
OS_ENUM(os_unfair_lock_options, uint32_t,
	OS_UNFAIR_LOCK_NONE
		OS_UNFAIR_LOCK_AVAILABILITY = 0x00000000,
	OS_UNFAIR_LOCK_DATA_SYNCHRONIZATION
		OS_UNFAIR_LOCK_AVAILABILITY = 0x00010000,
);

/*!
 * @function os_unfair_lock_lock_with_options
 *
 * @abstract
 * Locks an os_unfair_lock.
 *
 * @param lock
 * Pointer to an os_unfair_lock.
 *
 * @param options
 * Options to alter the behavior of the lock. See os_unfair_lock_options_t.
 */
OS_UNFAIR_LOCK_AVAILABILITY
OS_EXPORT OS_NOTHROW OS_NONNULL_ALL
void os_unfair_lock_lock_with_options(os_unfair_lock_t lock,
		os_unfair_lock_options_t options);

/*! @group os_unfair_lock no-TSD interfaces
 *
 * Like the above, but don't require being on a thread with valid TSD, so they
 * can be called from injected mach-threads.  The normal routines use the TSD
 * value for mach_thread_self(), these routines use MACH_PORT_DEAD for the
 * locked value instead.  As a result, they will be unable to resolve priority
 * inversions.
 *
 * This should only be used by libpthread.
 *
 */
OS_UNFAIR_LOCK_AVAILABILITY
OS_EXPORT OS_NOTHROW OS_NONNULL_ALL
void os_unfair_lock_lock_no_tsd_4libpthread(os_unfair_lock_t lock);

OS_UNFAIR_LOCK_AVAILABILITY
OS_EXPORT OS_NOTHROW OS_NONNULL_ALL
void os_unfair_lock_unlock_no_tsd_4libpthread(os_unfair_lock_t lock);

/*! @group os_unfair_recursive_lock SPI
 *
 * @abstract
 * Similar to os_unfair_lock, but recursive.
 *
 * @discussion
 * Must be initialized with OS_UNFAIR_RECURSIVE_LOCK_INIT
 */

#define OS_UNFAIR_RECURSIVE_LOCK_AVAILABILITY \
		__OSX_AVAILABLE(10.14) __IOS_AVAILABLE(12.0) \
		__TVOS_AVAILABLE(12.0) __WATCHOS_AVAILABLE(5.0)

#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
#define OS_UNFAIR_RECURSIVE_LOCK_INIT \
		((os_unfair_recursive_lock){OS_UNFAIR_LOCK_INIT, 0})
#elif defined(__cplusplus) && __cplusplus >= 201103L
#define OS_UNFAIR_RECURSIVE_LOCK_INIT \
		(os_unfair_recursive_lock{OS_UNFAIR_LOCK_INIT, 0})
#elif defined(__cplusplus)
#define OS_UNFAIR_RECURSIVE_LOCK_INIT (os_unfair_recursive_lock(\
		(os_unfair_recursive_lock){OS_UNFAIR_LOCK_INIT, 0}))
#else
#define OS_UNFAIR_RECURSIVE_LOCK_INIT \
		{OS_UNFAIR_LOCK_INIT, 0}
#endif // OS_UNFAIR_RECURSIVE_LOCK_INIT

/*!
 * @typedef os_unfair_recursive_lock
 *
 * @abstract
 * Low-level lock that allows waiters to block efficiently on contention.
 *
 * @discussion
 * See os_unfair_lock.
 *
 */
OS_UNFAIR_RECURSIVE_LOCK_AVAILABILITY
typedef struct os_unfair_recursive_lock_s {
	os_unfair_lock ourl_lock;
	uint32_t ourl_count;
} os_unfair_recursive_lock, *os_unfair_recursive_lock_t;

/*!
 * @function os_unfair_recursive_lock_lock_with_options
 *
 * @abstract
 * See os_unfair_lock_lock_with_options
 */
OS_UNFAIR_RECURSIVE_LOCK_AVAILABILITY
OS_EXPORT OS_NOTHROW OS_NONNULL_ALL
void os_unfair_recursive_lock_lock_with_options(os_unfair_recursive_lock_t lock,
		os_unfair_lock_options_t options);

/*!
 * @function os_unfair_recursive_lock_lock
 *
 * @abstract
 * See os_unfair_lock_lock
 */
OS_UNFAIR_RECURSIVE_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_NOTHROW OS_NONNULL_ALL
void
os_unfair_recursive_lock_lock(os_unfair_recursive_lock_t lock)
{
	os_unfair_recursive_lock_lock_with_options(lock, OS_UNFAIR_LOCK_NONE);
}

/*!
 * @function os_unfair_recursive_lock_trylock
 *
 * @abstract
 * See os_unfair_lock_trylock
 */
OS_UNFAIR_RECURSIVE_LOCK_AVAILABILITY
OS_EXPORT OS_NOTHROW OS_WARN_RESULT OS_NONNULL_ALL
bool os_unfair_recursive_lock_trylock(os_unfair_recursive_lock_t lock);

/*!
 * @function os_unfair_recursive_lock_unlock
 *
 * @abstract
 * See os_unfair_lock_unlock
 */
OS_UNFAIR_RECURSIVE_LOCK_AVAILABILITY
OS_EXPORT OS_NOTHROW OS_NONNULL_ALL
void os_unfair_recursive_lock_unlock(os_unfair_recursive_lock_t lock);

/*!
 * @function os_unfair_recursive_lock_tryunlock4objc
 *
 * @abstract
 * See os_unfair_lock_unlock
 */
OS_UNFAIR_RECURSIVE_LOCK_AVAILABILITY
OS_EXPORT OS_NOTHROW OS_NONNULL_ALL
bool os_unfair_recursive_lock_tryunlock4objc(os_unfair_recursive_lock_t lock);

/*!
 * @function os_unfair_recursive_lock_assert_owner
 *
 * @abstract
 * See os_unfair_lock_assert_owner
 */
OS_UNFAIR_RECURSIVE_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_NOTHROW OS_NONNULL_ALL
void
os_unfair_recursive_lock_assert_owner(os_unfair_recursive_lock_t lock)
{
	os_unfair_lock_assert_owner(&lock->ourl_lock);
}

/*!
 * @function os_unfair_recursive_lock_assert_not_owner
 *
 * @abstract
 * See os_unfair_lock_assert_not_owner
 */
OS_UNFAIR_RECURSIVE_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_NOTHROW OS_NONNULL_ALL
void
os_unfair_recursive_lock_assert_not_owner(os_unfair_recursive_lock_t lock)
{
	os_unfair_lock_assert_not_owner(&lock->ourl_lock);
}

#if __has_attribute(cleanup)

/*!
 * @function os_unfair_lock_scoped_guard_unlock
 *
 * @abstract
 * Used by os_unfair_lock_lock_scoped_guard
 */
OS_UNFAIR_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_NOTHROW OS_NONNULL_ALL
void
os_unfair_lock_scoped_guard_unlock(os_unfair_lock_t _Nonnull * _Nonnull lock)
{
	os_unfair_lock_unlock(*lock);
}

/*!
 * @function os_unfair_lock_lock_scoped_guard
 *
 * @abstract
 * Same as os_unfair_lock_lock() except that os_unfair_lock_unlock() is
 * automatically called when the enclosing C scope ends.
 *
 * @param name
 * Name for the variable holding the guard.
 *
 * @param lock
 * Pointer to an os_unfair_lock.
 *
 * @see os_unfair_lock_lock
 * @see os_unfair_lock_unlock
 */
#define os_unfair_lock_lock_scoped_guard(guard_name, lock) \
	os_unfair_lock_t \
		__attribute__((cleanup(os_unfair_lock_scoped_guard_unlock))) \
		guard_name = lock; \
	os_unfair_lock_lock(guard_name)

#endif // __has_attribute(cleanup)

__END_DECLS

OS_ASSUME_NONNULL_END

/*! @group Inline os_unfair_lock interfaces
 *
 * Inline versions of the os_unfair_lock fastpath.
 *
 * Intended exclusively for special highly performance-sensitive cases where the
 * function calls to the os_unfair_lock API entrypoints add measurable overhead.
 *
 * Do not use in frameworks to implement synchronization API primitives that are
 * exposed to developers, that would lead to false positives for that API from
 * tools such as ThreadSanitizer.
 *
 * !!!!!!!!!!!!!!!!!!!!! WARNING WARNING WARNING WARNING !!!!!!!!!!!!!!!!!!!!!
 * DO NOT USE IN CODE THAT IS NOT PART OF THE OPERATING SYSTEM OR THAT IS NOT
 *          REBUILT AS PART OF AN OS WORLDBUILD. YOU HAVE BEEN WARNED!
 * !!!!!!!!!!!!!!!!!!!!! WARNING WARNING WARNING WARNING !!!!!!!!!!!!!!!!!!!!!
 *
 * Define OS_UNFAIR_LOCK_INLINE=1 to indicate that you have read the warning
 * above and still wish to use these interfaces.
 */

#if defined(OS_UNFAIR_LOCK_INLINE) && OS_UNFAIR_LOCK_INLINE

#include <pthread/tsd_private.h>

#ifdef __cplusplus
extern "C++" {
#if !(__has_include(<atomic>) && __has_extension(cxx_atomic))
#error Cannot use inline os_unfair_lock without <atomic> and C++11 atomics
#endif
#include <atomic>
typedef std::atomic<os_unfair_lock> _os_atomic_unfair_lock;
#define OSLOCK_STD(_a) std::_a
__BEGIN_DECLS
#else
#if !(__has_include(<stdatomic.h>) && __has_extension(c_atomic))
#error Cannot use inline os_unfair_lock without <stdatomic.h> and C11 atomics
#endif
#include <stdatomic.h>
typedef _Atomic(os_unfair_lock) _os_atomic_unfair_lock;
#define OSLOCK_STD(_a) _a
#endif

OS_ASSUME_NONNULL_BEGIN

#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L
#define OS_UNFAIR_LOCK_UNLOCKED ((os_unfair_lock){0})
#elif defined(__cplusplus) && __cplusplus >= 201103L
#define OS_UNFAIR_LOCK_UNLOCKED (os_unfair_lock{})
#elif defined(__cplusplus)
#define OS_UNFAIR_LOCK_UNLOCKED (os_unfair_lock())
#else
#define OS_UNFAIR_LOCK_UNLOCKED {0}
#endif

/*!
 * @function os_unfair_lock_lock_inline
 *
 * @abstract
 * Locks an os_unfair_lock.
 *
 * @param lock
 * Pointer to an os_unfair_lock.
 */
OS_UNFAIR_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL
void
os_unfair_lock_lock_inline(os_unfair_lock_t lock)
{
	if (!_pthread_has_direct_tsd()) return os_unfair_lock_lock(lock);
	uint32_t mts = (uint32_t)(uintptr_t)_pthread_getspecific_direct(
			_PTHREAD_TSD_SLOT_MACH_THREAD_SELF);
	os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts };
	if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)(
			(_os_atomic_unfair_lock*)lock, &unlocked, locked,
			OSLOCK_STD(memory_order_acquire),
			OSLOCK_STD(memory_order_relaxed))) {
		return os_unfair_lock_lock(lock);
	}
}

/*!
 * @function os_unfair_lock_lock_with_options_inline
 *
 * @abstract
 * Locks an os_unfair_lock.
 *
 * @param lock
 * Pointer to an os_unfair_lock.
 *
 * @param options
 * Options to alter the behavior of the lock. See os_unfair_lock_options_t.
 */
OS_UNFAIR_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL
void
os_unfair_lock_lock_with_options_inline(os_unfair_lock_t lock,
		os_unfair_lock_options_t options)
{
	if (!_pthread_has_direct_tsd()) {
		return os_unfair_lock_lock_with_options(lock, options);
	}
	uint32_t mts = (uint32_t)(uintptr_t)_pthread_getspecific_direct(
			_PTHREAD_TSD_SLOT_MACH_THREAD_SELF);
	os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts };
	if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)(
			(_os_atomic_unfair_lock*)lock, &unlocked, locked,
			OSLOCK_STD(memory_order_acquire),
			OSLOCK_STD(memory_order_relaxed))) {
		return os_unfair_lock_lock_with_options(lock, options);
	}
}

/*!
 * @function os_unfair_lock_trylock_inline
 *
 * @abstract
 * Locks an os_unfair_lock if it is not already locked.
 *
 * @discussion
 * It is invalid to surround this function with a retry loop, if this function
 * returns false, the program must be able to proceed without having acquired
 * the lock, or it must call os_unfair_lock_lock_inline() instead.
 *
 * @param lock
 * Pointer to an os_unfair_lock.
 *
 * @result
 * Returns true if the lock was succesfully locked and false if the lock was
 * already locked.
 */
OS_UNFAIR_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_WARN_RESULT OS_NONNULL_ALL
bool
os_unfair_lock_trylock_inline(os_unfair_lock_t lock)
{
	if (!_pthread_has_direct_tsd()) return os_unfair_lock_trylock(lock);
	uint32_t mts = (uint32_t)(uintptr_t)_pthread_getspecific_direct(
			_PTHREAD_TSD_SLOT_MACH_THREAD_SELF);
	os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts };
	return OSLOCK_STD(atomic_compare_exchange_strong_explicit)(
			(_os_atomic_unfair_lock*)lock, &unlocked, locked,
			OSLOCK_STD(memory_order_acquire), OSLOCK_STD(memory_order_relaxed));
}

/*!
 * @function os_unfair_lock_unlock_inline
 *
 * @abstract
 * Unlocks an os_unfair_lock.
 *
 * @param lock
 * Pointer to an os_unfair_lock.
 */
OS_UNFAIR_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL
void
os_unfair_lock_unlock_inline(os_unfair_lock_t lock)
{
	if (!_pthread_has_direct_tsd()) return os_unfair_lock_unlock(lock);
	uint32_t mts = (uint32_t)(uintptr_t)_pthread_getspecific_direct(
			_PTHREAD_TSD_SLOT_MACH_THREAD_SELF);
	os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts };
	if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)(
			(_os_atomic_unfair_lock*)lock, &locked, unlocked,
			OSLOCK_STD(memory_order_release),
			OSLOCK_STD(memory_order_relaxed))) {
		return os_unfair_lock_unlock(lock);
	}
}

/*!
 * @function os_unfair_lock_lock_inline_no_tsd_4libpthread
 *
 * @abstract
 * Locks an os_unfair_lock, without requiring valid TSD.
 *
 * This should only be used by libpthread.
 *
 * @param lock
 * Pointer to an os_unfair_lock.
 */
OS_UNFAIR_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL
void
os_unfair_lock_lock_inline_no_tsd_4libpthread(os_unfair_lock_t lock)
{
	uint32_t mts = MACH_PORT_DEAD;
	os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts };
	if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)(
			(_os_atomic_unfair_lock*)lock, &unlocked, locked,
			OSLOCK_STD(memory_order_acquire),
			OSLOCK_STD(memory_order_relaxed))) {
		return os_unfair_lock_lock_no_tsd_4libpthread(lock);
	}
}

/*!
 * @function os_unfair_lock_unlock_inline_no_tsd_4libpthread
 *
 * @abstract
 * Unlocks an os_unfair_lock, without requiring valid TSD.
 *
 * This should only be used by libpthread.
 *
 * @param lock
 * Pointer to an os_unfair_lock.
 */
OS_UNFAIR_LOCK_AVAILABILITY
OS_INLINE OS_ALWAYS_INLINE OS_NONNULL_ALL
void
os_unfair_lock_unlock_inline_no_tsd_4libpthread(os_unfair_lock_t lock)
{
	uint32_t mts = MACH_PORT_DEAD;
	os_unfair_lock unlocked = OS_UNFAIR_LOCK_UNLOCKED, locked = { mts };
	if (!OSLOCK_STD(atomic_compare_exchange_strong_explicit)(
			(_os_atomic_unfair_lock*)lock, &locked, unlocked,
			OSLOCK_STD(memory_order_release),
			OSLOCK_STD(memory_order_relaxed))) {
		return os_unfair_lock_unlock_no_tsd_4libpthread(lock);
	}
}

OS_ASSUME_NONNULL_END

#undef OSLOCK_STD
#ifdef __cplusplus
__END_DECLS
} // extern "C++"
#endif

#endif // OS_UNFAIR_LOCK_INLINE

#endif // __OS_LOCK_PRIVATE__