[libc++] LWG2070: Use Allocator construction for objects created with allocate_shared
This patch updates `allocate_shared` to call `allocator_traits::construct`
when creating the object held inside the shared_pointer, and
`allocator_traits::destroy` when destroying it. This resolves
the part of P0674R1 that was originally filed as LWG2070.
This change is landed separately from the rest of P0674R1 because it is
incredibly tricky from an ABI perspective.
This is the reason why this change is so tricky is that we previously
used EBO in a compressed pair to store both the allocator and the object
type stored in the `shared_ptr`. However, starting in C++20, P0674
requires us to use Allocator construction for initializing the object type.
That requirement rules out the use of the EBO for the object type, since
using the EBO implies that the base will be initialized when the control
block is initialized (and hence we can't do it through Allocator construction).
Hence, supporting P0674 requires changing how we store the object type
inside the control block, which we do while being ABI compatible by using
some trickery with a properly aligned char buffer.
Fixes https://llvm.org/PR41900
Supersedes https://llvm.org/D62760
Differential Revision: https://reviews.llvm.org/D91201
GitOrigin-RevId: 955dd7b7f3f6df79f573508ffb567f3923e892f7
diff --git a/include/memory b/include/memory
index 4167f01..ef1a62b 100644
--- a/include/memory
+++ b/include/memory
@@ -1286,9 +1286,7 @@
template <class _T1, class _T2>
class __compressed_pair : private __compressed_pair_elem<_T1, 0>,
private __compressed_pair_elem<_T2, 1> {
- typedef _LIBCPP_NODEBUG_TYPE __compressed_pair_elem<_T1, 0> _Base1;
- typedef _LIBCPP_NODEBUG_TYPE __compressed_pair_elem<_T2, 1> _Base2;
-
+public:
// NOTE: This static assert should never fire because __compressed_pair
// is *almost never* used in a scenario where it's possible for T1 == T2.
// (The exception is std::function where it is possible that the function
@@ -1298,7 +1296,9 @@
"The current implementation is NOT ABI-compatible with the previous "
"implementation for this configuration");
-public:
+ typedef _LIBCPP_NODEBUG_TYPE __compressed_pair_elem<_T1, 0> _Base1;
+ typedef _LIBCPP_NODEBUG_TYPE __compressed_pair_elem<_T2, 1> _Base2;
+
template <bool _Dummy = true,
class = typename enable_if<
__dependent_type<is_default_constructible<_T1>, _Dummy>::value &&
@@ -1344,6 +1344,15 @@
return static_cast<_Base2 const&>(*this).__get();
}
+ _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+ static _Base1* __get_first_base(__compressed_pair* __pair) _NOEXCEPT {
+ return static_cast<_Base1*>(__pair);
+ }
+ _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+ static _Base2* __get_second_base(__compressed_pair* __pair) _NOEXCEPT {
+ return static_cast<_Base2*>(__pair);
+ }
+
_LIBCPP_INLINE_VISIBILITY
void swap(__compressed_pair& __x)
_NOEXCEPT_(__is_nothrow_swappable<_T1>::value &&
@@ -2574,43 +2583,81 @@
struct __shared_ptr_emplace
: __shared_weak_count
{
- _LIBCPP_HIDE_FROM_ABI
- explicit __shared_ptr_emplace(_Alloc __a)
- : __data_(_VSTD::move(__a), __value_init_tag())
- { }
-
- template <class ..._Args>
+ template<class ..._Args>
_LIBCPP_HIDE_FROM_ABI
explicit __shared_ptr_emplace(_Alloc __a, _Args&& ...__args)
-#ifndef _LIBCPP_CXX03_LANG
- : __data_(piecewise_construct, _VSTD::forward_as_tuple(__a),
- _VSTD::forward_as_tuple(_VSTD::forward<_Args>(__args)...))
+ : __storage_(_VSTD::move(__a))
+ {
+#if _LIBCPP_STD_VER > 17
+ using _TpAlloc = typename __allocator_traits_rebind<_Alloc, _Tp>::type;
+ _TpAlloc __tmp(*__get_alloc());
+ allocator_traits<_TpAlloc>::construct(__tmp, __get_elem(), _VSTD::forward<_Args>(__args)...);
#else
- : __data_(__a, _Tp(_VSTD::forward<_Args>(__args)...))
+ ::new ((void*)__get_elem()) _Tp(_VSTD::forward<_Args>(__args)...);
#endif
- { }
+ }
_LIBCPP_HIDE_FROM_ABI
- _Tp* __get_elem() _NOEXCEPT { return _VSTD::addressof(__data_.second()); }
+ _Alloc* __get_alloc() _NOEXCEPT { return __storage_.__get_alloc(); }
_LIBCPP_HIDE_FROM_ABI
- _Alloc* __get_alloc() _NOEXCEPT { return _VSTD::addressof(__data_.first()); }
+ _Tp* __get_elem() _NOEXCEPT { return __storage_.__get_elem(); }
private:
virtual void __on_zero_shared() _NOEXCEPT {
+#if _LIBCPP_STD_VER > 17
+ using _TpAlloc = typename __allocator_traits_rebind<_Alloc, _Tp>::type;
+ _TpAlloc __tmp(*__get_alloc());
+ allocator_traits<_TpAlloc>::destroy(__tmp, __get_elem());
+#else
__get_elem()->~_Tp();
+#endif
}
virtual void __on_zero_shared_weak() _NOEXCEPT {
using _ControlBlockAlloc = typename __allocator_traits_rebind<_Alloc, __shared_ptr_emplace>::type;
using _ControlBlockPointer = typename allocator_traits<_ControlBlockAlloc>::pointer;
_ControlBlockAlloc __tmp(*__get_alloc());
- __get_alloc()->~_Alloc();
+ __storage_.~_Storage();
allocator_traits<_ControlBlockAlloc>::deallocate(__tmp,
pointer_traits<_ControlBlockPointer>::pointer_to(*this), 1);
}
- __compressed_pair<_Alloc, _Tp> __data_;
+ // This class implements the control block for non-array shared pointers created
+ // through `std::allocate_shared` and `std::make_shared`.
+ //
+ // In previous versions of the library, we used a compressed pair to store
+ // both the _Alloc and the _Tp. This implies using EBO, which is incompatible
+ // with Allocator construction for _Tp. To allow implementing P0674 in C++20,
+ // we now use a properly aligned char buffer while making sure that we maintain
+ // the same layout that we had when we used a compressed pair.
+ using _CompressedPair = __compressed_pair<_Alloc, _Tp>;
+ struct _ALIGNAS_TYPE(_CompressedPair) _Storage {
+ char __blob_[sizeof(_CompressedPair)];
+
+ _LIBCPP_HIDE_FROM_ABI explicit _Storage(_Alloc&& __a) {
+ ::new ((void*)__get_alloc()) _Alloc(_VSTD::move(__a));
+ }
+ _LIBCPP_HIDE_FROM_ABI ~_Storage() {
+ __get_alloc()->~_Alloc();
+ }
+ _Alloc* __get_alloc() _NOEXCEPT {
+ _CompressedPair *__as_pair = reinterpret_cast<_CompressedPair*>(__blob_);
+ typename _CompressedPair::_Base1* __first = _CompressedPair::__get_first_base(__as_pair);
+ _Alloc *__alloc = reinterpret_cast<_Alloc*>(__first);
+ return __alloc;
+ }
+ _Tp* __get_elem() _NOEXCEPT {
+ _CompressedPair *__as_pair = reinterpret_cast<_CompressedPair*>(__blob_);
+ typename _CompressedPair::_Base2* __second = _CompressedPair::__get_second_base(__as_pair);
+ _Tp *__elem = reinterpret_cast<_Tp*>(__second);
+ return __elem;
+ }
+ };
+
+ static_assert(_LIBCPP_ALIGNOF(_Storage) == _LIBCPP_ALIGNOF(_CompressedPair), "");
+ static_assert(sizeof(_Storage) == sizeof(_CompressedPair), "");
+ _Storage __storage_;
};
struct __shared_ptr_dummy_rebind_allocator_type;