Skip to content
This repository was archived by the owner on Mar 22, 2023. It is now read-only.

Commit 7b9e9f0

Browse files
igchorigchor
authored
Merge pull request #1069 from JanDorniak99/radix_implement_atomic_operations
radix_tree: use std::atomic<tagged_pointer>
2 parents 04d16d0 + e8de081 commit 7b9e9f0

8 files changed

Lines changed: 848 additions & 435 deletions

File tree

‎doc/libpmemobj++.Doxyfile.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,6 @@ HTML_TIMESTAMP = NO
229229
# recursively expanded use the := operator instead of the = operator.
230230
# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
231231

232-
PREDEFINED = __cpp_lib_uncaught_exceptions _WIN32
232+
PREDEFINED = __cpp_lib_uncaught_exceptions _WIN32 DOXYGEN_SHOULD_SKIP_THIS
233233

234234
WARN_AS_ERROR = @DOXYGEN_WARN_AS_ERROR@

‎include/libpmemobj++/detail/common.hpp‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
// SPDX-License-Identifier: BSD-3-Clause
2-
/* Copyright 2016-2020, Intel Corporation */
2+
/* Copyright 2016-2021, Intel Corporation */
33

44
/**
55
* @file
@@ -65,6 +65,29 @@
6565
#include <drd.h>
6666
#endif
6767

68+
#if LIBPMEMOBJ_CPP_VG_HELGRIND_ENABLED
69+
70+
#define LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_BEFORE(order, ptr) \
71+
if (order == std::memory_order_release || \
72+
order == std::memory_order_acq_rel || \
73+
order == std::memory_order_seq_cst) { \
74+
ANNOTATE_HAPPENS_BEFORE(ptr); \
75+
}
76+
77+
#define LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_AFTER(order, ptr) \
78+
if (order == std::memory_order_consume || \
79+
order == std::memory_order_acquire || \
80+
order == std::memory_order_acq_rel || \
81+
order == std::memory_order_seq_cst) { \
82+
ANNOTATE_HAPPENS_AFTER(ptr); \
83+
}
84+
#else
85+
86+
#define LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_BEFORE(order, ptr)
87+
#define LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_AFTER(order, ptr)
88+
89+
#endif
90+
6891
/*
6992
* Workaround for missing "is_trivially_copyable" in gcc < 5.0.
7093
* Be aware of a difference between __has_trivial_copy and is_trivially_copyable

‎include/libpmemobj++/detail/tagged_ptr.hpp‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
// SPDX-License-Identifier: BSD-3-Clause
2+
/* Copyright 2021, Intel Corporation */
3+
4+
#ifndef LIBPMEMOBJ_CPP_TAGGED_PTR
5+
#define LIBPMEMOBJ_CPP_TAGGED_PTR
6+
7+
#include <cassert>
8+
9+
#include <libpmemobj++/detail/common.hpp>
10+
#include <libpmemobj++/experimental/atomic_self_relative_ptr.hpp>
11+
#include <libpmemobj++/experimental/self_relative_ptr.hpp>
12+
#include <libpmemobj++/persistent_ptr.hpp>
13+
14+
namespace pmem
15+
{
16+
namespace detail
17+
{
18+
19+
template <typename P1, typename P2, typename PointerType>
20+
struct tagged_ptr_impl {
21+
tagged_ptr_impl() = default;
22+
tagged_ptr_impl(const tagged_ptr_impl &rhs) = default;
23+
24+
tagged_ptr_impl(std::nullptr_t) : ptr(nullptr)
25+
{
26+
assert(!(bool)*this);
27+
}
28+
29+
tagged_ptr_impl(const PointerType &ptr) : ptr(ptr)
30+
{
31+
}
32+
33+
tagged_ptr_impl(const obj::persistent_ptr<P1> &ptr)
34+
: ptr(add_tag(ptr.get()))
35+
{
36+
assert(get<P1>() == ptr.get());
37+
}
38+
39+
tagged_ptr_impl(const obj::persistent_ptr<P2> &ptr) : ptr(ptr.get())
40+
{
41+
assert(get<P2>() == ptr.get());
42+
}
43+
44+
tagged_ptr_impl &operator=(const tagged_ptr_impl &rhs) = default;
45+
46+
tagged_ptr_impl &operator=(std::nullptr_t)
47+
{
48+
ptr = nullptr;
49+
assert(!(bool)*this);
50+
51+
return *this;
52+
}
53+
tagged_ptr_impl &
54+
operator=(const obj::persistent_ptr<P1> &rhs)
55+
{
56+
ptr = add_tag(rhs.get());
57+
assert(get<P1>() == rhs.get());
58+
59+
return *this;
60+
}
61+
tagged_ptr_impl &
62+
operator=(const obj::persistent_ptr<P2> &rhs)
63+
{
64+
ptr = rhs.get();
65+
assert(get<P2>() == rhs.get());
66+
67+
return *this;
68+
}
69+
70+
bool
71+
operator==(const tagged_ptr_impl &rhs) const
72+
{
73+
return ptr.to_byte_pointer() == rhs.ptr.to_byte_pointer();
74+
}
75+
bool
76+
operator!=(const tagged_ptr_impl &rhs) const
77+
{
78+
return !(*this == rhs);
79+
}
80+
81+
bool
82+
operator==(const P1 *rhs) const
83+
{
84+
return is_tagged() && get<P1>() == rhs;
85+
}
86+
87+
bool
88+
operator!=(const P2 *rhs) const
89+
{
90+
return !(*this == rhs);
91+
}
92+
93+
void
94+
swap(tagged_ptr_impl &rhs)
95+
{
96+
ptr.swap(rhs.ptr);
97+
}
98+
99+
template <typename T>
100+
typename std::enable_if<std::is_same<T, P1>::value, bool>::type
101+
is() const
102+
{
103+
return is_tagged();
104+
}
105+
106+
template <typename T>
107+
typename std::enable_if<!std::is_same<T, P1>::value, bool>::type
108+
is() const
109+
{
110+
return !is_tagged();
111+
}
112+
113+
template <typename T>
114+
typename std::enable_if<std::is_same<T, P1>::value, T *>::type
115+
get() const
116+
{
117+
assert(is_tagged());
118+
return static_cast<P1 *>(remove_tag(ptr.to_void_pointer()));
119+
}
120+
121+
template <typename T>
122+
typename std::enable_if<!std::is_same<T, P1>::value, T *>::type
123+
get() const
124+
{
125+
assert(!is_tagged());
126+
return static_cast<P2 *>(ptr.to_void_pointer());
127+
}
128+
129+
P2 *operator->() const
130+
{
131+
return get<P2>();
132+
}
133+
134+
explicit operator bool() const noexcept
135+
{
136+
return remove_tag(ptr.to_void_pointer()) != nullptr;
137+
}
138+
139+
private:
140+
static constexpr uintptr_t IS_TAGGED = 1;
141+
void *
142+
add_tag(P1 *ptr) const
143+
{
144+
auto tagged =
145+
reinterpret_cast<uintptr_t>(ptr) | uintptr_t(IS_TAGGED);
146+
return reinterpret_cast<P1 *>(tagged);
147+
}
148+
149+
void *
150+
remove_tag(void *ptr) const
151+
{
152+
auto untagged = reinterpret_cast<uintptr_t>(ptr) &
153+
~uintptr_t(IS_TAGGED);
154+
return reinterpret_cast<void *>(untagged);
155+
}
156+
157+
bool
158+
is_tagged() const
159+
{
160+
auto value = reinterpret_cast<uintptr_t>(ptr.to_void_pointer());
161+
return value & uintptr_t(IS_TAGGED);
162+
}
163+
164+
PointerType ptr;
165+
166+
#ifndef DOXYGEN_SHOULD_SKIP_THIS
167+
friend std::atomic<tagged_ptr_impl<
168+
P1, P2, obj::experimental::self_relative_ptr<void>>>;
169+
#endif /* DOXYGEN_SHOULD_SKIP_THIS */
170+
};
171+
172+
template <typename P1, typename P2>
173+
using tagged_ptr =
174+
tagged_ptr_impl<P1, P2, obj::experimental::self_relative_ptr<void>>;
175+
176+
} /* namespace detail */
177+
} /* namespace pmem */
178+
179+
namespace std
180+
{
181+
182+
template <typename P1, typename P2>
183+
struct atomic<pmem::detail::tagged_ptr<P1, P2>> {
184+
private:
185+
using ptr_type = pmem::detail::tagged_ptr_impl<
186+
P1, P2,
187+
atomic<pmem::obj::experimental::self_relative_ptr<void>>>;
188+
using value_type = pmem::detail::tagged_ptr<P1, P2>;
189+
190+
public:
191+
/*
192+
* Constructors
193+
*/
194+
constexpr atomic() noexcept = default;
195+
196+
atomic(value_type value) : ptr()
197+
{
198+
store(value);
199+
}
200+
201+
atomic(const atomic &) = delete;
202+
203+
void
204+
store(value_type desired,
205+
std::memory_order order = std::memory_order_seq_cst) noexcept
206+
{
207+
LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_BEFORE(order, &ptr.ptr);
208+
ptr.ptr.store(desired.ptr, order);
209+
}
210+
211+
void
212+
store_with_snapshot(value_type desired,
213+
std::memory_order order = std::memory_order_seq_cst)
214+
{
215+
LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_BEFORE(order, &ptr.ptr);
216+
pmem::obj::transaction::snapshot(&ptr.ptr);
217+
ptr.ptr.store(desired.ptr, order);
218+
}
219+
220+
void
221+
store_with_snapshot_release(value_type desired)
222+
{
223+
store_with_snapshot(desired, std::memory_order_release);
224+
}
225+
226+
value_type
227+
load(std::memory_order order = std::memory_order_seq_cst) const noexcept
228+
{
229+
#if LIBPMEMOBJ_CPP_VG_HELGRIND_ENABLED
230+
VALGRIND_HG_DISABLE_CHECKING(&ptr.ptr, sizeof(ptr.ptr));
231+
#endif
232+
auto ret = this->ptr.ptr.load(order);
233+
LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_AFTER(order, &ptr.ptr);
234+
return value_type(ret);
235+
}
236+
237+
value_type
238+
load_acquire() const noexcept
239+
{
240+
return load(std::memory_order_acquire);
241+
}
242+
243+
void
244+
swap(atomic<pmem::detail::tagged_ptr<P1, P2>> &rhs)
245+
{
246+
auto tmp = rhs.load();
247+
rhs.store_with_snapshot(this->load());
248+
this->store_with_snapshot(tmp);
249+
}
250+
251+
private:
252+
ptr_type ptr;
253+
};
254+
255+
} /* namespace std */
256+
257+
#endif /* LIBPMEMOBJ_CPP_TAGGED_PTR */

‎include/libpmemobj++/experimental/atomic_self_relative_ptr.hpp‎

Lines changed: 2 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,16 @@
11
// SPDX-License-Identifier: BSD-3-Clause
2-
/* Copyright 2020, Intel Corporation */
2+
/* Copyright 2020-2021, Intel Corporation */
33

44
#ifndef LIBPMEMOBJ_CPP_ATOMIC_SELF_RELATIVE_PTR_HPP
55
#define LIBPMEMOBJ_CPP_ATOMIC_SELF_RELATIVE_PTR_HPP
66

7+
#include <libpmemobj++/detail/common.hpp>
78
#include <libpmemobj++/detail/self_relative_ptr_base_impl.hpp>
89
#include <libpmemobj++/experimental/self_relative_ptr.hpp>
910
#include <libpmemobj++/transaction.hpp>
1011

1112
#include <atomic>
1213

13-
#if LIBPMEMOBJ_CPP_VG_HELGRIND_ENABLED
14-
15-
#define LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_BEFORE(order, ptr) \
16-
if (order == std::memory_order_release || \
17-
order == std::memory_order_acq_rel || \
18-
order == std::memory_order_seq_cst) { \
19-
ANNOTATE_HAPPENS_BEFORE(ptr); \
20-
}
21-
22-
#define LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_AFTER(order, ptr) \
23-
if (order == std::memory_order_consume || \
24-
order == std::memory_order_acquire || \
25-
order == std::memory_order_acq_rel || \
26-
order == std::memory_order_seq_cst) { \
27-
ANNOTATE_HAPPENS_AFTER(ptr); \
28-
}
29-
#else
30-
31-
#define LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_BEFORE(order, ptr)
32-
#define LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_AFTER(order, ptr)
33-
34-
#endif
35-
3614
namespace std
3715
{
3816
/**
@@ -252,9 +230,6 @@ struct atomic<pmem::obj::experimental::self_relative_ptr<T>> {
252230

253231
} /* namespace std */
254232

255-
#undef LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_BEFORE
256-
#undef LIBPMEMOBJ_CPP_ANNOTATE_HAPPENS_AFTER
257-
258233
namespace pmem
259234
{
260235

0 commit comments

Comments
 (0)