Skip to content

[libc++] Optimize __hash_table copy constructors and assignment #151951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
+299 −342
Draft

[libc++] Optimize __hash_table copy constructors and assignment #151951

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 117 additions & 14 deletions libcxx/include/__hash_table
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
#ifndef _LIBCPP___HASH_TABLE
#define _LIBCPP___HASH_TABLE

#include <__algorithm/fill_n.h>
#include <__algorithm/max.h>
#include <__algorithm/min.h>
#include <__assert>
Expand Down Expand Up @@ -700,6 +701,38 @@ private:

_LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __size_; }

_LIBCPP_HIDE_FROM_ABI void
__copy_construct(__next_pointer __other_iter, __next_pointer __own_iter, size_t __current_chash) {
auto __bucket_count = bucket_count();

for (; __other_iter; __other_iter = __other_iter->__next_) {
__node_holder __new_node = __construct_node_hash(__other_iter->__hash(), __other_iter->__upcast()->__get_value());

size_t __new_chash = std::__constrain_hash(__new_node->__hash(), __bucket_count);
if (__new_chash != __current_chash) {
__bucket_list_[__new_chash] = __own_iter;
__current_chash = __new_chash;
}

__own_iter->__next_ = static_cast<__next_pointer>(__new_node.release());
__own_iter = __own_iter->__next_;
}
}

_LIBCPP_HIDE_FROM_ABI void __copy_construct(__next_pointer __other_iter) {
__next_pointer __own_iter = __first_node_.__ptr();
{
__node_holder __new_node = __construct_node_hash(__other_iter->__hash(), __other_iter->__upcast()->__get_value());
__own_iter->__next_ = static_cast<__next_pointer>(__new_node.release());
}

size_t __current_chash = std::__constrain_hash(__own_iter->__next_->__hash(), bucket_count());
__bucket_list_[__current_chash] = __own_iter;
__other_iter = __other_iter->__next_;
__own_iter = __own_iter->__next_;
__copy_construct(__other_iter, __own_iter, __current_chash);
}

public:
_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }

Expand Down Expand Up @@ -1048,16 +1081,29 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const allocator_type& __a
__max_load_factor_(1.0f) {}

template <class _Tp, class _Hash, class _Equal, class _Alloc>
__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u)
__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __other)
: __bucket_list_(nullptr,
__bucket_list_deleter(allocator_traits<__pointer_allocator>::select_on_container_copy_construction(
__u.__bucket_list_.get_deleter().__alloc()),
__bucket_list_deleter(__pointer_alloc_traits::select_on_container_copy_construction(
__other.__bucket_list_.get_deleter().__alloc()),
0)),
__node_alloc_(allocator_traits<__node_allocator>::select_on_container_copy_construction(__u.__node_alloc())),
__node_alloc_(__node_traits::select_on_container_copy_construction(__other.__node_alloc())),
__size_(0),
__hasher_(__u.hash_function()),
__max_load_factor_(__u.__max_load_factor_),
__key_eq_(__u.__key_eq_) {}
__hasher_(__other.hash_function()),
__max_load_factor_(__other.__max_load_factor_),
__key_eq_(__other.__key_eq_) {
if (__other.size() == 0)
return;

auto& __bucket_list_del = __bucket_list_.get_deleter();
auto __bucket_count = __other.bucket_count();
__bucket_list_.reset(__pointer_alloc_traits::allocate(__bucket_list_del.__alloc(), __bucket_count));
__bucket_list_del.size() = __bucket_count;

std::fill_n(__bucket_list_.get(), __bucket_count, nullptr);

__copy_construct(__other.__first_node_.__next_);
__size_ = __other.size();
}

template <class _Tp, class _Hash, class _Equal, class _Alloc>
__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u, const allocator_type& __a)
Expand Down Expand Up @@ -1131,14 +1177,71 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__copy_assign_alloc(const __hash_
}

template <class _Tp, class _Hash, class _Equal, class _Alloc>
__hash_table<_Tp, _Hash, _Equal, _Alloc>& __hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __u) {
if (this != std::addressof(__u)) {
__copy_assign_alloc(__u);
hash_function() = __u.hash_function();
key_eq() = __u.key_eq();
max_load_factor() = __u.max_load_factor();
__assign_multi(__u.begin(), __u.end());
__hash_table<_Tp, _Hash, _Equal, _Alloc>&
__hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __other) {
if (this == std::addressof(__other))
return *this;

__copy_assign_alloc(__other);
hash_function() = __other.hash_function();
key_eq() = __other.key_eq();
max_load_factor() = __other.max_load_factor();

if (__other.size() == 0) {
clear();
return *this;
}

auto __bucket_count = __other.bucket_count();
if (__bucket_count != bucket_count()) {
auto& __bucket_list_del = __bucket_list_.get_deleter();
__bucket_list_.reset(__pointer_alloc_traits::allocate(__bucket_list_del.__alloc(), __bucket_count));
__bucket_list_del.size() = __bucket_count;
}
std::fill_n(__bucket_list_.get(), __bucket_count, nullptr);

if (!__first_node_.__next_) {
__copy_construct(__other.__first_node_.__next_);
__size_ = __other.size();
return *this;
}

__next_pointer __other_iter = __other.__first_node_.__next_;
__next_pointer __own_iter = __first_node_.__ptr();
{
__node_pointer __next = __own_iter->__next_->__upcast();
__assign_value(__next->__get_value(), __other_iter->__upcast()->__get_value());
__next->__hash_ = __other_iter->__hash();
}
size_t __current_chash = std::__constrain_hash(__own_iter->__next_->__hash(), __bucket_count);
__bucket_list_[__current_chash] = __own_iter;
__other_iter = __other_iter->__next_;
__own_iter = __own_iter->__next_;

while (__other_iter && __own_iter->__next_) {
__node_pointer __next = __own_iter->__next_->__upcast();
__assign_value(__next->__get_value(), __other_iter->__upcast()->__get_value());
__next->__hash_ = __other_iter->__hash();

size_t __new_chash = std::__constrain_hash(__next->__hash_, __bucket_count);
if (__new_chash != __current_chash) {
__bucket_list_[__new_chash] = __own_iter;
__current_chash = __new_chash;
}

__other_iter = __other_iter->__next_;
__own_iter = __own_iter->__next_;
}

if (__own_iter->__next_) {
__deallocate_node(__own_iter->__next_);
__own_iter->__next_ = nullptr;
} else {
__copy_construct(__other_iter, __own_iter, __current_chash);
}

__size_ = __other.size();

return *this;
}

Expand Down
68 changes: 8 additions & 60 deletions libcxx/include/unordered_map
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1046,10 +1046,11 @@ public:
# endif

_LIBCPP_HIDE_FROM_ABI explicit unordered_map(const allocator_type& __a);
_LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u);
_LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u) = default;
_LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u, const allocator_type& __a);
# ifndef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
_LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u)
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value) = default;
_LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u, const allocator_type& __a);
_LIBCPP_HIDE_FROM_ABI unordered_map(initializer_list<value_type> __il);
_LIBCPP_HIDE_FROM_ABI
Expand Down Expand Up @@ -1099,24 +1100,10 @@ public:
static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
}

_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) {
# ifndef _LIBCPP_CXX03_LANG
__table_ = __u.__table_;
# else
if (this != std::addressof(__u)) {
__table_.clear();
__table_.hash_function() = __u.__table_.hash_function();
__table_.key_eq() = __u.__table_.key_eq();
__table_.max_load_factor() = __u.__table_.max_load_factor();
__table_.__copy_assign_alloc(__u.__table_);
insert(__u.begin(), __u.end());
}
# endif
return *this;
}
_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) = default;
# ifndef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(unordered_map&& __u)
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) = default;
_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(initializer_list<value_type> __il);
# endif // _LIBCPP_CXX03_LANG

Expand Down Expand Up @@ -1563,12 +1550,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
insert(__first, __last);
}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u) : __table_(__u.__table_) {
__table_.__rehash_unique(__u.bucket_count());
insert(__u.begin(), __u.end());
}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u, const allocator_type& __a)
: __table_(__u.__table_, typename __table::allocator_type(__a)) {
Expand All @@ -1578,11 +1559,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_ma

# ifndef _LIBCPP_CXX03_LANG

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u)
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
: __table_(std::move(__u.__table_)) {}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u, const allocator_type& __a)
: __table_(std::move(__u.__table_), typename __table::allocator_type(__a)) {
Expand Down Expand Up @@ -1618,14 +1594,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
insert(__il.begin(), __il.end());
}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_map&& __u)
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
__table_ = std::move(__u.__table_);
return *this;
}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
Expand Down Expand Up @@ -1852,11 +1820,11 @@ public:
# endif

_LIBCPP_HIDE_FROM_ABI explicit unordered_multimap(const allocator_type& __a);
_LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u);
_LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u) = default;
_LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u, const allocator_type& __a);
# ifndef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u)
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value) = default;
_LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u, const allocator_type& __a);
_LIBCPP_HIDE_FROM_ABI unordered_multimap(initializer_list<value_type> __il);
_LIBCPP_HIDE_FROM_ABI unordered_multimap(
Expand Down Expand Up @@ -1923,7 +1891,7 @@ public:
}
# ifndef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(unordered_multimap&& __u)
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) = default;
_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(initializer_list<value_type> __il);
# endif // _LIBCPP_CXX03_LANG

Expand Down Expand Up @@ -2315,13 +2283,6 @@ template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const allocator_type& __a)
: __table_(typename __table::allocator_type(__a)) {}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const unordered_multimap& __u)
: __table_(__u.__table_) {
__table_.__rehash_multi(__u.bucket_count());
insert(__u.begin(), __u.end());
}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
const unordered_multimap& __u, const allocator_type& __a)
Expand All @@ -2332,11 +2293,6 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(

# ifndef _LIBCPP_CXX03_LANG

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(unordered_multimap&& __u)
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
: __table_(std::move(__u.__table_)) {}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
unordered_multimap&& __u, const allocator_type& __a)
Expand Down Expand Up @@ -2373,14 +2329,6 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
insert(__il.begin(), __il.end());
}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_multimap&& __u)
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
__table_ = std::move(__u.__table_);
return *this;
}

template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
Expand Down
Loading
Loading