libstdc++-v3/src/c++11/futex.cc

   1 // futex -*- C++ -*-
   2
   3 // Copyright (C) 2015-2024 Free Software Foundation, Inc.
   4 //
   5 // This file is part of the GNU ISO C++ Library.  This library is free
   6 // software; you can redistribute it and/or modify it under the
   7 // terms of the GNU General Public License as published by the
   8 // Free Software Foundation; either version 3, or (at your option)
   9 // any later version.
  10
  11 // This library is distributed in the hope that it will be useful,
  12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 // GNU General Public License for more details.
  15
  16 // Under Section 7 of GPL version 3, you are granted additional
  17 // permissions described in the GCC Runtime Library Exception, version
  18 // 3.1, as published by the Free Software Foundation.
  19
  20 // You should have received a copy of the GNU General Public License and
  21 // a copy of the GCC Runtime Library Exception along with this program;
  22 // see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  23 // <http://www.gnu.org/licenses/>.
  24
  25 #include <bits/atomic_futex.h>
  26 #ifdef _GLIBCXX_HAS_GTHREADS
  27 #if defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1
  28 #include <chrono>
  29 #include <climits>
  30 #include <syscall.h>
  31 #include <unistd.h>
  32 #include <sys/time.h>
  33 #include <errno.h>
  34 #include <ext/numeric_traits.h>
  35 #include <debug/debug.h>
  36
  37 #ifdef _GLIBCXX_USE_CLOCK_GETTIME_SYSCALL
  38 #include <unistd.h>
  39 #include <sys/syscall.h>
  40 #endif
  41
  42 // Constants for the wait/wake futex syscall operations
  43 const unsigned futex_wait_op = 0;
  44 const unsigned futex_wait_bitset_op = 9;
  45 const unsigned futex_clock_monotonic_flag = 0;
  46 const unsigned futex_clock_realtime_flag = 256;
  47 const unsigned futex_bitset_match_any = ~0;
  48 const unsigned futex_wake_op = 1;
  49
  50 namespace std _GLIBCXX_VISIBILITY(default)
  51 {
  52 _GLIBCXX_BEGIN_NAMESPACE_VERSION
  53
  54   using __gnu_cxx::__int_traits;
  55
  56 namespace
  57 {
  58   std::atomic<bool> futex_clock_realtime_unavailable;
  59   std::atomic<bool> futex_clock_monotonic_unavailable;
  60
  61 #if defined(SYS_futex_time64) && SYS_futex_time64 != SYS_futex
  62   // Userspace knows about the new time64 syscalls, so it's possible that
  63   // userspace has also updated timespec to use a 64-bit tv_sec.
  64   // The SYS_futex syscall still uses the old definition of timespec
  65   // where tv_sec is 32 bits, so define a type that matches that.
  66   struct syscall_timespec { long tv_sec; long tv_nsec; };
  67   using syscall_time_t = long;
  68 #else
  69   using syscall_timespec = ::timespec;
  70   using syscall_time_t = time_t;
  71 #endif
  72
  73   // Return the relative duration from (now_s + now_ns) to (abs_s + abs_ns)
  74   // as a timespec suitable for syscalls.
  75   syscall_timespec
  76   relative_timespec(chrono::seconds abs_s, chrono::nanoseconds abs_ns,
  77                     time_t now_s, long now_ns)
  78   {
  79     syscall_timespec rt;
  80
  81     // Did we already time out?
  82     if (now_s > abs_s.count())
  83       {
  84         rt.tv_sec = -1;
  85         return rt;
  86       }
  87
  88     const auto rel_s = abs_s.count() - now_s;
  89
  90     // Convert the absolute timeout to a relative timeout, without overflow.
  91     if (rel_s > __int_traits<syscall_time_t>::__max) [[unlikely]]
  92       {
  93         rt.tv_sec = __int_traits<syscall_time_t>::__max;
  94         rt.tv_nsec = 999999999;
  95       }
  96     else
  97       {
  98         rt.tv_sec = rel_s;
  99         rt.tv_nsec = abs_ns.count() - now_ns;
 100         if (rt.tv_nsec < 0)
 101           {
 102             rt.tv_nsec += 1000000000;
 103             --rt.tv_sec;
 104           }
 105       }
 106
 107     return rt;
 108   }
 109 } // namespace
 110
 111   bool
 112   __atomic_futex_unsigned_base::
 113   _M_futex_wait_until(unsigned *__addr, unsigned __val, bool __has_timeout,
 114                       chrono::seconds __s, chrono::nanoseconds __ns)
 115   {
 116     if (!__has_timeout)
 117       {
 118         // Ignore whether we actually succeeded to block because at worst,
 119         // we will fall back to spin-waiting.  The only thing we could do
 120         // here on errors is abort.
 121         int ret __attribute__((unused));
 122         ret = syscall (SYS_futex, __addr, futex_wait_op, __val, nullptr);
 123         __glibcxx_assert(ret == 0 || errno == EINTR || errno == EAGAIN);
 124         return true;
 125       }
 126     else
 127       {
 128         if (!futex_clock_realtime_unavailable.load(std::memory_order_relaxed))
 129           {
 130             // futex sets errno=EINVAL for absolute timeouts before the epoch.
 131             if (__s.count() < 0)
 132               return false;
 133
 134             syscall_timespec rt;
 135             if (__s.count() > __int_traits<syscall_time_t>::__max) [[unlikely]]
 136               rt.tv_sec = __int_traits<syscall_time_t>::__max;
 137             else
 138               rt.tv_sec = __s.count();
 139             rt.tv_nsec = __ns.count();
 140
 141             if (syscall (SYS_futex, __addr,
 142                          futex_wait_bitset_op | futex_clock_realtime_flag,
 143                          __val, &rt, nullptr, futex_bitset_match_any) == -1)
 144               {
 145                 __glibcxx_assert(errno == EINTR || errno == EAGAIN
 146                                 || errno == ETIMEDOUT || errno == ENOSYS);
 147                 if (errno == ETIMEDOUT)
 148                   return false;
 149                 if (errno == ENOSYS)
 150                   {
 151                     futex_clock_realtime_unavailable.store(true,
 152                                                     std::memory_order_relaxed);
 153                     // Fall through to legacy implementation if the system
 154                     // call is unavailable.
 155                   }
 156                 else
 157                   return true;
 158               }
 159             else
 160               return true;
 161           }
 162
 163         // We only get to here if futex_clock_realtime_unavailable was
 164         // true or has just been set to true.
 165         struct timeval tv;
 166         gettimeofday (&tv, NULL);
 167
 168         // Convert the absolute timeout value to a relative timeout
 169         auto rt = relative_timespec(__s, __ns, tv.tv_sec, tv.tv_usec * 1000);
 170
 171         // Did we already time out?
 172         if (rt.tv_sec < 0)
 173           return false;
 174
 175         if (syscall (SYS_futex, __addr, futex_wait_op, __val, &rt) == -1)
 176           {
 177             __glibcxx_assert(errno == EINTR || errno == EAGAIN
 178                              || errno == ETIMEDOUT);
 179             if (errno == ETIMEDOUT)
 180               return false;
 181           }
 182         return true;
 183       }
 184   }
 185
 186   bool
 187   __atomic_futex_unsigned_base::
 188   _M_futex_wait_until_steady(unsigned *__addr, unsigned __val,
 189                              bool __has_timeout,
 190                              chrono::seconds __s, chrono::nanoseconds __ns)
 191   {
 192     if (!__has_timeout)
 193       {
 194         // Ignore whether we actually succeeded to block because at worst,
 195         // we will fall back to spin-waiting.  The only thing we could do
 196         // here on errors is abort.
 197         int ret __attribute__((unused));
 198         ret = syscall (SYS_futex, __addr, futex_wait_op, __val, nullptr);
 199         __glibcxx_assert(ret == 0 || errno == EINTR || errno == EAGAIN);
 200         return true;
 201       }
 202     else
 203       {
 204         if (!futex_clock_monotonic_unavailable.load(std::memory_order_relaxed))
 205           {
 206             // futex sets errno=EINVAL for absolute timeouts before the epoch.
 207             if (__s.count() < 0) [[unlikely]]
 208               return false;
 209
 210             syscall_timespec rt;
 211             if (__s.count() > __int_traits<syscall_time_t>::__max) [[unlikely]]
 212               rt.tv_sec = __int_traits<syscall_time_t>::__max;
 213             else
 214               rt.tv_sec = __s.count();
 215             rt.tv_nsec = __ns.count();
 216
 217             if (syscall (SYS_futex, __addr,
 218                          futex_wait_bitset_op | futex_clock_monotonic_flag,
 219                          __val, &rt, nullptr, futex_bitset_match_any) == -1)
 220               {
 221                 __glibcxx_assert(errno == EINTR || errno == EAGAIN
 222                                  || errno == ETIMEDOUT || errno == ENOSYS);
 223                 if (errno == ETIMEDOUT)
 224                   return false;
 225                 else if (errno == ENOSYS)
 226                   {
 227                     futex_clock_monotonic_unavailable.store(true,
 228                                                     std::memory_order_relaxed);
 229                     // Fall through to legacy implementation if the system
 230                     // call is unavailable.
 231                   }
 232                 else
 233                   return true;
 234               }
 235           }
 236
 237         // We only get to here if futex_clock_monotonic_unavailable was
 238         // true or has just been set to true.
 239         struct timespec ts;
 240 #ifdef _GLIBCXX_USE_CLOCK_GETTIME_SYSCALL
 241         syscall(SYS_clock_gettime, CLOCK_MONOTONIC, &ts);
 242 #else
 243         clock_gettime(CLOCK_MONOTONIC, &ts);
 244 #endif
 245
 246         // Convert the absolute timeout value to a relative timeout
 247         auto rt = relative_timespec(__s, __ns, ts.tv_sec, ts.tv_nsec);
 248
 249         // Did we already time out?
 250         if (rt.tv_sec < 0)
 251           return false;
 252
 253         if (syscall (SYS_futex, __addr, futex_wait_op, __val, &rt) == -1)
 254           {
 255             __glibcxx_assert(errno == EINTR || errno == EAGAIN
 256                              || errno == ETIMEDOUT);
 257             if (errno == ETIMEDOUT)
 258               return false;
 259           }
 260         return true;
 261       }
 262   }
 263
 264   void
 265   __atomic_futex_unsigned_base::_M_futex_notify_all(unsigned* __addr)
 266   {
 267     // This syscall can fail for various reasons, including in situations
 268     // in which there is no real error.  Thus, we don't bother checking
 269     // the error codes.  See the futex documentation and glibc for background.
 270     syscall (SYS_futex, __addr, futex_wake_op, INT_MAX);
 271   }
 272
 273 _GLIBCXX_END_NAMESPACE_VERSION
 274 }
 275 #endif // defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1
 276 #endif // _GLIBCXX_HAS_GTHREADS