libitm/libitm_i.h

   1 /* Copyright (C) 2008-2024 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Transactional Memory Library (libitm).
   5
   6    Libitm is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
  12    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  13    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14    more details.
  15
  16    Under Section 7 of GPL version 3, you are granted additional
  17    permissions described in the GCC Runtime Library Exception, version
  18    3.1, as published by the Free Software Foundation.
  19
  20    You should have received a copy of the GNU General Public License and
  21    a copy of the GCC Runtime Library Exception along with this program;
  22    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  23    <http://www.gnu.org/licenses/>.  */
  24
  25 /* The following are internal implementation functions and definitions.
  26    To distinguish them from those defined by the Intel ABI, they all
  27    begin with GTM/gtm.  */
  28
  29 #ifndef LIBITM_I_H
  30 #define LIBITM_I_H 1
  31
  32 #include "libitm.h"
  33 #include "config.h"
  34
  35 #include <assert.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <unwind.h>
  39 #include "local_atomic"
  40
  41 /* Don't require libgcc_s.so for exceptions.  */
  42 extern void _Unwind_DeleteException (_Unwind_Exception*) __attribute__((weak));
  43
  44
  45 #include "common.h"
  46
  47 namespace GTM HIDDEN {
  48
  49 using namespace std;
  50
  51 typedef unsigned int gtm_word __attribute__((mode (word)));
  52
  53 // These values are given to GTM_restart_transaction and indicate the
  54 // reason for the restart.  The reason is used to decide what STM
  55 // implementation should be used during the next iteration.
  56 enum gtm_restart_reason
  57 {
  58   RESTART_REALLOCATE,
  59   RESTART_LOCKED_READ,
  60   RESTART_LOCKED_WRITE,
  61   RESTART_VALIDATE_READ,
  62   RESTART_VALIDATE_WRITE,
  63   RESTART_VALIDATE_COMMIT,
  64   RESTART_SERIAL_IRR,
  65   RESTART_NOT_READONLY,
  66   RESTART_CLOSED_NESTING,
  67   RESTART_INIT_METHOD_GROUP,
  68   NUM_RESTARTS,
  69   NO_RESTART = NUM_RESTARTS
  70 };
  71
  72 } // namespace GTM
  73
  74 #include "target.h"
  75 #include "rwlock.h"
  76 #include "aatree.h"
  77 #include "dispatch.h"
  78 #include "containers.h"
  79
  80 #ifdef __USER_LABEL_PREFIX__
  81 # define UPFX UPFX1(__USER_LABEL_PREFIX__)
  82 # define UPFX1(t) UPFX2(t)
  83 # define UPFX2(t) #t
  84 #else
  85 # define UPFX
  86 #endif
  87
  88 namespace GTM HIDDEN {
  89
  90 // A log of (de)allocation actions.  We defer handling of some actions until
  91 // a commit of the outermost transaction.  We also rely on potentially having
  92 // both an allocation and a deallocation for the same piece of memory in the
  93 // log; the order in which such entries are processed does not matter because
  94 // the actions are not in conflict (see below).
  95 // This type is private to alloc.c, but needs to be defined so that
  96 // the template used inside gtm_thread can instantiate.
  97 struct gtm_alloc_action
  98 {
  99   // Iff free_fn_sz is nonzero, it must be used instead of free_fn, and vice
 100   // versa.
 101   void (*free_fn)(void *);
 102   void (*free_fn_sz)(void *, size_t);
 103   size_t sz;
 104   // If true, this is an allocation; we discard the log entry on outermost
 105   // commit, and deallocate on abort.  If false, this is a deallocation and
 106   // we deallocate on outermost commit and discard the log entry on abort.
 107   bool allocated;
 108 };
 109
 110 struct gtm_thread;
 111
 112 // A transaction checkpoint: data that has to saved and restored when doing
 113 // closed nesting.
 114 struct gtm_transaction_cp
 115 {
 116   gtm_jmpbuf jb;
 117   size_t undolog_size;
 118   aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
 119   size_t user_actions_size;
 120   _ITM_transactionId_t id;
 121   uint32_t prop;
 122   uint32_t cxa_catch_count;
 123   unsigned int cxa_uncaught_count;
 124   // We might want to use a different but compatible dispatch method for
 125   // a nested transaction.
 126   abi_dispatch *disp;
 127   // Nesting level of this checkpoint (1 means that this is a checkpoint of
 128   // the outermost transaction).
 129   uint32_t nesting;
 130
 131   void save(gtm_thread* tx);
 132   void commit(gtm_thread* tx);
 133 };
 134
 135 // An undo log for writes.
 136 struct gtm_undolog
 137 {
 138   vector<gtm_word> undolog;
 139
 140   // Log the previous value at a certain address.
 141   // The easiest way to inline this is to just define this here.
 142   void log(const void *ptr, size_t len)
 143   {
 144     size_t words = (len + sizeof(gtm_word) - 1) / sizeof(gtm_word);
 145     gtm_word *undo = undolog.push(words + 2);
 146     memcpy(undo, ptr, len);
 147     undo[words] = len;
 148     undo[words + 1] = (gtm_word) ptr;
 149   }
 150
 151   void commit () { undolog.clear(); }
 152   size_t size() const { return undolog.size(); }
 153
 154   // In local.cc
 155   void rollback (gtm_thread* tx, size_t until_size = 0);
 156 };
 157
 158 // An entry of a read or write log.  Used by multi-lock TM methods.
 159 struct gtm_rwlog_entry
 160 {
 161   atomic<gtm_word> *orec;
 162   gtm_word value;
 163 };
 164
 165 // Contains all thread-specific data required by the entire library.
 166 // This includes all data relevant to a single transaction. Because most
 167 // thread-specific data is about the current transaction, we also refer to
 168 // the transaction-specific parts of gtm_thread as "the transaction" (the
 169 // same applies to names of variables and arguments).
 170 // All but the shared part of this data structure are thread-local data.
 171 // gtm_thread could be split into transaction-specific structures and other
 172 // per-thread data (with those parts then nested in gtm_thread), but this
 173 // would make it harder to later rearrange individual members to optimize data
 174 // accesses. Thus, for now we keep one flat object, and will only split it if
 175 // the code gets too messy.
 176 struct gtm_thread
 177 {
 178
 179   struct user_action
 180   {
 181     _ITM_userCommitFunction fn;
 182     void *arg;
 183     bool on_commit;
 184     _ITM_transactionId_t resuming_id;
 185   };
 186
 187   // The jump buffer by which GTM_longjmp restarts the transaction.
 188   // This field *must* be at the beginning of the transaction.
 189   gtm_jmpbuf jb;
 190
 191   // Data used by local.c for the undo log for both local and shared memory.
 192   gtm_undolog undolog;
 193
 194   // Read and write logs.  Used by multi-lock TM methods.
 195   vector<gtm_rwlog_entry> readlog;
 196   vector<gtm_rwlog_entry> writelog;
 197
 198   // Data used by alloc.c for the malloc/free undo log.
 199   aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
 200
 201   // Data used by useraction.c for the user-defined commit/abort handlers.
 202   vector<user_action> user_actions;
 203
 204   // A numerical identifier for this transaction.
 205   _ITM_transactionId_t id;
 206
 207   // The _ITM_codeProperties of this transaction as given by the compiler.
 208   uint32_t prop;
 209
 210   // The nesting depth for subsequently started transactions. This variable
 211   // will be set to 1 when starting an outermost transaction.
 212   uint32_t nesting;
 213
 214   // Set if this transaction owns the serial write lock.
 215   // Can be reset only when restarting the outermost transaction.
 216   static const uint32_t STATE_SERIAL            = 0x0001;
 217   // Set if the serial-irrevocable dispatch table is installed.
 218   // Implies that no logging is being done, and abort is not possible.
 219   // Can be reset only when restarting the outermost transaction.
 220   static const uint32_t STATE_IRREVOCABLE       = 0x0002;
 221
 222   // A bitmask of the above.
 223   uint32_t state;
 224
 225   // In order to reduce cacheline contention on global_tid during
 226   // beginTransaction, we allocate a block of 2**N ids to the thread
 227   // all at once.  This number is the next value to be allocated from
 228   // the block, or 0 % 2**N if no such block is allocated.
 229   _ITM_transactionId_t local_tid;
 230
 231   // Data used by eh_cpp.c for managing exceptions within the transaction.
 232   uint32_t cxa_catch_count;
 233   // If cxa_uncaught_count_ptr is 0, we don't need to roll back exceptions.
 234   unsigned int *cxa_uncaught_count_ptr;
 235   unsigned int cxa_uncaught_count;
 236   void *eh_in_flight;
 237
 238   // Checkpoints for closed nesting.
 239   vector<gtm_transaction_cp> parent_txns;
 240
 241   // Data used by retry.c for deciding what STM implementation should
 242   // be used for the next iteration of the transaction.
 243   // Only restart_total is reset to zero when the transaction commits, the
 244   // other counters are total values for all previously executed transactions.
 245   // restart_total is also used by the HTM fastpath in a different way.
 246   uint32_t restart_reason[NUM_RESTARTS];
 247   uint32_t restart_total;
 248
 249   // *** The shared part of gtm_thread starts here. ***
 250   // Shared state is on separate cachelines to avoid false sharing with
 251   // thread-local parts of gtm_thread.
 252
 253   // Points to the next thread in the list of all threads.
 254   gtm_thread *next_thread __attribute__((__aligned__(HW_CACHELINE_SIZE)));
 255
 256   // If this transaction is inactive, shared_state is ~0. Otherwise, this is
 257   // an active or serial transaction.
 258   atomic<gtm_word> shared_state;
 259
 260   // The lock that provides access to serial mode.  Non-serialized
 261   // transactions acquire read locks; a serialized transaction acquires
 262   // a write lock.
 263   // Accessed from assembly language, thus the "asm" specifier on
 264   // the name, avoiding complex name mangling.
 265   static gtm_rwlock serial_lock __asm__(UPFX "gtm_serial_lock");
 266
 267   // The head of the list of all threads' transactions.
 268   static gtm_thread *list_of_threads;
 269   // The number of all registered threads.
 270   static unsigned number_of_threads;
 271
 272   // In alloc.cc
 273   void commit_allocations (bool, aa_tree<uintptr_t, gtm_alloc_action>*);
 274   void record_allocation (void *, void (*)(void *));
 275   void forget_allocation (void *, void (*)(void *));
 276   void forget_allocation (void *, size_t, void (*)(void *, size_t));
 277   void discard_allocation (const void *ptr)
 278   {
 279     alloc_actions.erase((uintptr_t) ptr);
 280   }
 281
 282   // In beginend.cc
 283   void rollback (gtm_transaction_cp *cp = 0, bool aborting = false);
 284   bool trycommit ();
 285   void restart (gtm_restart_reason, bool finish_serial_upgrade = false)
 286         ITM_NORETURN;
 287
 288   gtm_thread();
 289   ~gtm_thread();
 290
 291   static void *operator new(size_t);
 292   static void operator delete(void *);
 293
 294   // Invoked from assembly language, thus the "asm" specifier on
 295   // the name, avoiding complex name mangling.
 296   static uint32_t begin_transaction(uint32_t, const gtm_jmpbuf *)
 297         __asm__(UPFX "GTM_begin_transaction") ITM_REGPARM;
 298   // In eh_cpp.cc
 299   void init_cpp_exceptions ();
 300   void revert_cpp_exceptions (gtm_transaction_cp *cp = 0);
 301
 302   // In retry.cc
 303   // Must be called outside of transactions (i.e., after rollback).
 304   void decide_retry_strategy (gtm_restart_reason);
 305   abi_dispatch* decide_begin_dispatch (uint32_t prop);
 306   void number_of_threads_changed(unsigned previous, unsigned now);
 307   // Must be called from serial mode. Does not call set_abi_disp().
 308   void set_default_dispatch(abi_dispatch* disp);
 309
 310   // In method-serial.cc
 311   void serialirr_mode ();
 312
 313   // In useraction.cc
 314   void rollback_user_actions (size_t until_size = 0);
 315   void commit_user_actions ();
 316 };
 317
 318 } // namespace GTM
 319
 320 #include "tls.h"
 321
 322 namespace GTM HIDDEN {
 323
 324 // An unscaled count of the number of times we should spin attempting to
 325 // acquire locks before we block the current thread and defer to the OS.
 326 // This variable isn't used when the standard POSIX lock implementations
 327 // are used.
 328 extern uint64_t gtm_spin_count_var;
 329
 330 extern "C" uint32_t GTM_longjmp (uint32_t, const gtm_jmpbuf *, uint32_t)
 331         ITM_NORETURN ITM_REGPARM;
 332
 333 extern "C" void GTM_LB (const void *, size_t) ITM_REGPARM;
 334
 335 extern void GTM_error (const char *fmt, ...)
 336         __attribute__((format (printf, 1, 2)));
 337 extern void GTM_fatal (const char *fmt, ...)
 338         __attribute__((noreturn, format (printf, 1, 2)));
 339
 340 extern abi_dispatch *dispatch_serial();
 341 extern abi_dispatch *dispatch_serialirr();
 342 extern abi_dispatch *dispatch_serialirr_onwrite();
 343 extern abi_dispatch *dispatch_gl_wt();
 344 extern abi_dispatch *dispatch_ml_wt();
 345 extern abi_dispatch *dispatch_htm();
 346
 347
 348 } // namespace GTM
 349
 350 #endif // LIBITM_I_H