libitm/libitm_i.h

   1 /* Copyright (C) 2008-2015 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Transactional Memory Library (libitm).
   5
   6    Libitm is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
  12    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  13    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14    more details.
  15
  16    Under Section 7 of GPL version 3, you are granted additional
  17    permissions described in the GCC Runtime Library Exception, version
  18    3.1, as published by the Free Software Foundation.
  19
  20    You should have received a copy of the GNU General Public License and
  21    a copy of the GCC Runtime Library Exception along with this program;
  22    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  23    <http://www.gnu.org/licenses/>.  */
  24
  25 /* The following are internal implementation functions and definitions.
  26    To distinguish them from those defined by the Intel ABI, they all
  27    begin with GTM/gtm.  */
  28
  29 #ifndef LIBITM_I_H
  30 #define LIBITM_I_H 1
  31
  32 #include "libitm.h"
  33 #include "config.h"
  34
  35 #include <assert.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <unwind.h>
  39 #include "local_type_traits"
  40 #include "local_atomic"
  41
  42 /* Don't require libgcc_s.so for exceptions.  */
  43 extern void _Unwind_DeleteException (_Unwind_Exception*) __attribute__((weak));
  44
  45
  46 #include "common.h"
  47
  48 namespace GTM HIDDEN {
  49
  50 using namespace std;
  51
  52 // A helper template for accessing an unsigned integral of SIZE bytes.
  53 template<size_t SIZE> struct sized_integral { };
  54 template<> struct sized_integral<1> { typedef uint8_t type; };
  55 template<> struct sized_integral<2> { typedef uint16_t type; };
  56 template<> struct sized_integral<4> { typedef uint32_t type; };
  57 template<> struct sized_integral<8> { typedef uint64_t type; };
  58
  59 typedef unsigned int gtm_word __attribute__((mode (word)));
  60
  61 // These values are given to GTM_restart_transaction and indicate the
  62 // reason for the restart.  The reason is used to decide what STM
  63 // implementation should be used during the next iteration.
  64 enum gtm_restart_reason
  65 {
  66   RESTART_REALLOCATE,
  67   RESTART_LOCKED_READ,
  68   RESTART_LOCKED_WRITE,
  69   RESTART_VALIDATE_READ,
  70   RESTART_VALIDATE_WRITE,
  71   RESTART_VALIDATE_COMMIT,
  72   RESTART_SERIAL_IRR,
  73   RESTART_NOT_READONLY,
  74   RESTART_CLOSED_NESTING,
  75   RESTART_INIT_METHOD_GROUP,
  76   NUM_RESTARTS,
  77   NO_RESTART = NUM_RESTARTS
  78 };
  79
  80 } // namespace GTM
  81
  82 #include "target.h"
  83 #include "rwlock.h"
  84 #include "aatree.h"
  85 #include "cacheline.h"
  86 #include "stmlock.h"
  87 #include "dispatch.h"
  88 #include "containers.h"
  89
  90 #ifdef __USER_LABEL_PREFIX__
  91 # define UPFX UPFX1(__USER_LABEL_PREFIX__)
  92 # define UPFX1(t) UPFX2(t)
  93 # define UPFX2(t) #t
  94 #else
  95 # define UPFX
  96 #endif
  97
  98 namespace GTM HIDDEN {
  99
 100 // A log of (de)allocation actions.  We defer handling of some actions until
 101 // a commit of the outermost transaction.  We also rely on potentially having
 102 // both an allocation and a deallocation for the same piece of memory in the
 103 // log; the order in which such entries are processed does not matter because
 104 // the actions are not in conflict (see below).
 105 // This type is private to alloc.c, but needs to be defined so that
 106 // the template used inside gtm_thread can instantiate.
 107 struct gtm_alloc_action
 108 {
 109   // Iff free_fn_sz is nonzero, it must be used instead of free_fn.
 110   union
 111   {
 112     void (*free_fn)(void *);
 113     void (*free_fn_sz)(void *, size_t);
 114   };
 115   size_t sz;
 116   // If true, this is an allocation; we discard the log entry on outermost
 117   // commit, and deallocate on abort.  If false, this is a deallocation and
 118   // we deallocate on outermost commit and discard the log entry on abort.
 119   bool allocated;
 120 };
 121
 122 struct gtm_thread;
 123
 124 // A transaction checkpoint: data that has to saved and restored when doing
 125 // closed nesting.
 126 struct gtm_transaction_cp
 127 {
 128   gtm_jmpbuf jb;
 129   size_t undolog_size;
 130   aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
 131   size_t user_actions_size;
 132   _ITM_transactionId_t id;
 133   uint32_t prop;
 134   uint32_t cxa_catch_count;
 135   void *cxa_unthrown;
 136   // We might want to use a different but compatible dispatch method for
 137   // a nested transaction.
 138   abi_dispatch *disp;
 139   // Nesting level of this checkpoint (1 means that this is a checkpoint of
 140   // the outermost transaction).
 141   uint32_t nesting;
 142
 143   void save(gtm_thread* tx);
 144   void commit(gtm_thread* tx);
 145 };
 146
 147 // An undo log for writes.
 148 struct gtm_undolog
 149 {
 150   vector<gtm_word> undolog;
 151
 152   // Log the previous value at a certain address.
 153   // The easiest way to inline this is to just define this here.
 154   void log(const void *ptr, size_t len)
 155   {
 156     size_t words = (len + sizeof(gtm_word) - 1) / sizeof(gtm_word);
 157     gtm_word *undo = undolog.push(words + 2);
 158     memcpy(undo, ptr, len);
 159     undo[words] = len;
 160     undo[words + 1] = (gtm_word) ptr;
 161   }
 162
 163   void commit () { undolog.clear(); }
 164   size_t size() const { return undolog.size(); }
 165
 166   // In local.cc
 167   void rollback (gtm_thread* tx, size_t until_size = 0);
 168 };
 169
 170 // An entry of a read or write log.  Used by multi-lock TM methods.
 171 struct gtm_rwlog_entry
 172 {
 173   atomic<gtm_word> *orec;
 174   gtm_word value;
 175 };
 176
 177 // Contains all thread-specific data required by the entire library.
 178 // This includes all data relevant to a single transaction. Because most
 179 // thread-specific data is about the current transaction, we also refer to
 180 // the transaction-specific parts of gtm_thread as "the transaction" (the
 181 // same applies to names of variables and arguments).
 182 // All but the shared part of this data structure are thread-local data.
 183 // gtm_thread could be split into transaction-specific structures and other
 184 // per-thread data (with those parts then nested in gtm_thread), but this
 185 // would make it harder to later rearrange individual members to optimize data
 186 // accesses. Thus, for now we keep one flat object, and will only split it if
 187 // the code gets too messy.
 188 struct gtm_thread
 189 {
 190
 191   struct user_action
 192   {
 193     _ITM_userCommitFunction fn;
 194     void *arg;
 195     bool on_commit;
 196     _ITM_transactionId_t resuming_id;
 197   };
 198
 199   // The jump buffer by which GTM_longjmp restarts the transaction.
 200   // This field *must* be at the beginning of the transaction.
 201   gtm_jmpbuf jb;
 202
 203   // Data used by local.c for the undo log for both local and shared memory.
 204   gtm_undolog undolog;
 205
 206   // Read and write logs.  Used by multi-lock TM methods.
 207   vector<gtm_rwlog_entry> readlog;
 208   vector<gtm_rwlog_entry> writelog;
 209
 210   // Data used by alloc.c for the malloc/free undo log.
 211   aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
 212
 213   // Data used by useraction.c for the user-defined commit/abort handlers.
 214   vector<user_action> user_actions;
 215
 216   // A numerical identifier for this transaction.
 217   _ITM_transactionId_t id;
 218
 219   // The _ITM_codeProperties of this transaction as given by the compiler.
 220   uint32_t prop;
 221
 222   // The nesting depth for subsequently started transactions. This variable
 223   // will be set to 1 when starting an outermost transaction.
 224   uint32_t nesting;
 225
 226   // Set if this transaction owns the serial write lock.
 227   // Can be reset only when restarting the outermost transaction.
 228   static const uint32_t STATE_SERIAL            = 0x0001;
 229   // Set if the serial-irrevocable dispatch table is installed.
 230   // Implies that no logging is being done, and abort is not possible.
 231   // Can be reset only when restarting the outermost transaction.
 232   static const uint32_t STATE_IRREVOCABLE       = 0x0002;
 233
 234   // A bitmask of the above.
 235   uint32_t state;
 236
 237   // In order to reduce cacheline contention on global_tid during
 238   // beginTransaction, we allocate a block of 2**N ids to the thread
 239   // all at once.  This number is the next value to be allocated from
 240   // the block, or 0 % 2**N if no such block is allocated.
 241   _ITM_transactionId_t local_tid;
 242
 243   // Data used by eh_cpp.c for managing exceptions within the transaction.
 244   uint32_t cxa_catch_count;
 245   void *cxa_unthrown;
 246   void *eh_in_flight;
 247
 248   // Checkpoints for closed nesting.
 249   vector<gtm_transaction_cp> parent_txns;
 250
 251   // Data used by retry.c for deciding what STM implementation should
 252   // be used for the next iteration of the transaction.
 253   // Only restart_total is reset to zero when the transaction commits, the
 254   // other counters are total values for all previously executed transactions.
 255   // restart_total is also used by the HTM fastpath in a different way.
 256   uint32_t restart_reason[NUM_RESTARTS];
 257   uint32_t restart_total;
 258
 259   // *** The shared part of gtm_thread starts here. ***
 260   // Shared state is on separate cachelines to avoid false sharing with
 261   // thread-local parts of gtm_thread.
 262
 263   // Points to the next thread in the list of all threads.
 264   gtm_thread *next_thread __attribute__((__aligned__(HW_CACHELINE_SIZE)));
 265
 266   // If this transaction is inactive, shared_state is ~0. Otherwise, this is
 267   // an active or serial transaction.
 268   atomic<gtm_word> shared_state;
 269
 270   // The lock that provides access to serial mode.  Non-serialized
 271   // transactions acquire read locks; a serialized transaction aquires
 272   // a write lock.
 273   // Accessed from assembly language, thus the "asm" specifier on
 274   // the name, avoiding complex name mangling.
 275   static gtm_rwlock serial_lock __asm__(UPFX "gtm_serial_lock");
 276
 277   // The head of the list of all threads' transactions.
 278   static gtm_thread *list_of_threads;
 279   // The number of all registered threads.
 280   static unsigned number_of_threads;
 281
 282   // In alloc.cc
 283   void commit_allocations (bool, aa_tree<uintptr_t, gtm_alloc_action>*);
 284   void record_allocation (void *, void (*)(void *));
 285   void forget_allocation (void *, void (*)(void *));
 286   void forget_allocation (void *, size_t, void (*)(void *, size_t));
 287   void drop_references_allocations (const void *ptr)
 288   {
 289     this->alloc_actions.erase((uintptr_t) ptr);
 290   }
 291
 292   // In beginend.cc
 293   void rollback (gtm_transaction_cp *cp = 0, bool aborting = false);
 294   bool trycommit ();
 295   void restart (gtm_restart_reason, bool finish_serial_upgrade = false)
 296         ITM_NORETURN;
 297
 298   gtm_thread();
 299   ~gtm_thread();
 300
 301   static void *operator new(size_t);
 302   static void operator delete(void *);
 303
 304   // Invoked from assembly language, thus the "asm" specifier on
 305   // the name, avoiding complex name mangling.
 306   static uint32_t begin_transaction(uint32_t, const gtm_jmpbuf *)
 307         __asm__(UPFX "GTM_begin_transaction") ITM_REGPARM;
 308   // In eh_cpp.cc
 309   void revert_cpp_exceptions (gtm_transaction_cp *cp = 0);
 310
 311   // In retry.cc
 312   // Must be called outside of transactions (i.e., after rollback).
 313   void decide_retry_strategy (gtm_restart_reason);
 314   abi_dispatch* decide_begin_dispatch (uint32_t prop);
 315   void number_of_threads_changed(unsigned previous, unsigned now);
 316   // Must be called from serial mode. Does not call set_abi_disp().
 317   void set_default_dispatch(abi_dispatch* disp);
 318
 319   // In method-serial.cc
 320   void serialirr_mode ();
 321
 322   // In useraction.cc
 323   void rollback_user_actions (size_t until_size = 0);
 324   void commit_user_actions ();
 325 };
 326
 327 } // namespace GTM
 328
 329 #include "tls.h"
 330
 331 namespace GTM HIDDEN {
 332
 333 // An unscaled count of the number of times we should spin attempting to
 334 // acquire locks before we block the current thread and defer to the OS.
 335 // This variable isn't used when the standard POSIX lock implementations
 336 // are used.
 337 extern uint64_t gtm_spin_count_var;
 338
 339 extern "C" uint32_t GTM_longjmp (uint32_t, const gtm_jmpbuf *, uint32_t)
 340         ITM_NORETURN ITM_REGPARM;
 341
 342 extern "C" void GTM_LB (const void *, size_t) ITM_REGPARM;
 343
 344 extern void GTM_error (const char *fmt, ...)
 345         __attribute__((format (printf, 1, 2)));
 346 extern void GTM_fatal (const char *fmt, ...)
 347         __attribute__((noreturn, format (printf, 1, 2)));
 348
 349 extern abi_dispatch *dispatch_serial();
 350 extern abi_dispatch *dispatch_serialirr();
 351 extern abi_dispatch *dispatch_serialirr_onwrite();
 352 extern abi_dispatch *dispatch_gl_wt();
 353 extern abi_dispatch *dispatch_ml_wt();
 354 extern abi_dispatch *dispatch_htm();
 355
 356 extern gtm_cacheline_mask gtm_mask_stack(gtm_cacheline *, gtm_cacheline_mask);
 357
 358 // Control variable for the HTM fastpath that uses serial mode as fallback.
 359 // Non-zero if the HTM fastpath is enabled. See gtm_thread::begin_transaction.
 360 // Accessed from assembly language, thus the "asm" specifier on
 361 // the name, avoiding complex name mangling.
 362 extern uint32_t htm_fastpath __asm__(UPFX "gtm_htm_fastpath");
 363
 364 } // namespace GTM
 365
 366 #endif // LIBITM_I_H