libgomp/team.c

   1 /* Copyright (C) 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU OpenMP Library (libgomp).
   5
   6    Libgomp is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU Lesser General Public License as published by
   8    the Free Software Foundation; either version 2.1 of the License, or
   9    (at your option) any later version.
  10
  11    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  12    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  13    FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
  14    more details.
  15
  16    You should have received a copy of the GNU Lesser General Public License
  17    along with libgomp; see the file COPYING.LIB.  If not, write to the
  18    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
  19    MA 02110-1301, USA.  */
  20
  21 /* As a special exception, if you link this library with other files, some
  22    of which are compiled with GCC, to produce an executable, this library
  23    does not by itself cause the resulting executable to be covered by the
  24    GNU General Public License.  This exception does not however invalidate
  25    any other reasons why the executable file might be covered by the GNU
  26    General Public License.  */
  27
  28 /* This file handles the maintainence of threads in response to team
  29    creation and termination.  */
  30
  31 #include "libgomp.h"
  32 #include <stdlib.h>
  33 #include <string.h>
  34
  35 /* This attribute contains PTHREAD_CREATE_DETACHED.  */
  36 pthread_attr_t gomp_thread_attr;
  37
  38 /* This key is for the thread destructor.  */
  39 pthread_key_t gomp_thread_destructor;
  40
  41
  42 /* This is the libgomp per-thread data structure.  */
  43 #ifdef HAVE_TLS
  44 __thread struct gomp_thread gomp_tls_data;
  45 #else
  46 pthread_key_t gomp_tls_key;
  47 #endif
  48
  49
  50 /* This structure is used to communicate across pthread_create.  */
  51
  52 struct gomp_thread_start_data
  53 {
  54   void (*fn) (void *);
  55   void *fn_data;
  56   struct gomp_team_state ts;
  57   struct gomp_task *task;
  58   struct gomp_thread_pool *thread_pool;
  59   bool nested;
  60 };
  61
  62
  63 /* This function is a pthread_create entry point.  This contains the idle
  64    loop in which a thread waits to be called up to become part of a team.  */
  65
  66 static void *
  67 gomp_thread_start (void *xdata)
  68 {
  69   struct gomp_thread_start_data *data = xdata;
  70   struct gomp_thread *thr;
  71   struct gomp_thread_pool *pool;
  72   void (*local_fn) (void *);
  73   void *local_data;
  74
  75 #ifdef HAVE_TLS
  76   thr = &gomp_tls_data;
  77 #else
  78   struct gomp_thread local_thr;
  79   thr = &local_thr;
  80   pthread_setspecific (gomp_tls_key, thr);
  81 #endif
  82   gomp_sem_init (&thr->release, 0);
  83
  84   /* Extract what we need from data.  */
  85   local_fn = data->fn;
  86   local_data = data->fn_data;
  87   thr->thread_pool = data->thread_pool;
  88   thr->ts = data->ts;
  89   thr->task = data->task;
  90
  91   thr->ts.team->ordered_release[thr->ts.team_id] = &thr->release;
  92
  93   /* Make thread pool local. */
  94   pool = thr->thread_pool;
  95
  96   if (data->nested)
  97     {
  98       struct gomp_team *team = thr->ts.team;
  99       struct gomp_task *task = thr->task;
 100
 101       gomp_barrier_wait (&team->barrier);
 102
 103       local_fn (local_data);
 104       gomp_team_barrier_wait (&team->barrier);
 105       gomp_finish_task (task);
 106       gomp_barrier_wait_last (&team->barrier);
 107     }
 108   else
 109     {
 110       pool->threads[thr->ts.team_id] = thr;
 111
 112       gomp_barrier_wait (&pool->threads_dock);
 113       do
 114         {
 115           struct gomp_team *team = thr->ts.team;
 116           struct gomp_task *task = thr->task;
 117
 118           local_fn (local_data);
 119           gomp_team_barrier_wait (&team->barrier);
 120           gomp_finish_task (task);
 121
 122           gomp_barrier_wait (&pool->threads_dock);
 123
 124           local_fn = thr->fn;
 125           local_data = thr->data;
 126           thr->fn = NULL;
 127         }
 128       while (local_fn);
 129     }
 130
 131   return NULL;
 132 }
 133
 134
 135 /* Create a new team data structure.  */
 136
 137 struct gomp_team *
 138 gomp_new_team (unsigned nthreads)
 139 {
 140   struct gomp_team *team;
 141   size_t size;
 142   int i;
 143
 144   size = sizeof (*team) + nthreads * (sizeof (team->ordered_release[0])
 145                                       + sizeof (team->implicit_task[0]));
 146   team = gomp_malloc (size);
 147
 148   team->work_share_chunk = 8;
 149 #ifdef HAVE_SYNC_BUILTINS
 150   team->single_count = 0;
 151 #else
 152   gomp_mutex_init (&team->work_share_list_free_lock);
 153 #endif
 154   gomp_init_work_share (&team->work_shares[0], false, nthreads);
 155   team->work_shares[0].next_alloc = NULL;
 156   team->work_share_list_free = NULL;
 157   team->work_share_list_alloc = &team->work_shares[1];
 158   for (i = 1; i < 7; i++)
 159     team->work_shares[i].next_free = &team->work_shares[i + 1];
 160   team->work_shares[i].next_free = NULL;
 161
 162   team->nthreads = nthreads;
 163   gomp_barrier_init (&team->barrier, nthreads);
 164
 165   gomp_sem_init (&team->master_release, 0);
 166   team->ordered_release = (void *) &team->implicit_task[nthreads];
 167   team->ordered_release[0] = &team->master_release;
 168
 169   gomp_mutex_init (&team->task_lock);
 170   team->task_queue = NULL;
 171   team->task_count = 0;
 172   team->task_running_count = 0;
 173
 174   return team;
 175 }
 176
 177
 178 /* Free a team data structure.  */
 179
 180 static void
 181 free_team (struct gomp_team *team)
 182 {
 183   gomp_barrier_destroy (&team->barrier);
 184   gomp_mutex_destroy (&team->task_lock);
 185   free (team);
 186 }
 187
 188 /* Allocate and initialize a thread pool. */
 189
 190 static struct gomp_thread_pool *gomp_new_thread_pool (void)
 191 {
 192   struct gomp_thread_pool *pool
 193     = gomp_malloc (sizeof(struct gomp_thread_pool));
 194   pool->threads = NULL;
 195   pool->threads_size = 0;
 196   pool->threads_used = 0;
 197   pool->last_team = NULL;
 198   return pool;
 199 }
 200
 201 static void
 202 gomp_free_pool_helper (void *thread_pool)
 203 {
 204   struct gomp_thread_pool *pool
 205     = (struct gomp_thread_pool *) thread_pool;
 206   gomp_barrier_wait_last (&pool->threads_dock);
 207   pthread_exit (NULL);
 208 }
 209
 210 /* Free a thread pool and release its threads. */
 211
 212 static void
 213 gomp_free_thread (void *arg __attribute__((unused)))
 214 {
 215   struct gomp_thread *thr = gomp_thread ();
 216   struct gomp_thread_pool *pool = thr->thread_pool;
 217   if (pool)
 218     {
 219       if (pool->threads_used > 0)
 220         {
 221           int i;
 222           for (i = 1; i < pool->threads_used; i++)
 223             {
 224               struct gomp_thread *nthr = pool->threads[i];
 225               nthr->fn = gomp_free_pool_helper;
 226               nthr->data = pool;
 227             }
 228           /* This barrier undocks threads docked on pool->threads_dock.  */
 229           gomp_barrier_wait (&pool->threads_dock);
 230           /* And this waits till all threads have called gomp_barrier_wait_last
 231              in gomp_free_pool_helper.  */
 232           gomp_barrier_wait (&pool->threads_dock);
 233           /* Now it is safe to destroy the barrier and free the pool.  */
 234           gomp_barrier_destroy (&pool->threads_dock);
 235         }
 236       free (pool->threads);
 237       if (pool->last_team)
 238         free_team (pool->last_team);
 239       free (pool);
 240       thr->thread_pool = NULL;
 241     }
 242   if (thr->task != NULL)
 243     {
 244       struct gomp_task *task = thr->task;
 245       gomp_end_task ();
 246       free (task);
 247     }
 248 }
 249
 250 /* Launch a team.  */
 251
 252 void
 253 gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
 254                  struct gomp_team *team)
 255 {
 256   struct gomp_thread_start_data *start_data;
 257   struct gomp_thread *thr, *nthr;
 258   struct gomp_task *task;
 259   struct gomp_task_icv *icv;
 260   bool nested;
 261   struct gomp_thread_pool *pool;
 262   unsigned i, n, old_threads_used = 0;
 263   pthread_attr_t thread_attr, *attr;
 264
 265   thr = gomp_thread ();
 266   nested = thr->ts.team != NULL;
 267   if (__builtin_expect (thr->thread_pool == NULL, 0))
 268     {
 269       thr->thread_pool = gomp_new_thread_pool ();
 270       pthread_setspecific (gomp_thread_destructor, thr);
 271     }
 272   pool = thr->thread_pool;
 273   task = thr->task;
 274   icv = task ? &task->icv : &gomp_global_icv;
 275
 276   /* Always save the previous state, even if this isn't a nested team.
 277      In particular, we should save any work share state from an outer
 278      orphaned work share construct.  */
 279   team->prev_ts = thr->ts;
 280
 281   thr->ts.team = team;
 282   thr->ts.team_id = 0;
 283   ++thr->ts.level;
 284   if (nthreads > 1)
 285     ++thr->ts.active_level;
 286   thr->ts.work_share = &team->work_shares[0];
 287   thr->ts.last_work_share = NULL;
 288 #ifdef HAVE_SYNC_BUILTINS
 289   thr->ts.single_count = 0;
 290 #endif
 291   thr->ts.static_trip = 0;
 292   thr->task = &team->implicit_task[0];
 293   gomp_init_task (thr->task, task, icv);
 294
 295   if (nthreads == 1)
 296     return;
 297
 298   i = 1;
 299
 300   /* We only allow the reuse of idle threads for non-nested PARALLEL
 301      regions.  This appears to be implied by the semantics of
 302      threadprivate variables, but perhaps that's reading too much into
 303      things.  Certainly it does prevent any locking problems, since
 304      only the initial program thread will modify gomp_threads.  */
 305   if (!nested)
 306     {
 307       old_threads_used = pool->threads_used;
 308
 309       if (nthreads <= old_threads_used)
 310         n = nthreads;
 311       else if (old_threads_used == 0)
 312         {
 313           n = 0;
 314           gomp_barrier_init (&pool->threads_dock, nthreads);
 315         }
 316       else
 317         {
 318           n = old_threads_used;
 319
 320           /* Increase the barrier threshold to make sure all new
 321              threads arrive before the team is released.  */
 322           gomp_barrier_reinit (&pool->threads_dock, nthreads);
 323         }
 324
 325       /* Not true yet, but soon will be.  We're going to release all
 326          threads from the dock, and those that aren't part of the
 327          team will exit.  */
 328       pool->threads_used = nthreads;
 329
 330       /* Release existing idle threads.  */
 331       for (; i < n; ++i)
 332         {
 333           nthr = pool->threads[i];
 334           nthr->ts.team = team;
 335           nthr->ts.work_share = &team->work_shares[0];
 336           nthr->ts.last_work_share = NULL;
 337           nthr->ts.team_id = i;
 338           nthr->ts.level = team->prev_ts.level + 1;
 339           nthr->ts.active_level = thr->ts.active_level;
 340 #ifdef HAVE_SYNC_BUILTINS
 341           nthr->ts.single_count = 0;
 342 #endif
 343           nthr->ts.static_trip = 0;
 344           nthr->task = &team->implicit_task[i];
 345           gomp_init_task (nthr->task, task, icv);
 346           nthr->fn = fn;
 347           nthr->data = data;
 348           team->ordered_release[i] = &nthr->release;
 349         }
 350
 351       if (i == nthreads)
 352         goto do_release;
 353
 354       /* If necessary, expand the size of the gomp_threads array.  It is
 355          expected that changes in the number of threads are rare, thus we
 356          make no effort to expand gomp_threads_size geometrically.  */
 357       if (nthreads >= pool->threads_size)
 358         {
 359           pool->threads_size = nthreads + 1;
 360           pool->threads
 361             = gomp_realloc (pool->threads,
 362                             pool->threads_size
 363                             * sizeof (struct gomp_thread_data *));
 364         }
 365     }
 366
 367   if (__builtin_expect (nthreads > old_threads_used, 0))
 368     {
 369       long diff = (long) nthreads - (long) old_threads_used;
 370
 371       if (old_threads_used == 0)
 372         --diff;
 373
 374 #ifdef HAVE_SYNC_BUILTINS
 375       __sync_fetch_and_add (&gomp_managed_threads, diff);
 376 #else
 377       gomp_mutex_lock (&gomp_remaining_threads_lock);
 378       gomp_managed_threads += diff;
 379       gomp_mutex_unlock (&gomp_remaining_threads_lock);
 380 #endif
 381     }
 382
 383   attr = &gomp_thread_attr;
 384   if (__builtin_expect (gomp_cpu_affinity != NULL, 0))
 385     {
 386       size_t stacksize;
 387       pthread_attr_init (&thread_attr);
 388       pthread_attr_setdetachstate (&thread_attr, PTHREAD_CREATE_DETACHED);
 389       if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize))
 390         pthread_attr_setstacksize (&thread_attr, stacksize);
 391       attr = &thread_attr;
 392     }
 393
 394   start_data = gomp_alloca (sizeof (struct gomp_thread_start_data)
 395                             * (nthreads-i));
 396
 397   /* Launch new threads.  */
 398   for (; i < nthreads; ++i, ++start_data)
 399     {
 400       pthread_t pt;
 401       int err;
 402
 403       start_data->fn = fn;
 404       start_data->fn_data = data;
 405       start_data->ts.team = team;
 406       start_data->ts.work_share = &team->work_shares[0];
 407       start_data->ts.last_work_share = NULL;
 408       start_data->ts.team_id = i;
 409       start_data->ts.level = team->prev_ts.level + 1;
 410       start_data->ts.active_level = thr->ts.active_level;
 411 #ifdef HAVE_SYNC_BUILTINS
 412       start_data->ts.single_count = 0;
 413 #endif
 414       start_data->ts.static_trip = 0;
 415       start_data->task = &team->implicit_task[i];
 416       gomp_init_task (start_data->task, task, icv);
 417       start_data->thread_pool = pool;
 418       start_data->nested = nested;
 419
 420       if (gomp_cpu_affinity != NULL)
 421         gomp_init_thread_affinity (attr);
 422
 423       err = pthread_create (&pt, attr, gomp_thread_start, start_data);
 424       if (err != 0)
 425         gomp_fatal ("Thread creation failed: %s", strerror (err));
 426     }
 427
 428   if (__builtin_expect (gomp_cpu_affinity != NULL, 0))
 429     pthread_attr_destroy (&thread_attr);
 430
 431  do_release:
 432   gomp_barrier_wait (nested ? &team->barrier : &pool->threads_dock);
 433
 434   /* Decrease the barrier threshold to match the number of threads
 435      that should arrive back at the end of this team.  The extra
 436      threads should be exiting.  Note that we arrange for this test
 437      to never be true for nested teams.  */
 438   if (__builtin_expect (nthreads < old_threads_used, 0))
 439     {
 440       long diff = (long) nthreads - (long) old_threads_used;
 441
 442       gomp_barrier_reinit (&pool->threads_dock, nthreads);
 443
 444 #ifdef HAVE_SYNC_BUILTINS
 445       __sync_fetch_and_add (&gomp_managed_threads, diff);
 446 #else
 447       gomp_mutex_lock (&gomp_remaining_threads_lock);
 448       gomp_managed_threads += diff;
 449       gomp_mutex_unlock (&gomp_remaining_threads_lock);
 450 #endif
 451     }
 452 }
 453
 454
 455 /* Terminate the current team.  This is only to be called by the master
 456    thread.  We assume that we must wait for the other threads.  */
 457
 458 void
 459 gomp_team_end (void)
 460 {
 461   struct gomp_thread *thr = gomp_thread ();
 462   struct gomp_team *team = thr->ts.team;
 463
 464   /* This barrier handles all pending explicit threads.  */
 465   gomp_team_barrier_wait (&team->barrier);
 466   gomp_fini_work_share (thr->ts.work_share);
 467
 468   gomp_end_task ();
 469   thr->ts = team->prev_ts;
 470
 471   if (__builtin_expect (thr->ts.team != NULL, 0))
 472     {
 473 #ifdef HAVE_SYNC_BUILTINS
 474       __sync_fetch_and_add (&gomp_managed_threads, 1L - team->nthreads);
 475 #else
 476       gomp_mutex_lock (&gomp_remaining_threads_lock);
 477       gomp_managed_threads -= team->nthreads - 1L;
 478       gomp_mutex_unlock (&gomp_remaining_threads_lock);
 479 #endif
 480       /* This barrier has gomp_barrier_wait_last counterparts
 481          and ensures the team can be safely destroyed.  */
 482       gomp_barrier_wait (&team->barrier);
 483     }
 484
 485   if (__builtin_expect (team->work_shares[0].next_alloc != NULL, 0))
 486     {
 487       struct gomp_work_share *ws = team->work_shares[0].next_alloc;
 488       do
 489         {
 490           struct gomp_work_share *next_ws = ws->next_alloc;
 491           free (ws);
 492           ws = next_ws;
 493         }
 494       while (ws != NULL);
 495     }
 496   gomp_sem_destroy (&team->master_release);
 497 #ifndef HAVE_SYNC_BUILTINS
 498   gomp_mutex_destroy (&team->work_share_list_free_lock);
 499 #endif
 500
 501   if (__builtin_expect (thr->ts.team != NULL, 0)
 502       || __builtin_expect (team->nthreads == 1, 0))
 503     free_team (team);
 504   else
 505     {
 506       struct gomp_thread_pool *pool = thr->thread_pool;
 507       if (pool->last_team)
 508         free_team (pool->last_team);
 509       pool->last_team = team;
 510     }
 511 }
 512
 513
 514 /* Constructors for this file.  */
 515
 516 static void __attribute__((constructor))
 517 initialize_team (void)
 518 {
 519   struct gomp_thread *thr;
 520
 521 #ifndef HAVE_TLS
 522   static struct gomp_thread initial_thread_tls_data;
 523
 524   pthread_key_create (&gomp_tls_key, NULL);
 525   pthread_setspecific (gomp_tls_key, &initial_thread_tls_data);
 526 #endif
 527
 528   if (pthread_key_create (&gomp_thread_destructor, gomp_free_thread) != 0)
 529     gomp_fatal ("could not create thread pool destructor.");
 530
 531 #ifdef HAVE_TLS
 532   thr = &gomp_tls_data;
 533 #else
 534   thr = &initial_thread_tls_data;
 535 #endif
 536   gomp_sem_init (&thr->release, 0);
 537 }
 538
 539 static void __attribute__((destructor))
 540 team_destructor (void)
 541 {
 542   /* Without this dlclose on libgomp could lead to subsequent
 543      crashes.  */
 544   pthread_key_delete (gomp_thread_destructor);
 545 }
 546
 547 struct gomp_task_icv *
 548 gomp_new_icv (void)
 549 {
 550   struct gomp_thread *thr = gomp_thread ();
 551   struct gomp_task *task = gomp_malloc (sizeof (struct gomp_task));
 552   gomp_init_task (task, NULL, &gomp_global_icv);
 553   thr->task = task;
 554   pthread_setspecific (gomp_thread_destructor, thr);
 555   return &task->icv;
 556 }