libgomp/team.c

   1 /* Copyright (C) 2005-2019 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file handles the maintainence of threads in response to team
  27    creation and termination.  */
  28
  29 #include "libgomp.h"
  30 #include "pool.h"
  31 #include <stdlib.h>
  32 #include <string.h>
  33
  34 #ifdef LIBGOMP_USE_PTHREADS
  35 pthread_attr_t gomp_thread_attr;
  36
  37 /* This key is for the thread destructor.  */
  38 pthread_key_t gomp_thread_destructor;
  39
  40
  41 /* This is the libgomp per-thread data structure.  */
  42 #if defined HAVE_TLS || defined USE_EMUTLS
  43 __thread struct gomp_thread gomp_tls_data;
  44 #else
  45 pthread_key_t gomp_tls_key;
  46 #endif
  47
  48
  49 /* This structure is used to communicate across pthread_create.  */
  50
  51 struct gomp_thread_start_data
  52 {
  53   void (*fn) (void *);
  54   void *fn_data;
  55   struct gomp_team_state ts;
  56   struct gomp_task *task;
  57   struct gomp_thread_pool *thread_pool;
  58   unsigned int place;
  59   bool nested;
  60   pthread_t handle;
  61 };
  62
  63
  64 /* This function is a pthread_create entry point.  This contains the idle
  65    loop in which a thread waits to be called up to become part of a team.  */
  66
  67 static void *
  68 gomp_thread_start (void *xdata)
  69 {
  70   struct gomp_thread_start_data *data = xdata;
  71   struct gomp_thread *thr;
  72   struct gomp_thread_pool *pool;
  73   void (*local_fn) (void *);
  74   void *local_data;
  75
  76 #if defined HAVE_TLS || defined USE_EMUTLS
  77   thr = &gomp_tls_data;
  78 #else
  79   struct gomp_thread local_thr;
  80   thr = &local_thr;
  81   pthread_setspecific (gomp_tls_key, thr);
  82 #endif
  83   gomp_sem_init (&thr->release, 0);
  84
  85   /* Extract what we need from data.  */
  86   local_fn = data->fn;
  87   local_data = data->fn_data;
  88   thr->thread_pool = data->thread_pool;
  89   thr->ts = data->ts;
  90   thr->task = data->task;
  91   thr->place = data->place;
  92 #ifdef GOMP_NEEDS_THREAD_HANDLE
  93   thr->handle = data->handle;
  94 #endif
  95
  96   thr->ts.team->ordered_release[thr->ts.team_id] = &thr->release;
  97
  98   /* Make thread pool local. */
  99   pool = thr->thread_pool;
 100
 101   if (data->nested)
 102     {
 103       struct gomp_team *team = thr->ts.team;
 104       struct gomp_task *task = thr->task;
 105
 106       gomp_barrier_wait (&team->barrier);
 107
 108       local_fn (local_data);
 109       gomp_team_barrier_wait_final (&team->barrier);
 110       gomp_finish_task (task);
 111       gomp_barrier_wait_last (&team->barrier);
 112     }
 113   else
 114     {
 115       pool->threads[thr->ts.team_id] = thr;
 116
 117       gomp_simple_barrier_wait (&pool->threads_dock);
 118       do
 119         {
 120           struct gomp_team *team = thr->ts.team;
 121           struct gomp_task *task = thr->task;
 122
 123           local_fn (local_data);
 124           gomp_team_barrier_wait_final (&team->barrier);
 125           gomp_finish_task (task);
 126
 127           gomp_simple_barrier_wait (&pool->threads_dock);
 128
 129           local_fn = thr->fn;
 130           local_data = thr->data;
 131           thr->fn = NULL;
 132         }
 133       while (local_fn);
 134     }
 135
 136   gomp_sem_destroy (&thr->release);
 137   pthread_detach (pthread_self ());
 138   thr->thread_pool = NULL;
 139   thr->task = NULL;
 140   return NULL;
 141 }
 142 #endif
 143
 144 static inline struct gomp_team *
 145 get_last_team (unsigned nthreads)
 146 {
 147   struct gomp_thread *thr = gomp_thread ();
 148   if (thr->ts.team == NULL)
 149     {
 150       struct gomp_thread_pool *pool = gomp_get_thread_pool (thr, nthreads);
 151       struct gomp_team *last_team = pool->last_team;
 152       if (last_team != NULL && last_team->nthreads == nthreads)
 153         {
 154           pool->last_team = NULL;
 155           return last_team;
 156         }
 157     }
 158   return NULL;
 159 }
 160
 161 /* Create a new team data structure.  */
 162
 163 struct gomp_team *
 164 gomp_new_team (unsigned nthreads)
 165 {
 166   struct gomp_team *team;
 167   int i;
 168
 169   team = get_last_team (nthreads);
 170   if (team == NULL)
 171     {
 172       size_t extra = sizeof (team->ordered_release[0])
 173                      + sizeof (team->implicit_task[0]);
 174       team = gomp_malloc (sizeof (*team) + nthreads * extra);
 175
 176 #ifndef HAVE_SYNC_BUILTINS
 177       gomp_mutex_init (&team->work_share_list_free_lock);
 178 #endif
 179       gomp_barrier_init (&team->barrier, nthreads);
 180       gomp_mutex_init (&team->task_lock);
 181
 182       team->nthreads = nthreads;
 183     }
 184
 185   team->work_share_chunk = 8;
 186 #ifdef HAVE_SYNC_BUILTINS
 187   team->single_count = 0;
 188 #endif
 189   team->work_shares_to_free = &team->work_shares[0];
 190   gomp_init_work_share (&team->work_shares[0], 0, nthreads);
 191   team->work_shares[0].next_alloc = NULL;
 192   team->work_share_list_free = NULL;
 193   team->work_share_list_alloc = &team->work_shares[1];
 194   for (i = 1; i < 7; i++)
 195     team->work_shares[i].next_free = &team->work_shares[i + 1];
 196   team->work_shares[i].next_free = NULL;
 197
 198   gomp_sem_init (&team->master_release, 0);
 199   team->ordered_release = (void *) &team->implicit_task[nthreads];
 200   team->ordered_release[0] = &team->master_release;
 201
 202   priority_queue_init (&team->task_queue);
 203   team->task_count = 0;
 204   team->task_queued_count = 0;
 205   team->task_running_count = 0;
 206   team->work_share_cancelled = 0;
 207   team->team_cancelled = 0;
 208
 209   return team;
 210 }
 211
 212
 213 /* Free a team data structure.  */
 214
 215 static void
 216 free_team (struct gomp_team *team)
 217 {
 218 #ifndef HAVE_SYNC_BUILTINS
 219   gomp_mutex_destroy (&team->work_share_list_free_lock);
 220 #endif
 221   gomp_barrier_destroy (&team->barrier);
 222   gomp_mutex_destroy (&team->task_lock);
 223   priority_queue_free (&team->task_queue);
 224   free (team);
 225 }
 226
 227 static void
 228 gomp_free_pool_helper (void *thread_pool)
 229 {
 230   struct gomp_thread *thr = gomp_thread ();
 231   struct gomp_thread_pool *pool
 232     = (struct gomp_thread_pool *) thread_pool;
 233   gomp_simple_barrier_wait_last (&pool->threads_dock);
 234   gomp_sem_destroy (&thr->release);
 235   thr->thread_pool = NULL;
 236   thr->task = NULL;
 237 #ifdef LIBGOMP_USE_PTHREADS
 238   pthread_detach (pthread_self ());
 239   pthread_exit (NULL);
 240 #elif defined(__nvptx__)
 241   asm ("exit;");
 242 #else
 243 #error gomp_free_pool_helper must terminate the thread
 244 #endif
 245 }
 246
 247 /* Free a thread pool and release its threads. */
 248
 249 void
 250 gomp_free_thread (void *arg __attribute__((unused)))
 251 {
 252   struct gomp_thread *thr = gomp_thread ();
 253   struct gomp_thread_pool *pool = thr->thread_pool;
 254   if (pool)
 255     {
 256       if (pool->threads_used > 0)
 257         {
 258           int i;
 259           for (i = 1; i < pool->threads_used; i++)
 260             {
 261               struct gomp_thread *nthr = pool->threads[i];
 262               nthr->fn = gomp_free_pool_helper;
 263               nthr->data = pool;
 264             }
 265           /* This barrier undocks threads docked on pool->threads_dock.  */
 266           gomp_simple_barrier_wait (&pool->threads_dock);
 267           /* And this waits till all threads have called gomp_barrier_wait_last
 268              in gomp_free_pool_helper.  */
 269           gomp_simple_barrier_wait (&pool->threads_dock);
 270           /* Now it is safe to destroy the barrier and free the pool.  */
 271           gomp_simple_barrier_destroy (&pool->threads_dock);
 272
 273 #ifdef HAVE_SYNC_BUILTINS
 274           __sync_fetch_and_add (&gomp_managed_threads,
 275                                 1L - pool->threads_used);
 276 #else
 277           gomp_mutex_lock (&gomp_managed_threads_lock);
 278           gomp_managed_threads -= pool->threads_used - 1L;
 279           gomp_mutex_unlock (&gomp_managed_threads_lock);
 280 #endif
 281         }
 282       if (pool->last_team)
 283         free_team (pool->last_team);
 284 #ifndef __nvptx__
 285       free (pool->threads);
 286       free (pool);
 287 #endif
 288       thr->thread_pool = NULL;
 289     }
 290   if (thr->ts.level == 0 && __builtin_expect (thr->ts.team != NULL, 0))
 291     gomp_team_end ();
 292   if (thr->task != NULL)
 293     {
 294       struct gomp_task *task = thr->task;
 295       gomp_end_task ();
 296       free (task);
 297     }
 298 }
 299
 300 /* Launch a team.  */
 301
 302 #ifdef LIBGOMP_USE_PTHREADS
 303 void
 304 gomp_team_start (void (*fn) (void *), void *data, unsigned nthreads,
 305                  unsigned flags, struct gomp_team *team,
 306                  struct gomp_taskgroup *taskgroup)
 307 {
 308   struct gomp_thread_start_data *start_data;
 309   struct gomp_thread *thr, *nthr;
 310   struct gomp_task *task;
 311   struct gomp_task_icv *icv;
 312   bool nested;
 313   struct gomp_thread_pool *pool;
 314   unsigned i, n, old_threads_used = 0;
 315   pthread_attr_t thread_attr, *attr;
 316   unsigned long nthreads_var;
 317   char bind, bind_var;
 318   unsigned int s = 0, rest = 0, p = 0, k = 0;
 319   unsigned int affinity_count = 0;
 320   struct gomp_thread **affinity_thr = NULL;
 321   bool force_display = false;
 322
 323   thr = gomp_thread ();
 324   nested = thr->ts.level;
 325   pool = thr->thread_pool;
 326   task = thr->task;
 327   icv = task ? &task->icv : &gomp_global_icv;
 328   if (__builtin_expect (gomp_places_list != NULL, 0) && thr->place == 0)
 329     {
 330       gomp_init_affinity ();
 331       if (__builtin_expect (gomp_display_affinity_var, 0) && nthreads == 1)
 332         gomp_display_affinity_thread (gomp_thread_self (), &thr->ts,
 333                                       thr->place);
 334     }
 335
 336   /* Always save the previous state, even if this isn't a nested team.
 337      In particular, we should save any work share state from an outer
 338      orphaned work share construct.  */
 339   team->prev_ts = thr->ts;
 340
 341   thr->ts.team = team;
 342   thr->ts.team_id = 0;
 343   ++thr->ts.level;
 344   if (nthreads > 1)
 345     ++thr->ts.active_level;
 346   thr->ts.work_share = &team->work_shares[0];
 347   thr->ts.last_work_share = NULL;
 348 #ifdef HAVE_SYNC_BUILTINS
 349   thr->ts.single_count = 0;
 350 #endif
 351   thr->ts.static_trip = 0;
 352   thr->task = &team->implicit_task[0];
 353 #ifdef GOMP_NEEDS_THREAD_HANDLE
 354   thr->handle = pthread_self ();
 355 #endif
 356   nthreads_var = icv->nthreads_var;
 357   if (__builtin_expect (gomp_nthreads_var_list != NULL, 0)
 358       && thr->ts.level < gomp_nthreads_var_list_len)
 359     nthreads_var = gomp_nthreads_var_list[thr->ts.level];
 360   bind_var = icv->bind_var;
 361   if (bind_var != omp_proc_bind_false && (flags & 7) != omp_proc_bind_false)
 362     bind_var = flags & 7;
 363   bind = bind_var;
 364   if (__builtin_expect (gomp_bind_var_list != NULL, 0)
 365       && thr->ts.level < gomp_bind_var_list_len)
 366     bind_var = gomp_bind_var_list[thr->ts.level];
 367   gomp_init_task (thr->task, task, icv);
 368   thr->task->taskgroup = taskgroup;
 369   team->implicit_task[0].icv.nthreads_var = nthreads_var;
 370   team->implicit_task[0].icv.bind_var = bind_var;
 371
 372   if (nthreads == 1)
 373     return;
 374
 375   i = 1;
 376
 377   if (__builtin_expect (gomp_places_list != NULL, 0))
 378     {
 379       /* Depending on chosen proc_bind model, set subpartition
 380          for the master thread and initialize helper variables
 381          P and optionally S, K and/or REST used by later place
 382          computation for each additional thread.  */
 383       p = thr->place - 1;
 384       switch (bind)
 385         {
 386         case omp_proc_bind_true:
 387         case omp_proc_bind_close:
 388           if (nthreads > thr->ts.place_partition_len)
 389             {
 390               /* T > P.  S threads will be placed in each place,
 391                  and the final REM threads placed one by one
 392                  into the already occupied places.  */
 393               s = nthreads / thr->ts.place_partition_len;
 394               rest = nthreads % thr->ts.place_partition_len;
 395             }
 396           else
 397             s = 1;
 398           k = 1;
 399           break;
 400         case omp_proc_bind_master:
 401           /* Each thread will be bound to master's place.  */
 402           break;
 403         case omp_proc_bind_spread:
 404           if (nthreads <= thr->ts.place_partition_len)
 405             {
 406               /* T <= P.  Each subpartition will have in between s
 407                  and s+1 places (subpartitions starting at or
 408                  after rest will have s places, earlier s+1 places),
 409                  each thread will be bound to the first place in
 410                  its subpartition (except for the master thread
 411                  that can be bound to another place in its
 412                  subpartition).  */
 413               s = thr->ts.place_partition_len / nthreads;
 414               rest = thr->ts.place_partition_len % nthreads;
 415               rest = (s + 1) * rest + thr->ts.place_partition_off;
 416               if (p < rest)
 417                 {
 418                   p -= (p - thr->ts.place_partition_off) % (s + 1);
 419                   thr->ts.place_partition_len = s + 1;
 420                 }
 421               else
 422                 {
 423                   p -= (p - rest) % s;
 424                   thr->ts.place_partition_len = s;
 425                 }
 426               thr->ts.place_partition_off = p;
 427             }
 428           else
 429             {
 430               /* T > P.  Each subpartition will have just a single
 431                  place and we'll place between s and s+1
 432                  threads into each subpartition.  */
 433               s = nthreads / thr->ts.place_partition_len;
 434               rest = nthreads % thr->ts.place_partition_len;
 435               thr->ts.place_partition_off = p;
 436               thr->ts.place_partition_len = 1;
 437               k = 1;
 438             }
 439           break;
 440         }
 441     }
 442   else
 443     bind = omp_proc_bind_false;
 444
 445   /* We only allow the reuse of idle threads for non-nested PARALLEL
 446      regions.  This appears to be implied by the semantics of
 447      threadprivate variables, but perhaps that's reading too much into
 448      things.  Certainly it does prevent any locking problems, since
 449      only the initial program thread will modify gomp_threads.  */
 450   if (!nested)
 451     {
 452       old_threads_used = pool->threads_used;
 453
 454       if (nthreads <= old_threads_used)
 455         n = nthreads;
 456       else if (old_threads_used == 0)
 457         {
 458           n = 0;
 459           gomp_simple_barrier_init (&pool->threads_dock, nthreads);
 460         }
 461       else
 462         {
 463           n = old_threads_used;
 464
 465           /* Increase the barrier threshold to make sure all new
 466              threads arrive before the team is released.  */
 467           gomp_simple_barrier_reinit (&pool->threads_dock, nthreads);
 468         }
 469
 470       /* Not true yet, but soon will be.  We're going to release all
 471          threads from the dock, and those that aren't part of the
 472          team will exit.  */
 473       pool->threads_used = nthreads;
 474
 475       /* If necessary, expand the size of the gomp_threads array.  It is
 476          expected that changes in the number of threads are rare, thus we
 477          make no effort to expand gomp_threads_size geometrically.  */
 478       if (nthreads >= pool->threads_size)
 479         {
 480           pool->threads_size = nthreads + 1;
 481           pool->threads
 482             = gomp_realloc (pool->threads,
 483                             pool->threads_size
 484                             * sizeof (struct gomp_thread *));
 485           /* Add current (master) thread to threads[].  */
 486           pool->threads[0] = thr;
 487         }
 488
 489       /* Release existing idle threads.  */
 490       for (; i < n; ++i)
 491         {
 492           unsigned int place_partition_off = thr->ts.place_partition_off;
 493           unsigned int place_partition_len = thr->ts.place_partition_len;
 494           unsigned int place = 0;
 495           if (__builtin_expect (gomp_places_list != NULL, 0))
 496             {
 497               switch (bind)
 498                 {
 499                 case omp_proc_bind_true:
 500                 case omp_proc_bind_close:
 501                   if (k == s)
 502                     {
 503                       ++p;
 504                       if (p == (team->prev_ts.place_partition_off
 505                                 + team->prev_ts.place_partition_len))
 506                         p = team->prev_ts.place_partition_off;
 507                       k = 1;
 508                       if (i == nthreads - rest)
 509                         s = 1;
 510                     }
 511                   else
 512                     ++k;
 513                   break;
 514                 case omp_proc_bind_master:
 515                   break;
 516                 case omp_proc_bind_spread:
 517                   if (k == 0)
 518                     {
 519                       /* T <= P.  */
 520                       if (p < rest)
 521                         p += s + 1;
 522                       else
 523                         p += s;
 524                       if (p == (team->prev_ts.place_partition_off
 525                                 + team->prev_ts.place_partition_len))
 526                         p = team->prev_ts.place_partition_off;
 527                       place_partition_off = p;
 528                       if (p < rest)
 529                         place_partition_len = s + 1;
 530                       else
 531                         place_partition_len = s;
 532                     }
 533                   else
 534                     {
 535                       /* T > P.  */
 536                       if (k == s)
 537                         {
 538                           ++p;
 539                           if (p == (team->prev_ts.place_partition_off
 540                                     + team->prev_ts.place_partition_len))
 541                             p = team->prev_ts.place_partition_off;
 542                           k = 1;
 543                           if (i == nthreads - rest)
 544                             s = 1;
 545                         }
 546                       else
 547                         ++k;
 548                       place_partition_off = p;
 549                       place_partition_len = 1;
 550                     }
 551                   break;
 552                 }
 553               if (affinity_thr != NULL
 554                   || (bind != omp_proc_bind_true
 555                       && pool->threads[i]->place != p + 1)
 556                   || pool->threads[i]->place <= place_partition_off
 557                   || pool->threads[i]->place > (place_partition_off
 558                                                 + place_partition_len))
 559                 {
 560                   unsigned int l;
 561                   force_display = true;
 562                   if (affinity_thr == NULL)
 563                     {
 564                       unsigned int j;
 565
 566                       if (team->prev_ts.place_partition_len > 64)
 567                         affinity_thr
 568                           = gomp_malloc (team->prev_ts.place_partition_len
 569                                          * sizeof (struct gomp_thread *));
 570                       else
 571                         affinity_thr
 572                           = gomp_alloca (team->prev_ts.place_partition_len
 573                                          * sizeof (struct gomp_thread *));
 574                       memset (affinity_thr, '\0',
 575                               team->prev_ts.place_partition_len
 576                               * sizeof (struct gomp_thread *));
 577                       for (j = i; j < old_threads_used; j++)
 578                         {
 579                           if (pool->threads[j]->place
 580                               > team->prev_ts.place_partition_off
 581                               && (pool->threads[j]->place
 582                                   <= (team->prev_ts.place_partition_off
 583                                       + team->prev_ts.place_partition_len)))
 584                             {
 585                               l = pool->threads[j]->place - 1
 586                                   - team->prev_ts.place_partition_off;
 587                               pool->threads[j]->data = affinity_thr[l];
 588                               affinity_thr[l] = pool->threads[j];
 589                             }
 590                           pool->threads[j] = NULL;
 591                         }
 592                       if (nthreads > old_threads_used)
 593                         memset (&pool->threads[old_threads_used],
 594                                 '\0', ((nthreads - old_threads_used)
 595                                        * sizeof (struct gomp_thread *)));
 596                       n = nthreads;
 597                       affinity_count = old_threads_used - i;
 598                     }
 599                   if (affinity_count == 0)
 600                     break;
 601                   l = p;
 602                   if (affinity_thr[l - team->prev_ts.place_partition_off]
 603                       == NULL)
 604                     {
 605                       if (bind != omp_proc_bind_true)
 606                         continue;
 607                       for (l = place_partition_off;
 608                            l < place_partition_off + place_partition_len;
 609                            l++)
 610                         if (affinity_thr[l - team->prev_ts.place_partition_off]
 611                             != NULL)
 612                           break;
 613                       if (l == place_partition_off + place_partition_len)
 614                         continue;
 615                     }
 616                   nthr = affinity_thr[l - team->prev_ts.place_partition_off];
 617                   affinity_thr[l - team->prev_ts.place_partition_off]
 618                     = (struct gomp_thread *) nthr->data;
 619                   affinity_count--;
 620                   pool->threads[i] = nthr;
 621                 }
 622               else
 623                 nthr = pool->threads[i];
 624               place = p + 1;
 625             }
 626           else
 627             nthr = pool->threads[i];
 628           nthr->ts.team = team;
 629           nthr->ts.work_share = &team->work_shares[0];
 630           nthr->ts.last_work_share = NULL;
 631           nthr->ts.team_id = i;
 632           nthr->ts.level = team->prev_ts.level + 1;
 633           nthr->ts.active_level = thr->ts.active_level;
 634           nthr->ts.place_partition_off = place_partition_off;
 635           nthr->ts.place_partition_len = place_partition_len;
 636 #ifdef HAVE_SYNC_BUILTINS
 637           nthr->ts.single_count = 0;
 638 #endif
 639           nthr->ts.static_trip = 0;
 640           nthr->task = &team->implicit_task[i];
 641           nthr->place = place;
 642           gomp_init_task (nthr->task, task, icv);
 643           team->implicit_task[i].icv.nthreads_var = nthreads_var;
 644           team->implicit_task[i].icv.bind_var = bind_var;
 645           nthr->task->taskgroup = taskgroup;
 646           nthr->fn = fn;
 647           nthr->data = data;
 648           team->ordered_release[i] = &nthr->release;
 649         }
 650
 651       if (__builtin_expect (affinity_thr != NULL, 0))
 652         {
 653           /* If AFFINITY_THR is non-NULL just because we had to
 654              permute some threads in the pool, but we've managed
 655              to find exactly as many old threads as we'd find
 656              without affinity, we don't need to handle this
 657              specially anymore.  */
 658           if (nthreads <= old_threads_used
 659               ? (affinity_count == old_threads_used - nthreads)
 660               : (i == old_threads_used))
 661             {
 662               if (team->prev_ts.place_partition_len > 64)
 663                 free (affinity_thr);
 664               affinity_thr = NULL;
 665               affinity_count = 0;
 666             }
 667           else
 668             {
 669               i = 1;
 670               /* We are going to compute the places/subpartitions
 671                  again from the beginning.  So, we need to reinitialize
 672                  vars modified by the switch (bind) above inside
 673                  of the loop, to the state they had after the initial
 674                  switch (bind).  */
 675               switch (bind)
 676                 {
 677                 case omp_proc_bind_true:
 678                 case omp_proc_bind_close:
 679                   if (nthreads > thr->ts.place_partition_len)
 680                     /* T > P.  S has been changed, so needs
 681                        to be recomputed.  */
 682                     s = nthreads / thr->ts.place_partition_len;
 683                   k = 1;
 684                   p = thr->place - 1;
 685                   break;
 686                 case omp_proc_bind_master:
 687                   /* No vars have been changed.  */
 688                   break;
 689                 case omp_proc_bind_spread:
 690                   p = thr->ts.place_partition_off;
 691                   if (k != 0)
 692                     {
 693                       /* T > P.  */
 694                       s = nthreads / team->prev_ts.place_partition_len;
 695                       k = 1;
 696                     }
 697                   break;
 698                 }
 699
 700               /* Increase the barrier threshold to make sure all new
 701                  threads and all the threads we're going to let die
 702                  arrive before the team is released.  */
 703               if (affinity_count)
 704                 gomp_simple_barrier_reinit (&pool->threads_dock,
 705                                             nthreads + affinity_count);
 706             }
 707         }
 708
 709       if (i == nthreads)
 710         goto do_release;
 711
 712     }
 713
 714   if (__builtin_expect (nthreads + affinity_count > old_threads_used, 0))
 715     {
 716       long diff = (long) (nthreads + affinity_count) - (long) old_threads_used;
 717
 718       if (old_threads_used == 0)
 719         --diff;
 720
 721 #ifdef HAVE_SYNC_BUILTINS
 722       __sync_fetch_and_add (&gomp_managed_threads, diff);
 723 #else
 724       gomp_mutex_lock (&gomp_managed_threads_lock);
 725       gomp_managed_threads += diff;
 726       gomp_mutex_unlock (&gomp_managed_threads_lock);
 727 #endif
 728     }
 729
 730   attr = &gomp_thread_attr;
 731   if (__builtin_expect (gomp_places_list != NULL, 0))
 732     {
 733       size_t stacksize;
 734       pthread_attr_init (&thread_attr);
 735       if (! pthread_attr_getstacksize (&gomp_thread_attr, &stacksize))
 736         pthread_attr_setstacksize (&thread_attr, stacksize);
 737       attr = &thread_attr;
 738     }
 739
 740   start_data = gomp_alloca (sizeof (struct gomp_thread_start_data)
 741                             * (nthreads - i));
 742
 743   /* Launch new threads.  */
 744   for (; i < nthreads; ++i)
 745     {
 746       int err;
 747
 748       start_data->ts.place_partition_off = thr->ts.place_partition_off;
 749       start_data->ts.place_partition_len = thr->ts.place_partition_len;
 750       start_data->place = 0;
 751       if (__builtin_expect (gomp_places_list != NULL, 0))
 752         {
 753           switch (bind)
 754             {
 755             case omp_proc_bind_true:
 756             case omp_proc_bind_close:
 757               if (k == s)
 758                 {
 759                   ++p;
 760                   if (p == (team->prev_ts.place_partition_off
 761                             + team->prev_ts.place_partition_len))
 762                     p = team->prev_ts.place_partition_off;
 763                   k = 1;
 764                   if (i == nthreads - rest)
 765                     s = 1;
 766                 }
 767               else
 768                 ++k;
 769               break;
 770             case omp_proc_bind_master:
 771               break;
 772             case omp_proc_bind_spread:
 773               if (k == 0)
 774                 {
 775                   /* T <= P.  */
 776                   if (p < rest)
 777                     p += s + 1;
 778                   else
 779                     p += s;
 780                   if (p == (team->prev_ts.place_partition_off
 781                             + team->prev_ts.place_partition_len))
 782                     p = team->prev_ts.place_partition_off;
 783                   start_data->ts.place_partition_off = p;
 784                   if (p < rest)
 785                     start_data->ts.place_partition_len = s + 1;
 786                   else
 787                     start_data->ts.place_partition_len = s;
 788                 }
 789               else
 790                 {
 791                   /* T > P.  */
 792                   if (k == s)
 793                     {
 794                       ++p;
 795                       if (p == (team->prev_ts.place_partition_off
 796                                 + team->prev_ts.place_partition_len))
 797                         p = team->prev_ts.place_partition_off;
 798                       k = 1;
 799                       if (i == nthreads - rest)
 800                         s = 1;
 801                     }
 802                   else
 803                     ++k;
 804                   start_data->ts.place_partition_off = p;
 805                   start_data->ts.place_partition_len = 1;
 806                 }
 807               break;
 808             }
 809           start_data->place = p + 1;
 810           if (affinity_thr != NULL && pool->threads[i] != NULL)
 811             continue;
 812           gomp_init_thread_affinity (attr, p);
 813         }
 814
 815       start_data->fn = fn;
 816       start_data->fn_data = data;
 817       start_data->ts.team = team;
 818       start_data->ts.work_share = &team->work_shares[0];
 819       start_data->ts.last_work_share = NULL;
 820       start_data->ts.team_id = i;
 821       start_data->ts.level = team->prev_ts.level + 1;
 822       start_data->ts.active_level = thr->ts.active_level;
 823 #ifdef HAVE_SYNC_BUILTINS
 824       start_data->ts.single_count = 0;
 825 #endif
 826       start_data->ts.static_trip = 0;
 827       start_data->task = &team->implicit_task[i];
 828       gomp_init_task (start_data->task, task, icv);
 829       team->implicit_task[i].icv.nthreads_var = nthreads_var;
 830       team->implicit_task[i].icv.bind_var = bind_var;
 831       start_data->task->taskgroup = taskgroup;
 832       start_data->thread_pool = pool;
 833       start_data->nested = nested;
 834
 835       attr = gomp_adjust_thread_attr (attr, &thread_attr);
 836       err = pthread_create (&start_data->handle, attr, gomp_thread_start,
 837                             start_data);
 838       start_data++;
 839       if (err != 0)
 840         gomp_fatal ("Thread creation failed: %s", strerror (err));
 841     }
 842
 843   if (__builtin_expect (attr == &thread_attr, 0))
 844     pthread_attr_destroy (&thread_attr);
 845
 846  do_release:
 847   if (nested)
 848     gomp_barrier_wait (&team->barrier);
 849   else
 850     gomp_simple_barrier_wait (&pool->threads_dock);
 851
 852   /* Decrease the barrier threshold to match the number of threads
 853      that should arrive back at the end of this team.  The extra
 854      threads should be exiting.  Note that we arrange for this test
 855      to never be true for nested teams.  If AFFINITY_COUNT is non-zero,
 856      the barrier as well as gomp_managed_threads was temporarily
 857      set to NTHREADS + AFFINITY_COUNT.  For NTHREADS < OLD_THREADS_COUNT,
 858      AFFINITY_COUNT if non-zero will be always at least
 859      OLD_THREADS_COUNT - NTHREADS.  */
 860   if (__builtin_expect (nthreads < old_threads_used, 0)
 861       || __builtin_expect (affinity_count, 0))
 862     {
 863       long diff = (long) nthreads - (long) old_threads_used;
 864
 865       if (affinity_count)
 866         diff = -affinity_count;
 867
 868       gomp_simple_barrier_reinit (&pool->threads_dock, nthreads);
 869
 870 #ifdef HAVE_SYNC_BUILTINS
 871       __sync_fetch_and_add (&gomp_managed_threads, diff);
 872 #else
 873       gomp_mutex_lock (&gomp_managed_threads_lock);
 874       gomp_managed_threads += diff;
 875       gomp_mutex_unlock (&gomp_managed_threads_lock);
 876 #endif
 877     }
 878   if (__builtin_expect (gomp_display_affinity_var, 0))
 879     {
 880       if (nested
 881           || nthreads != old_threads_used
 882           || force_display)
 883         {
 884           gomp_display_affinity_thread (gomp_thread_self (), &thr->ts,
 885                                         thr->place);
 886           if (nested)
 887             {
 888               start_data -= nthreads - 1;
 889               for (i = 1; i < nthreads; ++i)
 890                 {
 891                   gomp_display_affinity_thread (
 892 #ifdef LIBGOMP_USE_PTHREADS
 893                                                 start_data->handle,
 894 #else
 895                                                 gomp_thread_self (),
 896 #endif
 897                                                 &start_data->ts,
 898                                                 start_data->place);
 899                   start_data++;
 900                 }
 901             }
 902           else
 903             {
 904               for (i = 1; i < nthreads; ++i)
 905                 {
 906                   gomp_thread_handle handle
 907                     = gomp_thread_to_pthread_t (pool->threads[i]);
 908                   gomp_display_affinity_thread (handle, &pool->threads[i]->ts,
 909                                                 pool->threads[i]->place);
 910                 }
 911             }
 912         }
 913     }
 914   if (__builtin_expect (affinity_thr != NULL, 0)
 915       && team->prev_ts.place_partition_len > 64)
 916     free (affinity_thr);
 917 }
 918 #endif
 919
 920
 921 /* Terminate the current team.  This is only to be called by the master
 922    thread.  We assume that we must wait for the other threads.  */
 923
 924 void
 925 gomp_team_end (void)
 926 {
 927   struct gomp_thread *thr = gomp_thread ();
 928   struct gomp_team *team = thr->ts.team;
 929
 930   /* This barrier handles all pending explicit threads.
 931      As #pragma omp cancel parallel might get awaited count in
 932      team->barrier in a inconsistent state, we need to use a different
 933      counter here.  */
 934   gomp_team_barrier_wait_final (&team->barrier);
 935   if (__builtin_expect (team->team_cancelled, 0))
 936     {
 937       struct gomp_work_share *ws = team->work_shares_to_free;
 938       do
 939         {
 940           struct gomp_work_share *next_ws = gomp_ptrlock_get (&ws->next_ws);
 941           if (next_ws == NULL)
 942             gomp_ptrlock_set (&ws->next_ws, ws);
 943           gomp_fini_work_share (ws);
 944           ws = next_ws;
 945         }
 946       while (ws != NULL);
 947     }
 948   else
 949     gomp_fini_work_share (thr->ts.work_share);
 950
 951   gomp_end_task ();
 952   thr->ts = team->prev_ts;
 953
 954   if (__builtin_expect (thr->ts.level != 0, 0))
 955     {
 956 #ifdef HAVE_SYNC_BUILTINS
 957       __sync_fetch_and_add (&gomp_managed_threads, 1L - team->nthreads);
 958 #else
 959       gomp_mutex_lock (&gomp_managed_threads_lock);
 960       gomp_managed_threads -= team->nthreads - 1L;
 961       gomp_mutex_unlock (&gomp_managed_threads_lock);
 962 #endif
 963       /* This barrier has gomp_barrier_wait_last counterparts
 964          and ensures the team can be safely destroyed.  */
 965       gomp_barrier_wait (&team->barrier);
 966     }
 967
 968   if (__builtin_expect (team->work_shares[0].next_alloc != NULL, 0))
 969     {
 970       struct gomp_work_share *ws = team->work_shares[0].next_alloc;
 971       do
 972         {
 973           struct gomp_work_share *next_ws = ws->next_alloc;
 974           free (ws);
 975           ws = next_ws;
 976         }
 977       while (ws != NULL);
 978     }
 979   gomp_sem_destroy (&team->master_release);
 980
 981   if (__builtin_expect (thr->ts.team != NULL, 0)
 982       || __builtin_expect (team->nthreads == 1, 0))
 983     free_team (team);
 984   else
 985     {
 986       struct gomp_thread_pool *pool = thr->thread_pool;
 987       if (pool->last_team)
 988         free_team (pool->last_team);
 989       pool->last_team = team;
 990       gomp_release_thread_pool (pool);
 991     }
 992 }
 993
 994 #ifdef LIBGOMP_USE_PTHREADS
 995
 996 /* Constructors for this file.  */
 997
 998 static void __attribute__((constructor))
 999 initialize_team (void)
1000 {
1001 #if !defined HAVE_TLS && !defined USE_EMUTLS
1002   static struct gomp_thread initial_thread_tls_data;
1003
1004   pthread_key_create (&gomp_tls_key, NULL);
1005   pthread_setspecific (gomp_tls_key, &initial_thread_tls_data);
1006 #endif
1007
1008   if (pthread_key_create (&gomp_thread_destructor, gomp_free_thread) != 0)
1009     gomp_fatal ("could not create thread pool destructor.");
1010 }
1011
1012 static void __attribute__((destructor))
1013 team_destructor (void)
1014 {
1015   /* Without this dlclose on libgomp could lead to subsequent
1016      crashes.  */
1017   pthread_key_delete (gomp_thread_destructor);
1018 }
1019
1020 /* Similar to gomp_free_pool_helper, but don't detach itself,
1021    gomp_pause_host will pthread_join those threads.  */
1022
1023 static void
1024 gomp_pause_pool_helper (void *thread_pool)
1025 {
1026   struct gomp_thread *thr = gomp_thread ();
1027   struct gomp_thread_pool *pool
1028     = (struct gomp_thread_pool *) thread_pool;
1029   gomp_simple_barrier_wait_last (&pool->threads_dock);
1030   gomp_sem_destroy (&thr->release);
1031   thr->thread_pool = NULL;
1032   thr->task = NULL;
1033   pthread_exit (NULL);
1034 }
1035
1036 /* Free a thread pool and release its threads.  Return non-zero on
1037    failure.  */
1038
1039 int
1040 gomp_pause_host (void)
1041 {
1042   struct gomp_thread *thr = gomp_thread ();
1043   struct gomp_thread_pool *pool = thr->thread_pool;
1044   if (thr->ts.level)
1045     return -1;
1046   if (pool)
1047     {
1048       if (pool->threads_used > 0)
1049         {
1050           int i;
1051           pthread_t *thrs
1052             = gomp_alloca (sizeof (pthread_t) * pool->threads_used);
1053           for (i = 1; i < pool->threads_used; i++)
1054             {
1055               struct gomp_thread *nthr = pool->threads[i];
1056               nthr->fn = gomp_pause_pool_helper;
1057               nthr->data = pool;
1058               thrs[i] = gomp_thread_to_pthread_t (nthr);
1059             }
1060           /* This barrier undocks threads docked on pool->threads_dock.  */
1061           gomp_simple_barrier_wait (&pool->threads_dock);
1062           /* And this waits till all threads have called gomp_barrier_wait_last
1063              in gomp_pause_pool_helper.  */
1064           gomp_simple_barrier_wait (&pool->threads_dock);
1065           /* Now it is safe to destroy the barrier and free the pool.  */
1066           gomp_simple_barrier_destroy (&pool->threads_dock);
1067
1068 #ifdef HAVE_SYNC_BUILTINS
1069           __sync_fetch_and_add (&gomp_managed_threads,
1070                                 1L - pool->threads_used);
1071 #else
1072           gomp_mutex_lock (&gomp_managed_threads_lock);
1073           gomp_managed_threads -= pool->threads_used - 1L;
1074           gomp_mutex_unlock (&gomp_managed_threads_lock);
1075 #endif
1076           for (i = 1; i < pool->threads_used; i++)
1077             pthread_join (thrs[i], NULL);
1078         }
1079       if (pool->last_team)
1080         free_team (pool->last_team);
1081 #ifndef __nvptx__
1082       free (pool->threads);
1083       free (pool);
1084 #endif
1085       thr->thread_pool = NULL;
1086     }
1087   return 0;
1088 }
1089 #endif
1090
1091 struct gomp_task_icv *
1092 gomp_new_icv (void)
1093 {
1094   struct gomp_thread *thr = gomp_thread ();
1095   struct gomp_task *task = gomp_malloc (sizeof (struct gomp_task));
1096   gomp_init_task (task, NULL, &gomp_global_icv);
1097   thr->task = task;
1098 #ifdef LIBGOMP_USE_PTHREADS
1099   pthread_setspecific (gomp_thread_destructor, thr);
1100 #endif
1101   return &task->icv;
1102 }