libgomp/loop_ull.c

   1 /* Copyright (C) 2005-2023 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file handles the LOOP (FOR/DO) construct.  */
  27
  28 #include <limits.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include "libgomp.h"
  32
  33 ialias (GOMP_loop_ull_runtime_next)
  34 ialias_redirect (GOMP_taskgroup_reduction_register)
  35
  36 typedef unsigned long long gomp_ull;
  37
  38 /* Initialize the given work share construct from the given arguments.  */
  39
  40 static inline void
  41 gomp_loop_ull_init (struct gomp_work_share *ws, bool up, gomp_ull start,
  42                     gomp_ull end, gomp_ull incr, enum gomp_schedule_type sched,
  43                     gomp_ull chunk_size)
  44 {
  45   ws->sched = sched;
  46   ws->chunk_size_ull = chunk_size;
  47   /* Canonicalize loops that have zero iterations to ->next == ->end.  */
  48   ws->end_ull = ((up && start > end) || (!up && start < end))
  49                 ? start : end;
  50   ws->incr_ull = incr;
  51   ws->next_ull = start;
  52   ws->mode = 0;
  53   if (sched == GFS_DYNAMIC)
  54     {
  55       ws->chunk_size_ull *= incr;
  56
  57 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
  58       {
  59         /* For dynamic scheduling prepare things to make each iteration
  60            faster.  */
  61         struct gomp_thread *thr = gomp_thread ();
  62         struct gomp_team *team = thr->ts.team;
  63         long nthreads = team ? team->nthreads : 1;
  64
  65         if (__builtin_expect (up, 1))
  66           {
  67             /* Cheap overflow protection.  */
  68             if (__builtin_expect ((nthreads | ws->chunk_size_ull)
  69                                   < 1ULL << (sizeof (gomp_ull)
  70                                              * __CHAR_BIT__ / 2 - 1), 1))
  71               ws->mode = ws->end_ull < (__LONG_LONG_MAX__ * 2ULL + 1
  72                                         - (nthreads + 1) * ws->chunk_size_ull);
  73           }
  74         /* Cheap overflow protection.  */
  75         else if (__builtin_expect ((nthreads | -ws->chunk_size_ull)
  76                                    < 1ULL << (sizeof (gomp_ull)
  77                                               * __CHAR_BIT__ / 2 - 1), 1))
  78           ws->mode = ws->end_ull > ((nthreads + 1) * -ws->chunk_size_ull
  79                                     - (__LONG_LONG_MAX__ * 2ULL + 1));
  80       }
  81 #endif
  82     }
  83   if (!up)
  84     ws->mode |= 2;
  85 }
  86
  87 /* The *_start routines are called when first encountering a loop construct
  88    that is not bound directly to a parallel construct.  The first thread
  89    that arrives will create the work-share construct; subsequent threads
  90    will see the construct exists and allocate work from it.
  91
  92    START, END, INCR are the bounds of the loop; due to the restrictions of
  93    OpenMP, these values must be the same in every thread.  This is not
  94    verified (nor is it entirely verifiable, since START is not necessarily
  95    retained intact in the work-share data structure).  CHUNK_SIZE is the
  96    scheduling parameter; again this must be identical in all threads.
  97
  98    Returns true if there's any work for this thread to perform.  If so,
  99    *ISTART and *IEND are filled with the bounds of the iteration block
 100    allocated to this thread.  Returns false if all work was assigned to
 101    other threads prior to this thread's arrival.  */
 102
 103 static bool
 104 gomp_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
 105                             gomp_ull incr, gomp_ull chunk_size,
 106                             gomp_ull *istart, gomp_ull *iend)
 107 {
 108   struct gomp_thread *thr = gomp_thread ();
 109
 110   thr->ts.static_trip = 0;
 111   if (gomp_work_share_start (0))
 112     {
 113       gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
 114                           GFS_STATIC, chunk_size);
 115       gomp_work_share_init_done ();
 116     }
 117
 118   return !gomp_iter_ull_static_next (istart, iend);
 119 }
 120
 121 static bool
 122 gomp_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
 123                              gomp_ull incr, gomp_ull chunk_size,
 124                              gomp_ull *istart, gomp_ull *iend)
 125 {
 126   struct gomp_thread *thr = gomp_thread ();
 127   bool ret;
 128
 129   if (gomp_work_share_start (0))
 130     {
 131       gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
 132                           GFS_DYNAMIC, chunk_size);
 133       gomp_work_share_init_done ();
 134     }
 135
 136 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
 137   ret = gomp_iter_ull_dynamic_next (istart, iend);
 138 #else
 139   gomp_mutex_lock (&thr->ts.work_share->lock);
 140   ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
 141   gomp_mutex_unlock (&thr->ts.work_share->lock);
 142 #endif
 143
 144   return ret;
 145 }
 146
 147 static bool
 148 gomp_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
 149                             gomp_ull incr, gomp_ull chunk_size,
 150                             gomp_ull *istart, gomp_ull *iend)
 151 {
 152   struct gomp_thread *thr = gomp_thread ();
 153   bool ret;
 154
 155   if (gomp_work_share_start (0))
 156     {
 157       gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
 158                           GFS_GUIDED, chunk_size);
 159       gomp_work_share_init_done ();
 160     }
 161
 162 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
 163   ret = gomp_iter_ull_guided_next (istart, iend);
 164 #else
 165   gomp_mutex_lock (&thr->ts.work_share->lock);
 166   ret = gomp_iter_ull_guided_next_locked (istart, iend);
 167   gomp_mutex_unlock (&thr->ts.work_share->lock);
 168 #endif
 169
 170   return ret;
 171 }
 172
 173 bool
 174 GOMP_loop_ull_runtime_start (bool up, gomp_ull start, gomp_ull end,
 175                              gomp_ull incr, gomp_ull *istart, gomp_ull *iend)
 176 {
 177   struct gomp_task_icv *icv = gomp_icv (false);
 178   switch (icv->run_sched_var & ~GFS_MONOTONIC)
 179     {
 180     case GFS_STATIC:
 181       return gomp_loop_ull_static_start (up, start, end, incr,
 182                                          icv->run_sched_chunk_size,
 183                                          istart, iend);
 184     case GFS_DYNAMIC:
 185       return gomp_loop_ull_dynamic_start (up, start, end, incr,
 186                                           icv->run_sched_chunk_size,
 187                                           istart, iend);
 188     case GFS_GUIDED:
 189       return gomp_loop_ull_guided_start (up, start, end, incr,
 190                                          icv->run_sched_chunk_size,
 191                                          istart, iend);
 192     case GFS_AUTO:
 193       /* For now map to schedule(static), later on we could play with feedback
 194          driven choice.  */
 195       return gomp_loop_ull_static_start (up, start, end, incr,
 196                                          0, istart, iend);
 197     default:
 198       abort ();
 199     }
 200 }
 201
 202 static long
 203 gomp_adjust_sched (long sched, gomp_ull *chunk_size)
 204 {
 205   sched &= ~GFS_MONOTONIC;
 206   switch (sched)
 207     {
 208     case GFS_STATIC:
 209     case GFS_DYNAMIC:
 210     case GFS_GUIDED:
 211       return sched;
 212     /* GFS_RUNTIME is used for runtime schedule without monotonic
 213        or nonmonotonic modifiers on the clause.
 214        GFS_RUNTIME|GFS_MONOTONIC for runtime schedule with monotonic
 215        modifier.  */
 216     case GFS_RUNTIME:
 217     /* GFS_AUTO is used for runtime schedule with nonmonotonic
 218        modifier.  */
 219     case GFS_AUTO:
 220       {
 221         struct gomp_task_icv *icv = gomp_icv (false);
 222         sched = icv->run_sched_var & ~GFS_MONOTONIC;
 223         switch (sched)
 224           {
 225           case GFS_STATIC:
 226           case GFS_DYNAMIC:
 227           case GFS_GUIDED:
 228             *chunk_size = icv->run_sched_chunk_size;
 229             break;
 230           case GFS_AUTO:
 231             sched = GFS_STATIC;
 232             *chunk_size = 0;
 233             break;
 234           default:
 235             abort ();
 236           }
 237         return sched;
 238       }
 239     default:
 240       abort ();
 241     }
 242 }
 243
 244 bool
 245 GOMP_loop_ull_start (bool up, gomp_ull start, gomp_ull end,
 246                      gomp_ull incr, long sched, gomp_ull chunk_size,
 247                      gomp_ull *istart, gomp_ull *iend,
 248                      uintptr_t *reductions, void **mem)
 249 {
 250   struct gomp_thread *thr = gomp_thread ();
 251
 252   thr->ts.static_trip = 0;
 253   if (reductions)
 254     gomp_workshare_taskgroup_start ();
 255   if (gomp_work_share_start (0))
 256     {
 257       sched = gomp_adjust_sched (sched, &chunk_size);
 258       gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
 259                           sched, chunk_size);
 260       if (reductions)
 261         {
 262           GOMP_taskgroup_reduction_register (reductions);
 263           thr->task->taskgroup->workshare = true;
 264           thr->ts.work_share->task_reductions = reductions;
 265         }
 266       if (mem)
 267         {
 268           uintptr_t size = (uintptr_t) *mem;
 269 #define INLINE_ORDERED_TEAM_IDS_OFF \
 270   ((offsetof (struct gomp_work_share, inline_ordered_team_ids)          \
 271     + __alignof__ (long long) - 1) & ~(__alignof__ (long long) - 1))
 272           if (sizeof (struct gomp_work_share)
 273               <= INLINE_ORDERED_TEAM_IDS_OFF
 274               || __alignof__ (struct gomp_work_share) < __alignof__ (long long)
 275               || size > (sizeof (struct gomp_work_share)
 276                         - INLINE_ORDERED_TEAM_IDS_OFF))
 277             *mem
 278               = (void *) (thr->ts.work_share->ordered_team_ids
 279                           = gomp_malloc_cleared (size));
 280           else
 281             *mem = memset (((char *) thr->ts.work_share)
 282                            + INLINE_ORDERED_TEAM_IDS_OFF, '\0', size);
 283         }
 284       gomp_work_share_init_done ();
 285     }
 286   else
 287     {
 288       if (reductions)
 289         {
 290           uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
 291           gomp_workshare_task_reduction_register (reductions,
 292                                                   first_reductions);
 293         }
 294       if (mem)
 295         {
 296           if ((offsetof (struct gomp_work_share, inline_ordered_team_ids)
 297                & (__alignof__ (long long) - 1)) == 0)
 298             *mem = (void *) thr->ts.work_share->ordered_team_ids;
 299           else
 300             {
 301               uintptr_t p = (uintptr_t) thr->ts.work_share->ordered_team_ids;
 302               p += __alignof__ (long long) - 1;
 303               p &= ~(__alignof__ (long long) - 1);
 304               *mem = (void *) p;
 305             }
 306         }
 307     }
 308
 309   return ialias_call (GOMP_loop_ull_runtime_next) (istart, iend);
 310 }
 311
 312 /* The *_ordered_*_start routines are similar.  The only difference is that
 313    this work-share construct is initialized to expect an ORDERED section.  */
 314
 315 static bool
 316 gomp_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
 317                                     gomp_ull incr, gomp_ull chunk_size,
 318                                     gomp_ull *istart, gomp_ull *iend)
 319 {
 320   struct gomp_thread *thr = gomp_thread ();
 321
 322   thr->ts.static_trip = 0;
 323   if (gomp_work_share_start (1))
 324     {
 325       gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
 326                           GFS_STATIC, chunk_size);
 327       gomp_ordered_static_init ();
 328       gomp_work_share_init_done ();
 329     }
 330
 331   return !gomp_iter_ull_static_next (istart, iend);
 332 }
 333
 334 static bool
 335 gomp_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
 336                                      gomp_ull incr, gomp_ull chunk_size,
 337                                      gomp_ull *istart, gomp_ull *iend)
 338 {
 339   struct gomp_thread *thr = gomp_thread ();
 340   bool ret;
 341
 342   if (gomp_work_share_start (1))
 343     {
 344       gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
 345                           GFS_DYNAMIC, chunk_size);
 346       gomp_mutex_lock (&thr->ts.work_share->lock);
 347       gomp_work_share_init_done ();
 348     }
 349   else
 350     gomp_mutex_lock (&thr->ts.work_share->lock);
 351
 352   ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
 353   if (ret)
 354     gomp_ordered_first ();
 355   gomp_mutex_unlock (&thr->ts.work_share->lock);
 356
 357   return ret;
 358 }
 359
 360 static bool
 361 gomp_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
 362                                     gomp_ull incr, gomp_ull chunk_size,
 363                                     gomp_ull *istart, gomp_ull *iend)
 364 {
 365   struct gomp_thread *thr = gomp_thread ();
 366   bool ret;
 367
 368   if (gomp_work_share_start (1))
 369     {
 370       gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
 371                           GFS_GUIDED, chunk_size);
 372       gomp_mutex_lock (&thr->ts.work_share->lock);
 373       gomp_work_share_init_done ();
 374     }
 375   else
 376     gomp_mutex_lock (&thr->ts.work_share->lock);
 377
 378   ret = gomp_iter_ull_guided_next_locked (istart, iend);
 379   if (ret)
 380     gomp_ordered_first ();
 381   gomp_mutex_unlock (&thr->ts.work_share->lock);
 382
 383   return ret;
 384 }
 385
 386 bool
 387 GOMP_loop_ull_ordered_runtime_start (bool up, gomp_ull start, gomp_ull end,
 388                                      gomp_ull incr, gomp_ull *istart,
 389                                      gomp_ull *iend)
 390 {
 391   struct gomp_task_icv *icv = gomp_icv (false);
 392   switch (icv->run_sched_var & ~GFS_MONOTONIC)
 393     {
 394     case GFS_STATIC:
 395       return gomp_loop_ull_ordered_static_start (up, start, end, incr,
 396                                                  icv->run_sched_chunk_size,
 397                                                  istart, iend);
 398     case GFS_DYNAMIC:
 399       return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr,
 400                                                   icv->run_sched_chunk_size,
 401                                                   istart, iend);
 402     case GFS_GUIDED:
 403       return gomp_loop_ull_ordered_guided_start (up, start, end, incr,
 404                                                  icv->run_sched_chunk_size,
 405                                                  istart, iend);
 406     case GFS_AUTO:
 407       /* For now map to schedule(static), later on we could play with feedback
 408          driven choice.  */
 409       return gomp_loop_ull_ordered_static_start (up, start, end, incr,
 410                                                  0, istart, iend);
 411     default:
 412       abort ();
 413     }
 414 }
 415
 416 bool
 417 GOMP_loop_ull_ordered_start (bool up, gomp_ull start, gomp_ull end,
 418                              gomp_ull incr, long sched, gomp_ull chunk_size,
 419                              gomp_ull *istart, gomp_ull *iend,
 420                              uintptr_t *reductions, void **mem)
 421 {
 422   struct gomp_thread *thr = gomp_thread ();
 423   size_t ordered = 1;
 424   bool ret;
 425
 426   thr->ts.static_trip = 0;
 427   if (reductions)
 428     gomp_workshare_taskgroup_start ();
 429   if (mem)
 430     ordered += (uintptr_t) *mem;
 431   if (gomp_work_share_start (ordered))
 432     {
 433       sched = gomp_adjust_sched (sched, &chunk_size);
 434       gomp_loop_ull_init (thr->ts.work_share, up, start, end, incr,
 435                           sched, chunk_size);
 436       if (reductions)
 437         {
 438           GOMP_taskgroup_reduction_register (reductions);
 439           thr->task->taskgroup->workshare = true;
 440           thr->ts.work_share->task_reductions = reductions;
 441         }
 442       if (sched == GFS_STATIC)
 443         gomp_ordered_static_init ();
 444       else
 445         gomp_mutex_lock (&thr->ts.work_share->lock);
 446       gomp_work_share_init_done ();
 447     }
 448   else
 449     {
 450       if (reductions)
 451         {
 452           uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
 453           gomp_workshare_task_reduction_register (reductions,
 454                                                   first_reductions);
 455         }
 456       sched = thr->ts.work_share->sched;
 457       if (sched != GFS_STATIC)
 458         gomp_mutex_lock (&thr->ts.work_share->lock);
 459     }
 460
 461   if (mem)
 462     {
 463       uintptr_t p
 464         = (uintptr_t) (thr->ts.work_share->ordered_team_ids
 465                        + (thr->ts.team ? thr->ts.team->nthreads : 1));
 466       p += __alignof__ (long long) - 1;
 467       p &= ~(__alignof__ (long long) - 1);
 468       *mem = (void *) p;
 469     }
 470
 471   switch (sched)
 472     {
 473     case GFS_STATIC:
 474     case GFS_AUTO:
 475       return !gomp_iter_ull_static_next (istart, iend);
 476     case GFS_DYNAMIC:
 477       ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
 478       break;
 479     case GFS_GUIDED:
 480       ret = gomp_iter_ull_guided_next_locked (istart, iend);
 481       break;
 482     default:
 483       abort ();
 484     }
 485
 486   if (ret)
 487     gomp_ordered_first ();
 488   gomp_mutex_unlock (&thr->ts.work_share->lock);
 489   return ret;
 490 }
 491
 492 /* The *_doacross_*_start routines are similar.  The only difference is that
 493    this work-share construct is initialized to expect an ORDERED(N) - DOACROSS
 494    section, and the worksharing loop iterates always from 0 to COUNTS[0] - 1
 495    and other COUNTS array elements tell the library number of iterations
 496    in the ordered inner loops.  */
 497
 498 static bool
 499 gomp_loop_ull_doacross_static_start (unsigned ncounts, gomp_ull *counts,
 500                                      gomp_ull chunk_size, gomp_ull *istart,
 501                                      gomp_ull *iend)
 502 {
 503   struct gomp_thread *thr = gomp_thread ();
 504
 505   thr->ts.static_trip = 0;
 506   if (gomp_work_share_start (0))
 507     {
 508       gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
 509                           GFS_STATIC, chunk_size);
 510       gomp_doacross_ull_init (ncounts, counts, chunk_size, 0);
 511       gomp_work_share_init_done ();
 512     }
 513
 514   return !gomp_iter_ull_static_next (istart, iend);
 515 }
 516
 517 static bool
 518 gomp_loop_ull_doacross_dynamic_start (unsigned ncounts, gomp_ull *counts,
 519                                       gomp_ull chunk_size, gomp_ull *istart,
 520                                       gomp_ull *iend)
 521 {
 522   struct gomp_thread *thr = gomp_thread ();
 523   bool ret;
 524
 525   if (gomp_work_share_start (0))
 526     {
 527       gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
 528                           GFS_DYNAMIC, chunk_size);
 529       gomp_doacross_ull_init (ncounts, counts, chunk_size, 0);
 530       gomp_work_share_init_done ();
 531     }
 532
 533 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
 534   ret = gomp_iter_ull_dynamic_next (istart, iend);
 535 #else
 536   gomp_mutex_lock (&thr->ts.work_share->lock);
 537   ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
 538   gomp_mutex_unlock (&thr->ts.work_share->lock);
 539 #endif
 540
 541   return ret;
 542 }
 543
 544 static bool
 545 gomp_loop_ull_doacross_guided_start (unsigned ncounts, gomp_ull *counts,
 546                                      gomp_ull chunk_size, gomp_ull *istart,
 547                                      gomp_ull *iend)
 548 {
 549   struct gomp_thread *thr = gomp_thread ();
 550   bool ret;
 551
 552   if (gomp_work_share_start (0))
 553     {
 554       gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
 555                           GFS_GUIDED, chunk_size);
 556       gomp_doacross_ull_init (ncounts, counts, chunk_size, 0);
 557       gomp_work_share_init_done ();
 558     }
 559
 560 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
 561   ret = gomp_iter_ull_guided_next (istart, iend);
 562 #else
 563   gomp_mutex_lock (&thr->ts.work_share->lock);
 564   ret = gomp_iter_ull_guided_next_locked (istart, iend);
 565   gomp_mutex_unlock (&thr->ts.work_share->lock);
 566 #endif
 567
 568   return ret;
 569 }
 570
 571 bool
 572 GOMP_loop_ull_doacross_runtime_start (unsigned ncounts, gomp_ull *counts,
 573                                       gomp_ull *istart, gomp_ull *iend)
 574 {
 575   struct gomp_task_icv *icv = gomp_icv (false);
 576   switch (icv->run_sched_var & ~GFS_MONOTONIC)
 577     {
 578     case GFS_STATIC:
 579       return gomp_loop_ull_doacross_static_start (ncounts, counts,
 580                                                   icv->run_sched_chunk_size,
 581                                                   istart, iend);
 582     case GFS_DYNAMIC:
 583       return gomp_loop_ull_doacross_dynamic_start (ncounts, counts,
 584                                                    icv->run_sched_chunk_size,
 585                                                    istart, iend);
 586     case GFS_GUIDED:
 587       return gomp_loop_ull_doacross_guided_start (ncounts, counts,
 588                                                   icv->run_sched_chunk_size,
 589                                                   istart, iend);
 590     case GFS_AUTO:
 591       /* For now map to schedule(static), later on we could play with feedback
 592          driven choice.  */
 593       return gomp_loop_ull_doacross_static_start (ncounts, counts,
 594                                                   0, istart, iend);
 595     default:
 596       abort ();
 597     }
 598 }
 599
 600 bool
 601 GOMP_loop_ull_doacross_start (unsigned ncounts, gomp_ull *counts,
 602                               long sched, gomp_ull chunk_size,
 603                               gomp_ull *istart, gomp_ull *iend,
 604                               uintptr_t *reductions, void **mem)
 605 {
 606   struct gomp_thread *thr = gomp_thread ();
 607
 608   thr->ts.static_trip = 0;
 609   if (reductions)
 610     gomp_workshare_taskgroup_start ();
 611   if (gomp_work_share_start (0))
 612     {
 613       size_t extra = 0;
 614       if (mem)
 615         extra = (uintptr_t) *mem;
 616       sched = gomp_adjust_sched (sched, &chunk_size);
 617       gomp_loop_ull_init (thr->ts.work_share, true, 0, counts[0], 1,
 618                           sched, chunk_size);
 619       gomp_doacross_ull_init (ncounts, counts, chunk_size, extra);
 620       if (reductions)
 621         {
 622           GOMP_taskgroup_reduction_register (reductions);
 623           thr->task->taskgroup->workshare = true;
 624           thr->ts.work_share->task_reductions = reductions;
 625         }
 626       gomp_work_share_init_done ();
 627     }
 628   else
 629     {
 630       if (reductions)
 631         {
 632           uintptr_t *first_reductions = thr->ts.work_share->task_reductions;
 633           gomp_workshare_task_reduction_register (reductions,
 634                                                   first_reductions);
 635         }
 636       sched = thr->ts.work_share->sched;
 637     }
 638
 639   if (mem)
 640     *mem = thr->ts.work_share->doacross->extra;
 641
 642   return ialias_call (GOMP_loop_ull_runtime_next) (istart, iend);
 643 }
 644
 645 /* The *_next routines are called when the thread completes processing of
 646    the iteration block currently assigned to it.  If the work-share
 647    construct is bound directly to a parallel construct, then the iteration
 648    bounds may have been set up before the parallel.  In which case, this
 649    may be the first iteration for the thread.
 650
 651    Returns true if there is work remaining to be performed; *ISTART and
 652    *IEND are filled with a new iteration block.  Returns false if all work
 653    has been assigned.  */
 654
 655 static bool
 656 gomp_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
 657 {
 658   return !gomp_iter_ull_static_next (istart, iend);
 659 }
 660
 661 static bool
 662 gomp_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
 663 {
 664   bool ret;
 665
 666 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
 667   ret = gomp_iter_ull_dynamic_next (istart, iend);
 668 #else
 669   struct gomp_thread *thr = gomp_thread ();
 670   gomp_mutex_lock (&thr->ts.work_share->lock);
 671   ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
 672   gomp_mutex_unlock (&thr->ts.work_share->lock);
 673 #endif
 674
 675   return ret;
 676 }
 677
 678 static bool
 679 gomp_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
 680 {
 681   bool ret;
 682
 683 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
 684   ret = gomp_iter_ull_guided_next (istart, iend);
 685 #else
 686   struct gomp_thread *thr = gomp_thread ();
 687   gomp_mutex_lock (&thr->ts.work_share->lock);
 688   ret = gomp_iter_ull_guided_next_locked (istart, iend);
 689   gomp_mutex_unlock (&thr->ts.work_share->lock);
 690 #endif
 691
 692   return ret;
 693 }
 694
 695 bool
 696 GOMP_loop_ull_runtime_next (gomp_ull *istart, gomp_ull *iend)
 697 {
 698   struct gomp_thread *thr = gomp_thread ();
 699
 700   switch (thr->ts.work_share->sched)
 701     {
 702     case GFS_STATIC:
 703     case GFS_AUTO:
 704       return gomp_loop_ull_static_next (istart, iend);
 705     case GFS_DYNAMIC:
 706       return gomp_loop_ull_dynamic_next (istart, iend);
 707     case GFS_GUIDED:
 708       return gomp_loop_ull_guided_next (istart, iend);
 709     default:
 710       abort ();
 711     }
 712 }
 713
 714 /* The *_ordered_*_next routines are called when the thread completes
 715    processing of the iteration block currently assigned to it.
 716
 717    Returns true if there is work remaining to be performed; *ISTART and
 718    *IEND are filled with a new iteration block.  Returns false if all work
 719    has been assigned.  */
 720
 721 static bool
 722 gomp_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
 723 {
 724   struct gomp_thread *thr = gomp_thread ();
 725   int test;
 726
 727   gomp_ordered_sync ();
 728   gomp_mutex_lock (&thr->ts.work_share->lock);
 729   test = gomp_iter_ull_static_next (istart, iend);
 730   if (test >= 0)
 731     gomp_ordered_static_next ();
 732   gomp_mutex_unlock (&thr->ts.work_share->lock);
 733
 734   return test == 0;
 735 }
 736
 737 static bool
 738 gomp_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
 739 {
 740   struct gomp_thread *thr = gomp_thread ();
 741   bool ret;
 742
 743   gomp_ordered_sync ();
 744   gomp_mutex_lock (&thr->ts.work_share->lock);
 745   ret = gomp_iter_ull_dynamic_next_locked (istart, iend);
 746   if (ret)
 747     gomp_ordered_next ();
 748   else
 749     gomp_ordered_last ();
 750   gomp_mutex_unlock (&thr->ts.work_share->lock);
 751
 752   return ret;
 753 }
 754
 755 static bool
 756 gomp_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
 757 {
 758   struct gomp_thread *thr = gomp_thread ();
 759   bool ret;
 760
 761   gomp_ordered_sync ();
 762   gomp_mutex_lock (&thr->ts.work_share->lock);
 763   ret = gomp_iter_ull_guided_next_locked (istart, iend);
 764   if (ret)
 765     gomp_ordered_next ();
 766   else
 767     gomp_ordered_last ();
 768   gomp_mutex_unlock (&thr->ts.work_share->lock);
 769
 770   return ret;
 771 }
 772
 773 bool
 774 GOMP_loop_ull_ordered_runtime_next (gomp_ull *istart, gomp_ull *iend)
 775 {
 776   struct gomp_thread *thr = gomp_thread ();
 777
 778   switch (thr->ts.work_share->sched)
 779     {
 780     case GFS_STATIC:
 781     case GFS_AUTO:
 782       return gomp_loop_ull_ordered_static_next (istart, iend);
 783     case GFS_DYNAMIC:
 784       return gomp_loop_ull_ordered_dynamic_next (istart, iend);
 785     case GFS_GUIDED:
 786       return gomp_loop_ull_ordered_guided_next (istart, iend);
 787     default:
 788       abort ();
 789     }
 790 }
 791
 792 /* We use static functions above so that we're sure that the "runtime"
 793    function can defer to the proper routine without interposition.  We
 794    export the static function with a strong alias when possible, or with
 795    a wrapper function otherwise.  */
 796
 797 #ifdef HAVE_ATTRIBUTE_ALIAS
 798 extern __typeof(gomp_loop_ull_static_start) GOMP_loop_ull_static_start
 799         __attribute__((alias ("gomp_loop_ull_static_start")));
 800 extern __typeof(gomp_loop_ull_dynamic_start) GOMP_loop_ull_dynamic_start
 801         __attribute__((alias ("gomp_loop_ull_dynamic_start")));
 802 extern __typeof(gomp_loop_ull_guided_start) GOMP_loop_ull_guided_start
 803         __attribute__((alias ("gomp_loop_ull_guided_start")));
 804 extern __typeof(gomp_loop_ull_dynamic_start) GOMP_loop_ull_nonmonotonic_dynamic_start
 805         __attribute__((alias ("gomp_loop_ull_dynamic_start")));
 806 extern __typeof(gomp_loop_ull_guided_start) GOMP_loop_ull_nonmonotonic_guided_start
 807         __attribute__((alias ("gomp_loop_ull_guided_start")));
 808 extern __typeof(GOMP_loop_ull_runtime_start) GOMP_loop_ull_nonmonotonic_runtime_start
 809         __attribute__((alias ("GOMP_loop_ull_runtime_start")));
 810 extern __typeof(GOMP_loop_ull_runtime_start) GOMP_loop_ull_maybe_nonmonotonic_runtime_start
 811         __attribute__((alias ("GOMP_loop_ull_runtime_start")));
 812
 813 extern __typeof(gomp_loop_ull_ordered_static_start) GOMP_loop_ull_ordered_static_start
 814         __attribute__((alias ("gomp_loop_ull_ordered_static_start")));
 815 extern __typeof(gomp_loop_ull_ordered_dynamic_start) GOMP_loop_ull_ordered_dynamic_start
 816         __attribute__((alias ("gomp_loop_ull_ordered_dynamic_start")));
 817 extern __typeof(gomp_loop_ull_ordered_guided_start) GOMP_loop_ull_ordered_guided_start
 818         __attribute__((alias ("gomp_loop_ull_ordered_guided_start")));
 819
 820 extern __typeof(gomp_loop_ull_doacross_static_start) GOMP_loop_ull_doacross_static_start
 821         __attribute__((alias ("gomp_loop_ull_doacross_static_start")));
 822 extern __typeof(gomp_loop_ull_doacross_dynamic_start) GOMP_loop_ull_doacross_dynamic_start
 823         __attribute__((alias ("gomp_loop_ull_doacross_dynamic_start")));
 824 extern __typeof(gomp_loop_ull_doacross_guided_start) GOMP_loop_ull_doacross_guided_start
 825         __attribute__((alias ("gomp_loop_ull_doacross_guided_start")));
 826
 827 extern __typeof(gomp_loop_ull_static_next) GOMP_loop_ull_static_next
 828         __attribute__((alias ("gomp_loop_ull_static_next")));
 829 extern __typeof(gomp_loop_ull_dynamic_next) GOMP_loop_ull_dynamic_next
 830         __attribute__((alias ("gomp_loop_ull_dynamic_next")));
 831 extern __typeof(gomp_loop_ull_guided_next) GOMP_loop_ull_guided_next
 832         __attribute__((alias ("gomp_loop_ull_guided_next")));
 833 extern __typeof(gomp_loop_ull_dynamic_next) GOMP_loop_ull_nonmonotonic_dynamic_next
 834         __attribute__((alias ("gomp_loop_ull_dynamic_next")));
 835 extern __typeof(gomp_loop_ull_guided_next) GOMP_loop_ull_nonmonotonic_guided_next
 836         __attribute__((alias ("gomp_loop_ull_guided_next")));
 837 extern __typeof(GOMP_loop_ull_runtime_next) GOMP_loop_ull_nonmonotonic_runtime_next
 838         __attribute__((alias ("GOMP_loop_ull_runtime_next")));
 839 extern __typeof(GOMP_loop_ull_runtime_next) GOMP_loop_ull_maybe_nonmonotonic_runtime_next
 840         __attribute__((alias ("GOMP_loop_ull_runtime_next")));
 841
 842 extern __typeof(gomp_loop_ull_ordered_static_next) GOMP_loop_ull_ordered_static_next
 843         __attribute__((alias ("gomp_loop_ull_ordered_static_next")));
 844 extern __typeof(gomp_loop_ull_ordered_dynamic_next) GOMP_loop_ull_ordered_dynamic_next
 845         __attribute__((alias ("gomp_loop_ull_ordered_dynamic_next")));
 846 extern __typeof(gomp_loop_ull_ordered_guided_next) GOMP_loop_ull_ordered_guided_next
 847         __attribute__((alias ("gomp_loop_ull_ordered_guided_next")));
 848 #else
 849 bool
 850 GOMP_loop_ull_static_start (bool up, gomp_ull start, gomp_ull end,
 851                             gomp_ull incr, gomp_ull chunk_size,
 852                             gomp_ull *istart, gomp_ull *iend)
 853 {
 854   return gomp_loop_ull_static_start (up, start, end, incr, chunk_size, istart,
 855                                      iend);
 856 }
 857
 858 bool
 859 GOMP_loop_ull_dynamic_start (bool up, gomp_ull start, gomp_ull end,
 860                              gomp_ull incr, gomp_ull chunk_size,
 861                              gomp_ull *istart, gomp_ull *iend)
 862 {
 863   return gomp_loop_ull_dynamic_start (up, start, end, incr, chunk_size, istart,
 864                                       iend);
 865 }
 866
 867 bool
 868 GOMP_loop_ull_guided_start (bool up, gomp_ull start, gomp_ull end,
 869                             gomp_ull incr, gomp_ull chunk_size,
 870                             gomp_ull *istart, gomp_ull *iend)
 871 {
 872   return gomp_loop_ull_guided_start (up, start, end, incr, chunk_size, istart,
 873                                      iend);
 874 }
 875
 876 bool
 877 GOMP_loop_ull_nonmonotonic_dynamic_start (bool up, gomp_ull start,
 878                                           gomp_ull end, gomp_ull incr,
 879                                           gomp_ull chunk_size,
 880                                           gomp_ull *istart, gomp_ull *iend)
 881 {
 882   return gomp_loop_ull_dynamic_start (up, start, end, incr, chunk_size, istart,
 883                                       iend);
 884 }
 885
 886 bool
 887 GOMP_loop_ull_nonmonotonic_guided_start (bool up, gomp_ull start, gomp_ull end,
 888                                          gomp_ull incr, gomp_ull chunk_size,
 889                                          gomp_ull *istart, gomp_ull *iend)
 890 {
 891   return gomp_loop_ull_guided_start (up, start, end, incr, chunk_size, istart,
 892                                      iend);
 893 }
 894
 895 bool
 896 GOMP_loop_ull_nonmonotonic_runtime_start (bool up, gomp_ull start,
 897                                           gomp_ull end, gomp_ull incr,
 898                                           gomp_ull *istart, gomp_ull *iend)
 899 {
 900   return GOMP_loop_ull_runtime_start (up, start, end, incr, istart, iend);
 901 }
 902
 903 bool
 904 GOMP_loop_ull_maybe_nonmonotonic_runtime_start (bool up, gomp_ull start,
 905                                                 gomp_ull end, gomp_ull incr,
 906                                                 gomp_ull *istart,
 907                                                 gomp_ull *iend)
 908 {
 909   return GOMP_loop_ull_runtime_start (up, start, end, incr, istart, iend);
 910 }
 911
 912 bool
 913 GOMP_loop_ull_ordered_static_start (bool up, gomp_ull start, gomp_ull end,
 914                                     gomp_ull incr, gomp_ull chunk_size,
 915                                     gomp_ull *istart, gomp_ull *iend)
 916 {
 917   return gomp_loop_ull_ordered_static_start (up, start, end, incr, chunk_size,
 918                                              istart, iend);
 919 }
 920
 921 bool
 922 GOMP_loop_ull_ordered_dynamic_start (bool up, gomp_ull start, gomp_ull end,
 923                                      gomp_ull incr, gomp_ull chunk_size,
 924                                      gomp_ull *istart, gomp_ull *iend)
 925 {
 926   return gomp_loop_ull_ordered_dynamic_start (up, start, end, incr, chunk_size,
 927                                               istart, iend);
 928 }
 929
 930 bool
 931 GOMP_loop_ull_ordered_guided_start (bool up, gomp_ull start, gomp_ull end,
 932                                     gomp_ull incr, gomp_ull chunk_size,
 933                                     gomp_ull *istart, gomp_ull *iend)
 934 {
 935   return gomp_loop_ull_ordered_guided_start (up, start, end, incr, chunk_size,
 936                                              istart, iend);
 937 }
 938
 939 bool
 940 GOMP_loop_ull_doacross_static_start (unsigned ncounts, gomp_ull *counts,
 941                                      gomp_ull chunk_size, gomp_ull *istart,
 942                                      gomp_ull *iend)
 943 {
 944   return gomp_loop_ull_doacross_static_start (ncounts, counts, chunk_size,
 945                                               istart, iend);
 946 }
 947
 948 bool
 949 GOMP_loop_ull_doacross_dynamic_start (unsigned ncounts, gomp_ull *counts,
 950                                       gomp_ull chunk_size, gomp_ull *istart,
 951                                       gomp_ull *iend)
 952 {
 953   return gomp_loop_ull_doacross_dynamic_start (ncounts, counts, chunk_size,
 954                                                istart, iend);
 955 }
 956
 957 bool
 958 GOMP_loop_ull_doacross_guided_start (unsigned ncounts, gomp_ull *counts,
 959                                      gomp_ull chunk_size, gomp_ull *istart,
 960                                      gomp_ull *iend)
 961 {
 962   return gomp_loop_ull_doacross_guided_start (ncounts, counts, chunk_size,
 963                                               istart, iend);
 964 }
 965
 966 bool
 967 GOMP_loop_ull_static_next (gomp_ull *istart, gomp_ull *iend)
 968 {
 969   return gomp_loop_ull_static_next (istart, iend);
 970 }
 971
 972 bool
 973 GOMP_loop_ull_dynamic_next (gomp_ull *istart, gomp_ull *iend)
 974 {
 975   return gomp_loop_ull_dynamic_next (istart, iend);
 976 }
 977
 978 bool
 979 GOMP_loop_ull_guided_next (gomp_ull *istart, gomp_ull *iend)
 980 {
 981   return gomp_loop_ull_guided_next (istart, iend);
 982 }
 983
 984 bool
 985 GOMP_loop_ull_nonmonotonic_dynamic_next (gomp_ull *istart, gomp_ull *iend)
 986 {
 987   return gomp_loop_ull_dynamic_next (istart, iend);
 988 }
 989
 990 bool
 991 GOMP_loop_ull_nonmonotonic_guided_next (gomp_ull *istart, gomp_ull *iend)
 992 {
 993   return gomp_loop_ull_guided_next (istart, iend);
 994 }
 995
 996 bool
 997 GOMP_loop_ull_nonmonotonic_runtime_next (gomp_ull *istart, gomp_ull *iend)
 998 {
 999   return GOMP_loop_ull_runtime_next (istart, iend);
1000 }
1001
1002 bool
1003 GOMP_loop_ull_maybe_nonmonotonic_runtime_next (gomp_ull *istart,
1004                                                gomp_ull *iend)
1005 {
1006   return GOMP_loop_ull_runtime_next (istart, iend);
1007 }
1008
1009 bool
1010 GOMP_loop_ull_ordered_static_next (gomp_ull *istart, gomp_ull *iend)
1011 {
1012   return gomp_loop_ull_ordered_static_next (istart, iend);
1013 }
1014
1015 bool
1016 GOMP_loop_ull_ordered_dynamic_next (gomp_ull *istart, gomp_ull *iend)
1017 {
1018   return gomp_loop_ull_ordered_dynamic_next (istart, iend);
1019 }
1020
1021 bool
1022 GOMP_loop_ull_ordered_guided_next (gomp_ull *istart, gomp_ull *iend)
1023 {
1024   return gomp_loop_ull_ordered_guided_next (istart, iend);
1025 }
1026 #endif