libgomp/iter.c

   1 /* Copyright (C) 2005-2023 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains routines for managing work-share iteration, both
  27    for loops and sections.  */
  28
  29 #include "libgomp.h"
  30 #include <stdlib.h>
  31
  32
  33 /* This function implements the STATIC scheduling method.  The caller should
  34    iterate *pstart <= x < *pend.  Return zero if there are more iterations
  35    to perform; nonzero if not.  Return less than 0 if this thread had
  36    received the absolutely last iteration.  */
  37
  38 int
  39 gomp_iter_static_next (long *pstart, long *pend)
  40 {
  41   struct gomp_thread *thr = gomp_thread ();
  42   struct gomp_team *team = thr->ts.team;
  43   struct gomp_work_share *ws = thr->ts.work_share;
  44   unsigned long nthreads = team ? team->nthreads : 1;
  45
  46   if (thr->ts.static_trip == -1)
  47     return -1;
  48
  49   /* Quick test for degenerate teams and orphaned constructs.  */
  50   if (nthreads == 1)
  51     {
  52       *pstart = ws->next;
  53       *pend = ws->end;
  54       thr->ts.static_trip = -1;
  55       return ws->next == ws->end;
  56     }
  57
  58   /* We interpret chunk_size zero as "unspecified", which means that we
  59      should break up the iterations such that each thread makes only one
  60      trip through the outer loop.  */
  61   if (ws->chunk_size == 0)
  62     {
  63       unsigned long n, q, i, t;
  64       unsigned long s0, e0;
  65       long s, e;
  66
  67       if (thr->ts.static_trip > 0)
  68         return 1;
  69
  70       /* Compute the total number of iterations.  */
  71       s = ws->incr + (ws->incr > 0 ? -1 : 1);
  72       n = (ws->end - ws->next + s) / ws->incr;
  73       i = thr->ts.team_id;
  74
  75       /* Compute the "zero-based" start and end points.  That is, as
  76          if the loop began at zero and incremented by one.  */
  77       q = n / nthreads;
  78       t = n % nthreads;
  79       if (i < t)
  80         {
  81           t = 0;
  82           q++;
  83         }
  84       s0 = q * i + t;
  85       e0 = s0 + q;
  86
  87       /* Notice when no iterations allocated for this thread.  */
  88       if (s0 >= e0)
  89         {
  90           thr->ts.static_trip = 1;
  91           return 1;
  92         }
  93
  94       /* Transform these to the actual start and end numbers.  */
  95       s = (long)s0 * ws->incr + ws->next;
  96       e = (long)e0 * ws->incr + ws->next;
  97
  98       *pstart = s;
  99       *pend = e;
 100       thr->ts.static_trip = (e0 == n ? -1 : 1);
 101       return 0;
 102     }
 103   else
 104     {
 105       unsigned long n, s0, e0, i, c;
 106       long s, e;
 107
 108       /* Otherwise, each thread gets exactly chunk_size iterations
 109          (if available) each time through the loop.  */
 110
 111       s = ws->incr + (ws->incr > 0 ? -1 : 1);
 112       n = (ws->end - ws->next + s) / ws->incr;
 113       i = thr->ts.team_id;
 114       c = ws->chunk_size;
 115
 116       /* Initial guess is a C sized chunk positioned nthreads iterations
 117          in, offset by our thread number.  */
 118       s0 = (thr->ts.static_trip * nthreads + i) * c;
 119       e0 = s0 + c;
 120
 121       /* Detect overflow.  */
 122       if (s0 >= n)
 123         return 1;
 124       if (e0 > n)
 125         e0 = n;
 126
 127       /* Transform these to the actual start and end numbers.  */
 128       s = (long)s0 * ws->incr + ws->next;
 129       e = (long)e0 * ws->incr + ws->next;
 130
 131       *pstart = s;
 132       *pend = e;
 133
 134       if (e0 == n)
 135         thr->ts.static_trip = -1;
 136       else
 137         thr->ts.static_trip++;
 138       return 0;
 139     }
 140 }
 141
 142
 143 /* This function implements the DYNAMIC scheduling method.  Arguments are
 144    as for gomp_iter_static_next.  This function must be called with ws->lock
 145    held.  */
 146
 147 bool
 148 gomp_iter_dynamic_next_locked (long *pstart, long *pend)
 149 {
 150   struct gomp_thread *thr = gomp_thread ();
 151   struct gomp_work_share *ws = thr->ts.work_share;
 152   long start, end, chunk, left;
 153
 154   start = ws->next;
 155   if (start == ws->end)
 156     return false;
 157
 158   chunk = ws->chunk_size;
 159   left = ws->end - start;
 160   if (ws->incr < 0)
 161     {
 162       if (chunk < left)
 163         chunk = left;
 164     }
 165   else
 166     {
 167       if (chunk > left)
 168         chunk = left;
 169     }
 170   end = start + chunk;
 171
 172   ws->next = end;
 173   *pstart = start;
 174   *pend = end;
 175   return true;
 176 }
 177
 178
 179 #ifdef HAVE_SYNC_BUILTINS
 180 /* Similar, but doesn't require the lock held, and uses compare-and-swap
 181    instead.  Note that the only memory value that changes is ws->next.  */
 182
 183 bool
 184 gomp_iter_dynamic_next (long *pstart, long *pend)
 185 {
 186   struct gomp_thread *thr = gomp_thread ();
 187   struct gomp_work_share *ws = thr->ts.work_share;
 188   long start, end, nend, chunk, incr;
 189
 190   end = ws->end;
 191   incr = ws->incr;
 192   chunk = ws->chunk_size;
 193
 194   if (__builtin_expect (ws->mode, 1))
 195     {
 196       long tmp = __sync_fetch_and_add (&ws->next, chunk);
 197       if (incr > 0)
 198         {
 199           if (tmp >= end)
 200             return false;
 201           nend = tmp + chunk;
 202           if (nend > end)
 203             nend = end;
 204           *pstart = tmp;
 205           *pend = nend;
 206           return true;
 207         }
 208       else
 209         {
 210           if (tmp <= end)
 211             return false;
 212           nend = tmp + chunk;
 213           if (nend < end)
 214             nend = end;
 215           *pstart = tmp;
 216           *pend = nend;
 217           return true;
 218         }
 219     }
 220
 221   start = __atomic_load_n (&ws->next, MEMMODEL_RELAXED);
 222   while (1)
 223     {
 224       long left = end - start;
 225       long tmp;
 226
 227       if (start == end)
 228         return false;
 229
 230       if (incr < 0)
 231         {
 232           if (chunk < left)
 233             chunk = left;
 234         }
 235       else
 236         {
 237           if (chunk > left)
 238             chunk = left;
 239         }
 240       nend = start + chunk;
 241
 242       tmp = __sync_val_compare_and_swap (&ws->next, start, nend);
 243       if (__builtin_expect (tmp == start, 1))
 244         break;
 245
 246       start = tmp;
 247     }
 248
 249   *pstart = start;
 250   *pend = nend;
 251   return true;
 252 }
 253 #endif /* HAVE_SYNC_BUILTINS */
 254
 255
 256 /* This function implements the GUIDED scheduling method.  Arguments are
 257    as for gomp_iter_static_next.  This function must be called with the
 258    work share lock held.  */
 259
 260 bool
 261 gomp_iter_guided_next_locked (long *pstart, long *pend)
 262 {
 263   struct gomp_thread *thr = gomp_thread ();
 264   struct gomp_work_share *ws = thr->ts.work_share;
 265   struct gomp_team *team = thr->ts.team;
 266   unsigned long nthreads = team ? team->nthreads : 1;
 267   unsigned long n, q;
 268   long start, end;
 269
 270   if (ws->next == ws->end)
 271     return false;
 272
 273   start = ws->next;
 274   n = (ws->end - start) / ws->incr;
 275   q = (n + nthreads - 1) / nthreads;
 276
 277   if (q < ws->chunk_size)
 278     q = ws->chunk_size;
 279   if (q <= n)
 280     end = start + q * ws->incr;
 281   else
 282     end = ws->end;
 283
 284   ws->next = end;
 285   *pstart = start;
 286   *pend = end;
 287   return true;
 288 }
 289
 290 #ifdef HAVE_SYNC_BUILTINS
 291 /* Similar, but doesn't require the lock held, and uses compare-and-swap
 292    instead.  Note that the only memory value that changes is ws->next.  */
 293
 294 bool
 295 gomp_iter_guided_next (long *pstart, long *pend)
 296 {
 297   struct gomp_thread *thr = gomp_thread ();
 298   struct gomp_work_share *ws = thr->ts.work_share;
 299   struct gomp_team *team = thr->ts.team;
 300   unsigned long nthreads = team ? team->nthreads : 1;
 301   long start, end, nend, incr;
 302   unsigned long chunk_size;
 303
 304   start = __atomic_load_n (&ws->next, MEMMODEL_RELAXED);
 305   end = ws->end;
 306   incr = ws->incr;
 307   chunk_size = ws->chunk_size;
 308
 309   while (1)
 310     {
 311       unsigned long n, q;
 312       long tmp;
 313
 314       if (start == end)
 315         return false;
 316
 317       n = (end - start) / incr;
 318       q = (n + nthreads - 1) / nthreads;
 319
 320       if (q < chunk_size)
 321         q = chunk_size;
 322       if (__builtin_expect (q <= n, 1))
 323         nend = start + q * incr;
 324       else
 325         nend = end;
 326
 327       tmp = __sync_val_compare_and_swap (&ws->next, start, nend);
 328       if (__builtin_expect (tmp == start, 1))
 329         break;
 330
 331       start = tmp;
 332     }
 333
 334   *pstart = start;
 335   *pend = nend;
 336   return true;
 337 }
 338 #endif /* HAVE_SYNC_BUILTINS */