libgomp/taskloop.c

   1 /* Copyright (C) 2015-2018 Free Software Foundation, Inc.
   2    Contributed by Jakub Jelinek <jakub@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file handles the taskloop construct.  It is included twice, once
  27    for the long and once for unsigned long long variant.  */
  28
  29 /* Called when encountering an explicit task directive.  If IF_CLAUSE is
  30    false, then we must not delay in executing the task.  If UNTIED is true,
  31    then the task may be executed by any member of the team.  */
  32
  33 void
  34 GOMP_taskloop (void (*fn) (void *), void *data, void (*cpyfn) (void *, void *),
  35                long arg_size, long arg_align, unsigned flags,
  36                unsigned long num_tasks, int priority,
  37                TYPE start, TYPE end, TYPE step)
  38 {
  39   struct gomp_thread *thr = gomp_thread ();
  40   struct gomp_team *team = thr->ts.team;
  41
  42 #ifdef HAVE_BROKEN_POSIX_SEMAPHORES
  43   /* If pthread_mutex_* is used for omp_*lock*, then each task must be
  44      tied to one thread all the time.  This means UNTIED tasks must be
  45      tied and if CPYFN is non-NULL IF(0) must be forced, as CPYFN
  46      might be running on different thread than FN.  */
  47   if (cpyfn)
  48     flags &= ~GOMP_TASK_FLAG_IF;
  49   flags &= ~GOMP_TASK_FLAG_UNTIED;
  50 #endif
  51
  52   /* If parallel or taskgroup has been cancelled, don't start new tasks.  */
  53   if (team && gomp_team_barrier_cancelled (&team->barrier))
  54     return;
  55
  56 #ifdef TYPE_is_long
  57   TYPE s = step;
  58   if (step > 0)
  59     {
  60       if (start >= end)
  61         return;
  62       s--;
  63     }
  64   else
  65     {
  66       if (start <= end)
  67         return;
  68       s++;
  69     }
  70   UTYPE n = (end - start + s) / step;
  71 #else
  72   UTYPE n;
  73   if (flags & GOMP_TASK_FLAG_UP)
  74     {
  75       if (start >= end)
  76         return;
  77       n = (end - start + step - 1) / step;
  78     }
  79   else
  80     {
  81       if (start <= end)
  82         return;
  83       n = (start - end - step - 1) / -step;
  84     }
  85 #endif
  86
  87   TYPE task_step = step;
  88   unsigned long nfirst = n;
  89   if (flags & GOMP_TASK_FLAG_GRAINSIZE)
  90     {
  91       unsigned long grainsize = num_tasks;
  92 #ifdef TYPE_is_long
  93       num_tasks = n / grainsize;
  94 #else
  95       UTYPE ndiv = n / grainsize;
  96       num_tasks = ndiv;
  97       if (num_tasks != ndiv)
  98         num_tasks = ~0UL;
  99 #endif
 100       if (num_tasks <= 1)
 101         {
 102           num_tasks = 1;
 103           task_step = end - start;
 104         }
 105       else if (num_tasks >= grainsize
 106 #ifndef TYPE_is_long
 107                && num_tasks != ~0UL
 108 #endif
 109               )
 110         {
 111           UTYPE mul = num_tasks * grainsize;
 112           task_step = (TYPE) grainsize * step;
 113           if (mul != n)
 114             {
 115               task_step += step;
 116               nfirst = n - mul - 1;
 117             }
 118         }
 119       else
 120         {
 121           UTYPE div = n / num_tasks;
 122           UTYPE mod = n % num_tasks;
 123           task_step = (TYPE) div * step;
 124           if (mod)
 125             {
 126               task_step += step;
 127               nfirst = mod - 1;
 128             }
 129         }
 130     }
 131   else
 132     {
 133       if (num_tasks == 0)
 134         num_tasks = team ? team->nthreads : 1;
 135       if (num_tasks >= n)
 136         num_tasks = n;
 137       else
 138         {
 139           UTYPE div = n / num_tasks;
 140           UTYPE mod = n % num_tasks;
 141           task_step = (TYPE) div * step;
 142           if (mod)
 143             {
 144               task_step += step;
 145               nfirst = mod - 1;
 146             }
 147         }
 148     }
 149
 150   if (flags & GOMP_TASK_FLAG_NOGROUP)
 151     {
 152       if (thr->task && thr->task->taskgroup && thr->task->taskgroup->cancelled)
 153         return;
 154     }
 155   else
 156     ialias_call (GOMP_taskgroup_start) ();
 157
 158   if (priority > gomp_max_task_priority_var)
 159     priority = gomp_max_task_priority_var;
 160
 161   if ((flags & GOMP_TASK_FLAG_IF) == 0 || team == NULL
 162       || (thr->task && thr->task->final_task)
 163       || team->task_count + num_tasks > 64 * team->nthreads)
 164     {
 165       unsigned long i;
 166       if (__builtin_expect (cpyfn != NULL, 0))
 167         {
 168           struct gomp_task task[num_tasks];
 169           struct gomp_task *parent = thr->task;
 170           arg_size = (arg_size + arg_align - 1) & ~(arg_align - 1);
 171           char buf[num_tasks * arg_size + arg_align - 1];
 172           char *arg = (char *) (((uintptr_t) buf + arg_align - 1)
 173                                 & ~(uintptr_t) (arg_align - 1));
 174           char *orig_arg = arg;
 175           for (i = 0; i < num_tasks; i++)
 176             {
 177               gomp_init_task (&task[i], parent, gomp_icv (false));
 178               task[i].priority = priority;
 179               task[i].kind = GOMP_TASK_UNDEFERRED;
 180               task[i].final_task = (thr->task && thr->task->final_task)
 181                                    || (flags & GOMP_TASK_FLAG_FINAL);
 182               if (thr->task)
 183                 {
 184                   task[i].in_tied_task = thr->task->in_tied_task;
 185                   task[i].taskgroup = thr->task->taskgroup;
 186                 }
 187               thr->task = &task[i];
 188               cpyfn (arg, data);
 189               arg += arg_size;
 190             }
 191           arg = orig_arg;
 192           for (i = 0; i < num_tasks; i++)
 193             {
 194               thr->task = &task[i];
 195               ((TYPE *)arg)[0] = start;
 196               start += task_step;
 197               ((TYPE *)arg)[1] = start;
 198               if (i == nfirst)
 199                 task_step -= step;
 200               fn (arg);
 201               arg += arg_size;
 202               if (!priority_queue_empty_p (&task[i].children_queue,
 203                                            MEMMODEL_RELAXED))
 204                 {
 205                   gomp_mutex_lock (&team->task_lock);
 206                   gomp_clear_parent (&task[i].children_queue);
 207                   gomp_mutex_unlock (&team->task_lock);
 208                 }
 209               gomp_end_task ();
 210             }
 211         }
 212       else
 213         for (i = 0; i < num_tasks; i++)
 214           {
 215             struct gomp_task task;
 216
 217             gomp_init_task (&task, thr->task, gomp_icv (false));
 218             task.priority = priority;
 219             task.kind = GOMP_TASK_UNDEFERRED;
 220             task.final_task = (thr->task && thr->task->final_task)
 221                               || (flags & GOMP_TASK_FLAG_FINAL);
 222             if (thr->task)
 223               {
 224                 task.in_tied_task = thr->task->in_tied_task;
 225                 task.taskgroup = thr->task->taskgroup;
 226               }
 227             thr->task = &task;
 228             ((TYPE *)data)[0] = start;
 229             start += task_step;
 230             ((TYPE *)data)[1] = start;
 231             if (i == nfirst)
 232               task_step -= step;
 233             fn (data);
 234             if (!priority_queue_empty_p (&task.children_queue,
 235                                          MEMMODEL_RELAXED))
 236               {
 237                 gomp_mutex_lock (&team->task_lock);
 238                 gomp_clear_parent (&task.children_queue);
 239                 gomp_mutex_unlock (&team->task_lock);
 240               }
 241             gomp_end_task ();
 242           }
 243     }
 244   else
 245     {
 246       struct gomp_task *tasks[num_tasks];
 247       struct gomp_task *parent = thr->task;
 248       struct gomp_taskgroup *taskgroup = parent->taskgroup;
 249       char *arg;
 250       int do_wake;
 251       unsigned long i;
 252
 253       for (i = 0; i < num_tasks; i++)
 254         {
 255           struct gomp_task *task
 256             = gomp_malloc (sizeof (*task) + arg_size + arg_align - 1);
 257           tasks[i] = task;
 258           arg = (char *) (((uintptr_t) (task + 1) + arg_align - 1)
 259                           & ~(uintptr_t) (arg_align - 1));
 260           gomp_init_task (task, parent, gomp_icv (false));
 261           task->priority = priority;
 262           task->kind = GOMP_TASK_UNDEFERRED;
 263           task->in_tied_task = parent->in_tied_task;
 264           task->taskgroup = taskgroup;
 265           thr->task = task;
 266           if (cpyfn)
 267             {
 268               cpyfn (arg, data);
 269               task->copy_ctors_done = true;
 270             }
 271           else
 272             memcpy (arg, data, arg_size);
 273           ((TYPE *)arg)[0] = start;
 274           start += task_step;
 275           ((TYPE *)arg)[1] = start;
 276           if (i == nfirst)
 277             task_step -= step;
 278           thr->task = parent;
 279           task->kind = GOMP_TASK_WAITING;
 280           task->fn = fn;
 281           task->fn_data = arg;
 282           task->final_task = (flags & GOMP_TASK_FLAG_FINAL) >> 1;
 283         }
 284       gomp_mutex_lock (&team->task_lock);
 285       /* If parallel or taskgroup has been cancelled, don't start new
 286          tasks.  */
 287       if (__builtin_expect ((gomp_team_barrier_cancelled (&team->barrier)
 288                              || (taskgroup && taskgroup->cancelled))
 289                             && cpyfn == NULL, 0))
 290         {
 291           gomp_mutex_unlock (&team->task_lock);
 292           for (i = 0; i < num_tasks; i++)
 293             {
 294               gomp_finish_task (tasks[i]);
 295               free (tasks[i]);
 296             }
 297           if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0)
 298             ialias_call (GOMP_taskgroup_end) ();
 299           return;
 300         }
 301       if (taskgroup)
 302         taskgroup->num_children += num_tasks;
 303       for (i = 0; i < num_tasks; i++)
 304         {
 305           struct gomp_task *task = tasks[i];
 306           priority_queue_insert (PQ_CHILDREN, &parent->children_queue,
 307                                  task, priority,
 308                                  PRIORITY_INSERT_BEGIN,
 309                                  /*last_parent_depends_on=*/false,
 310                                  task->parent_depends_on);
 311           if (taskgroup)
 312             priority_queue_insert (PQ_TASKGROUP, &taskgroup->taskgroup_queue,
 313                                    task, priority, PRIORITY_INSERT_BEGIN,
 314                                    /*last_parent_depends_on=*/false,
 315                                    task->parent_depends_on);
 316           priority_queue_insert (PQ_TEAM, &team->task_queue, task, priority,
 317                                  PRIORITY_INSERT_END,
 318                                  /*last_parent_depends_on=*/false,
 319                                  task->parent_depends_on);
 320           ++team->task_count;
 321           ++team->task_queued_count;
 322         }
 323       gomp_team_barrier_set_task_pending (&team->barrier);
 324       if (team->task_running_count + !parent->in_tied_task
 325           < team->nthreads)
 326         {
 327           do_wake = team->nthreads - team->task_running_count
 328                     - !parent->in_tied_task;
 329           if ((unsigned long) do_wake > num_tasks)
 330             do_wake = num_tasks;
 331         }
 332       else
 333         do_wake = 0;
 334       gomp_mutex_unlock (&team->task_lock);
 335       if (do_wake)
 336         gomp_team_barrier_wake (&team->barrier, do_wake);
 337     }
 338   if ((flags & GOMP_TASK_FLAG_NOGROUP) == 0)
 339     ialias_call (GOMP_taskgroup_end) ();
 340 }