libgomp/config/linux/affinity.c

   1 /* Copyright (C) 2006-2024 Free Software Foundation, Inc.
   2    Contributed by Jakub Jelinek <jakub@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This is a Linux specific implementation of a CPU affinity setting.  */
  27
  28 #ifndef _GNU_SOURCE
  29 #define _GNU_SOURCE 1
  30 #endif
  31 #include "libgomp.h"
  32 #include "proc.h"
  33 #include <errno.h>
  34 #include <stdlib.h>
  35 #include <stdio.h>
  36 #include <string.h>
  37 #include <unistd.h>
  38 #include <limits.h>
  39
  40 #ifdef HAVE_PTHREAD_AFFINITY_NP
  41
  42 #ifndef CPU_ALLOC_SIZE
  43 #define CPU_ISSET_S(idx, size, set) CPU_ISSET(idx, set)
  44 #define CPU_ZERO_S(size, set) CPU_ZERO(set)
  45 #define CPU_SET_S(idx, size, set) CPU_SET(idx, set)
  46 #define CPU_CLR_S(idx, size, set) CPU_CLR(idx, set)
  47 #endif
  48
  49 void
  50 gomp_init_affinity (void)
  51 {
  52   if (gomp_places_list == NULL)
  53     {
  54       if (!gomp_affinity_init_level (1, ULONG_MAX, true))
  55         return;
  56     }
  57
  58   struct gomp_thread *thr = gomp_thread ();
  59   pthread_setaffinity_np (pthread_self (), gomp_cpuset_size,
  60                           (cpu_set_t *) gomp_places_list[0]);
  61   thr->place = 1;
  62   thr->ts.place_partition_off = 0;
  63   thr->ts.place_partition_len = gomp_places_list_len;
  64 }
  65
  66 void
  67 gomp_init_thread_affinity (pthread_attr_t *attr, unsigned int place)
  68 {
  69   pthread_attr_setaffinity_np (attr, gomp_cpuset_size,
  70                                (cpu_set_t *) gomp_places_list[place]);
  71 }
  72
  73 void **
  74 gomp_affinity_alloc (unsigned long count, bool quiet)
  75 {
  76   unsigned long i;
  77   void **ret;
  78   char *p;
  79
  80   if (gomp_cpusetp == NULL)
  81     {
  82       if (!quiet)
  83         gomp_error ("Could not get CPU affinity set");
  84       return NULL;
  85     }
  86
  87   ret = malloc (count * sizeof (void *) + count * gomp_cpuset_size);
  88   if (ret == NULL)
  89     {
  90       if (!quiet)
  91         gomp_error ("Out of memory trying to allocate places list");
  92       return NULL;
  93     }
  94
  95   p = (char *) (ret + count);
  96   for (i = 0; i < count; i++, p += gomp_cpuset_size)
  97     ret[i] = p;
  98   return ret;
  99 }
 100
 101 void
 102 gomp_affinity_init_place (void *p)
 103 {
 104   cpu_set_t *cpusetp = (cpu_set_t *) p;
 105   CPU_ZERO_S (gomp_cpuset_size, cpusetp);
 106 }
 107
 108 bool
 109 gomp_affinity_add_cpus (void *p, unsigned long num,
 110                         unsigned long len, long stride, bool quiet)
 111 {
 112   cpu_set_t *cpusetp = (cpu_set_t *) p;
 113   unsigned long max = 8 * gomp_cpuset_size;
 114   for (;;)
 115     {
 116       if (num >= max)
 117         {
 118           if (!quiet)
 119             gomp_error ("Logical CPU number %lu out of range", num);
 120           return false;
 121         }
 122       CPU_SET_S (num, gomp_cpuset_size, cpusetp);
 123       if (--len == 0)
 124         return true;
 125       if ((stride < 0 && num + stride > num)
 126           || (stride > 0 && num + stride < num))
 127         {
 128           if (!quiet)
 129             gomp_error ("Logical CPU number %lu+%ld out of range",
 130                         num, stride);
 131           return false;
 132         }
 133       num += stride;
 134     }
 135 }
 136
 137 bool
 138 gomp_affinity_remove_cpu (void *p, unsigned long num)
 139 {
 140   cpu_set_t *cpusetp = (cpu_set_t *) p;
 141   if (num >= 8 * gomp_cpuset_size)
 142     {
 143       gomp_error ("Logical CPU number %lu out of range", num);
 144       return false;
 145     }
 146   if (!CPU_ISSET_S (num, gomp_cpuset_size, cpusetp))
 147     {
 148       gomp_error ("Logical CPU %lu to be removed is not in the set", num);
 149       return false;
 150     }
 151   CPU_CLR_S (num, gomp_cpuset_size, cpusetp);
 152   return true;
 153 }
 154
 155 bool
 156 gomp_affinity_copy_place (void *p, void *q, long stride)
 157 {
 158   unsigned long i, max = 8 * gomp_cpuset_size;
 159   cpu_set_t *destp = (cpu_set_t *) p;
 160   cpu_set_t *srcp = (cpu_set_t *) q;
 161
 162   CPU_ZERO_S (gomp_cpuset_size, destp);
 163   for (i = 0; i < max; i++)
 164     if (CPU_ISSET_S (i, gomp_cpuset_size, srcp))
 165       {
 166         if ((stride < 0 && i + stride > i)
 167             || (stride > 0 && (i + stride < i || i + stride >= max)))
 168           {
 169             gomp_error ("Logical CPU number %lu+%ld out of range", i, stride);
 170             return false;
 171           }
 172         CPU_SET_S (i + stride, gomp_cpuset_size, destp);
 173       }
 174   return true;
 175 }
 176
 177 bool
 178 gomp_affinity_same_place (void *p, void *q)
 179 {
 180 #ifdef CPU_EQUAL_S
 181   return CPU_EQUAL_S (gomp_cpuset_size, (cpu_set_t *) p, (cpu_set_t *) q);
 182 #else
 183   return memcmp (p, q, gomp_cpuset_size) == 0;
 184 #endif
 185 }
 186
 187 bool
 188 gomp_affinity_finalize_place_list (bool quiet)
 189 {
 190   unsigned long i, j;
 191
 192   for (i = 0, j = 0; i < gomp_places_list_len; i++)
 193     {
 194       cpu_set_t *cpusetp = (cpu_set_t *) gomp_places_list[i];
 195       bool nonempty = false;
 196 #ifdef CPU_AND_S
 197       CPU_AND_S (gomp_cpuset_size, cpusetp, cpusetp, gomp_cpusetp);
 198       nonempty = gomp_cpuset_popcount (gomp_cpuset_size, cpusetp) != 0;
 199 #else
 200       unsigned long k, max = gomp_cpuset_size / sizeof (cpusetp->__bits[0]);
 201       for (k = 0; k < max; k++)
 202         if ((cpusetp->__bits[k] &= gomp_cpusetp->__bits[k]) != 0)
 203           nonempty = true;
 204 #endif
 205       if (nonempty)
 206         gomp_places_list[j++] = gomp_places_list[i];
 207     }
 208
 209   if (j == 0)
 210     {
 211       if (!quiet)
 212         gomp_error ("None of the places contain usable logical CPUs");
 213       return false;
 214     }
 215   else if (j < gomp_places_list_len)
 216     {
 217       if (!quiet)
 218         gomp_error ("Number of places reduced from %ld to %ld because some "
 219                     "places didn't contain any usable logical CPUs",
 220                     gomp_places_list_len, j);
 221       gomp_places_list_len = j;
 222     }
 223   return true;
 224 }
 225
 226 /* Find the index of the last level cache.  We assume the index
 227    of the last level cache is the same for all logical CPUs.
 228    Also, if there are multiple caches with the same highest level,
 229    assume they have the same shared_cpu_list and pick the last one
 230    from them (highest index number).  */
 231
 232 static int
 233 gomp_affinity_find_last_cache_level (char *name, size_t prefix_len,
 234                                      unsigned long cpu)
 235 {
 236   int ret = -1;
 237   unsigned long maxval = 0;
 238   char *line = NULL;
 239   size_t linelen = 0;
 240   FILE *f;
 241
 242   for (int l = 0; l < 128; l++)
 243     {
 244       sprintf (name + prefix_len, "%lu/cache/index%u/level", cpu, l);
 245       f = fopen (name, "r");
 246       if (f == NULL)
 247         break;
 248       if (getline (&line, &linelen, f) > 0)
 249         {
 250           unsigned long val;
 251           char *p;
 252           errno = 0;
 253           val = strtoul (line, &p, 10);
 254           if (!errno && p > line && val >= maxval)
 255             {
 256               ret = l;
 257               maxval = val;
 258             }
 259         }
 260       fclose (f);
 261     }
 262   free (line);
 263   return ret;
 264 }
 265
 266 static void
 267 gomp_affinity_init_level_1 (int level, int this_level, unsigned long count,
 268                             cpu_set_t *copy, char *name, bool quiet)
 269 {
 270   size_t prefix_len = sizeof ("/sys/devices/system/cpu/cpu") - 1;
 271   FILE *f;
 272   char *line = NULL;
 273   size_t linelen = 0;
 274   unsigned long i, max = 8 * gomp_cpuset_size;
 275   int init = -1;
 276
 277   for (i = 0; i < max && gomp_places_list_len < count; i++)
 278     if (CPU_ISSET_S (i, gomp_cpuset_size, copy))
 279       {
 280         if (level == 4)
 281           {
 282             if (init == -1)
 283               {
 284                 init = gomp_affinity_find_last_cache_level (name, prefix_len,
 285                                                             i);
 286                 if (init == -1)
 287                   {
 288                     CPU_CLR_S (i, gomp_cpuset_size, copy);
 289                     continue;
 290                   }
 291                 sprintf (name + prefix_len,
 292                          "%lu/cache/index%u/shared_cpu_list", i, init);
 293               }
 294           }
 295         else
 296           sprintf (name + prefix_len, "%lu/topology/%s_siblings_list",
 297                    i, this_level == 3 ? "core" : "thread");
 298         f = fopen (name, "r");
 299         if (f == NULL)
 300           {
 301             CPU_CLR_S (i, gomp_cpuset_size, copy);
 302             continue;
 303           }
 304         if (getline (&line, &linelen, f) > 0)
 305           {
 306             char *p = line, *end;
 307             void *pl = gomp_places_list[gomp_places_list_len];
 308             if (level == this_level)
 309               gomp_affinity_init_place (pl);
 310             while (*p && *p != '\n')
 311               {
 312                 unsigned long first, last;
 313                 errno = 0;
 314                 first = strtoul (p, &end, 10);
 315                 if (errno || end == p)
 316                   break;
 317                 p = end;
 318                 last = first;
 319                 if (*p == '-')
 320                   {
 321                     errno = 0;
 322                     last = strtoul (p + 1, &end, 10);
 323                     if (errno || end == p + 1 || last < first)
 324                       break;
 325                     p = end;
 326                   }
 327                 for (; first <= last; first++)
 328                   if (!CPU_ISSET_S (first, gomp_cpuset_size, copy))
 329                     continue;
 330                   else if (this_level == 3 && level < this_level)
 331                     gomp_affinity_init_level_1 (level, 2, count, copy,
 332                                                 name, quiet);
 333                   else
 334                     {
 335                       if (level == 1)
 336                         {
 337                           pl = gomp_places_list[gomp_places_list_len];
 338                           gomp_affinity_init_place (pl);
 339                         }
 340                       if (gomp_affinity_add_cpus (pl, first, 1, 0, true))
 341                         {
 342                           CPU_CLR_S (first, gomp_cpuset_size, copy);
 343                           if (level == 1
 344                               && ++gomp_places_list_len >= count)
 345                             {
 346                               fclose (f);
 347                               free (line);
 348                               return;
 349                             }
 350                         }
 351                     }
 352                 if (*p == ',')
 353                   ++p;
 354               }
 355             if (level == this_level
 356                 && !CPU_ISSET_S (i, gomp_cpuset_size, copy))
 357               gomp_places_list_len++;
 358             CPU_CLR_S (i, gomp_cpuset_size, copy);
 359           }
 360         fclose (f);
 361       }
 362   free (line);
 363 }
 364
 365 static void
 366 gomp_affinity_init_numa_domains (unsigned long count, cpu_set_t *copy,
 367                                  char *name)
 368 {
 369   FILE *f;
 370   char *nline = NULL, *line = NULL;
 371   size_t nlinelen = 0, linelen = 0;
 372   char *q;
 373   size_t prefix_len = sizeof ("/sys/devices/system/node/") - 1;
 374
 375   strcpy (name, "/sys/devices/system/node/online");
 376   f = fopen (name, "r");
 377   if (f == NULL || getline (&nline, &nlinelen, f) <= 0)
 378     {
 379       if (f)
 380         fclose (f);
 381       return;
 382     }
 383   fclose (f);
 384   q = nline;
 385   while (*q && *q != '\n' && gomp_places_list_len < count)
 386     {
 387       unsigned long nfirst, nlast;
 388       char *end;
 389
 390       errno = 0;
 391       nfirst = strtoul (q, &end, 10);
 392       if (errno || end == q)
 393         break;
 394       q = end;
 395       nlast = nfirst;
 396       if (*q == '-')
 397         {
 398           errno = 0;
 399           nlast = strtoul (q + 1, &end, 10);
 400           if (errno || end == q + 1 || nlast < nfirst)
 401             break;
 402           q = end;
 403         }
 404       for (; nfirst <= nlast && gomp_places_list_len < count; nfirst++)
 405         {
 406           sprintf (name + prefix_len, "node%lu/cpulist", nfirst);
 407           f = fopen (name, "r");
 408           if (f == NULL)
 409             continue;
 410           if (getline (&line, &linelen, f) > 0)
 411             {
 412               char *p = line;
 413               void *pl = NULL;
 414               bool seen = false;
 415
 416               while (*p && *p != '\n')
 417                 {
 418                   unsigned long first, last;
 419
 420                   errno = 0;
 421                   first = strtoul (p, &end, 10);
 422                   if (errno || end == p)
 423                     break;
 424                   p = end;
 425                   last = first;
 426                   if (*p == '-')
 427                     {
 428                       errno = 0;
 429                       last = strtoul (p + 1, &end, 10);
 430                       if (errno || end == p + 1 || last < first)
 431                         break;
 432                       p = end;
 433                     }
 434                   for (; first <= last; first++)
 435                     {
 436                       if (!CPU_ISSET_S (first, gomp_cpuset_size, copy))
 437                         continue;
 438                       if (pl == NULL)
 439                         {
 440                           pl = gomp_places_list[gomp_places_list_len];
 441                           gomp_affinity_init_place (pl);
 442                         }
 443                       if (gomp_affinity_add_cpus (pl, first, 1, 0, true))
 444                         {
 445                           CPU_CLR_S (first, gomp_cpuset_size, copy);
 446                           if (!seen)
 447                             {
 448                               gomp_places_list_len++;
 449                               seen = true;
 450                             }
 451                         }
 452                     }
 453                   if (*p == ',')
 454                     ++p;
 455                 }
 456             }
 457           fclose (f);
 458         }
 459       if (*q == ',')
 460         ++q;
 461     }
 462   free (line);
 463   free (nline);
 464 }
 465
 466 bool
 467 gomp_affinity_init_level (int level, unsigned long count, bool quiet)
 468 {
 469   char name[sizeof ("/sys/devices/system/cpu/cpu/topology/"
 470                     "thread_siblings_list") + 6 * sizeof (unsigned long)];
 471   cpu_set_t *copy;
 472
 473   if (gomp_cpusetp)
 474     {
 475       unsigned long maxcount
 476         = gomp_cpuset_popcount (gomp_cpuset_size, gomp_cpusetp);
 477       if (count > maxcount)
 478         count = maxcount;
 479     }
 480   gomp_places_list = gomp_affinity_alloc (count, quiet);
 481   gomp_places_list_len = 0;
 482   if (gomp_places_list == NULL)
 483     return false;
 484
 485   copy = gomp_alloca (gomp_cpuset_size);
 486   strcpy (name, "/sys/devices/system/cpu/cpu");
 487   memcpy (copy, gomp_cpusetp, gomp_cpuset_size);
 488   if (level == 5)
 489     gomp_affinity_init_numa_domains (count, copy, name);
 490   else
 491     gomp_affinity_init_level_1 (level, level > 3 ? level : 3, count, copy,
 492                                 name, quiet);
 493   if (gomp_places_list_len == 0)
 494     {
 495       if (!quiet)
 496         gomp_error ("Error reading core/socket topology");
 497       free (gomp_places_list);
 498       gomp_places_list = NULL;
 499       return false;
 500     }
 501   return true;
 502 }
 503
 504 void
 505 gomp_affinity_print_place (void *p)
 506 {
 507   unsigned long i, max = 8 * gomp_cpuset_size, len;
 508   cpu_set_t *cpusetp = (cpu_set_t *) p;
 509   bool notfirst = false;
 510
 511   for (i = 0, len = 0; i < max; i++)
 512     if (CPU_ISSET_S (i, gomp_cpuset_size, cpusetp))
 513       {
 514         if (len == 0)
 515           {
 516             if (notfirst)
 517               fputc (',', stderr);
 518             notfirst = true;
 519             fprintf (stderr, "%lu", i);
 520           }
 521         ++len;
 522       }
 523     else
 524       {
 525         if (len > 1)
 526           fprintf (stderr, ":%lu", len);
 527         len = 0;
 528       }
 529   if (len > 1)
 530     fprintf (stderr, ":%lu", len);
 531 }
 532
 533 int
 534 omp_get_place_num_procs (int place_num)
 535 {
 536   if (place_num < 0 || place_num >= gomp_places_list_len)
 537     return 0;
 538
 539   cpu_set_t *cpusetp = (cpu_set_t *) gomp_places_list[place_num];
 540   return gomp_cpuset_popcount (gomp_cpuset_size, cpusetp);
 541 }
 542
 543 void
 544 omp_get_place_proc_ids (int place_num, int *ids)
 545 {
 546   if (place_num < 0 || place_num >= gomp_places_list_len)
 547     return;
 548
 549   cpu_set_t *cpusetp = (cpu_set_t *) gomp_places_list[place_num];
 550   unsigned long i, max = 8 * gomp_cpuset_size;
 551   for (i = 0; i < max; i++)
 552     if (CPU_ISSET_S (i, gomp_cpuset_size, cpusetp))
 553       *ids++ = i;
 554 }
 555
 556 void
 557 gomp_get_place_proc_ids_8 (int place_num, int64_t *ids)
 558 {
 559   if (place_num < 0 || place_num >= gomp_places_list_len)
 560     return;
 561
 562   cpu_set_t *cpusetp = (cpu_set_t *) gomp_places_list[place_num];
 563   unsigned long i, max = 8 * gomp_cpuset_size;
 564   for (i = 0; i < max; i++)
 565     if (CPU_ISSET_S (i, gomp_cpuset_size, cpusetp))
 566       *ids++ = i;
 567 }
 568
 569 void
 570 gomp_display_affinity_place (char *buffer, size_t size, size_t *ret,
 571                              int place)
 572 {
 573   cpu_set_t *cpusetp;
 574   char buf[sizeof (long) * 3 + 4];
 575   if (place >= 0 && place < gomp_places_list_len)
 576     cpusetp = (cpu_set_t *) gomp_places_list[place];
 577   else if (gomp_cpusetp)
 578     cpusetp = gomp_cpusetp;
 579   else
 580     {
 581       if (gomp_available_cpus > 1)
 582         sprintf (buf, "0-%lu", gomp_available_cpus - 1);
 583       else
 584         strcpy (buf, "0");
 585       gomp_display_string (buffer, size, ret, buf, strlen (buf));
 586       return;
 587     }
 588
 589   unsigned long i, max = 8 * gomp_cpuset_size, start;
 590   bool prev_set = false;
 591   start = max;
 592   for (i = 0; i <= max; i++)
 593     {
 594       bool this_set;
 595       if (i == max)
 596         this_set = false;
 597       else
 598         this_set = CPU_ISSET_S (i, gomp_cpuset_size, cpusetp);
 599       if (this_set != prev_set)
 600         {
 601           prev_set = this_set;
 602           if (this_set)
 603             {
 604               char *p = buf;
 605               if (start != max)
 606                 *p++ = ',';
 607               sprintf (p, "%lu", i);
 608               start = i;
 609             }
 610           else if (i == start + 1)
 611             continue;
 612           else
 613             sprintf (buf, "-%lu", i - 1);
 614           gomp_display_string (buffer, size, ret, buf, strlen (buf));
 615         }
 616     }
 617 }
 618
 619 ialias(omp_get_place_num_procs)
 620 ialias(omp_get_place_proc_ids)
 621
 622 #else
 623
 624 #include "../../affinity.c"
 625
 626 #endif