contrib/hwloc/src/topology-pci.c

   1 /*
   2  * Copyright © 2009 CNRS
   3  * Copyright © 2009-2018 Inria.  All rights reserved.
   4  * Copyright © 2009-2011, 2013 Université Bordeaux
   5  * Copyright © 2014-2018 Cisco Systems, Inc.  All rights reserved.
   6  * Copyright © 2015      Research Organization for Information Science
   7  *                       and Technology (RIST). All rights reserved.
   8  * See COPYING in top-level directory.
   9  */
  10
  11 #include <private/autogen/config.h>
  12 #include <hwloc.h>
  13 #include <hwloc/helper.h>
  14 #include <hwloc/plugins.h>
  15
  16 /* private headers allowed for convenience because this plugin is built within hwloc */
  17 #include <private/debug.h>
  18 #include <private/misc.h>
  19
  20 #include <stdio.h>
  21 #include <fcntl.h>
  22 #include <string.h>
  23 #include <assert.h>
  24 #include <stdarg.h>
  25 #ifdef HWLOC_LINUX_SYS
  26 #include <dirent.h>
  27 #endif
  28
  29 #include <pciaccess.h>
  30
  31 #ifndef PCI_HEADER_TYPE
  32 #define PCI_HEADER_TYPE 0x0e
  33 #endif
  34 #ifndef PCI_HEADER_TYPE_BRIDGE
  35 #define PCI_HEADER_TYPE_BRIDGE 1
  36 #endif
  37
  38 #ifndef PCI_CLASS_DEVICE
  39 #define PCI_CLASS_DEVICE 0x0a
  40 #endif
  41 #ifndef PCI_CLASS_BRIDGE_PCI
  42 #define PCI_CLASS_BRIDGE_PCI 0x0604
  43 #endif
  44
  45 #ifndef PCI_REVISION_ID
  46 #define PCI_REVISION_ID 0x08
  47 #endif
  48
  49 #ifndef PCI_SUBSYSTEM_VENDOR_ID
  50 #define PCI_SUBSYSTEM_VENDOR_ID 0x2c
  51 #endif
  52 #ifndef PCI_SUBSYSTEM_ID
  53 #define PCI_SUBSYSTEM_ID 0x2e
  54 #endif
  55
  56 #ifndef PCI_PRIMARY_BUS
  57 #define PCI_PRIMARY_BUS 0x18
  58 #endif
  59 #ifndef PCI_SECONDARY_BUS
  60 #define PCI_SECONDARY_BUS 0x19
  61 #endif
  62 #ifndef PCI_SUBORDINATE_BUS
  63 #define PCI_SUBORDINATE_BUS 0x1a
  64 #endif
  65
  66 #ifndef PCI_CAP_ID_EXP
  67 #define PCI_CAP_ID_EXP 0x10
  68 #endif
  69
  70 #ifndef PCI_CAP_NORMAL
  71 #define PCI_CAP_NORMAL 1
  72 #endif
  73
  74 #define CONFIG_SPACE_CACHESIZE 256
  75
  76 #ifdef HWLOC_WIN_SYS
  77 #error pciaccess locking currently not implemented on Windows
  78
  79 #elif defined HWLOC_HAVE_PTHREAD_MUTEX
  80 /* pthread mutex if available (except on windows) */
  81 #include <pthread.h>
  82 static pthread_mutex_t hwloc_pciaccess_mutex = PTHREAD_MUTEX_INITIALIZER;
  83 #define HWLOC_PCIACCESS_LOCK() pthread_mutex_lock(&hwloc_pciaccess_mutex)
  84 #define HWLOC_PCIACCESS_UNLOCK() pthread_mutex_unlock(&hwloc_pciaccess_mutex)
  85
  86 #else /* HWLOC_WIN_SYS || HWLOC_HAVE_PTHREAD_MUTEX */
  87 #error No mutex implementation available
  88 #endif
  89
  90 static int
  91 hwloc_look_pci(struct hwloc_backend *backend)
  92 {
  93   struct hwloc_topology *topology = backend->topology;
  94   struct hwloc_obj *first_obj = NULL, *last_obj = NULL;
  95   int ret;
  96   struct pci_device_iterator *iter;
  97   struct pci_device *pcidev;
  98 #ifdef HWLOC_LINUX_SYS
  99   DIR *dir;
 100 #endif
 101
 102   if (!(hwloc_topology_get_flags(topology) & (HWLOC_TOPOLOGY_FLAG_IO_DEVICES|HWLOC_TOPOLOGY_FLAG_WHOLE_IO)))
 103     return 0;
 104
 105   if (hwloc_get_next_pcidev(topology, NULL)) {
 106     hwloc_debug("%s", "PCI objects already added, ignoring pci backend.\n");
 107     return 0;
 108   }
 109
 110   if (!hwloc_topology_is_thissystem(topology)) {
 111     hwloc_debug("%s", "\nno PCI detection (not thissystem)\n");
 112     return 0;
 113   }
 114
 115   hwloc_debug("%s", "\nScanning PCI buses...\n");
 116
 117   /* pciaccess isn't thread-safe. it uses a single global variable that doesn't have
 118    * refcounting, and is dynamically reallocated when vendor/device names are needed, etc.
 119    */
 120   HWLOC_PCIACCESS_LOCK();
 121
 122   /* initialize PCI scanning */
 123   ret = pci_system_init();
 124   if (ret) {
 125     HWLOC_PCIACCESS_UNLOCK();
 126     hwloc_debug("%s", "Can not initialize libpciaccess\n");
 127     return -1;
 128   }
 129
 130   iter = pci_slot_match_iterator_create(NULL);
 131
 132   /* iterate over devices */
 133   for (pcidev = pci_device_next(iter);
 134        pcidev;
 135        pcidev = pci_device_next(iter))
 136   {
 137     const char *vendorname, *devicename, *fullname;
 138     unsigned char config_space_cache[CONFIG_SPACE_CACHESIZE];
 139     struct hwloc_obj *obj;
 140     unsigned os_index;
 141     unsigned domain;
 142     unsigned device_class;
 143     unsigned short tmp16;
 144     char name[128];
 145     unsigned offset;
 146
 147     /* initialize the config space in case we fail to read it (missing permissions, etc). */
 148     memset(config_space_cache, 0xff, CONFIG_SPACE_CACHESIZE);
 149     pci_device_probe(pcidev);
 150     pci_device_cfg_read(pcidev, config_space_cache, 0, CONFIG_SPACE_CACHESIZE, NULL);
 151
 152     /* try to read the domain */
 153     domain = pcidev->domain;
 154
 155     /* try to read the device_class */
 156     device_class = pcidev->device_class >> 8;
 157
 158     /* fixup SR-IOV buggy VF device/vendor IDs */
 159     if (0xffff == pcidev->vendor_id && 0xffff == pcidev->device_id) {
 160       /* SR-IOV puts ffff:ffff in Virtual Function config space.
 161        * The actual VF device ID is stored at a special (dynamic) location in the Physical Function config space.
 162        * VF and PF have the same vendor ID.
 163        *
 164        * libpciaccess just returns ffff:ffff, needs to be fixed.
 165        * linuxpci is OK because sysfs files are already fixed the kernel.
 166        * (pciutils is OK when it uses those Linux sysfs files.)
 167        *
 168        * Reading these files is an easy way to work around the libpciaccess issue on Linux,
 169        * but we have no way to know if this is caused by SR-IOV or not.
 170        *
 171        * TODO:
 172        *  If PF has CAP_ID_PCIX or CAP_ID_EXP (offset>0),
 173        *  look for extended capability PCI_EXT_CAP_ID_SRIOV (need extended config space (more than 256 bytes)),
 174        *  then read the VF device ID after it (PCI_IOV_DID bytes later).
 175        *  Needs access to extended config space (needs root on Linux).
 176        * TODO:
 177        *  Add string info attributes in VF and PF objects?
 178        */
 179 #ifdef HWLOC_LINUX_SYS
 180       /* Workaround for Linux (the kernel returns the VF device/vendor IDs). */
 181       char path[64];
 182       char value[16];
 183       FILE *file;
 184       size_t bytes_read;
 185
 186       snprintf(path, sizeof(path), "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/vendor",
 187                domain, pcidev->bus, pcidev->dev, pcidev->func);
 188       file = fopen(path, "r");
 189       if (file) {
 190         bytes_read = fread(value, 1, sizeof(value), file);
 191         fclose(file);
 192         if (bytes_read)
 193           /* fixup the pciaccess struct so that pci_device_get_vendor_name() is correct later. */
 194           pcidev->vendor_id = strtoul(value, NULL, 16);
 195       }
 196
 197       snprintf(path, sizeof(path), "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/device",
 198                domain, pcidev->bus, pcidev->dev, pcidev->func);
 199       file = fopen(path, "r");
 200       if (file) {
 201         bytes_read = fread(value, 1, sizeof(value), file);
 202         fclose(file);
 203         if (bytes_read)
 204           /* fixup the pciaccess struct so that pci_device_get_device_name() is correct later. */
 205           pcidev->device_id = strtoul(value, NULL, 16);
 206       }
 207 #endif
 208     }
 209
 210     /* might be useful for debugging (note that domain might be truncated) */
 211     os_index = (domain << 20) + (pcidev->bus << 12) + (pcidev->dev << 4) + pcidev->func;
 212
 213     obj = hwloc_alloc_setup_object(HWLOC_OBJ_PCI_DEVICE, os_index);
 214     obj->attr->pcidev.domain = domain;
 215     obj->attr->pcidev.bus = pcidev->bus;
 216     obj->attr->pcidev.dev = pcidev->dev;
 217     obj->attr->pcidev.func = pcidev->func;
 218     obj->attr->pcidev.vendor_id = pcidev->vendor_id;
 219     obj->attr->pcidev.device_id = pcidev->device_id;
 220     obj->attr->pcidev.class_id = device_class;
 221     obj->attr->pcidev.revision = config_space_cache[PCI_REVISION_ID];
 222
 223     obj->attr->pcidev.linkspeed = 0; /* unknown */
 224     offset = hwloc_pci_find_cap(config_space_cache, PCI_CAP_ID_EXP);
 225
 226     if (offset > 0 && offset + 20 /* size of PCI express block up to link status */ <= CONFIG_SPACE_CACHESIZE) {
 227       hwloc_pci_find_linkspeed(config_space_cache, offset, &obj->attr->pcidev.linkspeed);
 228 #ifdef HWLOC_LINUX_SYS
 229     } else {
 230       /* if not available from config-space (extended part is root-only), look in Linux sysfs files added in 4.13 */
 231       char path[64];
 232       char value[16];
 233       FILE *file;
 234       size_t bytes_read;
 235       float speed = 0.f;
 236       unsigned width = 0;
 237       snprintf(path, sizeof(path), "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/current_link_speed",
 238                domain, pcidev->bus, pcidev->dev, pcidev->func);
 239       file = fopen(path, "r");
 240       if (file) {
 241         bytes_read = fread(value, 1, sizeof(value), file);
 242         fclose(file);
 243         if (bytes_read)
 244           speed = hwloc_linux_pci_link_speed_from_string(value);
 245       }
 246       snprintf(path, sizeof(path), "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/current_link_width",
 247                domain, pcidev->bus, pcidev->dev, pcidev->func);
 248       file = fopen(path, "r");
 249       if (file) {
 250         bytes_read = fread(value, 1, sizeof(value), file);
 251         fclose(file);
 252         if (bytes_read)
 253           width = atoi(value);
 254       }
 255       obj->attr->pcidev.linkspeed = speed*width/8;
 256 #endif
 257     }
 258
 259     if (hwloc_pci_prepare_bridge(obj, config_space_cache) < 0)
 260       continue;
 261
 262     if (obj->type == HWLOC_OBJ_PCI_DEVICE) {
 263       memcpy(&tmp16, &config_space_cache[PCI_SUBSYSTEM_VENDOR_ID], sizeof(tmp16));
 264       obj->attr->pcidev.subvendor_id = tmp16;
 265       memcpy(&tmp16, &config_space_cache[PCI_SUBSYSTEM_ID], sizeof(tmp16));
 266       obj->attr->pcidev.subdevice_id = tmp16;
 267     } else {
 268       /* TODO:
 269        * bridge must lookup PCI_CAP_ID_SSVID and then look at offset+PCI_SSVID_VENDOR/DEVICE_ID
 270        * cardbus must look at PCI_CB_SUBSYSTEM_VENDOR_ID and PCI_CB_SUBSYSTEM_ID
 271        */
 272     }
 273
 274     /* get the vendor name */
 275     vendorname = pci_device_get_vendor_name(pcidev);
 276     if (vendorname && *vendorname)
 277       hwloc_obj_add_info(obj, "PCIVendor", vendorname);
 278
 279     /* get the device name */
 280     devicename = pci_device_get_device_name(pcidev);
 281     if (devicename && *devicename)
 282       hwloc_obj_add_info(obj, "PCIDevice", devicename);
 283
 284     /* generate or get the fullname */
 285     snprintf(name, sizeof(name), "%s%s%s",
 286              vendorname ? vendorname : "",
 287              vendorname && devicename ? " " : "",
 288              devicename ? devicename : "");
 289     fullname = name;
 290     if (*name)
 291       obj->name = strdup(name);
 292     hwloc_debug("  %04x:%02x:%02x.%01x %04x %04x:%04x %s\n",
 293                 domain, pcidev->bus, pcidev->dev, pcidev->func,
 294                 device_class, pcidev->vendor_id, pcidev->device_id,
 295                 fullname && *fullname ? fullname : "??");
 296
 297     /* queue the object for now */
 298     if (first_obj)
 299       last_obj->next_sibling = obj;
 300     else
 301       first_obj = obj;
 302     last_obj = obj;
 303   }
 304
 305   /* finalize device scanning */
 306   pci_iterator_destroy(iter);
 307   pci_system_cleanup();
 308   HWLOC_PCIACCESS_UNLOCK();
 309
 310 #ifdef HWLOC_LINUX_SYS
 311   dir = opendir("/sys/bus/pci/slots/");
 312   if (dir) {
 313     struct dirent *dirent;
 314     while ((dirent = readdir(dir)) != NULL) {
 315       char path[64];
 316       FILE *file;
 317       int err;
 318       if (dirent->d_name[0] == '.')
 319         continue;
 320       err = snprintf(path, sizeof(path), "/sys/bus/pci/slots/%s/address", dirent->d_name);
 321       if ((size_t) err < sizeof(path)) {
 322         file = fopen(path, "r");
 323         if (file) {
 324           unsigned domain, bus, dev;
 325           if (fscanf(file, "%x:%x:%x", &domain, &bus, &dev) == 3) {
 326             hwloc_obj_t obj = first_obj;
 327             while (obj) {
 328               if (obj->attr->pcidev.domain == domain
 329                   && obj->attr->pcidev.bus == bus
 330                   && obj->attr->pcidev.dev == dev) {
 331                 hwloc_obj_add_info(obj, "PCISlot", dirent->d_name);
 332               }
 333               obj = obj->next_sibling;
 334             }
 335           }
 336           fclose(file);
 337         }
 338       }
 339     }
 340     closedir(dir);
 341   }
 342 #endif
 343
 344   return hwloc_insert_pci_device_list(backend, first_obj);
 345 }
 346
 347 static struct hwloc_backend *
 348 hwloc_pci_component_instantiate(struct hwloc_disc_component *component,
 349                                    const void *_data1 __hwloc_attribute_unused,
 350                                    const void *_data2 __hwloc_attribute_unused,
 351                                    const void *_data3 __hwloc_attribute_unused)
 352 {
 353   struct hwloc_backend *backend;
 354
 355   /* thissystem may not be fully initialized yet, we'll check flags in discover() */
 356
 357   backend = hwloc_backend_alloc(component);
 358   if (!backend)
 359     return NULL;
 360   backend->flags = HWLOC_BACKEND_FLAG_NEED_LEVELS;
 361 #ifdef HWLOC_SOLARIS_SYS
 362   if ((uid_t)0 != geteuid())
 363     backend->discover = NULL;
 364   else
 365 #endif
 366     backend->discover = hwloc_look_pci;
 367   return backend;
 368 }
 369
 370 static struct hwloc_disc_component hwloc_pci_disc_component = {
 371   HWLOC_DISC_COMPONENT_TYPE_MISC,
 372   "pci",
 373   HWLOC_DISC_COMPONENT_TYPE_GLOBAL,
 374   hwloc_pci_component_instantiate,
 375   20,
 376   NULL
 377 };
 378
 379 static int
 380 hwloc_pci_component_init(unsigned long flags)
 381 {
 382   if (flags)
 383     return -1;
 384   if (hwloc_plugin_check_namespace("pci", "hwloc_backend_alloc") < 0)
 385     return -1;
 386   return 0;
 387 }
 388
 389 #ifdef HWLOC_INSIDE_PLUGIN
 390 HWLOC_DECLSPEC extern const struct hwloc_component hwloc_pci_component;
 391 #endif
 392
 393 const struct hwloc_component hwloc_pci_component = {
 394   HWLOC_COMPONENT_ABI,
 395   hwloc_pci_component_init, NULL,
 396   HWLOC_COMPONENT_TYPE_DISC,
 397   0,
 398   &hwloc_pci_disc_component
 399 };