dev.c

   1 /*
   2  * Copyright (C) 2012-2020 all contributors <cmogstored-public@yhbt.net>
   3  * License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
   4  */
   5 #include "cmogstored.h"
   6
   7 static int stat_prefix(struct mog_svc *svc, struct stat *sb, uint32_t mog_devid)
   8 {
   9         char prefix[sizeof("/dev" MOG_STR(MOG_DEVID_MAX) "/")];
  10         int rc;
  11
  12         assert(mog_devid <= MOG_DEVID_MAX && "mog_devid not filtered");
  13         rc = snprintf(prefix, sizeof(prefix), "/dev%u/", mog_devid);
  14         assert(rc > 0 && rc < (int)sizeof(prefix) && "we suck at snprintf");
  15
  16         return mog_stat(svc, prefix, sb);
  17 }
  18
  19 static struct mog_dev *mog_dev_new(struct mog_svc *svc, uint32_t mog_devid)
  20 {
  21         struct mog_dev *dev;
  22         struct stat sb;
  23
  24         /* we have no space on stack */
  25         if (mog_devid > MOG_DEVID_MAX)
  26                 return NULL;
  27
  28         if (stat_prefix(svc, &sb, mog_devid) < 0)
  29                 return NULL;
  30
  31         dev = mog_cachealign(sizeof(struct mog_dev));
  32         dev->devid = mog_devid;
  33         dev->st_dev = sb.st_dev;
  34         mog_ioq_init(&dev->fsckq, svc, 1);
  35         mog_ioq_init(&dev->ioq, svc, svc->thr_per_dev);
  36         dev->no_me_warned = 0;
  37         dev->usage_txt = 0;
  38         dev->usage_len = 0;
  39         dev->usage_mtime = 0;
  40         CHECK(int, 0, pthread_mutex_init(&dev->usage_lock, NULL));
  41
  42         return dev;
  43 }
  44
  45 struct mog_dev *
  46 mog_dev_for(struct mog_svc *svc, uint32_t mog_devid, bool update)
  47 {
  48         struct mog_dev finder;
  49         struct mog_dev *ret;
  50         bool need_refresh = false;
  51
  52         finder.devid = mog_devid;
  53
  54         CHECK(int, 0, pthread_mutex_lock(&svc->by_mog_devid_lock));
  55         ret = hash_lookup(svc->by_mog_devid, &finder);
  56         if (ret) {
  57                 struct stat sb;
  58
  59                 if (!update)
  60                         goto out;
  61
  62                 /*
  63                  * devXXX dir existed before, but is no longer readable
  64                  * Possible FS/device error, it could come back, so do
  65                  * not remove here.
  66                  */
  67                 if (stat_prefix(svc, &sb, ret->devid) < 0)
  68                         goto out;
  69
  70                 /* st_dev may change due to remount, update if needed */
  71                 ret->st_dev = sb.st_dev;
  72         } else { /* create a new dev */
  73                 ret = mog_dev_new(svc, mog_devid);
  74
  75                 if (!ret)
  76                         goto out; /* could not stat */
  77
  78                 switch (hash_insert_if_absent(svc->by_mog_devid, ret, NULL)) {
  79                 case 0:
  80                         assert(0 && "mog_dev existed while adding");
  81                         abort();
  82                 case 1:
  83                         if (!update)
  84                                 need_refresh = true;
  85                         break; /* OK, inserted */
  86                 default: mog_oom();
  87                 }
  88         }
  89 out:
  90         CHECK(int, 0, pthread_mutex_unlock(&svc->by_mog_devid_lock));
  91
  92         /* we need to get the notify thread to create new worker threads */
  93         if (need_refresh)
  94                 mog_notify(MOG_NOTIFY_DEVICE_REFRESH);
  95
  96         return ret;
  97 }
  98
  99
 100 size_t mog_dev_hash(const void *x, size_t tablesize)
 101 {
 102         const struct mog_dev *dev = x;
 103
 104         return dev->devid % tablesize;
 105 }
 106
 107 bool mog_dev_cmp(const void *a, const void *b)
 108 {
 109         const struct mog_dev *dev_a = a;
 110         const struct mog_dev *dev_b = b;
 111
 112         return dev_a->devid == dev_b->devid;
 113 }
 114
 115 static int
 116 emit_usage(struct mog_dev *dev, struct mog_svc *svc, int fd, struct statvfs *v)
 117 {
 118         int rc = -1;
 119         unsigned long long available = v->f_bavail;
 120         unsigned long long total = v->f_blocks - (v->f_bfree - v->f_bavail);
 121         unsigned long long used = v->f_blocks - v->f_bfree;
 122         unsigned use = (used * 100) / total + !!((used * 100) % total);
 123         long long now = (long long)time(NULL);
 124         long double mb = v->f_frsize / (long double)1024.0;
 125         const struct mount_entry *me;
 126         char *usage_txt;
 127         static const char usage_fmt[] =
 128                 "available: %llu\n"
 129                 "device: %s\n"
 130                 "disk: %s/dev%u\n"
 131                 "time: %lld\n"
 132                 "total: %llu\n"
 133                 "use: %u%%\n"
 134                 "used: %llu\n";
 135
 136         available *= mb;
 137         total *= mb;
 138         used *= mb;
 139
 140         if (use > 100)
 141                 use = 100;
 142
 143         me = mog_mnt_acquire(dev->st_dev);
 144         if (!me && !dev->no_me_warned) {
 145                 syslog(LOG_ERR, "mount entry not found for %s/dev%u",
 146                        svc->docroot, (unsigned)dev->devid);
 147                 dev->no_me_warned = 1;
 148         }
 149         errno = 0;
 150         rc = asprintf(&usage_txt, usage_fmt, available,
 151                         /* device:, MogileFS::Worker::Monitor doesn't care: */
 152                         me ? me->me_devname : "(?)",
 153                         svc->docroot,
 154                         (unsigned)dev->devid, now, total, use, used);
 155         if (rc > 0) {
 156                 char *old_usage;
 157
 158                 CHECK(int, 0, pthread_mutex_lock(&dev->usage_lock));
 159                 old_usage = dev->usage_txt;
 160                 dev->usage_txt = usage_txt;
 161                 dev->usage_len = rc;
 162                 dev->usage_mtime = (time_t)now;
 163                 CHECK(int, 0, pthread_mutex_unlock(&dev->usage_lock));
 164
 165                 free(old_usage);
 166                 if (fd >= 0) {
 167                         ssize_t w = write(fd, usage_txt, rc);
 168
 169                         if (w >= 0 && w != rc)
 170                                 errno = ENOSPC;
 171                         else if (w < 0)
 172                                 rc = -1;
 173                 }
 174         }
 175         if (me)
 176                 PRESERVE_ERRNO( mog_mnt_release(me) );
 177         if (rc < 0 || errno == ENOSPC) {
 178                 PRESERVE_ERRNO(do {
 179                         syslog(LOG_ERR, "write(%s/dev%u/usage): %m",
 180                                svc->docroot, (unsigned)dev->devid);
 181                 } while (0));
 182         }
 183
 184         return rc;
 185 }
 186
 187 static void
 188 dev_usage_update(struct mog_dev *dev, struct mog_svc *svc, struct statvfs *v)
 189 {
 190         if (mog_statvfs(svc, dev, v) < 0) {
 191                 syslog(LOG_ERR, "statvfs error: %s/dev%u/usage (%m)",
 192                         svc->docroot, (unsigned)dev->devid);
 193                 return;
 194         }
 195         (void)emit_usage(dev, svc, -1, v);
 196 }
 197
 198 void
 199 mog_dev_usage_update(struct mog_dev *dev, struct mog_svc *svc)
 200 {
 201         struct statvfs v;
 202
 203         dev_usage_update(dev, svc, &v);
 204 }
 205
 206 int mog_dev_mkusage(struct mog_dev *dev, struct mog_svc *svc)
 207 {
 208         struct statvfs v;
 209         char *usage_path;
 210         char *tmp_path;
 211         int fd = -1;
 212         struct stat sb;
 213
 214         if (!svc->mgmt_mfd)
 215                 return 0;
 216
 217         if (asprintf(&usage_path, "/dev%u/usage", (unsigned)dev->devid) < 0) {
 218                 syslog(LOG_ERR, "error generating path: /dev%u/usage (%m)",
 219                         (unsigned)dev->devid);
 220                 return 0;
 221         }
 222
 223         /*
 224          * allow chmod 0000 on devNNN/usage files to prevent us from
 225          * overwriting them
 226          */
 227         if (mog_stat(svc, usage_path, &sb) == 0 &&
 228             ((sb.st_mode & (S_IRWXU|S_IRWXG|S_IRWXO)) == 0)) {
 229                 dev_usage_update(dev, svc, &v);
 230                 tmp_path = 0;
 231                 goto out;
 232         }
 233         if (asprintf(&tmp_path, "%s.%x", usage_path, (unsigned)getpid()) < 0) {
 234                 syslog(LOG_ERR, "error generating path: /dev%u/usage.%u (%m)",
 235                         (unsigned)dev->devid, (unsigned)getpid());
 236                 free(usage_path);
 237                 return 0;
 238         }
 239
 240         if (mog_unlink(svc, tmp_path) < 0 && errno != ENOENT) goto out;
 241
 242         errno = 0;
 243         fd = mog_open_put(svc, tmp_path, O_EXCL|O_CREAT);
 244         if (fd < 0) {
 245                 PRESERVE_ERRNO(do {
 246                         syslog(LOG_ERR, "open(%s%s): %m",
 247                                svc->docroot, tmp_path);
 248                 } while (0));
 249                 dev_usage_update(dev, svc, &v);
 250                 goto out;
 251         }
 252         if (fstatvfs(fd, &v) < 0) {
 253                 PRESERVE_ERRNO(do {
 254                         syslog(LOG_ERR, "fstatvfs(%s%s): %m",
 255                                svc->docroot, tmp_path);
 256                 } while (0));
 257                 goto out;
 258         }
 259         if (emit_usage(dev, svc, fd, &v) < 0) goto out;
 260         if (fchmod(fd, svc->put_perms) < 0) {
 261                 PRESERVE_ERRNO(do {
 262                         syslog(LOG_ERR, "fchmod(%s%s): %m",
 263                                svc->docroot, tmp_path);
 264                 } while (0));
 265                 goto out;
 266         }
 267
 268         /* skip rename on EIO if close() fails */
 269         if (close(fd) != 0) {
 270                 assert(errno != EBADF && "attempted to close bad FD");
 271                 fd = -1;
 272                 if (errno != EINTR) {
 273                         /* possible EIO */
 274                         PRESERVE_ERRNO(do {
 275                                 syslog(LOG_ERR, "close(%s%s) failed: %m",
 276                                        svc->docroot, tmp_path);
 277                         } while (0));
 278                         goto out;
 279                 }
 280         }
 281
 282         fd = -1;
 283         errno = 0;
 284         if (mog_rename(svc, tmp_path, usage_path) != 0) {
 285                 PRESERVE_ERRNO(do {
 286                         syslog(LOG_ERR, "rename(%s(%s => %s))",
 287                                svc->docroot, tmp_path, usage_path);
 288                 } while (0));
 289         }
 290 out:
 291         PRESERVE_ERRNO(do {
 292                 if (errno)
 293                         (void)mog_unlink(svc, tmp_path);
 294                 if (fd >= 0)
 295                         (void)mog_close(fd);
 296                 free(tmp_path);
 297                 free(usage_path);
 298         } while (0));
 299         return errno ? -1 : 0;
 300 }
 301
 302 void mog_dev_free(void *ptr)
 303 {
 304         struct mog_dev *dev = ptr;
 305
 306         mog_ioq_destroy(&dev->fsckq);
 307         mog_ioq_destroy(&dev->ioq);
 308         free(dev->usage_txt);
 309         CHECK(int, 0, pthread_mutex_destroy(&dev->usage_lock));
 310         free(dev);
 311 }
 312
 313 /*
 314  * Only called by the main/notify thread as a hash iterator function
 315  * This increases or decreases the capacity of a given device if a
 316  * a sidechannel user changes the worker thread pool size.
 317  */
 318 bool mog_dev_user_rescale_i(void *devp, void *svcp)
 319 {
 320         struct mog_dev *dev = devp;
 321         struct mog_svc *svc = svcp;
 322
 323         mog_ioq_adjust(&dev->ioq, svc->thr_per_dev);
 324
 325         return true; /* continue iteration */
 326 }
 327
 328 bool mog_dev_requeue_prepare(void *devp, void *ign)
 329 {
 330         struct mog_dev *dev = devp;
 331
 332         mog_ioq_requeue_prepare(&dev->ioq);
 333         mog_ioq_requeue_prepare(&dev->fsckq);
 334
 335         return true; /* continue iteration */
 336 }