nix/libstore/optimise-store.cc

   1 #include "config.h"
   2
   3 #include "util.hh"
   4 #include "local-store.hh"
   5 #include "globals.hh"
   6
   7 #include <cstdlib>
   8 #include <cstring>
   9 #include <sys/types.h>
  10 #include <sys/stat.h>
  11 #include <unistd.h>
  12 #include <errno.h>
  13 #include <stdio.h>
  14
  15
  16 namespace nix {
  17
  18
  19 static void makeWritable(const Path & path)
  20 {
  21     struct stat st;
  22     if (lstat(path.c_str(), &st))
  23         throw SysError(format("getting attributes of path `%1%'") % path);
  24     if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1)
  25         throw SysError(format("changing writability of `%1%'") % path);
  26 }
  27
  28
  29 struct MakeReadOnly
  30 {
  31     Path path;
  32     MakeReadOnly(const Path & path) : path(path) { }
  33     ~MakeReadOnly()
  34     {
  35         try {
  36             /* This will make the path read-only. */
  37             if (path != "") canonicaliseTimestampAndPermissions(path);
  38         } catch (...) {
  39             ignoreException();
  40         }
  41     }
  42 };
  43
  44
  45 LocalStore::InodeHash LocalStore::loadInodeHash()
  46 {
  47     printMsg(lvlDebug, "loading hash inodes in memory");
  48     InodeHash inodeHash;
  49
  50     AutoCloseDir dir = opendir(linksDir.c_str());
  51     if (!dir) throw SysError(format("opening directory `%1%'") % linksDir);
  52
  53     struct dirent * dirent;
  54     while (errno = 0, dirent = readdir(dir)) { /* sic */
  55         checkInterrupt();
  56         // We don't care if we hit non-hash files, anything goes
  57         inodeHash.insert(dirent->d_ino);
  58     }
  59     if (errno) throw SysError(format("reading directory `%1%'") % linksDir);
  60
  61     printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size());
  62
  63     return inodeHash;
  64 }
  65
  66
  67 Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash)
  68 {
  69     Strings names;
  70
  71     AutoCloseDir dir = opendir(path.c_str());
  72     if (!dir) throw SysError(format("opening directory `%1%'") % path);
  73
  74     struct dirent * dirent;
  75     while (errno = 0, dirent = readdir(dir)) { /* sic */
  76         checkInterrupt();
  77
  78         if (inodeHash.count(dirent->d_ino)) {
  79             printMsg(lvlDebug, format("`%1%' is already linked") % dirent->d_name);
  80             continue;
  81         }
  82
  83         string name = dirent->d_name;
  84         if (name == "." || name == "..") continue;
  85         names.push_back(name);
  86     }
  87     if (errno) throw SysError(format("reading directory `%1%'") % path);
  88
  89     return names;
  90 }
  91
  92
  93 void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash)
  94 {
  95     checkInterrupt();
  96
  97     struct stat st;
  98     if (lstat(path.c_str(), &st))
  99         throw SysError(format("getting attributes of path `%1%'") % path);
 100
 101     if (S_ISDIR(st.st_mode)) {
 102         Strings names = readDirectoryIgnoringInodes(path, inodeHash);
 103         foreach (Strings::iterator, i, names)
 104             optimisePath_(stats, path + "/" + *i, inodeHash);
 105         return;
 106     }
 107
 108     /* We can hard link regular files and maybe symlinks. */
 109     if (!S_ISREG(st.st_mode)
 110 #if CAN_LINK_SYMLINK
 111         && !S_ISLNK(st.st_mode)
 112 #endif
 113         ) return;
 114
 115     /* Sometimes SNAFUs can cause files in the store to be
 116        modified, in particular when running programs as root under
 117        Guix System (example: $fontconfig/var/cache being modified).  Skip
 118        those files.  FIXME: check the modification time. */
 119     if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
 120         printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path);
 121         return;
 122     }
 123
 124     /* This can still happen on top-level files. */
 125     if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
 126         printMsg(lvlDebug, format("`%1%' is already linked, with %2% other file(s).") % path % (st.st_nlink - 2));
 127         return;
 128     }
 129
 130     /* Hash the file.  Note that hashPath() returns the hash over the
 131        NAR serialisation, which includes the execute bit on the file.
 132        Thus, executable and non-executable files with the same
 133        contents *won't* be linked (which is good because otherwise the
 134        permissions would be screwed up).
 135
 136        Also note that if `path' is a symlink, then we're hashing the
 137        contents of the symlink (i.e. the result of readlink()), not
 138        the contents of the target (which may not even exist). */
 139     Hash hash = hashPath(htSHA256, path).first;
 140     printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash));
 141
 142     /* Check if this is a known hash. */
 143     Path linkPath = linksDir + "/" + printHash32(hash);
 144
 145  retry:
 146     if (!pathExists(linkPath)) {
 147         /* Nope, create a hard link in the links directory. */
 148         if (link(path.c_str(), linkPath.c_str()) == 0) {
 149             inodeHash.insert(st.st_ino);
 150             return;
 151         }
 152
 153         switch (errno) {
 154         case EEXIST:
 155             /* Fall through if another process created ‘linkPath’ before
 156                we did. */
 157             break;
 158
 159         case ENOSPC:
 160             /* On ext4, that probably means the directory index is full.  When
 161                that happens, it's fine to ignore it: we just effectively
 162                disable deduplication of this file.  */
 163             printMsg(lvlInfo, format("cannot link `%1%' to `%2%': %3%")
 164                      % linkPath % path % strerror(ENOSPC));
 165             return;
 166
 167         default:
 168             throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path);
 169         }
 170     }
 171
 172     /* Yes!  We've seen a file with the same contents.  Replace the
 173        current file with a hard link to that file. */
 174     struct stat stLink;
 175     if (lstat(linkPath.c_str(), &stLink))
 176         throw SysError(format("getting attributes of path `%1%'") % linkPath);
 177
 178     if (st.st_ino == stLink.st_ino) {
 179         printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath);
 180         return;
 181     }
 182
 183     if (st.st_size != stLink.st_size) {
 184         printMsg(lvlError, format("removing corrupted link ‘%1%’") % linkPath);
 185         unlink(linkPath.c_str());
 186         goto retry;
 187     }
 188
 189     printMsg(lvlTalkative, format("linking ‘%1%’ to ‘%2%’") % path % linkPath);
 190
 191     /* Make the containing directory writable, but only if it's not
 192        the store itself (we don't want or need to mess with its
 193        permissions). */
 194     bool mustToggle = !isStorePath(path);
 195     if (mustToggle) makeWritable(dirOf(path));
 196
 197     /* When we're done, make the directory read-only again and reset
 198        its timestamp back to 0. */
 199     MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : "");
 200
 201     Path tempLink = (format("%1%/.tmp-link-%2%-%3%")
 202         % settings.nixStore % getpid() % rand()).str();
 203
 204     if (link(linkPath.c_str(), tempLink.c_str()) == -1) {
 205         if (errno == EMLINK) {
 206             /* Too many links to the same file (>= 32000 on most file
 207                systems).  This is likely to happen with empty files.
 208                Just shrug and ignore. */
 209             if (st.st_size)
 210                 printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
 211             return;
 212         }
 213             throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath);
 214         }
 215
 216     /* Atomically replace the old file with the new hard link. */
 217     if (rename(tempLink.c_str(), path.c_str()) == -1) {
 218         if (unlink(tempLink.c_str()) == -1)
 219             printMsg(lvlError, format("unable to unlink `%1%'") % tempLink);
 220         if (errno == EMLINK) {
 221             /* Some filesystems generate too many links on the rename,
 222                rather than on the original link.  (Probably it
 223                temporarily increases the st_nlink field before
 224                decreasing it again.) */
 225             if (st.st_size)
 226                 printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath);
 227             return;
 228         }
 229         throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path);
 230     }
 231
 232     stats.filesLinked++;
 233     stats.bytesFreed += st.st_size;
 234     stats.blocksFreed += st.st_blocks;
 235 }
 236
 237
 238 void LocalStore::optimiseStore(OptimiseStats & stats)
 239 {
 240     PathSet paths = queryAllValidPaths();
 241     InodeHash inodeHash = loadInodeHash();
 242
 243     foreach (PathSet::iterator, i, paths) {
 244         addTempRoot(*i);
 245         if (!isValidPath(*i)) continue; /* path was GC'ed, probably */
 246         startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i);
 247         optimisePath_(stats, *i, inodeHash);
 248     }
 249 }
 250
 251 static string showBytes(unsigned long long bytes)
 252 {
 253     return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
 254 }
 255
 256 void LocalStore::optimiseStore()
 257 {
 258     OptimiseStats stats;
 259
 260     optimiseStore(stats);
 261
 262     printMsg(lvlError,
 263         format("%1% freed by hard-linking %2% files")
 264         % showBytes(stats.bytesFreed)
 265         % stats.filesLinked);
 266 }
 267
 268 void LocalStore::optimisePath(const Path & path)
 269 {
 270     OptimiseStats stats;
 271     InodeHash inodeHash;
 272
 273     if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash);
 274 }
 275
 276
 277 }