hphp/runtime/base/file_repository.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-2013 Facebook, Inc. (http://www.facebook.com)     |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16
  17 #include "hphp/runtime/base/file_repository.h"
  18 #include "hphp/runtime/base/runtime_option.h"
  19 #include "hphp/runtime/base/zend/zend_string.h"
  20 #include "hphp/util/process.h"
  21 #include "hphp/util/trace.h"
  22 #include "hphp/runtime/base/stat_cache.h"
  23 #include "hphp/runtime/base/server/source_root_info.h"
  24
  25 #include "hphp/runtime/vm/jit/targetcache.h"
  26 #include "hphp/runtime/vm/jit/translator-x64.h"
  27 #include "hphp/runtime/vm/bytecode.h"
  28 #include "hphp/runtime/vm/pendq.h"
  29 #include "hphp/runtime/vm/repo.h"
  30 #include "hphp/runtime/vm/runtime.h"
  31
  32 using std::endl;
  33
  34 namespace HPHP {
  35
  36 static const Trace::Module TRACEMOD = Trace::fr;
  37 extern bool (*file_dump)(const char *filename);
  38
  39 namespace Eval {
  40 ///////////////////////////////////////////////////////////////////////////////
  41
  42 std::set<string> FileRepository::s_names;
  43
  44 PhpFile::PhpFile(const string &fileName, const string &srcRoot,
  45                  const string &relPath, const string &md5,
  46                  HPHP::Unit* unit)
  47     : m_refCount(0), m_id(0),
  48       m_profName(string("run_init::") + string(fileName)),
  49       m_fileName(fileName), m_srcRoot(srcRoot), m_relPath(relPath), m_md5(md5),
  50       m_unit(unit) {
  51 }
  52
  53 PhpFile::~PhpFile() {
  54   always_assert(getRef() == 0);
  55   if (m_unit != nullptr) {
  56     // Deleting a Unit can grab a low-ranked lock and we're probably
  57     // at a high rank right now
  58     PendQ::defer(new DeferredDeleter<Unit>(m_unit));
  59     m_unit = nullptr;
  60   }
  61 }
  62
  63 void PhpFile::incRef() {
  64   UNUSED int ret = m_refCount.fetch_add(1, std::memory_order_acq_rel);
  65   TRACE(4, "PhpFile: %s incRef() %d -> %d %p called by %p\n",
  66         m_fileName.c_str(), ret - 1, ret, this, __builtin_return_address(0));
  67 }
  68
  69 int PhpFile::decRef(int n) {
  70   int ret = m_refCount.fetch_sub(n, std::memory_order_acq_rel);
  71   TRACE(4, "PhpFile: %s decRef() %d -> %d %p called by %p\n",
  72         m_fileName.c_str(), ret, ret - n, this, __builtin_return_address(0));
  73   assert(ret >= n); // fetch_sub returns the old value
  74   return ret - n;
  75 }
  76
  77 void PhpFile::decRefAndDelete() {
  78   if (decRef() == 0) {
  79     FileRepository::onDelete(this);
  80   }
  81 }
  82
  83 void PhpFile::setId(int id) {
  84   m_id = id;
  85   m_unit->setCacheId(id);
  86 }
  87
  88 ReadWriteMutex FileRepository::s_md5Lock(RankFileMd5);
  89 ParsedFilesMap FileRepository::s_files;
  90 Md5FileMap FileRepository::s_md5Files;
  91 UnitMd5Map FileRepository::s_unitMd5Map;
  92
  93 static class FileDumpInitializer {
  94   public: FileDumpInitializer() {
  95     file_dump = FileRepository::fileDump;
  96   }
  97 } s_fileDumpInitializer;
  98
  99 bool FileRepository::fileDump(const char *filename) {
 100   std::ofstream out(filename);
 101   if (out.fail()) return false;
 102   out << "s_files: " << s_files.size() << endl;
 103   for (ParsedFilesMap::const_iterator it =
 104        s_files.begin(); it != s_files.end(); it++) {
 105     out << it->first->data() << endl;
 106   }
 107   {
 108     ReadLock lock(s_md5Lock);
 109     out << "s_md5Files: " << s_md5Files.size() << endl;
 110     for (Md5FileMap::const_iterator it = s_md5Files.begin();
 111          it != s_md5Files.end(); it++) {
 112       out << it->second->getMd5().c_str() << " "
 113           << it->second->getFileName().c_str() << endl;
 114     }
 115   }
 116   out.close();
 117   return true;
 118 }
 119
 120 void FileRepository::onDelete(PhpFile *f) {
 121   assert(f->getRef() == 0);
 122   if (md5Enabled()) {
 123     WriteLock lock(s_md5Lock);
 124     s_md5Files.erase(f->getMd5());
 125   }
 126   delete f;
 127 }
 128
 129 void FileRepository::forEachUnit(UnitVisitor& uit) {
 130   ReadLock lock(s_md5Lock);
 131   for (Md5FileMap::const_iterator it = s_md5Files.begin();
 132        it != s_md5Files.end(); ++it) {
 133     uit(it->second->unit());
 134   }
 135 }
 136
 137 size_t FileRepository::getLoadedFiles() {
 138   ReadLock lock(s_md5Lock);
 139   return s_md5Files.size();
 140 }
 141
 142 PhpFile *FileRepository::checkoutFile(StringData *rname,
 143                                       const struct stat &s) {
 144   FileInfo fileInfo;
 145   PhpFile *ret = nullptr;
 146   String name(rname);
 147   if (rname->data()[0] != '/') {
 148     name = String(SourceRootInfo::GetCurrentSourceRoot()) + name;
 149   }
 150
 151   {
 152     // Get the common fast path out of the way with as little locking
 153     // as possible: it's in the map and has not changed on disk
 154     ParsedFilesMap::const_accessor acc;
 155     if (s_files.find(acc, name.get()) && !acc->second->isChanged(s)) {
 156       TRACE(1, "FR fast path hit %s\n", rname->data());
 157       ret = acc->second->getPhpFile();
 158       ret->incRef();
 159       return ret;
 160     }
 161   }
 162
 163   TRACE(1, "FR fast path miss: %s\n", rname->data());
 164   const StringData *n = StringData::GetStaticString(name.get());
 165   ParsedFilesMap::accessor acc;
 166   bool isNew = s_files.insert(acc, n);
 167   assert(isNew || acc->second); // We don't leave null entries around.
 168   bool isChanged = !isNew && acc->second->isChanged(s);
 169
 170   if (isNew || isChanged) {
 171     if (!readFile(n, s, fileInfo)) {
 172       // Be sure to get rid of the new reference to it.
 173       s_files.erase(acc);
 174       TRACE(1, "File disappeared between stat and FR::readNewFile: %s\n",
 175             rname->data());
 176       return nullptr;
 177     }
 178     ret = fileInfo.m_phpFile;
 179     if (isChanged && ret == acc->second->getPhpFile()) {
 180       // The file changed but had the same contents.
 181       if (debug && md5Enabled()) {
 182         ReadLock lock(s_md5Lock);
 183         assert(s_md5Files.find(ret->getMd5())->second == ret);
 184       }
 185       ret->incRef();
 186       return ret;
 187     }
 188   } else if (!isNew) {
 189     // Somebody might have loaded the file between when we dropped
 190     // our read lock and got the write lock
 191     ret = acc->second->getPhpFile();
 192     ret->incRef();
 193     return ret;
 194   }
 195
 196   // If we get here the file was not in s_files or has changed on disk
 197   if (!ret) {
 198     // Try to read Unit from .hhbc repo.
 199     ret = readHhbc(n->data(), fileInfo);
 200   }
 201   if (!ret) {
 202     if (isAuthoritativeRepo()) {
 203       raise_error("Tried to parse %s in repo authoritative mode", n->data());
 204     }
 205     ret = parseFile(n->data(), fileInfo);
 206     if (!ret) {
 207       s_files.erase(acc);
 208       return nullptr;
 209     }
 210   }
 211   assert(ret != nullptr);
 212
 213   if (isNew) {
 214     acc->second = new PhpFileWrapper(s, ret);
 215     ret->incRef();
 216     ret->setId(Transl::TargetCache::allocBit());
 217   } else {
 218     PhpFile *f = acc->second->getPhpFile();
 219     if (f != ret) {
 220       ret->setId(f->getId());
 221       tx64->invalidateFile(f); // f has changed
 222     }
 223     f->decRefAndDelete();
 224     delete acc->second;
 225     acc->second = new PhpFileWrapper(s, ret);
 226     ret->incRef();
 227   }
 228
 229   if (md5Enabled()) {
 230     WriteLock lock(s_md5Lock);
 231     s_md5Files[ret->getMd5()] = ret;
 232   }
 233   ret->incRef();
 234   return ret;
 235 }
 236
 237 bool FileRepository::findFile(const StringData *path, struct stat *s) {
 238   if (isAuthoritativeRepo()) {
 239     {
 240       UnitMd5Map::const_accessor acc;
 241       if (s_unitMd5Map.find(acc, path)) {
 242         return acc->second.m_present;
 243       }
 244     }
 245     MD5 md5;
 246     const StringData* spath = StringData::GetStaticString(path);
 247     UnitMd5Map::accessor acc;
 248     if (s_unitMd5Map.insert(acc, spath)) {
 249       bool present = Repo::get().findFile(
 250         path->data(), SourceRootInfo::GetCurrentSourceRoot(), md5);
 251       acc->second.m_present = present;
 252       acc->second.m_unitMd5 = md5;
 253     }
 254     return acc->second.m_present;
 255   }
 256   return fileStat(path->data(), s) && !S_ISDIR(s->st_mode);
 257 }
 258
 259 String FileRepository::translateFileName(StringData *file) {
 260   ParsedFilesMap::const_accessor acc;
 261   if (!s_files.find(acc, file)) return file;
 262   string srcRoot(SourceRootInfo::GetCurrentSourceRoot());
 263   if (srcRoot.empty()) return file;
 264   PhpFile *f = acc->second->getPhpFile();
 265   const string &parsedSrcRoot = f->getSrcRoot();
 266   if (srcRoot == parsedSrcRoot) return file;
 267   int len = parsedSrcRoot.size();
 268   if (len > 0 && file->size() > len &&
 269       strncmp(file->data(), parsedSrcRoot.c_str(), len) == 0) {
 270     return srcRoot + (file->data() + len);
 271   }
 272   return file;
 273 }
 274
 275 string FileRepository::unitMd5(const string& fileMd5) {
 276   // Incorporate relevant options into the unit md5 (there will be more)
 277   char* md5str;
 278   int md5len;
 279   std::ostringstream opts;
 280   string t = fileMd5 + '\0'
 281     + (RuntimeOption::EnableHipHopSyntax ? '1' : '0')
 282     + (RuntimeOption::EnableEmitSwitch ? '1' : '0')
 283     + (RuntimeOption::EvalJitEnableRenameFunction ? '1' : '0')
 284     + (RuntimeOption::EvalAllowHhas ? '1' : '0');
 285   md5str = string_md5(t.c_str(), t.size(), false, md5len);
 286   string s = string(md5str, md5len);
 287   free(md5str);
 288   return s;
 289 }
 290
 291 void FileRepository::setFileInfo(const StringData *name,
 292                                  const string& md5,
 293                                  FileInfo &fileInfo,
 294                                  bool fromRepo) {
 295   int md5len;
 296   char* md5str;
 297   // Incorporate the path into the md5 that is used as the key for file
 298   // repository lookups.  This assures that even if two PHP files have
 299   // identical content, separate units exist for them (so that
 300   // Unit::filepath() and Unit::dirpath() work correctly).
 301   string s = md5 + '\0' + name->data();
 302   md5str = string_md5(s.c_str(), s.size(), false, md5len);
 303   fileInfo.m_md5 = string(md5str, md5len);
 304   free(md5str);
 305
 306   if (fromRepo) {
 307     fileInfo.m_unitMd5 = md5;
 308   } else {
 309     fileInfo.m_unitMd5 = unitMd5(md5);
 310   }
 311
 312   fileInfo.m_srcRoot = SourceRootInfo::GetCurrentSourceRoot();
 313   int srcRootLen = fileInfo.m_srcRoot.size();
 314   if (srcRootLen) {
 315     if (!strncmp(name->data(), fileInfo.m_srcRoot.c_str(), srcRootLen)) {
 316       fileInfo.m_relPath = string(name->data() + srcRootLen);
 317     }
 318   }
 319
 320   ReadLock lock(s_md5Lock);
 321   Md5FileMap::iterator it = s_md5Files.find(fileInfo.m_md5);
 322   if (it != s_md5Files.end()) {
 323     PhpFile *f = it->second;
 324     if (!fileInfo.m_relPath.empty() &&
 325         fileInfo.m_relPath == f->getRelPath()) {
 326       assert(fileInfo.m_md5 == f->getMd5());
 327       fileInfo.m_phpFile = f;
 328     }
 329   }
 330 }
 331
 332 bool FileRepository::readActualFile(const StringData *name,
 333                                     const struct stat &s,
 334                                     FileInfo &fileInfo) {
 335   if (s.st_size > StringData::MaxSize) {
 336     throw FatalErrorException(0, "file %s is too big", name->data());
 337   }
 338   int fileSize = s.st_size;
 339   if (!fileSize) return false;
 340   int fd = open(name->data(), O_RDONLY);
 341   if (!fd) return false; // ignore file open exception
 342   String str = String(fileSize, ReserveString);
 343   char *input = str.mutableSlice().ptr;
 344   if (!input) return false;
 345   int nbytes = read(fd, input, fileSize);
 346   close(fd);
 347   str.setSize(fileSize);
 348   fileInfo.m_inputString = str;
 349   if (nbytes != fileSize) return false;
 350
 351   if (md5Enabled()) {
 352     string md5 = StringUtil::MD5(fileInfo.m_inputString).c_str();
 353     setFileInfo(name, md5, fileInfo, false);
 354   }
 355   return true;
 356 }
 357
 358 bool FileRepository::readRepoMd5(const StringData *path,
 359                                  FileInfo& fileInfo) {
 360   MD5 md5;
 361   bool found;
 362   {
 363     UnitMd5Map::const_accessor acc;
 364     found = s_unitMd5Map.find(acc, path);
 365     if (found) {
 366       if (!acc->second.m_present) {
 367         return false;
 368       }
 369       md5 = acc->second.m_unitMd5;
 370       path = acc->first;
 371     }
 372   }
 373   if (!found) {
 374     UnitMd5Map::accessor acc;
 375     path = StringData::GetStaticString(path);
 376     if (s_unitMd5Map.insert(acc, path)) {
 377       if (!Repo::get().findFile(path->data(),
 378                                     SourceRootInfo::GetCurrentSourceRoot(),
 379                                     md5)) {
 380         acc->second.m_present = false;
 381         return false;
 382       }
 383       acc->second.m_present = true;
 384       acc->second.m_unitMd5 = md5;
 385     } else {
 386       md5 = acc->second.m_unitMd5;
 387       path = acc->first;
 388     }
 389   }
 390   setFileInfo(path, md5.toString(), fileInfo, true);
 391   return true;
 392 }
 393
 394 bool FileRepository::readFile(const StringData *name,
 395                               const struct stat &s,
 396                               FileInfo &fileInfo) {
 397   if (!isAuthoritativeRepo()) {
 398     TRACE(1, "read initial \"%s\"\n", name->data());
 399     return readActualFile(name, s, fileInfo);
 400   }
 401
 402   if (!readRepoMd5(name, fileInfo)) {
 403     return false;
 404   }
 405   return true;
 406 }
 407
 408 PhpFile *FileRepository::readHhbc(const std::string &name,
 409                                   const FileInfo &fileInfo) {
 410   MD5 md5 = MD5(fileInfo.m_unitMd5.c_str());
 411   Unit* u = Repo::get().loadUnit(name, md5);
 412   if (u != nullptr) {
 413     PhpFile *p = new PhpFile(name, fileInfo.m_srcRoot, fileInfo.m_relPath,
 414                              fileInfo.m_md5, u);
 415     return p;
 416   }
 417
 418   return nullptr;
 419 }
 420
 421 PhpFile *FileRepository::parseFile(const std::string &name,
 422                                    const FileInfo &fileInfo) {
 423   MD5 md5 = MD5(fileInfo.m_unitMd5.c_str());
 424   Unit* unit = compile_file(fileInfo.m_inputString->data(),
 425                                 fileInfo.m_inputString->size(),
 426                                 md5, name.c_str());
 427   PhpFile *p = new PhpFile(name, fileInfo.m_srcRoot, fileInfo.m_relPath,
 428                            fileInfo.m_md5, unit);
 429   return p;
 430 }
 431
 432 bool FileRepository::fileStat(const string &name, struct stat *s) {
 433   return StatCache::stat(name, s) == 0;
 434 }
 435
 436 struct ResolveIncludeContext {
 437   String path; // translated path of the file
 438   struct stat* s; // stat for the file
 439 };
 440
 441 static bool findFileWrapper(CStrRef file, void* ctx) {
 442   ResolveIncludeContext* context = (ResolveIncludeContext*)ctx;
 443   assert(context->path.isNull());
 444   // TranslatePath() will canonicalize the path and also check
 445   // whether the file is in an allowed directory.
 446   String translatedPath = File::TranslatePath(file, false, true);
 447   if (file[0] != '/') {
 448     if (HPHP::Eval::FileRepository::findFile(translatedPath.get(),
 449                                              context->s)) {
 450       context->path = translatedPath;
 451       return true;
 452     }
 453     return false;
 454   }
 455   if (RuntimeOption::SandboxMode || !RuntimeOption::AlwaysUseRelativePath) {
 456     if (HPHP::Eval::FileRepository::findFile(translatedPath.get(),
 457                                              context->s)) {
 458       context->path = translatedPath;
 459       return true;
 460     }
 461   }
 462   string server_root(SourceRootInfo::GetCurrentSourceRoot());
 463   if (server_root.empty()) {
 464     server_root = string(g_vmContext->getCwd()->data());
 465     if (server_root.empty() || server_root[server_root.size() - 1] != '/') {
 466       server_root += "/";
 467     }
 468   }
 469   String rel_path(Util::relativePath(server_root, translatedPath.data()));
 470   if (HPHP::Eval::FileRepository::findFile(rel_path.get(),
 471                                            context->s)) {
 472     context->path = rel_path;
 473     return true;
 474   }
 475   return false;
 476 }
 477
 478 String resolveVmInclude(StringData* path, const char* currentDir,
 479                         struct stat *s) {
 480   ResolveIncludeContext ctx;
 481   ctx.s = s;
 482   resolve_include(path, currentDir, findFileWrapper,
 483                   (void*)&ctx);
 484   // If resolve_include() could not find the file, return NULL
 485   return ctx.path;
 486 }
 487
 488 ///////////////////////////////////////////////////////////////////////////////
 489 }
 490 }