libgo: permit loff_t and off_t to be macros

[official-gcc.git] / gcc / go / gofrontend / embed.cc

// embed.cc -- Go frontend go:embed handling.

// Copyright 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

#include "go-system.h"

#include "operator.h"
#include "go-diagnostics.h"
#include "lex.h"
#include "types.h"
#include "expressions.h"
#include "gogo.h"

#ifndef O_BINARY
#define O_BINARY 0
#endif

// Read a file into *DATA.  Returns false on error.

static bool
read_file(const char* filename, Location loc, std::string* data)
{
  int fd = open(filename, O_RDONLY | O_BINARY);
  if (fd < 0)
    {
      go_error_at(loc, "%s: %m", filename);
      return false;
    }

  struct stat st;
  if (fstat(fd, &st) < 0)
    {
      go_error_at(loc, "%s: %m", filename);
      return false;
    }
  off_t want = st.st_size;

  // Most files read here are going to be incorporated into the object file
  // and then the executable.  Set a limit on the size we will accept.
  if (want > 2000000000)
    {
      go_error_at(loc, "%s: file too large", filename);
      return false;
    }

  data->resize(want);
  off_t got = 0;
  while (want > 0)
    {
      // C++11 requires that std::string use contiguous bytes, so this
      // is safe.
      ssize_t n = read(fd, &(*data)[got], want);
      if (n < 0)
        {
          close(fd);
          go_error_at(loc, "%s: %m", filename);
          return false;
        }
      if (n == 0)
        {
          data->resize(got);
          break;
        }
      got += n;
      want -= n;
    }

  close(fd);
  return true;
}

// A JSON value as read from an embedcfg file.  For our purposes a
// JSON value is a string, or a list of strings, or a mapping from
// strings to values.  We don't expect any numbers.  We also don't
// expect an array of anything other than strings; that is, we don't
// accept an array of general JSON values.

class Json_value
{
 public:
  // The types of values.
  enum Json_value_classification
    {
      JSON_VALUE_UNKNOWN,
      JSON_VALUE_STRING,
      JSON_VALUE_ARRAY,
      JSON_VALUE_MAP
    };

  Json_value()
    : classification_(JSON_VALUE_UNKNOWN), string_(), array_(), map_()
  { }

  ~Json_value();

  Json_value_classification
  classification() const
  { return this->classification_; }

  // Set to a string value.
  void
  set_string(const std::string& str)
  {
    go_assert(this->classification_ == JSON_VALUE_UNKNOWN);
    this->classification_ = JSON_VALUE_STRING;
    this->string_ = str;
  }

  // Start an array value.
  void
  start_array()
  {
    go_assert(this->classification_ == JSON_VALUE_UNKNOWN);
    this->classification_ = JSON_VALUE_ARRAY;
  }

  // Add an array entry.
  void
  add_array_entry(const std::string& s)
  {
    go_assert(this->classification_ == JSON_VALUE_ARRAY);
    this->array_.push_back(s);
  }

  // Start a map value.
  void
  start_map()
  {
    go_assert(this->classification_ == JSON_VALUE_UNKNOWN);
    this->classification_ = JSON_VALUE_MAP;
  }

  // Add a map entry.
  void
  add_map_entry(const std::string& key, Json_value* val)
  {
    go_assert(this->classification_ == JSON_VALUE_MAP);
    this->map_[key] = val;
  }

  // Return the strings from a string value.
  const std::string&
  to_string() const
  {
    go_assert(this->classification_ == JSON_VALUE_STRING);
    return this->string_;
  }

  // Fetch a vector of strings, and drop them from the JSON value.
  void
  get_and_clear_array(std::vector<std::string>* v)
  {
    go_assert(this->classification_ == JSON_VALUE_ARRAY);
    std::swap(*v, this->array_);
  }

  // Look up a map entry.  Returns NULL if not found.
  Json_value*
  lookup_map_entry(const std::string& key);

  // Iterate over a map.
  typedef Unordered_map(std::string, Json_value*)::iterator map_iterator;

  map_iterator
  map_begin()
  {
    go_assert(this->classification_ == JSON_VALUE_MAP);
    return this->map_.begin();
  }

  map_iterator
  map_end()
  { return this->map_.end(); }

 private:
  // Classification.
  Json_value_classification classification_;
  // A string, for JSON_VALUE_STRING.
  std::string string_;
  // Array, for JSON_VALUE_ARRAY.
  std::vector<std::string> array_;
  // Mapping, for JSON_VALUE_MAP.
  Unordered_map(std::string, Json_value*) map_;
};

// Delete a JSON value.

Json_value::~Json_value()
{
  if (this->classification_ == JSON_VALUE_MAP)
    {
      for (map_iterator p = this->map_begin();
           p != this->map_end();
           ++p)
        delete p->second;
    }
}

// Look up a map entry in a JSON value.

Json_value*
Json_value::lookup_map_entry(const std::string& key)
{
  go_assert(this->classification_ == JSON_VALUE_MAP);
  Unordered_map(std::string, Json_value*)::iterator p = this->map_.find(key);
  if (p == this->map_.end())
    return NULL;
  return p->second;
}

// Manage reading the embedcfg file.

class Embedcfg_reader
{
 public:
  Embedcfg_reader(const char* filename)
    : filename_(filename), data_(), p_(NULL), pend_(NULL)
  {}

  // Read the contents of FILENAME.  Return whether it succeeded.
  bool
  initialize_from_file();

  // Read a JSON object.
  bool
  read_object(Json_value*);

  // Report an error if not at EOF.
  void
  check_eof();

  // Report an error for the embedcfg file.
  void
  error(const char* msg);

 private:
  bool
  read_value(Json_value*);

  bool
  read_array(Json_value*);

  bool
  read_string(std::string*);

  bool
  skip_whitespace(bool eof_ok);

  // File name.
  const char* filename_;
  // File contents.
  std::string data_;
  // Next character to process.
  const char *p_;
  // End of data.
  const char *pend_;
};

// Read the embedcfg file.

void
Gogo::read_embedcfg(const char *filename)
{
  class Embedcfg_reader r(filename);
  if (!r.initialize_from_file())
    return;

  Json_value val;
  if (!r.read_object(&val))
    return;

  r.check_eof();

  if (val.classification() != Json_value::JSON_VALUE_MAP)
    {
      r.error("invalid embedcfg: not a JSON object");
      return;
    }

  Json_value* patterns = val.lookup_map_entry("Patterns");
  if (patterns == NULL)
    {
      r.error("invalid embedcfg: missing Patterns");
      return;
    }
  if (patterns->classification() != Json_value::JSON_VALUE_MAP)
    {
      r.error("invalid embedcfg: Patterns is not a JSON object");
      return;
    }

  Json_value* files = val.lookup_map_entry("Files");
  if (files == NULL)
    {
      r.error("invalid embedcfg: missing Files");
      return;
    }
  if (files->classification() != Json_value::JSON_VALUE_MAP)
    {
      r.error("invalid embedcfg: Files is not a JSON object");
      return;
    }

  for (Json_value::map_iterator p = patterns->map_begin();
       p != patterns->map_end();
       ++p)
    {
      if (p->second->classification() != Json_value::JSON_VALUE_ARRAY)
        {
          r.error("invalid embedcfg: Patterns entry is not an array");
          return;
        }
      std::vector<std::string> files;
      p->second->get_and_clear_array(&files);

      std::pair<std::string, std::vector<std::string> > val;
      val.first = p->first;
      std::pair<Embed_patterns::iterator, bool> ins =
        this->embed_patterns_.insert(val);
      if (!ins.second)
        {
          r.error("invalid embedcfg: duplicate Patterns entry");
          return;
        }
      std::swap(ins.first->second, files);
    }

  for (Json_value::map_iterator p = files->map_begin();
       p != files->map_end();
       ++p)
    {
      if (p->second->classification() != Json_value::JSON_VALUE_STRING)
        {
          r.error("invalid embedcfg: Files entry is not a string");
          return;
        }
      this->embed_files_[p->first] = p->second->to_string();
    }
}

// Read the contents of FILENAME into this->data_.  Returns whether it
// succeeded.

bool
Embedcfg_reader::initialize_from_file()
{
  if (!read_file(this->filename_, Linemap::unknown_location(), &this->data_))
    return false;
  if (this->data_.empty())
    {
      this->error("empty file");
      return false;
    }
  this->p_ = this->data_.data();
  this->pend_ = this->p_ + this->data_.size();
  return true;
}

// Read a JSON object into VAL.  Return whether it succeeded.

bool
Embedcfg_reader::read_object(Json_value* val)
{
  if (!this->skip_whitespace(false))
    return false;
  if (*this->p_ != '{')
    {
      this->error("expected %<{%>");
      return false;
    }
  ++this->p_;

  val->start_map();

  if (!this->skip_whitespace(false))
    return false;
  if (*this->p_ == '}')
    {
      ++this->p_;
      return true;
    }

  while (true)
    {
      if (!this->skip_whitespace(false))
        return false;
      if (*this->p_ != '"')
        {
          this->error("expected %<\"%>");
          return false;
        }

      std::string key;
      if (!this->read_string(&key))
        return false;

      if (!this->skip_whitespace(false))
        return false;
      if (*this->p_ != ':')
        {
          this->error("expected %<:%>");
          return false;
        }
      ++this->p_;

      Json_value* subval = new Json_value();
      if (!this->read_value(subval))
        return false;

      val->add_map_entry(key, subval);

      if (!this->skip_whitespace(false))
        return false;
      if (*this->p_ == '}')
        {
          ++this->p_;
          return true;
        }
      if (*this->p_ != ',')
        {
          this->error("expected %<,%> or %<}%>");
          return false;
        }
      ++this->p_;
    }
}

// Read a JSON array into VAL.  Return whether it succeeded.

bool
Embedcfg_reader::read_array(Json_value* val)
{
  if (!this->skip_whitespace(false))
    return false;
  if (*this->p_ != '[')
    {
      this->error("expected %<[%>");
      return false;
    }
  ++this->p_;

  val->start_array();

  if (!this->skip_whitespace(false))
    return false;
  if (*this->p_ == ']')
    {
      ++this->p_;
      return true;
    }

  while (true)
    {
      // If we were parsing full JSON we would call read_value here,
      // not read_string.

      std::string s;
      if (!this->read_string(&s))
        return false;

      val->add_array_entry(s);

      if (!this->skip_whitespace(false))
        return false;
      if (*this->p_ == ']')
        {
          ++this->p_;
          return true;
        }
      if (*this->p_ != ',')
        {
          this->error("expected %<,%> or %<]%>");
          return false;
        }
      ++this->p_;
    }
}

// Read a JSON value into VAL.  Return whether it succeeded.

bool
Embedcfg_reader::read_value(Json_value* val)
{
  if (!this->skip_whitespace(false))
    return false;
  switch (*this->p_)
    {
    case '"':
      {
        std::string s;
        if (!this->read_string(&s))
          return false;
        val->set_string(s);
        return true;
      }

    case '{':
      return this->read_object(val);

    case '[':
      return this->read_array(val);

    default:
      this->error("invalid JSON syntax");
      return false;
    }
}

// Read a JSON string.  Return whether it succeeded.

bool
Embedcfg_reader::read_string(std::string* str)
{
  if (!this->skip_whitespace(false))
    return false;
  if (*this->p_ != '"')
    {
      this->error("expected %<\"%>");
      return false;
    }
  ++this->p_;

  str->clear();
  while (this->p_ < this->pend_ && *this->p_ != '"')
    {
      if (*this->p_ != '\\')
        {
          str->push_back(*this->p_);
          ++this->p_;
          continue;
        }

      ++this->p_;
      if (this->p_ >= this->pend_)
        {
          this->error("unterminated string");
          return false;
        }
      switch (*this->p_)
        {
        case '"': case '\\': case '/':
          str->push_back(*this->p_);
          ++this->p_;
          break;

        case 'b':
          str->push_back('\b');
          ++this->p_;
          break;

        case 'f':
          str->push_back('\f');
          ++this->p_;
          break;

        case 'n':
          str->push_back('\n');
          ++this->p_;
          break;

        case 'r':
          str->push_back('\r');
          ++this->p_;
          break;

        case 't':
          str->push_back('\t');
          ++this->p_;
          break;

        case 'u':
          {
            ++this->p_;
            unsigned int rune = 0;
            for (int i = 0; i < 4; i++)
              {
                if (this->p_ >= this->pend_)
                  {
                    this->error("unterminated string");
                    return false;
                  }
                unsigned char c = *this->p_;
                ++this->p_;
                rune <<= 4;
                if (c >= '0' && c <= '9')
                  rune += c - '0';
                else if (c >= 'A' && c <= 'F')
                  rune += c - 'A' + 10;
                else if (c >= 'a' && c <= 'f')
                  rune += c - 'a' + 10;
                else
                  {
                    this->error("invalid hex digit");
                    return false;
                  }
              }
            Lex::append_char(rune, false, str, Linemap::unknown_location());
          }
          break;

        default:
          this->error("unrecognized string escape");
          return false;
        }
    }

  if (*this->p_ == '"')
    {
      ++this->p_;
      return true;
    }

  this->error("unterminated string");
  return false;
}

// Report an error if not at EOF.

void
Embedcfg_reader::check_eof()
{
  if (this->skip_whitespace(true))
    this->error("extraneous data at end of file");
}

// Skip whitespace.  Return whether there is more to read.

bool
Embedcfg_reader::skip_whitespace(bool eof_ok)
{
  while (this->p_ < this->pend_)
    {
      switch (*this->p_)
        {
        case ' ': case '\t': case '\n': case '\r':
          ++this->p_;
          break;
        default:
          return true;
        }
    }
  if (!eof_ok)
    this->error("unexpected EOF");
  return false;
}

// Report an error.

void
Embedcfg_reader::error(const char* msg)
{
  if (!this->data_.empty() && this->p_ != NULL)
    go_error_at(Linemap::unknown_location(),
                "%<-fgo-embedcfg%>: %s: %lu: %s",
                this->filename_,
                static_cast<unsigned long>(this->p_ - this->data_.data()),
                msg);
  else
    go_error_at(Linemap::unknown_location(),
                "%<-fgo-embedcfg%>: %s: %s",
                this->filename_, msg);
}

// Implement the sort order for a list of embedded files, as discussed
// at the docs for embed.FS.

class Embedfs_sort
{
 public:
  bool
  operator()(const std::string& p1, const std::string& p2) const;

 private:
  void
  split(const std::string&, size_t*, size_t*, size_t*) const;
};

bool
Embedfs_sort::operator()(const std::string& p1, const std::string& p2) const
{
  size_t dirlen1, elem1, elemlen1;
  this->split(p1, &dirlen1, &elem1, &elemlen1);
  size_t dirlen2, elem2, elemlen2;
  this->split(p2, &dirlen2, &elem2, &elemlen2);

  if (dirlen1 == 0)
    {
      if (dirlen2 > 0)
        {
          int i = p2.compare(0, dirlen2, ".");
          if (i != 0)
            return i > 0;
        }
    }
  else if (dirlen2 == 0)
    {
      int i = p1.compare(0, dirlen1, ".");
      if (i != 0)
        return i < 0;
    }
  else
    {
      int i = p1.compare(0, dirlen1, p2, 0, dirlen2);
      if (i != 0)
        return i < 0;
    }

  int i = p1.compare(elem1, elemlen1, p2, elem2, elemlen2);
  return i < 0;
}

// Pick out the directory and file name components for comparison.

void
Embedfs_sort::split(const std::string& s, size_t* dirlen, size_t* elem,
                    size_t* elemlen) const
{
  size_t len = s.size();
  if (len > 0 && s[len - 1] == '/')
    --len;
  size_t slash = s.rfind('/', len - 1);
  if (slash == std::string::npos)
    {
      *dirlen = 0;
      *elem = 0;
      *elemlen = len;
    }
  else
    {
      *dirlen = slash;
      *elem = slash + 1;
      *elemlen = len - (slash + 1);
    }
}

// Convert the go:embed directives for a variable into an initializer
// for that variable.

Expression*
Gogo::initializer_for_embeds(Type* type,
                             const std::vector<std::string>* embeds,
                             Location loc)
{
  if (this->embed_patterns_.empty())
    {
      go_error_at(loc,
                  ("invalid go:embed: build system did not "
                   "supply embed configuration"));
      return Expression::make_error(loc);
    }

  type = type->unalias();

  enum {
    EMBED_STRING = 0,
    EMBED_BYTES = 1,
    EMBED_FS = 2
  } embed_kind;

  const Named_type* nt = type->named_type();
  if (nt != NULL
      && nt->named_object()->package() != NULL
      && nt->named_object()->package()->pkgpath() == "embed"
      && nt->name() == "FS")
    embed_kind = EMBED_FS;
  else if (type->is_string_type())
    embed_kind = EMBED_STRING;
  else if (type->is_slice_type()
           && type->array_type()->element_type()->integer_type() != NULL
           && type->array_type()->element_type()->integer_type()->is_byte())
    embed_kind = EMBED_BYTES;
  else
    {
      go_error_at(loc, "invalid type for go:embed");
      return Expression::make_error(loc);
    }

  // The patterns in the go:embed directive(s) are in EMBEDS.  Find
  // them in the patterns in the embedcfg file.

  Unordered_set(std::string) have;
  std::vector<std::string> paths;
  for (std::vector<std::string>::const_iterator pe = embeds->begin();
       pe != embeds->end();
       pe++)
    {
      Embed_patterns::const_iterator pp = this->embed_patterns_.find(*pe);
      if (pp == this->embed_patterns_.end())
        {
          go_error_at(loc,
                      ("invalid go:embed: build system did not "
                       "map pattern %<%s%>"),
                      pe->c_str());
          continue;
        }

      // Each pattern in the embedcfg file maps to a list of file
      // names.  Add those file names to PATHS.
      for (std::vector<std::string>::const_iterator pf = pp->second.begin();
           pf != pp->second.end();
           pf++)
        {
          if (this->embed_files_.find(*pf) == this->embed_files_.end())
            {
              go_error_at(loc,
                          ("invalid go:embed: build system did not "
                           "map file %<%s%>"),
                          pf->c_str());
              continue;
            }

          std::pair<Unordered_set(std::string)::iterator, bool> ins
            = have.insert(*pf);
          if (ins.second)
            {
              const std::string& path(*pf);
              paths.push_back(path);

              if (embed_kind == EMBED_FS)
                {
                  // Add each required directory, with a trailing slash.
                  size_t i = std::string::npos;
                  while (i > 0)
                    {
                      i = path.rfind('/', i);
                      if (i == std::string::npos)
                        break;
                      std::string dir = path.substr(0, i + 1);
                      ins = have.insert(dir);
                      if (ins.second)
                        paths.push_back(dir);
                      --i;
                    }
                }
            }
        }
    }

  if (embed_kind == EMBED_STRING || embed_kind == EMBED_BYTES)
    {
      if (paths.size() > 1)
        {
          go_error_at(loc,
                      ("invalid go:embed: multiple files for "
                       "string or byte slice"));;
          return Expression::make_error(loc);
        }

      std::string data;
      if (!read_file(this->embed_files_[paths[0]].c_str(), loc, &data))
        return Expression::make_error(loc);

      Expression* e = Expression::make_string(data, loc);
      if (embed_kind == EMBED_BYTES)
        e = Expression::make_cast(type, e, loc);
      return e;
    }

  std::sort(paths.begin(), paths.end(), Embedfs_sort());

  if (type->struct_type() == NULL
      || type->struct_type()->field_count() != 1)
    {
      go_error_at(loc,
                  ("internal error: embed.FS should be struct type "
                   "with one field"));
      return Expression::make_error(loc);
    }

  Type* ptr_type = type->struct_type()->field(0)->type();
  if (ptr_type->points_to() == NULL)
    {
      go_error_at(loc,
                  "internal error: embed.FS struct field should be pointer");
      return Expression::make_error(loc);
    }

  Type* slice_type = ptr_type->points_to();
  if (!slice_type->is_slice_type())
    {
      go_error_at(loc,
                  ("internal error: embed.FS struct field should be "
                   "pointer to slice"));
      return Expression::make_error(loc);
    }

  Type* file_type = slice_type->array_type()->element_type();
  if (file_type->struct_type() == NULL
      || (file_type->struct_type()->find_local_field(".embed.name", NULL)
          == NULL)
      || (file_type->struct_type()->find_local_field(".embed.data", NULL)
          == NULL))
    {
      go_error_at(loc,
                  ("internal error: embed.FS slice element should be struct "
                   "with name and data fields"));
      return Expression::make_error(loc);
    }

  const Struct_field_list* file_fields = file_type->struct_type()->fields();
  Expression_list* file_vals = new(Expression_list);
  file_vals->reserve(paths.size());
  for (std::vector<std::string>::const_iterator pp = paths.begin();
       pp != paths.end();
       ++pp)
    {
      std::string data;
      if ((*pp)[pp->size() - 1] != '/')
        {
          if (!read_file(this->embed_files_[*pp].c_str(), loc, &data))
            return Expression::make_error(loc);
        }

      Expression_list* field_vals = new(Expression_list);
      for (Struct_field_list::const_iterator pf = file_fields->begin();
           pf != file_fields->end();
           ++pf)
        {
          if (pf->is_field_name(".embed.name"))
            field_vals->push_back(Expression::make_string(*pp, loc));
          else if (pf->is_field_name(".embed.data"))
            field_vals->push_back(Expression::make_string(data, loc));
          else
            {
              // FIXME: The embed.file type has a hash field, which is
              // currently unused.  We should fill it in, but don't.
              // The hash is a SHA256, and we don't have convenient
              // SHA256 code.  Do this later when the field is
              // actually used.
              field_vals->push_back(NULL);
            }
        }

      Expression* file_val =
        Expression::make_struct_composite_literal(file_type, field_vals, loc);
      file_vals->push_back(file_val);
    }

  Expression* slice_init =
    Expression::make_slice_composite_literal(slice_type, file_vals, loc);
  Expression* fs_init = Expression::make_heap_expression(slice_init, loc);
  Expression_list* fs_vals = new Expression_list();
  fs_vals->push_back(fs_init);
  return Expression::make_struct_composite_literal(type, fs_vals, loc);
}