1 // embed.cc -- Go frontend go:embed handling.
3 // Copyright 2021 The Go Authors. All rights reserved.
4 // Use of this source code is governed by a BSD-style
5 // license that can be found in the LICENSE file.
10 #include "go-diagnostics.h"
13 #include "expressions.h"
20 // Read a file into *DATA. Returns false on error.
23 Gogo::read_file(const char* filename
, Location loc
, std::string
* data
)
25 int fd
= open(filename
, O_RDONLY
| O_BINARY
);
28 go_error_at(loc
, "%s: %m", filename
);
33 if (fstat(fd
, &st
) < 0)
35 go_error_at(loc
, "%s: %m", filename
);
38 off_t want
= st
.st_size
;
40 // Most files read here are going to be incorporated into the object file
41 // and then the executable. Set a limit on the size we will accept.
42 if (want
> 2000000000)
44 go_error_at(loc
, "%s: file too large", filename
);
52 // C++11 requires that std::string use contiguous bytes, so this
54 ssize_t n
= read(fd
, &(*data
)[got
], want
);
58 go_error_at(loc
, "%s: %m", filename
);
74 // A JSON value as read from an embedcfg file. For our purposes a
75 // JSON value is a string, or a list of strings, or a mapping from
76 // strings to values. We don't expect any numbers. We also don't
77 // expect an array of anything other than strings; that is, we don't
78 // accept an array of general JSON values.
83 // The types of values.
84 enum Json_value_classification
93 : classification_(JSON_VALUE_UNKNOWN
), string_(), array_(), map_()
98 Json_value_classification
99 classification() const
100 { return this->classification_
; }
102 // Set to a string value.
104 set_string(const std::string
& str
)
106 go_assert(this->classification_
== JSON_VALUE_UNKNOWN
);
107 this->classification_
= JSON_VALUE_STRING
;
111 // Start an array value.
115 go_assert(this->classification_
== JSON_VALUE_UNKNOWN
);
116 this->classification_
= JSON_VALUE_ARRAY
;
119 // Add an array entry.
121 add_array_entry(const std::string
& s
)
123 go_assert(this->classification_
== JSON_VALUE_ARRAY
);
124 this->array_
.push_back(s
);
127 // Start a map value.
131 go_assert(this->classification_
== JSON_VALUE_UNKNOWN
);
132 this->classification_
= JSON_VALUE_MAP
;
137 add_map_entry(const std::string
& key
, Json_value
* val
)
139 go_assert(this->classification_
== JSON_VALUE_MAP
);
140 this->map_
[key
] = val
;
143 // Return the strings from a string value.
147 go_assert(this->classification_
== JSON_VALUE_STRING
);
148 return this->string_
;
151 // Fetch a vector of strings, and drop them from the JSON value.
153 get_and_clear_array(std::vector
<std::string
>* v
)
155 go_assert(this->classification_
== JSON_VALUE_ARRAY
);
156 std::swap(*v
, this->array_
);
159 // Look up a map entry. Returns NULL if not found.
161 lookup_map_entry(const std::string
& key
);
163 // Iterate over a map.
164 typedef Unordered_map(std::string
, Json_value
*)::iterator map_iterator
;
169 go_assert(this->classification_
== JSON_VALUE_MAP
);
170 return this->map_
.begin();
175 { return this->map_
.end(); }
179 Json_value_classification classification_
;
180 // A string, for JSON_VALUE_STRING.
182 // Array, for JSON_VALUE_ARRAY.
183 std::vector
<std::string
> array_
;
184 // Mapping, for JSON_VALUE_MAP.
185 Unordered_map(std::string
, Json_value
*) map_
;
188 // Delete a JSON value.
190 Json_value::~Json_value()
192 if (this->classification_
== JSON_VALUE_MAP
)
194 for (map_iterator p
= this->map_begin();
195 p
!= this->map_end();
201 // Look up a map entry in a JSON value.
204 Json_value::lookup_map_entry(const std::string
& key
)
206 go_assert(this->classification_
== JSON_VALUE_MAP
);
207 Unordered_map(std::string
, Json_value
*)::iterator p
= this->map_
.find(key
);
208 if (p
== this->map_
.end())
213 // Manage reading the embedcfg file.
215 class Embedcfg_reader
218 Embedcfg_reader(const char* filename
)
219 : filename_(filename
), data_(), p_(NULL
), pend_(NULL
)
222 // Read the contents of FILENAME. Return whether it succeeded.
224 initialize_from_file();
226 // Read a JSON object.
228 read_object(Json_value
*);
230 // Report an error if not at EOF.
234 // Report an error for the embedcfg file.
236 error(const char* msg
);
240 read_value(Json_value
*);
243 read_array(Json_value
*);
246 read_string(std::string
*);
249 skip_whitespace(bool eof_ok
);
252 const char* filename_
;
255 // Next character to process.
261 // Read the embedcfg file.
264 Gogo::read_embedcfg(const char *filename
)
266 class Embedcfg_reader
r(filename
);
267 if (!r
.initialize_from_file())
271 if (!r
.read_object(&val
))
276 if (val
.classification() != Json_value::JSON_VALUE_MAP
)
278 r
.error("invalid embedcfg: not a JSON object");
282 Json_value
* patterns
= val
.lookup_map_entry("Patterns");
283 if (patterns
== NULL
)
285 r
.error("invalid embedcfg: missing Patterns");
288 if (patterns
->classification() != Json_value::JSON_VALUE_MAP
)
290 r
.error("invalid embedcfg: Patterns is not a JSON object");
294 Json_value
* files
= val
.lookup_map_entry("Files");
297 r
.error("invalid embedcfg: missing Files");
300 if (files
->classification() != Json_value::JSON_VALUE_MAP
)
302 r
.error("invalid embedcfg: Files is not a JSON object");
306 for (Json_value::map_iterator p
= patterns
->map_begin();
307 p
!= patterns
->map_end();
310 if (p
->second
->classification() != Json_value::JSON_VALUE_ARRAY
)
312 r
.error("invalid embedcfg: Patterns entry is not an array");
315 std::vector
<std::string
> files
;
316 p
->second
->get_and_clear_array(&files
);
318 std::pair
<std::string
, std::vector
<std::string
> > val
;
319 val
.first
= p
->first
;
320 std::pair
<Embed_patterns::iterator
, bool> ins
=
321 this->embed_patterns_
.insert(val
);
324 r
.error("invalid embedcfg: duplicate Patterns entry");
327 std::swap(ins
.first
->second
, files
);
330 for (Json_value::map_iterator p
= files
->map_begin();
331 p
!= files
->map_end();
334 if (p
->second
->classification() != Json_value::JSON_VALUE_STRING
)
336 r
.error("invalid embedcfg: Files entry is not a string");
339 this->embed_files_
[p
->first
] = p
->second
->to_string();
343 // Read the contents of FILENAME into this->data_. Returns whether it
347 Embedcfg_reader::initialize_from_file()
349 if (!Gogo::read_file(this->filename_
, Linemap::unknown_location(),
352 if (this->data_
.empty())
354 this->error("empty file");
357 this->p_
= this->data_
.data();
358 this->pend_
= this->p_
+ this->data_
.size();
362 // Read a JSON object into VAL. Return whether it succeeded.
365 Embedcfg_reader::read_object(Json_value
* val
)
367 if (!this->skip_whitespace(false))
369 if (*this->p_
!= '{')
371 this->error("expected %<{%>");
378 if (!this->skip_whitespace(false))
380 if (*this->p_
== '}')
388 if (!this->skip_whitespace(false))
390 if (*this->p_
!= '"')
392 this->error("expected %<\"%>");
397 if (!this->read_string(&key
))
400 if (!this->skip_whitespace(false))
402 if (*this->p_
!= ':')
404 this->error("expected %<:%>");
409 Json_value
* subval
= new Json_value();
410 if (!this->read_value(subval
))
413 val
->add_map_entry(key
, subval
);
415 if (!this->skip_whitespace(false))
417 if (*this->p_
== '}')
422 if (*this->p_
!= ',')
424 this->error("expected %<,%> or %<}%>");
431 // Read a JSON array into VAL. Return whether it succeeded.
434 Embedcfg_reader::read_array(Json_value
* val
)
436 if (!this->skip_whitespace(false))
438 if (*this->p_
!= '[')
440 this->error("expected %<[%>");
447 if (!this->skip_whitespace(false))
449 if (*this->p_
== ']')
457 // If we were parsing full JSON we would call read_value here,
461 if (!this->read_string(&s
))
464 val
->add_array_entry(s
);
466 if (!this->skip_whitespace(false))
468 if (*this->p_
== ']')
473 if (*this->p_
!= ',')
475 this->error("expected %<,%> or %<]%>");
482 // Read a JSON value into VAL. Return whether it succeeded.
485 Embedcfg_reader::read_value(Json_value
* val
)
487 if (!this->skip_whitespace(false))
494 if (!this->read_string(&s
))
501 return this->read_object(val
);
504 return this->read_array(val
);
507 this->error("invalid JSON syntax");
512 // Read a JSON string. Return whether it succeeded.
515 Embedcfg_reader::read_string(std::string
* str
)
517 if (!this->skip_whitespace(false))
519 if (*this->p_
!= '"')
521 this->error("expected %<\"%>");
527 while (this->p_
< this->pend_
&& *this->p_
!= '"')
529 if (*this->p_
!= '\\')
531 str
->push_back(*this->p_
);
537 if (this->p_
>= this->pend_
)
539 this->error("unterminated string");
544 case '"': case '\\': case '/':
545 str
->push_back(*this->p_
);
550 str
->push_back('\b');
555 str
->push_back('\f');
560 str
->push_back('\n');
565 str
->push_back('\r');
570 str
->push_back('\t');
577 unsigned int rune
= 0;
578 for (int i
= 0; i
< 4; i
++)
580 if (this->p_
>= this->pend_
)
582 this->error("unterminated string");
585 unsigned char c
= *this->p_
;
588 if (c
>= '0' && c
<= '9')
590 else if (c
>= 'A' && c
<= 'F')
591 rune
+= c
- 'A' + 10;
592 else if (c
>= 'a' && c
<= 'f')
593 rune
+= c
- 'a' + 10;
596 this->error("invalid hex digit");
600 Lex::append_char(rune
, false, str
, Linemap::unknown_location());
605 this->error("unrecognized string escape");
610 if (*this->p_
== '"')
616 this->error("unterminated string");
620 // Report an error if not at EOF.
623 Embedcfg_reader::check_eof()
625 if (this->skip_whitespace(true))
626 this->error("extraneous data at end of file");
629 // Skip whitespace. Return whether there is more to read.
632 Embedcfg_reader::skip_whitespace(bool eof_ok
)
634 while (this->p_
< this->pend_
)
638 case ' ': case '\t': case '\n': case '\r':
646 this->error("unexpected EOF");
653 Embedcfg_reader::error(const char* msg
)
655 if (!this->data_
.empty() && this->p_
!= NULL
)
656 go_error_at(Linemap::unknown_location(),
657 "%<-fgo-embedcfg%>: %s: %lu: %s",
659 static_cast<unsigned long>(this->p_
- this->data_
.data()),
662 go_error_at(Linemap::unknown_location(),
663 "%<-fgo-embedcfg%>: %s: %s",
664 this->filename_
, msg
);
667 // Implement the sort order for a list of embedded files, as discussed
668 // at the docs for embed.FS.
674 operator()(const std::string
& p1
, const std::string
& p2
) const;
678 split(const std::string
&, size_t*, size_t*, size_t*) const;
682 Embedfs_sort::operator()(const std::string
& p1
, const std::string
& p2
) const
684 size_t dirlen1
, elem1
, elemlen1
;
685 this->split(p1
, &dirlen1
, &elem1
, &elemlen1
);
686 size_t dirlen2
, elem2
, elemlen2
;
687 this->split(p2
, &dirlen2
, &elem2
, &elemlen2
);
693 int i
= p2
.compare(0, dirlen2
, ".");
698 else if (dirlen2
== 0)
700 int i
= p1
.compare(0, dirlen1
, ".");
706 int i
= p1
.compare(0, dirlen1
, p2
, 0, dirlen2
);
711 int i
= p1
.compare(elem1
, elemlen1
, p2
, elem2
, elemlen2
);
715 // Pick out the directory and file name components for comparison.
718 Embedfs_sort::split(const std::string
& s
, size_t* dirlen
, size_t* elem
,
719 size_t* elemlen
) const
721 size_t len
= s
.size();
722 if (len
> 0 && s
[len
- 1] == '/')
724 size_t slash
= s
.rfind('/', len
- 1);
725 if (slash
== std::string::npos
)
735 *elemlen
= len
- (slash
+ 1);
739 // Convert the go:embed directives for a variable into an initializer
740 // for that variable.
743 Gogo::initializer_for_embeds(Type
* type
,
744 const std::vector
<std::string
>* embeds
,
747 if (this->embed_patterns_
.empty())
750 ("invalid go:embed: build system did not "
751 "supply embed configuration"));
752 return Expression::make_error(loc
);
755 type
= type
->unalias();
763 const Named_type
* nt
= type
->named_type();
765 && nt
->named_object()->package() != NULL
766 && nt
->named_object()->package()->pkgpath() == "embed"
767 && nt
->name() == "FS")
768 embed_kind
= EMBED_FS
;
769 else if (type
->is_string_type())
770 embed_kind
= EMBED_STRING
;
771 else if (type
->is_slice_type()
772 && type
->array_type()->element_type()->integer_type() != NULL
773 && type
->array_type()->element_type()->integer_type()->is_byte())
774 embed_kind
= EMBED_BYTES
;
777 go_error_at(loc
, "invalid type for go:embed");
778 return Expression::make_error(loc
);
781 // The patterns in the go:embed directive(s) are in EMBEDS. Find
782 // them in the patterns in the embedcfg file.
784 Unordered_set(std::string
) have
;
785 std::vector
<std::string
> paths
;
786 for (std::vector
<std::string
>::const_iterator pe
= embeds
->begin();
790 Embed_patterns::const_iterator pp
= this->embed_patterns_
.find(*pe
);
791 if (pp
== this->embed_patterns_
.end())
794 ("invalid go:embed: build system did not "
795 "map pattern %<%s%>"),
800 // Each pattern in the embedcfg file maps to a list of file
801 // names. Add those file names to PATHS.
802 for (std::vector
<std::string
>::const_iterator pf
= pp
->second
.begin();
803 pf
!= pp
->second
.end();
806 if (this->embed_files_
.find(*pf
) == this->embed_files_
.end())
809 ("invalid go:embed: build system did not "
815 std::pair
<Unordered_set(std::string
)::iterator
, bool> ins
819 const std::string
& path(*pf
);
820 paths
.push_back(path
);
822 if (embed_kind
== EMBED_FS
)
824 // Add each required directory, with a trailing slash.
825 size_t i
= std::string::npos
;
828 i
= path
.rfind('/', i
);
829 if (i
== std::string::npos
)
831 std::string dir
= path
.substr(0, i
+ 1);
832 ins
= have
.insert(dir
);
834 paths
.push_back(dir
);
842 if (embed_kind
== EMBED_STRING
|| embed_kind
== EMBED_BYTES
)
844 if (paths
.size() > 1)
847 ("invalid go:embed: multiple files for "
848 "string or byte slice"));;
849 return Expression::make_error(loc
);
853 if (!Gogo::read_file(this->embed_files_
[paths
[0]].c_str(), loc
, &data
))
854 return Expression::make_error(loc
);
856 Expression
* e
= Expression::make_string(data
, loc
);
857 if (embed_kind
== EMBED_BYTES
)
858 e
= Expression::make_cast(type
, e
, loc
);
862 std::sort(paths
.begin(), paths
.end(), Embedfs_sort());
864 if (type
->struct_type() == NULL
865 || type
->struct_type()->field_count() != 1)
868 ("internal error: embed.FS should be struct type "
870 return Expression::make_error(loc
);
873 Type
* ptr_type
= type
->struct_type()->field(0)->type();
874 if (ptr_type
->points_to() == NULL
)
877 "internal error: embed.FS struct field should be pointer");
878 return Expression::make_error(loc
);
881 Type
* slice_type
= ptr_type
->points_to();
882 if (!slice_type
->is_slice_type())
885 ("internal error: embed.FS struct field should be "
886 "pointer to slice"));
887 return Expression::make_error(loc
);
890 Type
* file_type
= slice_type
->array_type()->element_type();
891 if (file_type
->struct_type() == NULL
892 || (file_type
->struct_type()->find_local_field(".embed.name", NULL
)
894 || (file_type
->struct_type()->find_local_field(".embed.data", NULL
)
898 ("internal error: embed.FS slice element should be struct "
899 "with name and data fields"));
900 return Expression::make_error(loc
);
903 const Struct_field_list
* file_fields
= file_type
->struct_type()->fields();
904 Expression_list
* file_vals
= new(Expression_list
);
905 file_vals
->reserve(paths
.size());
906 for (std::vector
<std::string
>::const_iterator pp
= paths
.begin();
911 if ((*pp
)[pp
->size() - 1] != '/')
913 if (!Gogo::read_file(this->embed_files_
[*pp
].c_str(), loc
, &data
))
914 return Expression::make_error(loc
);
917 Expression_list
* field_vals
= new(Expression_list
);
918 for (Struct_field_list::const_iterator pf
= file_fields
->begin();
919 pf
!= file_fields
->end();
922 if (pf
->is_field_name(".embed.name"))
923 field_vals
->push_back(Expression::make_string(*pp
, loc
));
924 else if (pf
->is_field_name(".embed.data"))
925 field_vals
->push_back(Expression::make_string(data
, loc
));
928 // FIXME: The embed.file type has a hash field, which is
929 // currently unused. We should fill it in, but don't.
930 // The hash is a SHA256, and we don't have convenient
931 // SHA256 code. Do this later when the field is
933 field_vals
->push_back(NULL
);
937 Expression
* file_val
=
938 Expression::make_struct_composite_literal(file_type
, field_vals
, loc
);
939 file_vals
->push_back(file_val
);
942 Expression
* slice_init
=
943 Expression::make_slice_composite_literal(slice_type
, file_vals
, loc
);
944 Expression
* fs_init
= Expression::make_heap_expression(slice_init
, loc
);
945 Expression_list
* fs_vals
= new Expression_list();
946 fs_vals
->push_back(fs_init
);
947 return Expression::make_struct_composite_literal(type
, fs_vals
, loc
);