hphp/hack/src/parser/schema/full_fidelity_schema.ml

   1 (*
   2  * Copyright (c) 2016, Facebook, Inc.
   3  * All rights reserved.
   4  *
   5  * This source code is licensed under the MIT license found in the
   6  * LICENSE file in the "hack" directory of this source tree.
   7  *
   8  *)
   9
  10 (* If you make changes to the schema that cause it to serialize / deserialize
  11 differently, please update this version number *)
  12 let full_fidelity_schema_version_number = "2021-03-10-0001"
  13
  14 (* TODO: Consider basing the version number on an auto-generated
  15 hash of a file rather than relying on people remembering to update it. *)
  16 (* TODO: It may be worthwhile to investigate how Thrift describes data types
  17 and use that standard. *)
  18
  19 include Operator_schema_definition
  20 include Token_schema_definition
  21 include Schema_definition
  22
  23 let schema_map =
  24   let add map ({ kind_name; _ } as schema_node) =
  25     SMap.add kind_name schema_node map
  26   in
  27   List.fold_left add SMap.empty
  28   @@ {
  29        kind_name = "Token";
  30        type_name = "Token.t";
  31        func_name = "token";
  32        description = "token";
  33        prefix = "token";
  34        aggregates = [Expression; Name];
  35        fields = [];
  36      }
  37      :: {
  38           kind_name = "error";
  39           type_name = "error";
  40           func_name = "error";
  41           description = "error";
  42           prefix = "error";
  43           aggregates = [];
  44           fields = [];
  45         }
  46      :: schema
  47
  48 type trivia_node = {
  49   trivia_kind: string;
  50   trivia_text: string;
  51 }
  52
  53 type transformation = {
  54   pattern: string;
  55   func: schema_node -> string;
  56 }
  57
  58 type token_transformation = {
  59   token_pattern: string;
  60   token_func: token_node list -> string;
  61 }
  62
  63 type trivia_transformation = {
  64   trivia_pattern: string;
  65   trivia_func: trivia_node list -> string;
  66 }
  67
  68 type aggregate_transformation = {
  69   aggregate_pattern: string;
  70   aggregate_func: aggregate_type -> string;
  71 }
  72
  73 type operator_transformation = {
  74   operator_pattern: string;
  75   operator_func: operator_node list -> string;
  76 }
  77
  78 type template_file = {
  79   filename: string;
  80   template: string;
  81   transformations: transformation list;
  82   token_transformations: token_transformation list;
  83   token_no_text_transformations: token_transformation list;
  84   token_variable_text_transformations: token_transformation list;
  85   token_given_text_transformations: token_transformation list;
  86   trivia_transformations: trivia_transformation list;
  87   aggregate_transformations: aggregate_transformation list;
  88   operator_transformations: operator_transformation list;
  89 }
  90
  91 let make_template_file
  92     ?(transformations = [])
  93     ?(token_transformations = [])
  94     ?(token_no_text_transformations = [])
  95     ?(token_given_text_transformations = [])
  96     ?(token_variable_text_transformations = [])
  97     ?(trivia_transformations = [])
  98     ?(aggregate_transformations = [])
  99     ?(operator_transformations = [])
 100     ~filename
 101     ~template
 102     () =
 103   {
 104     filename;
 105     template;
 106     transformations;
 107     token_transformations;
 108     token_no_text_transformations;
 109     token_given_text_transformations;
 110     token_variable_text_transformations;
 111     trivia_transformations;
 112     aggregate_transformations;
 113     operator_transformations;
 114   }
 115
 116 module Language_flags = struct
 117   let php_and_hack = "php_and_hack"
 118
 119   let hack_only = "hack_only"
 120
 121   let is_hack_only : string -> bool = function
 122     | "php_and_hack" -> false
 123     | "hack_only" -> true
 124     | f -> failwith ("Unknown language flag " ^ f ^ " for token.")
 125 end
 126
 127 module LF = Language_flags
 128
 129 module Optional_flags = struct
 130   let xhp = "xhp"
 131
 132   (* See documentation of token_node.allowed_as_identifier. *)
 133   let allowed_as_identifier = "allowed_as_identifier"
 134
 135   let is_recognized : string -> bool = function
 136     | "xhp"
 137     | "allowed_as_identifier" ->
 138       true
 139     | _ -> false
 140
 141   let is_xhp : string list -> bool = (fun flags -> List.mem xhp flags)
 142
 143   let is_allowed_as_identifier : string list -> bool = function
 144     | flags -> List.mem allowed_as_identifier flags
 145 end
 146
 147 module OF = Optional_flags
 148
 149 let trivia_node_from_list l =
 150   match l with
 151   | [trivia_kind; trivia_text] -> { trivia_kind; trivia_text }
 152   | _ -> failwith "bad trivia schema"
 153
 154 let trivia_kinds =
 155   List.map
 156     trivia_node_from_list
 157     [
 158       ["WhiteSpace"; "whitespace"];
 159       ["EndOfLine"; "end_of_line"];
 160       ["DelimitedComment"; "delimited_comment"];
 161       ["SingleLineComment"; "single_line_comment"];
 162       ["FixMe"; "fix_me"];
 163       ["IgnoreError"; "ignore_error"];
 164       ["FallThrough"; "fall_through"];
 165       ["ExtraTokenError"; "extra_token_error"];
 166     ]
 167
 168 let escape_token_text t =
 169   (* add one extra backslash because
 170      it is removed by Str.replace_first downstream *)
 171   if t = "\\" then
 172     "\\\\\\"
 173   else
 174     t
 175
 176 let map_and_concat_separated separator f items =
 177   String.concat separator (List.map f items)
 178
 179 let map_and_concat f items = map_and_concat_separated "" f items
 180
 181 let filter_map_concat p f items = map_and_concat f (List.filter p items)
 182
 183 let transform_schema f = map_and_concat f schema
 184
 185 let transform_aggregate f = map_and_concat f generated_aggregate_types
 186
 187 let replace pattern new_text source =
 188   Str.replace_first (Str.regexp pattern) new_text source
 189
 190 let generate_string template =
 191   let syntax_folder s x = replace x.pattern (transform_schema x.func) s in
 192   let tokens_folder token_list s x =
 193     replace x.token_pattern (x.token_func token_list) s
 194   in
 195   let trivia_folder trivia_list s x =
 196     replace x.trivia_pattern (x.trivia_func trivia_list) s
 197   in
 198   let aggregate_folder s x =
 199     replace x.aggregate_pattern (transform_aggregate x.aggregate_func) s
 200   in
 201   let operator_folder operators s x =
 202     replace x.operator_pattern (x.operator_func operators) s
 203   in
 204   let result =
 205     List.fold_left syntax_folder template.template template.transformations
 206   in
 207   let result =
 208     List.fold_left (tokens_folder tokens) result template.token_transformations
 209   in
 210   let result =
 211     List.fold_left
 212       (tokens_folder no_text_tokens)
 213       result
 214       template.token_no_text_transformations
 215   in
 216   let result =
 217     List.fold_left
 218       (tokens_folder given_text_tokens)
 219       result
 220       template.token_given_text_transformations
 221   in
 222   let result =
 223     List.fold_left
 224       (tokens_folder variable_text_tokens)
 225       result
 226       template.token_variable_text_transformations
 227   in
 228   let result =
 229     List.fold_left
 230       (trivia_folder trivia_kinds)
 231       result
 232       template.trivia_transformations
 233   in
 234   let result =
 235     List.fold_left
 236       (operator_folder operators)
 237       result
 238       template.operator_transformations
 239   in
 240   let result =
 241     List.fold_left aggregate_folder result template.aggregate_transformations
 242   in
 243   result
 244
 245 let format_ocaml src path : string =
 246   (* Write the string to a temporary file. *)
 247   let tmp_filename = Filename.temp_file "" (Filename.basename path) in
 248
 249   let open Core_kernel in
 250   let file = Out_channel.create tmp_filename in
 251   Printf.fprintf file "%s" src;
 252   Out_channel.close file;
 253
 254   let ocamlformat_path =
 255     Option.value
 256       (Sys.getenv_opt "OCAMLFORMAT_PATH")
 257       ~default:"../tools/third-party/ocamlformat/ocamlformat"
 258   in
 259
 260   (* Run ocamlformat on the file. *)
 261   let cmd =
 262     Printf.sprintf "%s -i --name=%s %s" ocamlformat_path path tmp_filename
 263   in
 264   ignore (Sys.command cmd);
 265
 266   (* Read the formatted file, then delete it. *)
 267   let res =
 268     In_channel.with_file tmp_filename ~f:(fun channel ->
 269         In_channel.input_all channel)
 270   in
 271   Sys.remove tmp_filename;
 272   res
 273
 274 let generate_formatted_string (template : template_file) : string =
 275   let open Core_kernel in
 276   let s = generate_string template in
 277   let has_suffix s = String.is_suffix template.filename ~suffix:s in
 278   if has_suffix ".ml" || has_suffix ".mli" then
 279     format_ocaml s template.filename
 280   else
 281     s
 282
 283 let generate_file (template : template_file) : unit =
 284   let open Core_kernel in
 285   let filename = template.filename in
 286   let file = Out_channel.create filename in
 287   let s = generate_formatted_string template in
 288   Printf.fprintf file "%s" s;
 289   Out_channel.close file
 290
 291 module GenerateFFJSONSchema = struct
 292   let to_json_trivia { trivia_kind; trivia_text } =
 293     Printf.sprintf
 294       "    { \"trivia_kind_name\" : \"%s\",
 295       \"trivia_type_name\" : \"%s\" }"
 296       trivia_kind
 297       trivia_text
 298
 299   let to_json_given_text x =
 300     Printf.sprintf
 301       "    { \"token_kind\" : \"%s\",
 302       \"token_text\" : \"%s\" },
 303 "
 304       x.token_kind
 305       (escape_token_text x.token_text)
 306
 307   let to_json_variable_text x =
 308     Printf.sprintf
 309       "    { \"token_kind\" : \"%s\",
 310       \"token_text\" : null },
 311 "
 312       x.token_kind
 313
 314   let to_json_ast_nodes x =
 315     let mapper (f, _) = Printf.sprintf "{ \"field_name\" : \"%s\" }" f in
 316     let fields = String.concat ",\n        " (List.map mapper x.fields) in
 317     Printf.sprintf
 318       "    { \"kind_name\" : \"%s\",
 319       \"type_name\" : \"%s\",
 320       \"description\" : \"%s\",
 321       \"prefix\" : \"%s\",
 322       \"fields\" : [
 323         %s
 324       ] },
 325 "
 326       x.kind_name
 327       x.type_name
 328       x.description
 329       x.prefix
 330       fields
 331
 332   let full_fidelity_json_schema_template =
 333     "{ \"description\" :
 334   \"@"
 335     ^ "generated JSON schema of the Hack Full Fidelity Parser AST\",
 336   \"version\" : \""
 337     ^ full_fidelity_schema_version_number
 338     ^ "\",
 339   \"trivia\" : [
 340 TRIVIA_KINDS ],
 341   \"tokens\" : [
 342 GIVEN_TEXT_TOKENS
 343 VARIABLE_TEXT_TOKENS
 344     { \"token_kind\" : \"EndOfFile\",
 345       \"token_text\" : null } ],
 346   \"AST\" : [
 347 AST_NODES
 348     { \"kind_name\" : \"Token\",
 349       \"type_name\" : \"token\",
 350       \"description\" : \"token\",
 351       \"prefix\" : \"\",
 352       \"fields\" : [
 353         { \"field_name\" : \"leading\" },
 354         { \"field_name\" : \"trailing\" } ] },
 355     { \"kind_name\" : \"Missing\",
 356       \"type_name\" : \"missing\",
 357       \"description\" : \"missing\",
 358       \"prefix\" : \"\",
 359       \"fields\" : [ ] },
 360     { \"kind_name\" : \"SyntaxList\",
 361       \"type_name\" : \"syntax_list\",
 362       \"description\" : \"syntax_list\",
 363       \"prefix\" : \"\",
 364       \"fields\" : [ ] } ] }"
 365
 366   let full_fidelity_json_schema =
 367     make_template_file
 368       ~transformations:[{ pattern = "AST_NODES"; func = to_json_ast_nodes }]
 369       ~token_given_text_transformations:
 370         [
 371           {
 372             token_pattern = "GIVEN_TEXT_TOKENS";
 373             token_func = map_and_concat to_json_given_text;
 374           };
 375         ]
 376       ~token_variable_text_transformations:
 377         [
 378           {
 379             token_pattern = "VARIABLE_TEXT_TOKENS";
 380             token_func = map_and_concat to_json_variable_text;
 381           };
 382         ]
 383       ~trivia_transformations:
 384         [
 385           {
 386             trivia_pattern = "TRIVIA_KINDS";
 387             trivia_func = map_and_concat_separated ",\n" to_json_trivia;
 388           };
 389         ]
 390       ~template:full_fidelity_json_schema_template
 391       ~filename:"hphp/hack/src/parser/js/full_fidelity_schema.json"
 392       ()
 393 end
 394
 395 let schema_as_json () =
 396   generate_string GenerateFFJSONSchema.full_fidelity_json_schema