hphp/hack/src/parser/schema/full_fidelity_schema.ml

   1 (*
   2  * Copyright (c) 2016, Facebook, Inc.
   3  * All rights reserved.
   4  *
   5  * This source code is licensed under the MIT license found in the
   6  * LICENSE file in the "hack" directory of this source tree.
   7  *
   8  *)
   9
  10 (* If you make changes to the schema that cause it to serialize / deserialize
  11 differently, please update this version number *)
  12 let full_fidelity_schema_version_number = "2019-08-22-0001"
  13
  14 (* TODO: Consider basing the version number on an auto-generated
  15 hash of a file rather than relying on people remembering to update it. *)
  16 (* TODO: It may be worthwhile to investigate how Thrift describes data types
  17 and use that standard. *)
  18
  19 include Operator_schema_definition
  20 include Token_schema_definition
  21 include Schema_definition
  22
  23 let schema_map =
  24   let add map ({ kind_name; _ } as schema_node) =
  25     SMap.add kind_name schema_node map
  26   in
  27   List.fold_left add SMap.empty
  28   @@ {
  29        kind_name = "Token";
  30        type_name = "Token.t";
  31        func_name = "token";
  32        description = "token";
  33        prefix = "token";
  34        aggregates = [Expression; Name];
  35        fields = [];
  36      }
  37      :: {
  38           kind_name = "error";
  39           type_name = "error";
  40           func_name = "error";
  41           description = "error";
  42           prefix = "error";
  43           aggregates = [];
  44           fields = [];
  45         }
  46      :: schema
  47
  48 type trivia_node = {
  49   trivia_kind: string;
  50   trivia_text: string;
  51 }
  52
  53 type transformation = {
  54   pattern: string;
  55   func: schema_node -> string;
  56 }
  57
  58 type token_transformation = {
  59   token_pattern: string;
  60   token_func: token_node list -> string;
  61 }
  62
  63 type trivia_transformation = {
  64   trivia_pattern: string;
  65   trivia_func: trivia_node list -> string;
  66 }
  67
  68 type aggregate_transformation = {
  69   aggregate_pattern: string;
  70   aggregate_func: aggregate_type -> string;
  71 }
  72
  73 type operator_transformation = {
  74   operator_pattern: string;
  75   operator_func: operator_node list -> string;
  76 }
  77
  78 type template_file = {
  79   filename: string;
  80   template: string;
  81   transformations: transformation list;
  82   token_transformations: token_transformation list;
  83   token_no_text_transformations: token_transformation list;
  84   token_variable_text_transformations: token_transformation list;
  85   token_given_text_transformations: token_transformation list;
  86   trivia_transformations: trivia_transformation list;
  87   aggregate_transformations: aggregate_transformation list;
  88   operator_transformations: operator_transformation list;
  89 }
  90
  91 let make_template_file
  92     ?(transformations = [])
  93     ?(token_transformations = [])
  94     ?(token_no_text_transformations = [])
  95     ?(token_given_text_transformations = [])
  96     ?(token_variable_text_transformations = [])
  97     ?(trivia_transformations = [])
  98     ?(aggregate_transformations = [])
  99     ?(operator_transformations = [])
 100     ~filename
 101     ~template
 102     () =
 103   {
 104     filename;
 105     template;
 106     transformations;
 107     token_transformations;
 108     token_no_text_transformations;
 109     token_given_text_transformations;
 110     token_variable_text_transformations;
 111     trivia_transformations;
 112     aggregate_transformations;
 113     operator_transformations;
 114   }
 115
 116 module Language_flags = struct
 117   let php_and_hack = "php_and_hack"
 118
 119   let hack_only = "hack_only"
 120
 121   let is_hack_only : string -> bool = function
 122     | "php_and_hack" -> false
 123     | "hack_only" -> true
 124     | f -> failwith ("Unknown language flag " ^ f ^ " for token.")
 125 end
 126
 127 module LF = Language_flags
 128
 129 module Optional_flags = struct
 130   let xhp = "xhp"
 131
 132   (* See documentation of token_node.allowed_as_identifier. *)
 133   let allowed_as_identifier = "allowed_as_identifier"
 134
 135   let is_recognized : string -> bool = function
 136     | "xhp"
 137     | "allowed_as_identifier" ->
 138       true
 139     | _ -> false
 140
 141   let is_xhp : string list -> bool = (fun flags -> List.mem xhp flags)
 142
 143   let is_allowed_as_identifier : string list -> bool = function
 144     | flags -> List.mem allowed_as_identifier flags
 145 end
 146
 147 module OF = Optional_flags
 148
 149 let trivia_node_from_list l =
 150   match l with
 151   | [trivia_kind; trivia_text] -> { trivia_kind; trivia_text }
 152   | _ -> failwith "bad trivia schema"
 153
 154 let trivia_kinds =
 155   List.map
 156     trivia_node_from_list
 157     [
 158       ["WhiteSpace"; "whitespace"];
 159       ["EndOfLine"; "end_of_line"];
 160       ["DelimitedComment"; "delimited_comment"];
 161       ["SingleLineComment"; "single_line_comment"];
 162       ["FixMe"; "fix_me"];
 163       ["IgnoreError"; "ignore_error"];
 164       ["FallThrough"; "fall_through"];
 165       ["ExtraTokenError"; "extra_token_error"];
 166       ["AfterHaltCompiler"; "after_halt_compiler"];
 167     ]
 168
 169 let escape_token_text t =
 170   (* add one extra backslash because
 171      it is removed by Str.replace_first downstream *)
 172   if t = "\\" then
 173     "\\\\\\"
 174   else
 175     t
 176
 177 let map_and_concat_separated separator f items =
 178   String.concat separator (List.map f items)
 179
 180 let map_and_concat f items = map_and_concat_separated "" f items
 181
 182 let filter_map_concat p f items = map_and_concat f (List.filter p items)
 183
 184 let transform_schema f = map_and_concat f schema
 185
 186 let transform_aggregate f = map_and_concat f generated_aggregate_types
 187
 188 let replace pattern new_text source =
 189   Str.replace_first (Str.regexp pattern) new_text source
 190
 191 let generate_string template =
 192   let syntax_folder s x = replace x.pattern (transform_schema x.func) s in
 193   let tokens_folder token_list s x =
 194     replace x.token_pattern (x.token_func token_list) s
 195   in
 196   let trivia_folder trivia_list s x =
 197     replace x.trivia_pattern (x.trivia_func trivia_list) s
 198   in
 199   let aggregate_folder s x =
 200     replace x.aggregate_pattern (transform_aggregate x.aggregate_func) s
 201   in
 202   let operator_folder operators s x =
 203     replace x.operator_pattern (x.operator_func operators) s
 204   in
 205   let result =
 206     List.fold_left syntax_folder template.template template.transformations
 207   in
 208   let result =
 209     List.fold_left (tokens_folder tokens) result template.token_transformations
 210   in
 211   let result =
 212     List.fold_left
 213       (tokens_folder no_text_tokens)
 214       result
 215       template.token_no_text_transformations
 216   in
 217   let result =
 218     List.fold_left
 219       (tokens_folder given_text_tokens)
 220       result
 221       template.token_given_text_transformations
 222   in
 223   let result =
 224     List.fold_left
 225       (tokens_folder variable_text_tokens)
 226       result
 227       template.token_variable_text_transformations
 228   in
 229   let result =
 230     List.fold_left
 231       (trivia_folder trivia_kinds)
 232       result
 233       template.trivia_transformations
 234   in
 235   let result =
 236     List.fold_left
 237       (operator_folder operators)
 238       result
 239       template.operator_transformations
 240   in
 241   let result =
 242     List.fold_left aggregate_folder result template.aggregate_transformations
 243   in
 244   result
 245
 246 let generate_file template =
 247   let file = open_out template.filename in
 248   let s = generate_string template in
 249   Printf.fprintf file "%s" s;
 250   close_out file
 251
 252 module GenerateFFJSONSchema = struct
 253   let to_json_trivia { trivia_kind; trivia_text } =
 254     Printf.sprintf
 255       "    { \"trivia_kind_name\" : \"%s\",
 256       \"trivia_type_name\" : \"%s\" }"
 257       trivia_kind
 258       trivia_text
 259
 260   let to_json_given_text x =
 261     Printf.sprintf
 262       "    { \"token_kind\" : \"%s\",
 263       \"token_text\" : \"%s\" },
 264 "
 265       x.token_kind
 266       (escape_token_text x.token_text)
 267
 268   let to_json_variable_text x =
 269     Printf.sprintf
 270       "    { \"token_kind\" : \"%s\",
 271       \"token_text\" : null },
 272 "
 273       x.token_kind
 274
 275   let to_json_ast_nodes x =
 276     let mapper (f, _) = Printf.sprintf "{ \"field_name\" : \"%s\" }" f in
 277     let fields = String.concat ",\n        " (List.map mapper x.fields) in
 278     Printf.sprintf
 279       "    { \"kind_name\" : \"%s\",
 280       \"type_name\" : \"%s\",
 281       \"description\" : \"%s\",
 282       \"prefix\" : \"%s\",
 283       \"fields\" : [
 284         %s
 285       ] },
 286 "
 287       x.kind_name
 288       x.type_name
 289       x.description
 290       x.prefix
 291       fields
 292
 293   let full_fidelity_json_schema_template =
 294     "{ \"description\" :
 295   \"@"
 296     ^ "generated JSON schema of the Hack Full Fidelity Parser AST\",
 297   \"version\" : \""
 298     ^ full_fidelity_schema_version_number
 299     ^ "\",
 300   \"trivia\" : [
 301 TRIVIA_KINDS ],
 302   \"tokens\" : [
 303 GIVEN_TEXT_TOKENS
 304 VARIABLE_TEXT_TOKENS
 305     { \"token_kind\" : \"EndOfFile\",
 306       \"token_text\" : null } ],
 307   \"AST\" : [
 308 AST_NODES
 309     { \"kind_name\" : \"Token\",
 310       \"type_name\" : \"token\",
 311       \"description\" : \"token\",
 312       \"prefix\" : \"\",
 313       \"fields\" : [
 314         { \"field_name\" : \"leading\" },
 315         { \"field_name\" : \"trailing\" } ] },
 316     { \"kind_name\" : \"Missing\",
 317       \"type_name\" : \"missing\",
 318       \"description\" : \"missing\",
 319       \"prefix\" : \"\",
 320       \"fields\" : [ ] },
 321     { \"kind_name\" : \"SyntaxList\",
 322       \"type_name\" : \"syntax_list\",
 323       \"description\" : \"syntax_list\",
 324       \"prefix\" : \"\",
 325       \"fields\" : [ ] } ] }"
 326
 327   let full_fidelity_json_schema =
 328     make_template_file
 329       ~transformations:[{ pattern = "AST_NODES"; func = to_json_ast_nodes }]
 330       ~token_given_text_transformations:
 331         [
 332           {
 333             token_pattern = "GIVEN_TEXT_TOKENS";
 334             token_func = map_and_concat to_json_given_text;
 335           };
 336         ]
 337       ~token_variable_text_transformations:
 338         [
 339           {
 340             token_pattern = "VARIABLE_TEXT_TOKENS";
 341             token_func = map_and_concat to_json_variable_text;
 342           };
 343         ]
 344       ~trivia_transformations:
 345         [
 346           {
 347             trivia_pattern = "TRIVIA_KINDS";
 348             trivia_func = map_and_concat_separated ",\n" to_json_trivia;
 349           };
 350         ]
 351       ~template:full_fidelity_json_schema_template
 352       ~filename:"hphp/hack/src/parser/js/full_fidelity_schema.json"
 353       ()
 354 end
 355
 356 let schema_as_json () =
 357   generate_string GenerateFFJSONSchema.full_fidelity_json_schema