hphp/hack/src/parser/schema/full_fidelity_schema.ml

   1 (**
   2  * Copyright (c) 2016, Facebook, Inc.
   3  * All rights reserved.
   4  *
   5  * This source code is licensed under the BSD-style license found in the
   6  * LICENSE file in the "hack" directory of this source tree. An additional grant
   7  * of patent rights can be found in the PATENTS file in the same directory.
   8  *
   9  *)
  10
  11 (* If you make changes to the schema that cause it to serialize / deserialize
  12 differently, please update this version number *)
  13 let full_fidelity_schema_version_number = "2018-02-07-0002"
  14 (* TODO: Consider basing the version number on an auto-generated
  15 hash of a file rather than relying on people remembering to update it. *)
  16 (* TODO: It may be worthwhile to investigate how Thrift describes data types
  17 and use that standard. *)
  18
  19 include Schema_definition
  20 let schema_map =
  21   let add map ({ kind_name; _ } as schema_node) =
  22     SMap.add kind_name schema_node map
  23   in
  24   List.fold_left add SMap.empty @@
  25     { kind_name   = "Token"
  26     ; type_name   = "Token.t"
  27     ; func_name   = "token"
  28     ; description = "token"
  29     ; prefix      = "token"
  30     ; aggregates  = [ Expression; Name ]
  31     ; fields      = []
  32     } ::
  33     { kind_name   = "error"
  34     ; type_name   = "error"
  35     ; func_name   = "error"
  36     ; description = "error"
  37     ; prefix      = "error"
  38     ; aggregates  = []
  39     ; fields      = []
  40     } ::
  41     schema
  42
  43 type trivia_node = {
  44   trivia_kind : string;
  45   trivia_text : string
  46 }
  47
  48 type token_node = {
  49     token_kind : string;
  50     token_text : string
  51 }
  52
  53 type transformation =
  54 {
  55   pattern : string;
  56   func : schema_node -> string
  57 }
  58
  59 type token_transformation =
  60 {
  61   token_pattern : string;
  62   token_func : token_node list -> string
  63 }
  64
  65 type trivia_transformation =
  66 {
  67   trivia_pattern : string;
  68   trivia_func : trivia_node list -> string
  69 }
  70
  71 type aggregate_transformation =
  72 {
  73   aggregate_pattern : string;
  74   aggregate_func : aggregate_type -> string
  75 }
  76
  77 type template_file =
  78 {
  79   filename : string;
  80   template : string;
  81   transformations : transformation list;
  82   token_no_text_transformations : token_transformation list;
  83   token_variable_text_transformations : token_transformation list;
  84   token_given_text_transformations : token_transformation list;
  85   trivia_transformations : trivia_transformation list;
  86   aggregate_transformations : aggregate_transformation list;
  87 }
  88
  89 let token_node_from_list l =
  90   match l with
  91   | [ token_kind; token_text ] ->
  92     { token_kind; token_text }
  93   | _ -> failwith "bad token schema"
  94
  95 let trivia_node_from_list l =
  96   match l with
  97   | [ trivia_kind; trivia_text ] ->
  98     { trivia_kind; trivia_text }
  99   | _ -> failwith "bad trivia schema"
 100
 101 let variable_text_tokens = List.map token_node_from_list [
 102   [ "ErrorToken"; "error_token" ];
 103   [ "Name"; "name" ];
 104   [ "Variable"; "variable" ];
 105   [ "DecimalLiteral"; "decimal_literal" ];
 106   [ "OctalLiteral"; "octal_literal" ];
 107   [ "HexadecimalLiteral"; "hexadecimal_literal" ];
 108   [ "BinaryLiteral"; "binary_literal" ];
 109   [ "FloatingLiteral"; "floating_literal" ];
 110   [ "ExecutionStringLiteral"; "execution_string_literal" ];
 111   [ "ExecutionStringLiteralHead"; "execution_string_literal_head" ];
 112   [ "ExecutionStringLiteralTail"; "execution_string_literal_tail" ];
 113   [ "SingleQuotedStringLiteral"; "single_quoted_string_literal" ];
 114   [ "DoubleQuotedStringLiteral"; "double_quoted_string_literal" ];
 115   [ "DoubleQuotedStringLiteralHead"; "double_quoted_string_literal_head" ];
 116   [ "StringLiteralBody"; "string_literal_body" ];
 117   [ "DoubleQuotedStringLiteralTail"; "double_quoted_string_literal_tail" ];
 118   [ "HeredocStringLiteral"; "heredoc_string_literal" ];
 119   [ "HeredocStringLiteralHead"; "heredoc_string_literal_head" ];
 120   [ "HeredocStringLiteralTail"; "heredoc_string_literal_tail" ];
 121   [ "NowdocStringLiteral"; "nowdoc_string_literal" ];
 122   [ "BooleanLiteral"; "boolean_literal" ];
 123   [ "XHPCategoryName"; "XHP_category_name" ];
 124   [ "XHPElementName"; "XHP_element_name" ];
 125   [ "XHPClassName"; "XHP_class_name" ];
 126   [ "XHPStringLiteral"; "XHP_string_literal" ];
 127   [ "XHPBody"; "XHP_body" ];
 128   [ "XHPComment"; "XHP_comment" ];
 129   [ "Markup"; "markup" ]]
 130
 131 let no_text_tokens = List.map token_node_from_list [
 132   [ "EndOfFile"; "end_of_file" ]]
 133
 134 let given_text_tokens = List.map token_node_from_list [
 135   [ "Abstract"; "abstract" ];
 136   [ "And"; "and" ];
 137   [ "Array"; "array" ];
 138   [ "Arraykey"; "arraykey" ];
 139   [ "As"; "as" ];
 140   [ "Async"; "async" ];
 141   [ "Attribute"; "attribute" ];
 142   [ "Await"; "await" ];
 143   [ "Backslash"; "\\"];
 144   [ "Bool"; "bool" ];
 145   [ "Break"; "break" ];
 146   [ "Case"; "case" ];
 147   [ "Catch"; "catch" ];
 148   [ "Category"; "category" ];
 149   [ "Children"; "children" ];
 150   [ "Class"; "class" ];
 151   [ "Classname"; "classname" ];
 152   [ "Clone"; "clone" ];
 153   [ "Const"; "const" ];
 154   [ "Construct"; "__construct" ];
 155   [ "Continue"; "continue" ];
 156   [ "Coroutine"; "coroutine" ];
 157   [ "Darray"; "darray" ];
 158   [ "Declare"; "declare"];
 159   [ "Default"; "default" ];
 160   [ "Define"; "define"];
 161   [ "Destruct"; "__destruct" ];
 162   [ "Dict"; "dict" ];
 163   [ "Do"; "do" ];
 164   [ "Double"; "double" ];
 165   [ "Echo"; "echo" ];
 166   [ "Else"; "else" ];
 167   [ "Elseif"; "elseif" ];
 168   [ "Empty"; "empty" ];
 169   [ "Endif"; "endif" ];
 170   [ "Endfor"; "endfor" ];
 171   [ "Endforeach"; "endforeach" ];
 172   [ "Enum"; "enum" ];
 173   [ "Eval"; "eval" ];
 174   [ "Extends"; "extends" ];
 175   [ "Fallthrough"; "fallthrough" ];
 176   [ "Float"; "float" ];
 177   [ "Final"; "final" ];
 178   [ "Finally"; "finally" ];
 179   [ "For"; "for" ];
 180   [ "Foreach"; "foreach" ];
 181   [ "From"; "from"];
 182   [ "Function"; "function" ];
 183   [ "Global"; "global" ];
 184   [ "Goto"; "goto" ];
 185   [ "If"; "if" ];
 186   [ "Implements"; "implements" ];
 187   [ "Include"; "include" ];
 188   [ "Include_once"; "include_once" ];
 189   [ "Inout"; "inout" ];
 190   [ "Instanceof"; "instanceof" ];
 191   [ "Insteadof"; "insteadof" ];
 192   [ "Int"; "int" ];
 193   [ "Interface"; "interface" ];
 194   [ "Is"; "is"];
 195   [ "Isset"; "isset" ];
 196   [ "Keyset"; "keyset" ];
 197   [ "List"; "list" ];
 198   [ "Mixed"; "mixed" ];
 199   [ "Namespace"; "namespace" ];
 200   [ "New"; "new" ];
 201   [ "Newtype"; "newtype" ];
 202   [ "Noreturn"; "noreturn" ];
 203   [ "Num"; "num" ];
 204   [ "Object"; "object" ];
 205   [ "Or"; "or" ];
 206   [ "Parent"; "parent" ];
 207   [ "Print"; "print" ];
 208   [ "Private"; "private" ];
 209   [ "Protected"; "protected" ];
 210   [ "Public"; "public" ];
 211   [ "Require"; "require" ];
 212   [ "Require_once"; "require_once" ];
 213   [ "Required"; "required" ];
 214   [ "Resource"; "resource" ];
 215   [ "Return"; "return" ];
 216   [ "Self"; "self" ];
 217   [ "Shape"; "shape" ];
 218   [ "Static"; "static" ];
 219   [ "String"; "string" ];
 220   [ "Super"; "super" ];
 221   [ "Suspend"; "suspend" ];
 222   [ "Switch"; "switch" ];
 223   [ "This"; "this" ];
 224   [ "Throw"; "throw" ];
 225   [ "Trait"; "trait" ];
 226   [ "Try"; "try" ];
 227   [ "Tuple"; "tuple" ];
 228   [ "Type"; "type" ];
 229   [ "Unset"; "unset" ];
 230   [ "Use"; "use" ];
 231   [ "Using"; "using" ];
 232   [ "Var"; "var" ];
 233   [ "Varray"; "varray" ];
 234   [ "Vec"; "vec" ];
 235   [ "Void"; "void" ];
 236   [ "Where"; "where" ];
 237   [ "While"; "while" ];
 238   [ "Xor"; "xor" ];
 239   [ "Yield"; "yield" ];
 240   [ "LeftBracket"; "[" ];
 241   [ "RightBracket"; "]" ];
 242   [ "LeftParen"; "(" ];
 243   [ "RightParen"; ")" ];
 244   [ "LeftBrace"; "{" ];
 245   [ "RightBrace"; "}" ];
 246   [ "Dot"; "." ];
 247   [ "MinusGreaterThan"; "->" ];
 248   [ "PlusPlus"; "++" ];
 249   [ "MinusMinus"; "--" ];
 250   [ "StarStar"; "**" ];
 251   [ "Star"; "*" ];
 252   [ "Plus"; "+" ];
 253   [ "Minus"; "-" ];
 254   [ "Tilde"; "~" ];
 255   [ "Exclamation"; "!" ];
 256   [ "Dollar"; "$" ];
 257   [ "Slash"; "/" ];
 258   [ "Percent"; "%" ];
 259   [ "LessThanGreaterThan"; "<>"];
 260   [ "LessThanEqualGreaterThan"; "<=>"];
 261   [ "LessThanLessThan"; "<<" ];
 262   [ "GreaterThanGreaterThan"; ">>" ];
 263   [ "LessThan"; "<" ];
 264   [ "GreaterThan"; ">" ];
 265   [ "LessThanEqual"; "<=" ];
 266   [ "GreaterThanEqual"; ">=" ];
 267   [ "EqualEqual"; "==" ];
 268   [ "EqualEqualEqual"; "===" ];
 269   [ "ExclamationEqual"; "!=" ];
 270   [ "ExclamationEqualEqual"; "!==" ];
 271   [ "Carat"; "^" ];
 272   [ "Bar"; "|" ];
 273   [ "Ampersand"; "&" ];
 274   [ "AmpersandAmpersand"; "&&" ];
 275   [ "BarBar"; "||" ];
 276   [ "Question"; "?" ];
 277   [ "QuestionColon"; "?:" ];
 278   [ "QuestionQuestion"; "??" ];
 279   [ "Colon"; ":" ];
 280   [ "Semicolon"; ";" ];
 281   [ "Equal"; "=" ];
 282   [ "StarStarEqual"; "**=" ];
 283   [ "StarEqual"; "*=" ];
 284   [ "SlashEqual"; "/=" ];
 285   [ "PercentEqual"; "%=" ];
 286   [ "PlusEqual"; "+=" ];
 287   [ "MinusEqual"; "-=" ];
 288   [ "DotEqual"; ".=" ];
 289   [ "LessThanLessThanEqual"; "<<=" ];
 290   [ "GreaterThanGreaterThanEqual"; ">>=" ];
 291   [ "AmpersandEqual"; "&=" ];
 292   [ "CaratEqual"; "^=" ];
 293   [ "BarEqual"; "|=" ];
 294   [ "Comma"; "," ];
 295   [ "At"; "@" ];
 296   [ "ColonColon"; "::" ];
 297   [ "EqualGreaterThan"; "=>" ];
 298   [ "EqualEqualGreaterThan"; "==>" ];
 299   [ "QuestionMinusGreaterThan"; "?->" ];
 300   [ "DotDotDot"; "..." ];
 301   [ "DollarDollar"; "$$" ];
 302   [ "BarGreaterThan"; "|>" ];
 303   [ "NullLiteral"; "null" ];
 304   [ "SlashGreaterThan"; "/>" ];
 305   [ "LessThanSlash"; "</" ];
 306   [ "LessThanQuestion";"<?" ];
 307   [ "QuestionGreaterThan"; "?>" ];
 308   [ "HaltCompiler"; "__halt_compiler" ]]
 309
 310 let trivia_kinds = List.map trivia_node_from_list [
 311   [ "WhiteSpace"; "whitespace" ];
 312   [ "EndOfLine"; "end_of_line" ];
 313   [ "DelimitedComment"; "delimited_comment" ];
 314   [ "SingleLineComment"; "single_line_comment" ];
 315   [ "Unsafe"; "unsafe" ];
 316   [ "UnsafeExpression"; "unsafe_expression" ];
 317   [ "FixMe"; "fix_me" ];
 318   [ "IgnoreError"; "ignore_error" ];
 319   [ "FallThrough"; "fall_through" ];
 320   [ "ExtraTokenError"; "extra_token_error"];
 321   [ "AfterHaltCompiler"; "after_halt_compiler" ]]
 322
 323 let escape_token_text t =
 324   (* add one extra backslash because
 325      it is removed by Str.replace_first downstream *)
 326   if t = "\\" then "\\\\\\" else t
 327
 328 let map_and_concat_separated separator f items =
 329   String.concat separator (List.map f items)
 330
 331 let map_and_concat f items =
 332   map_and_concat_separated "" f items
 333
 334 let transform_schema f =
 335   map_and_concat f schema
 336
 337 let transform_aggregate f =
 338   map_and_concat f generated_aggregate_types
 339
 340 let replace pattern new_text source =
 341   Str.replace_first (Str.regexp pattern) new_text source
 342
 343 let generate_string template =
 344   let syntax_folder s x =
 345     replace x.pattern (transform_schema x.func) s in
 346   let tokens_folder token_list s x =
 347     replace x.token_pattern (x.token_func token_list) s in
 348   let trivia_folder trivia_list s x =
 349     replace x.trivia_pattern (x.trivia_func trivia_list) s in
 350   let aggregate_folder s x =
 351     replace x.aggregate_pattern (transform_aggregate x.aggregate_func) s in
 352
 353   let result = List.fold_left
 354     syntax_folder template.template template.transformations in
 355
 356   let result = List.fold_left (tokens_folder no_text_tokens)
 357     result template.token_no_text_transformations in
 358   let result = List.fold_left (tokens_folder variable_text_tokens)
 359     result template.token_variable_text_transformations in
 360   let result = List.fold_left (tokens_folder given_text_tokens)
 361     result template.token_given_text_transformations in
 362   let result = List.fold_left (trivia_folder trivia_kinds)
 363     result template.trivia_transformations in
 364   let result = List.fold_left
 365     aggregate_folder result template.aggregate_transformations in
 366
 367   result
 368
 369 let generate_file template =
 370   let file = open_out template.filename in
 371   let s = generate_string template in
 372   Printf.fprintf file "%s" s;
 373   close_out file
 374
 375
 376 module GenerateFFJSONSchema = struct
 377   let to_json_trivia { trivia_kind; trivia_text } =
 378     Printf.sprintf
 379 "    { \"trivia_kind_name\" : \"%s\",
 380       \"trivia_type_name\" : \"%s\" }"
 381     trivia_kind trivia_text
 382
 383   let to_json_given_text x =
 384     Printf.sprintf
 385 "    { \"token_kind\" : \"%s\",
 386       \"token_text\" : \"%s\" },
 387 "
 388     x.token_kind (escape_token_text x.token_text)
 389
 390   let to_json_variable_text x =
 391     Printf.sprintf
 392 "    { \"token_kind\" : \"%s\",
 393       \"token_text\" : null },
 394 "
 395     x.token_kind
 396
 397   let to_json_ast_nodes x =
 398     let mapper (f,_) = Printf.sprintf
 399 "{ \"field_name\" : \"%s\" }" f in
 400     let fields = String.concat ",\n        " (List.map mapper x.fields) in
 401     Printf.sprintf
 402 "    { \"kind_name\" : \"%s\",
 403       \"type_name\" : \"%s\",
 404       \"description\" : \"%s\",
 405       \"prefix\" : \"%s\",
 406       \"fields\" : [
 407         %s
 408       ] },
 409 "  x.kind_name x.type_name x.description x.prefix fields
 410
 411   let full_fidelity_json_schema_template =
 412 "{ \"description\" :
 413   \"@"^"generated JSON schema of the Hack Full Fidelity Parser AST\",
 414   \"version\" : \"" ^ full_fidelity_schema_version_number ^ "\",
 415   \"trivia\" : [
 416 TRIVIA_KINDS ],
 417   \"tokens\" : [
 418 GIVEN_TEXT_TOKENS
 419 VARIABLE_TEXT_TOKENS
 420     { \"token_kind\" : \"EndOfFile\",
 421       \"token_text\" : null } ],
 422   \"AST\" : [
 423 AST_NODES
 424     { \"kind_name\" : \"Token\",
 425       \"type_name\" : \"token\",
 426       \"description\" : \"token\",
 427       \"prefix\" : \"\",
 428       \"fields\" : [
 429         { \"field_name\" : \"leading\" },
 430         { \"field_name\" : \"trailing\" } ] },
 431     { \"kind_name\" : \"Missing\",
 432       \"type_name\" : \"missing\",
 433       \"description\" : \"missing\",
 434       \"prefix\" : \"\",
 435       \"fields\" : [ ] },
 436     { \"kind_name\" : \"SyntaxList\",
 437       \"type_name\" : \"syntax_list\",
 438       \"description\" : \"syntax_list\",
 439       \"prefix\" : \"\",
 440       \"fields\" : [ ] } ] }"
 441
 442   let full_fidelity_json_schema =
 443   {
 444     filename = "hphp/hack/src/parser/js/full_fidelity_schema.json";
 445     template = full_fidelity_json_schema_template;
 446     transformations = [
 447       { pattern = "AST_NODES"; func = to_json_ast_nodes }
 448     ];
 449     token_no_text_transformations = [ ];
 450     token_given_text_transformations = [
 451       { token_pattern = "GIVEN_TEXT_TOKENS";
 452         token_func = map_and_concat to_json_given_text } ];
 453     token_variable_text_transformations = [
 454       { token_pattern = "VARIABLE_TEXT_TOKENS";
 455         token_func = map_and_concat to_json_variable_text }];
 456     trivia_transformations = [
 457       { trivia_pattern = "TRIVIA_KINDS";
 458         trivia_func = map_and_concat_separated ",\n" to_json_trivia }
 459     ];
 460     aggregate_transformations = [];
 461   }
 462
 463 end (* GenerateFFJSONSchema *)
 464
 465 let schema_as_json () =
 466   generate_string GenerateFFJSONSchema.full_fidelity_json_schema