hphp/hack/src/generate_full_fidelity_data.ml

   1 (*
   2  * Copyright (c) 2016, Facebook, Inc.
   3  * All rights reserved.
   4  *
   5  * This source code is licensed under the MIT license found in the
   6  * LICENSE file in the "hack" directory of this source tree.
   7  *
   8  *)
   9
  10 module OcamlPrintf = Printf
  11 open Hh_prelude
  12 open Printf
  13 open Full_fidelity_schema
  14
  15 let full_fidelity_path_prefix = "hphp/hack/src/parser/"
  16
  17 let rust_keywords =
  18   [ "as"; "break"; "const"; "continue"; "crate"; "else"; "enum"; "extern";
  19     "false"; "fn"; "for"; "if"; "impl"; "in"; "let"; "loop"; "match"; "mod";
  20     "move"; "mut"; "pub"; "ref"; "return"; "self"; "Self"; "static"; "struct";
  21     "super"; "trait"; "true"; "type"; "unsafe"; "use"; "where"; "while";
  22     "async"; "await"; "dyn" ]
  23   [@@ocamlformat "disable"]
  24
  25 let escape_rust_keyword field_name =
  26   if List.mem rust_keywords field_name ~equal:String.equal then
  27     sprintf "%s_" field_name
  28   else
  29     field_name
  30
  31 type comment_style =
  32   | CStyle
  33   | MLStyle
  34 [@@deriving show]
  35
  36 let make_header comment_style (header_comment : string) : string =
  37   let (open_char, close_char) =
  38     match comment_style with
  39     | CStyle -> ("/*", '/')
  40     | MLStyle -> ("(", ')')
  41   in
  42   sprintf
  43     "%s*
  44  * Copyright (c) 2016, Facebook, Inc.
  45  * All rights reserved.
  46  *
  47  * This source code is licensed under the MIT license found in the
  48  * LICENSE file in the \"hack\" directory of this source tree. An additional
  49  * directory.
  50  *
  51  **
  52  *
  53  * THIS FILE IS @%s; DO NOT EDIT IT
  54  * To regenerate this file, run
  55  *
  56  *   buck run //hphp/hack/src:generate_full_fidelity
  57  *
  58  **
  59  *%s
  60  *%c"
  61     open_char
  62     (* Any file containing '@' followed by the word 'generated' is considered a
  63      * generated file in Phabricator. Cheeky trick to avoid making this script
  64      * being seen as generated. *)
  65     "generated"
  66     header_comment
  67     close_char
  68
  69 type valign =
  70   (string -> string -> string, unit, string, string -> string) format4
  71
  72 let all_tokens = given_text_tokens @ variable_text_tokens @ no_text_tokens
  73
  74 let align_fmt : 'a. ('a -> string) -> 'a list -> valign =
  75  fun f xs ->
  76   let folder acc x = max acc (String.length (f x)) in
  77   let width = List.fold_left ~f:folder ~init:0 xs in
  78   Scanf.format_from_string (sprintf "%%-%ds" width) "%-1s"
  79
  80 let kind_name_fmt = align_fmt (fun x -> x.kind_name) schema
  81
  82 let type_name_fmt = align_fmt (fun x -> x.type_name) schema
  83
  84 let trivia_kind_fmt = align_fmt (fun x -> x.trivia_kind) trivia_kinds
  85
  86 let token_kind_fmt = align_fmt (fun x -> x.token_kind) all_tokens
  87
  88 let omit_syntax_record =
  89   let names =
  90     SSet.of_list
  91       [
  92         "anonymous_function";
  93         "closure_type_specifier";
  94         "function_declaration";
  95         "function_declaration_header";
  96         "lambda_expression";
  97         "lambda_signature";
  98         "methodish_declaration";
  99       ]
 100   in
 101   (fun x -> not (SSet.mem x.type_name names))
 102
 103 module GenerateFFSyntaxType = struct
 104   let to_parse_tree x =
 105     if omit_syntax_record x then
 106       ""
 107     else
 108       let field_width = 50 - String.length x.prefix in
 109       let fmt = sprintf "%s_%%-%ds: t\n" x.prefix field_width in
 110       let mapper (f, _) = sprintf (Scanf.format_from_string fmt "%-1s") f in
 111       let fields = map_and_concat_separated "    ; " mapper x.fields in
 112       sprintf "  and %s =\n    { %s    }\n" x.type_name fields
 113
 114   let to_syntax x =
 115     let field_width = 50 - String.length x.prefix in
 116     let fmt = sprintf "%s_%%-%ds: t\n" x.prefix field_width in
 117     let mapper (f, _) = sprintf (Scanf.format_from_string fmt "%-1s") f in
 118     let fields = map_and_concat_separated "    ; " mapper x.fields in
 119     sprintf
 120       ("  | " ^^ kind_name_fmt ^^ " of\n    { %s    }\n")
 121       x.kind_name
 122       fields
 123
 124   let to_aggregate_type x =
 125     let aggregated_types = aggregation_of x in
 126     let (prefix, trim) = aggregate_type_pfx_trim x in
 127     let compact = Str.global_replace (Str.regexp trim) "" in
 128     let valign = align_fmt (fun x -> compact x.kind_name) aggregated_types in
 129     let type_name = aggregate_type_name x in
 130     let make_constructor ty =
 131       sprintf
 132         ("%s" ^^ valign ^^ " of %s")
 133         prefix
 134         (compact ty.kind_name)
 135         ty.type_name
 136     in
 137     let type_body = List.map ~f:make_constructor aggregated_types in
 138     sprintf
 139       "  and %s =\n  | %s\n"
 140       type_name
 141       (String.concat ~sep:"\n  | " type_body)
 142
 143   let full_fidelity_syntax_template : string =
 144     make_header
 145       MLStyle
 146       "
 147  * This module contains the type describing the structure of a syntax tree.
 148  *
 149  * The structure of the syntax tree is described by the collection of recursive
 150  * types that makes up the bulk of this file. The type `t` is the type of a node
 151  * in the syntax tree; each node has associated with it an arbitrary value of
 152  * type `SyntaxValue.t`, and syntax node proper, which has structure given by
 153  * the `syntax` type.
 154  *
 155  * Note that every child in the syntax tree is of type `t`, except for the
 156  * `Token.t` type. This should be the *only* child of a type other than `t`.
 157  * We are explicitly NOT attempting to impose a type structure on the parse
 158  * tree beyond what is already implied by the types here. For example,
 159  * we are not attempting to put into the type system here the restriction that
 160  * the children of a binary operator must be expressions. The reason for this
 161  * is because we are potentially parsing code as it is being typed, and we
 162  * do not want to restrict our ability to make good error recovery by imposing
 163  * a restriction that will only be valid in correct program text.
 164  *
 165  * That said, it would of course be ideal if the only children of a compound
 166  * statement were statements, and so on. But those invariants should be
 167  * imposed by the design of the parser, not by the type system of the syntax
 168  * tree code.
 169  *
 170  * We want to be able to use different kinds of tokens, with different
 171  * performance characteristics. Moreover, we want to associate arbitrary values
 172  * with the syntax nodes, so that we can construct syntax trees with various
 173  * properties -- trees that only know their widths and are thereby cheap to
 174  * serialize, trees that have full position data for each node, trees where the
 175  * tokens know their text and can therefore be edited, trees that have name
 176  * annotations or type annotations, and so on.
 177  *
 178  * We wish to associate arbitrary values with the syntax nodes so that we can
 179  * construct syntax trees with various properties -- trees that only know
 180  * their widths and are thereby cheap to serialize, trees that have full
 181  * position data for each node, trees where the tokens know their text and
 182  * can therefore be edited, trees that have name annotations or type
 183  * annotations, and so on.
 184  *
 185  * Therefore this module is functorized by the types for token and value to be
 186  * associated with the node."
 187     ^ "
 188 open Sexplib.Std
 189
 190 module type TokenType = sig
 191   module Trivia : Lexable_trivia_sig.LexableTrivia_S
 192   type t [@@deriving show, eq, sexp_of]
 193   val kind: t -> Full_fidelity_token_kind.t
 194   val to_json: t -> Hh_json.json
 195   val leading : t -> Trivia.t list
 196 end
 197
 198 module type SyntaxValueType = sig
 199   type t [@@deriving show, eq, sexp_of]
 200   val to_json: t -> Hh_json.json
 201 end
 202
 203 (* This functor describe the shape of a parse tree that has a particular kind of
 204  * token in the leaves, and a particular kind of value associated with each
 205  * node.
 206  *)
 207 module MakeSyntaxType(Token : TokenType)(SyntaxValue : SyntaxValueType) = struct
 208   type value = SyntaxValue.t [@@deriving show, eq, sexp_of]
 209   type t = { syntax : syntax ; value : value } [@@deriving show, eq, sexp_of]
 210 PARSE_TREE   and syntax =
 211   | Token                             of Token.t
 212   | Missing
 213   | SyntaxList                        of t list
 214 SYNTAX
 215     [@@deriving sexp_of]
 216 end
 217
 218 "
 219
 220   let full_fidelity_syntax_type =
 221     Full_fidelity_schema.make_template_file
 222       ~transformations:
 223         [
 224           { pattern = "PARSE_TREE"; func = to_parse_tree };
 225           { pattern = "SYNTAX"; func = to_syntax };
 226         ]
 227       ~aggregate_transformations:
 228         [
 229           {
 230             aggregate_pattern = "AGGREGATE_TYPES";
 231             aggregate_func = to_aggregate_type;
 232           };
 233         ]
 234       ~filename:(full_fidelity_path_prefix ^ "full_fidelity_syntax_type.ml")
 235       ~template:full_fidelity_syntax_template
 236       ()
 237 end
 238
 239 module GenerateFFRustSyntaxImplByRef = struct
 240   let to_kind x =
 241     sprintf
 242       "            SyntaxVariant::%s {..} => SyntaxKind::%s,\n"
 243       x.kind_name
 244       x.kind_name
 245
 246   let template =
 247     make_header CStyle ""
 248     ^ "
 249 use crate::{syntax_kind::SyntaxKind, lexable_token::LexableToken};
 250 use super::{syntax::Syntax, syntax_variant_generated::SyntaxVariant};
 251
 252 impl<T: LexableToken, V> Syntax<'_, T, V> {
 253     pub fn kind(&self) -> SyntaxKind {
 254         match &self.children {
 255             SyntaxVariant::Missing => SyntaxKind::Missing,
 256             SyntaxVariant::Token (t) => SyntaxKind::Token(t.kind()),
 257             SyntaxVariant::SyntaxList (_) => SyntaxKind::SyntaxList,
 258 TO_KIND        }
 259     }
 260 }
 261     "
 262
 263   let full_fidelity_syntax =
 264     Full_fidelity_schema.make_template_file
 265       ~transformations:[{ pattern = "TO_KIND"; func = to_kind }]
 266       ~filename:
 267         (full_fidelity_path_prefix ^ "syntax_by_ref/syntax_impl_generated.rs")
 268       ~template
 269       ()
 270 end
 271
 272 module GenerateSyntaxSerialize = struct
 273   let match_arm x =
 274     let get_field x = escape_rust_keyword (fst x) in
 275     let serialize_fields =
 276       map_and_concat_separated
 277         "\n"
 278         (fun y ->
 279           sprintf
 280             "ss.serialize_field(\"%s_%s\", &self.with(%s))?;"
 281             x.prefix
 282             (fst y)
 283             (get_field y))
 284         x.fields
 285     in
 286     let fields = map_and_concat_separated "," get_field x.fields in
 287     sprintf
 288       "SyntaxVariant::%s (%sChildren{%s} ) => {
 289       let mut ss = s.serialize_struct(\"\", %d)?;
 290       ss.serialize_field(\"kind\", \"%s\")?;
 291       %s
 292       ss.end()
 293 } \n"
 294       x.kind_name
 295       x.kind_name
 296       fields
 297       (1 + List.length x.fields)
 298       x.description
 299       serialize_fields
 300
 301   let template =
 302     make_header CStyle ""
 303     ^ "
 304 use super::{serialize::WithContext, syntax::Syntax, syntax_variant_generated::*};
 305 use serde::{ser::SerializeStruct, Serialize, Serializer};
 306
 307 impl<'a, T, V> Serialize for WithContext<'a, Syntax<'a, T, V>>
 308 where
 309     T: 'a,
 310     WithContext<'a, T>: Serialize,
 311 {
 312     fn serialize<S: Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
 313         match self.1.children {
 314             SyntaxVariant::Missing => {
 315                 let mut ss = s.serialize_struct(\"\", 1)?;
 316                 ss.serialize_field(\"kind\", \"missing\")?;
 317                 ss.end()
 318             }
 319             SyntaxVariant::Token(ref t) => {
 320                 let mut ss = s.serialize_struct(\"\", 2)?;
 321                 ss.serialize_field(\"kind\", \"token\")?;
 322                 ss.serialize_field(\"token\", &self.with(t))?;
 323                 ss.end()
 324             }
 325             SyntaxVariant::SyntaxList(l) => {
 326                 let mut ss = s.serialize_struct(\"\", 2)?;
 327                 ss.serialize_field(\"kind\", \"list\")?;
 328                 ss.serialize_field(\"elements\", &self.with(l))?;
 329                 ss.end()
 330             }
 331             MATCH_ARMS
 332         }
 333     }
 334 }
 335 "
 336
 337   let gen =
 338     Full_fidelity_schema.make_template_file
 339       ~transformations:[{ pattern = "MATCH_ARMS"; func = match_arm }]
 340       ~filename:
 341         (full_fidelity_path_prefix
 342         ^ "syntax_by_ref/syntax_serialize_generated.rs")
 343       ~template
 344       ()
 345 end
 346
 347 module GenerateFFRustSyntaxVariantByRef = struct
 348   let to_syntax_variant_children x =
 349     let mapper (f, _) =
 350       sprintf "    pub %s: Syntax<'a, T, V>," (escape_rust_keyword f)
 351     in
 352     let fields = map_and_concat_separated "\n" mapper x.fields in
 353     sprintf
 354       "#[derive(Debug, Clone)]\npub struct %sChildren<'a, T, V> {\n%s\n}\n\n"
 355       x.kind_name
 356       fields
 357
 358   let to_syntax_variant x =
 359     sprintf "    %s(&'a %sChildren<'a, T, V>),\n" x.kind_name x.kind_name
 360
 361   let full_fidelity_syntax_template =
 362     make_header CStyle ""
 363     ^ "
 364 use super::{
 365     syntax::Syntax,
 366     syntax_children_iterator::SyntaxChildrenIterator,
 367 };
 368
 369 #[derive(Debug, Clone)]
 370 pub enum SyntaxVariant<'a, T, V> {
 371     Token(T),
 372     Missing,
 373     SyntaxList(&'a [Syntax<'a, T, V>]),
 374 SYNTAX_VARIANT}
 375
 376 SYNTAX_CHILDREN
 377
 378 impl<'a, T, V> SyntaxVariant<'a, T, V> {
 379     pub fn iter_children(&'a self) -> SyntaxChildrenIterator<'a, T, V> {
 380         SyntaxChildrenIterator {
 381             syntax: self,
 382             index: 0,
 383             index_back: 0,
 384         }
 385     }
 386 }
 387 "
 388
 389   let full_fidelity_syntax =
 390     Full_fidelity_schema.make_template_file
 391       ~transformations:
 392         [
 393           { pattern = "SYNTAX_VARIANT"; func = to_syntax_variant };
 394           { pattern = "SYNTAX_CHILDREN"; func = to_syntax_variant_children };
 395         ]
 396       ~filename:
 397         (full_fidelity_path_prefix ^ "syntax_by_ref/syntax_variant_generated.rs")
 398       ~template:full_fidelity_syntax_template
 399       ()
 400 end
 401
 402 module GenerateSyntaxChildrenIterator = struct
 403   let to_iter_children x =
 404     let index = ref 0 in
 405     let mapper (f, _) =
 406       let res = sprintf "%d => Some(&x.%s)," !index (escape_rust_keyword f) in
 407       let () = incr index in
 408       res
 409     in
 410     let fields =
 411       map_and_concat_separated "\n                    " mapper x.fields
 412     in
 413     sprintf
 414       "            %s(x) => {
 415                 get_index(%d).and_then(|index| { match index {
 416                         %s
 417                         _ => None,
 418                     }
 419                 })
 420             },\n"
 421       x.kind_name
 422       (List.length x.fields)
 423       fields
 424
 425   let full_fidelity_syntax_template =
 426     make_header CStyle ""
 427     ^ "
 428 use super::{
 429     syntax_children_iterator::*,
 430     syntax_variant_generated::*,
 431     syntax::*
 432 };
 433
 434 #[allow(clippy::assign_op_pattern)]
 435 #[allow(clippy::let_and_return)]
 436
 437 impl<'a, T, V> SyntaxChildrenIterator<'a, T, V> {
 438     pub fn next_impl(&mut self, direction : bool) -> Option<&'a Syntax<'a, T, V>> {
 439         use SyntaxVariant::*;
 440         let get_index = |len| {
 441             let back_index_plus_1 = len - self.index_back;
 442             if back_index_plus_1 <= self.index {
 443                 return None
 444             }
 445             if direction {
 446                 Some (self.index)
 447             } else {
 448                 Some (back_index_plus_1 - 1)
 449             }
 450         };
 451         let res = match self.syntax {
 452             Missing => None,
 453             Token (_) => None,
 454             SyntaxList(elems) => {
 455                 get_index(elems.len()).and_then(|x| elems.get(x))
 456             },
 457 ITER_CHILDREN
 458         };
 459         if res.is_some() {
 460             if direction {
 461                 self.index = self.index + 1
 462             } else {
 463                 self.index_back = self.index_back + 1
 464             }
 465         }
 466         res
 467     }
 468 }
 469     "
 470
 471   let full_fidelity_syntax =
 472     Full_fidelity_schema.make_template_file
 473       ~transformations:[{ pattern = "ITER_CHILDREN"; func = to_iter_children }]
 474       ~filename:
 475         (full_fidelity_path_prefix
 476         ^ "syntax_by_ref/syntax_children_iterator_generated.rs")
 477       ~template:full_fidelity_syntax_template
 478       ()
 479 end
 480
 481 module GenerateSyntaxTypeImpl = struct
 482   let to_syntax_constructors x =
 483     let mapper (f, _) = sprintf "%s: Self" (escape_rust_keyword f) in
 484     let args = map_and_concat_separated ", " mapper x.fields in
 485     let mapper (f, _) = sprintf "%s" (escape_rust_keyword f) in
 486     let fields = map_and_concat_separated ",\n            " mapper x.fields in
 487     sprintf
 488       "    fn make_%s(ctx: &C, %s) -> Self {
 489         let syntax = SyntaxVariant::%s(ctx.get_arena().alloc(%sChildren {
 490             %s,
 491         }));
 492         let value = V::from_values(syntax.iter_children().map(|child| &child.value));
 493         Self::make(syntax, value)
 494     }\n\n"
 495       x.type_name
 496       args
 497       x.kind_name
 498       x.kind_name
 499       fields
 500
 501   let full_fidelity_syntax_template =
 502     make_header CStyle ""
 503     ^ "
 504 use super::{
 505     has_arena::HasArena,
 506     syntax::*, syntax_variant_generated::*,
 507 };
 508 use crate::{
 509     lexable_token::LexableToken,
 510     syntax::{SyntaxType, SyntaxValueType},
 511 };
 512
 513 #[allow(clippy::assign_op_pattern)]
 514 #[allow(clippy::let_and_return)]
 515
 516 impl<'a, C, T, V> SyntaxType<C> for Syntax<'a, T, V>
 517 where
 518     T: LexableToken + Copy,
 519     V: SyntaxValueType<T>,
 520     C: HasArena<'a>,
 521 {
 522 SYNTAX_CONSTRUCTORS }
 523 "
 524
 525   let full_fidelity_syntax =
 526     Full_fidelity_schema.make_template_file
 527       ~transformations:
 528         [{ pattern = "SYNTAX_CONSTRUCTORS"; func = to_syntax_constructors }]
 529       ~filename:
 530         (full_fidelity_path_prefix
 531         ^ "syntax_by_ref/syntax_type_impl_generated.rs")
 532       ~template:full_fidelity_syntax_template
 533       ()
 534 end
 535
 536 module GenerateFFRustSyntax = struct
 537   let from_children x =
 538     let mapper prefix (f, _) =
 539       sprintf "%s_%s : (Box::new(ts.pop().unwrap()))" prefix f
 540     in
 541     let fields =
 542       map_and_concat_separated ",\n       " (mapper x.prefix) x.fields
 543     in
 544     sprintf
 545       "SyntaxKind::%s => SyntaxVariant::%s {
 546         %s
 547     },"
 548       x.kind_name
 549       x.kind_name
 550       fields
 551
 552   let to_kind x =
 553     sprintf
 554       "            SyntaxVariant::%s {..} => SyntaxKind::%s,\n"
 555       x.kind_name
 556       x.kind_name
 557
 558   let to_children x =
 559     let mapper prefix (f, _) = sprintf "&*%s_%s" prefix f in
 560     let fields2 =
 561       map_and_concat_separated ",\n       " (mapper x.prefix) x.fields
 562     in
 563     let mapper prefix (f, _) = sprintf "ref %s_%s" prefix f in
 564     let fields =
 565       map_and_concat_separated ",\n       " (mapper x.prefix) x.fields
 566     in
 567     sprintf
 568       "SyntaxVariant::%s {
 569         %s
 570     } => vec![%s],"
 571       x.kind_name
 572       fields
 573       fields2
 574
 575   let to_iter_children x =
 576     let index = ref 0 in
 577     let mapper prefix (f, _) =
 578       let res = sprintf "%d => Some(&x.%s_%s)," !index prefix f in
 579       let () = incr index in
 580       res
 581     in
 582     let fields =
 583       map_and_concat_separated
 584         "\n                    "
 585         (mapper x.prefix)
 586         x.fields
 587     in
 588     sprintf
 589       "            %s(x) => {
 590                 get_index(%d).and_then(|index| { match index {
 591                         %s
 592                         _ => None,
 593                     }
 594                 })
 595             },\n"
 596       x.kind_name
 597       (List.length x.fields)
 598       fields
 599
 600   let syntax_as_slice x =
 601     (* SAFETY: We have `#[repr(C)]` on all the children structs, and all their
 602        fields are of type Syntax. So a child struct with 3 fields should have
 603        the same layout as the array `[Syntax; 3]`, and it should be valid to
 604        take a slice of length 3 from a pointer to its first field. *)
 605     sprintf
 606       "            SyntaxVariant::%s(x) => unsafe { std::slice::from_raw_parts(&x.%s_%s, %d) },\n"
 607       x.kind_name
 608       x.prefix
 609       (fst (List.hd_exn x.fields))
 610       (List.length x.fields)
 611
 612   let syntax_as_slice_mut x =
 613     (* SAFETY: As above in `syntax_as_slice` *)
 614     sprintf
 615       "            SyntaxVariant::%s(x) => unsafe { std::slice::from_raw_parts_mut(&mut x.%s_%s, %d) },\n"
 616       x.kind_name
 617       x.prefix
 618       (fst (List.hd_exn x.fields))
 619       (List.length x.fields)
 620
 621   let fold_over_owned x =
 622     let fold_mapper prefix (f, _) =
 623       sprintf "let acc = f(%s_%s, acc)" prefix f
 624     in
 625     let fold_fields =
 626       map_and_concat_separated
 627         ";\n                "
 628         (fold_mapper x.prefix)
 629         x.fields
 630     in
 631     let destructure_mapper prefix (f, _) = sprintf "%s_%s" prefix f in
 632     let destructure_fields =
 633       map_and_concat_separated ", " (destructure_mapper x.prefix) x.fields
 634     in
 635     sprintf
 636       "            SyntaxVariant::%s(x) => {
 637                 let %sChildren { %s } = *x;
 638                 %s;
 639                 acc
 640             },\n"
 641       x.kind_name
 642       x.kind_name
 643       destructure_fields
 644       fold_fields
 645
 646   let to_syntax_variant x =
 647     sprintf "    %s(Box<%sChildren<T, V>>),\n" x.kind_name x.kind_name
 648
 649   let to_syntax_variant_children x =
 650     let mapper prefix (f, _) =
 651       sprintf "    pub %s_%s: Syntax<T, V>," prefix f
 652     in
 653     let fields = map_and_concat_separated "\n" (mapper x.prefix) x.fields in
 654     (* NB: The `#[repr(C)]` here is required for the `from_raw_parts` and
 655        `from_raw_parts_mut` above (in `syntax_as_slice` and
 656        `syntax_as_slice_mut`) to be valid. *)
 657     sprintf
 658       "#[derive(Debug, Clone)]\n#[repr(C)]\npub struct %sChildren<T, V> {\n%s\n}\n\n"
 659       x.kind_name
 660       fields
 661
 662   let to_syntax_constructors x =
 663     let mapper prefix (f, _) = sprintf "%s_%s: Self" prefix f in
 664     let args = map_and_concat_separated ", " (mapper x.prefix) x.fields in
 665     let mapper prefix (f, _) = sprintf "%s_%s" prefix f in
 666     let fields =
 667       map_and_concat_separated ",\n            " (mapper x.prefix) x.fields
 668     in
 669     sprintf
 670       "    fn make_%s(_: &C, %s) -> Self {
 671         let syntax = SyntaxVariant::%s(Box::new(%sChildren {
 672             %s,
 673         }));
 674         let value = V::from_values(syntax.iter_children().map(|child| &child.value));
 675         Self::make(syntax, value)
 676     }\n\n"
 677       x.type_name
 678       args
 679       x.kind_name
 680       x.kind_name
 681       fields
 682
 683   let to_syntax_from_children x =
 684     let mapper (f, _) =
 685       sprintf "%s_%s: ts.pop().unwrap(),\n                 " x.prefix f
 686     in
 687     let fields = map_and_concat mapper (List.rev x.fields) in
 688     sprintf
 689       "             (SyntaxKind::%s, %d) => SyntaxVariant::%s(Box::new(%sChildren {
 690                  %s
 691              })),\n"
 692       x.kind_name
 693       (List.length x.fields)
 694       x.kind_name
 695       x.kind_name
 696       fields
 697
 698   let full_fidelity_syntax_template =
 699     make_header CStyle ""
 700     ^ "
 701 use crate::lexable_token::LexableToken;
 702 use crate::syntax::*;
 703 use crate::syntax_kind::SyntaxKind;
 704
 705 #[allow(clippy::assign_op_pattern)]
 706 #[allow(clippy::let_and_return)]
 707
 708 impl<T, V, C> SyntaxType<C> for Syntax<T, V>
 709 where
 710     T: LexableToken,
 711     V: SyntaxValueType<T>,
 712 {
 713 SYNTAX_CONSTRUCTORS }
 714
 715 #[allow(clippy::assign_op_pattern)]
 716 #[allow(clippy::let_and_return)]
 717
 718 impl<T, V> Syntax<T, V>
 719 where
 720     T: LexableToken,
 721 {
 722     pub fn fold_over_children_owned<U>(
 723         f: &dyn Fn(Self, U) -> U,
 724         acc: U,
 725         syntax: SyntaxVariant<T, V>,
 726     ) -> U {
 727         match syntax {
 728             SyntaxVariant::Missing => acc,
 729             SyntaxVariant::Token (_) => acc,
 730             SyntaxVariant::SyntaxList(elems) => {
 731                 let mut acc = acc;
 732                 for item in elems {
 733                     acc = f(item, acc);
 734                 }
 735                 acc
 736             },
 737 FOLD_OVER_CHILDREN_OWNED
 738         }
 739     }
 740
 741     pub fn kind(&self) -> SyntaxKind {
 742         match &self.syntax {
 743             SyntaxVariant::Missing => SyntaxKind::Missing,
 744             SyntaxVariant::Token (t) => SyntaxKind::Token(t.kind()),
 745             SyntaxVariant::SyntaxList (_) => SyntaxKind::SyntaxList,
 746 TO_KIND        }
 747     }
 748
 749     pub fn from_children(kind : SyntaxKind, mut ts : Vec<Self>) -> SyntaxVariant<T, V> {
 750          match (kind, ts.len()) {
 751              (SyntaxKind::Missing, 0) => SyntaxVariant::Missing,
 752              (SyntaxKind::SyntaxList, _) => SyntaxVariant::SyntaxList(ts),
 753 SYNTAX_FROM_CHILDREN             _ => panic!(\"from_children called with wrong number of children\"),
 754          }
 755     }
 756
 757     pub fn children(&self) -> &[Self] {
 758         match &self.syntax {
 759             SyntaxVariant::Missing => &[],
 760             SyntaxVariant::Token(..) => &[],
 761             SyntaxVariant::SyntaxList(l) => l.as_slice(),
 762 SYNTAX_AS_SLICE        }
 763     }
 764
 765     pub fn children_mut(&mut self) -> &mut [Self] {
 766         match &mut self.syntax {
 767             SyntaxVariant::Missing => &mut [],
 768             SyntaxVariant::Token(..) => &mut [],
 769             SyntaxVariant::SyntaxList(l) => l.as_mut_slice(),
 770 SYNTAX_AS_SLICE_MUT        }
 771     }
 772 }
 773
 774 SYNTAX_CHILDREN
 775 #[derive(Debug, Clone)]
 776 pub enum SyntaxVariant<T, V> {
 777     Token(Box<T>),
 778     Missing,
 779     SyntaxList(Vec<Syntax<T, V>>),
 780 SYNTAX_VARIANT}
 781
 782 #[allow(clippy::assign_op_pattern)]
 783 #[allow(clippy::let_and_return)]
 784
 785 impl<'a, T, V> SyntaxChildrenIterator<'a, T, V> {
 786     pub fn next_impl(&mut self, direction : bool) -> Option<&'a Syntax<T, V>> {
 787         use SyntaxVariant::*;
 788         let get_index = |len| {
 789             let back_index_plus_1 = len - self.index_back;
 790             if back_index_plus_1 <= self.index {
 791                 return None
 792             }
 793             if direction {
 794                 Some (self.index)
 795             } else {
 796                 Some (back_index_plus_1 - 1)
 797             }
 798         };
 799         let res = match &self.syntax {
 800             Missing => None,
 801             Token (_) => None,
 802             SyntaxList(elems) => {
 803                 get_index(elems.len()).and_then(|x| elems.get(x))
 804             },
 805 ITER_CHILDREN
 806         };
 807         if res.is_some() {
 808             if direction {
 809                 self.index = self.index + 1
 810             } else {
 811                 self.index_back = self.index_back + 1
 812             }
 813         }
 814         res
 815     }
 816 }
 817 "
 818
 819   let full_fidelity_syntax =
 820     Full_fidelity_schema.make_template_file
 821       ~transformations:
 822         [
 823           { pattern = "SYNTAX_VARIANT"; func = to_syntax_variant };
 824           { pattern = "SYNTAX_CHILDREN"; func = to_syntax_variant_children };
 825           { pattern = "SYNTAX_CONSTRUCTORS"; func = to_syntax_constructors };
 826           { pattern = "ITER_CHILDREN"; func = to_iter_children };
 827           { pattern = "FOLD_OVER_CHILDREN_OWNED"; func = fold_over_owned };
 828           { pattern = "TO_KIND"; func = to_kind };
 829           { pattern = "SYNTAX_FROM_CHILDREN"; func = to_syntax_from_children };
 830           { pattern = "SYNTAX_AS_SLICE"; func = syntax_as_slice };
 831           { pattern = "SYNTAX_AS_SLICE_MUT"; func = syntax_as_slice_mut };
 832         ]
 833       ~filename:(full_fidelity_path_prefix ^ "syntax_generated.rs")
 834       ~template:full_fidelity_syntax_template
 835       ()
 836 end
 837
 838 (* GenerateFFRustSyntax *)
 839
 840 module GenerateFFRustSyntaxType = struct
 841   let to_kind x =
 842     sprintf
 843       "            SyntaxVariant::%s {..} => SyntaxKind::%s,\n"
 844       x.kind_name
 845       x.kind_name
 846
 847   let to_children x =
 848     let mapper prefix (f, _) = sprintf "&*%s_%s" prefix f in
 849     let fields2 =
 850       map_and_concat_separated ",\n       " (mapper x.prefix) x.fields
 851     in
 852     let mapper prefix (f, _) = sprintf "ref %s_%s" prefix f in
 853     let fields =
 854       map_and_concat_separated ",\n       " (mapper x.prefix) x.fields
 855     in
 856     sprintf
 857       "SyntaxVariant::%s {
 858         %s
 859     } => vec![%s],"
 860       x.kind_name
 861       fields
 862       fields2
 863
 864   let into_children x =
 865     let mapper prefix (f, _) = sprintf "x.%s_%s" prefix f in
 866     let fields =
 867       map_and_concat_separated ",\n                " (mapper x.prefix) x.fields
 868     in
 869     sprintf
 870       "            SyntaxVariant::%s (x) => { vec!(
 871                 %s
 872             )},\n"
 873       x.kind_name
 874       fields
 875
 876   let fold_over x =
 877     let mapper prefix (f, _) = sprintf "let acc = f(&x.%s_%s, acc)" prefix f in
 878     let fields =
 879       map_and_concat_separated ";\n                " (mapper x.prefix) x.fields
 880     in
 881     sprintf
 882       "            SyntaxVariant::%s(x) => {
 883                 %s;
 884                 acc
 885             },\n"
 886       x.kind_name
 887       fields
 888
 889   let to_syntax_constructors x =
 890     let mapper prefix (f, _) = sprintf "%s_%s: Self" prefix f in
 891     let args = map_and_concat_separated ", " (mapper x.prefix) x.fields in
 892     sprintf "    fn make_%s(ctx: &C, %s) -> Self;\n" x.type_name args
 893
 894   let full_fidelity_syntax_template =
 895     make_header CStyle ""
 896     ^ "
 897 use crate::syntax::*;
 898
 899 pub trait SyntaxType<C>: SyntaxTypeBase<C>
 900 {
 901 SYNTAX_CONSTRUCTORS
 902 }
 903 "
 904
 905   let full_fidelity_syntax =
 906     Full_fidelity_schema.make_template_file
 907       ~transformations:
 908         [{ pattern = "SYNTAX_CONSTRUCTORS"; func = to_syntax_constructors }]
 909       ~filename:(full_fidelity_path_prefix ^ "syntax_type.rs")
 910       ~template:full_fidelity_syntax_template
 911       ()
 912 end
 913
 914 (* GenerateFFRustSyntaxType *)
 915
 916 module GenerateFFSyntaxSig = struct
 917   let to_constructor_methods x =
 918     let mapper1 (_f, _) = " t ->" in
 919     let fields1 = map_and_concat mapper1 x.fields in
 920     sprintf "  val make_%s :%s t\n" x.type_name fields1
 921
 922   let to_type_tests x = sprintf "  val is_%s : t -> bool\n" x.type_name
 923
 924   let to_syntax x =
 925     let field_width = 50 - String.length x.prefix in
 926     let fmt = sprintf "%s_%%-%ds: t\n" x.prefix field_width in
 927     let mapper (f, _) = sprintf (Scanf.format_from_string fmt "%-1s") f in
 928     let fields = map_and_concat_separated "    ; " mapper x.fields in
 929     sprintf
 930       ("  | " ^^ kind_name_fmt ^^ " of\n    { %s    }\n")
 931       x.kind_name
 932       fields
 933
 934   let full_fidelity_syntax_template : string =
 935     make_header
 936       MLStyle
 937       "
 938 * This module contains a signature which can be used to describe the public
 939 * surface area of a constructable syntax tree.
 940 "
 941     ^ "
 942
 943 module TriviaKind = Full_fidelity_trivia_kind
 944 module TokenKind = Full_fidelity_token_kind
 945
 946 module type Syntax_S = sig
 947   module Token : Lexable_token_sig.LexableToken_S
 948   type value [@@deriving show, eq, sexp_of]
 949   type t = { syntax : syntax ; value : value } [@@deriving show, eq, sexp_of]
 950   and syntax =
 951   | Token                             of Token.t
 952   | Missing
 953   | SyntaxList                        of t list
 954 SYNTAX
 955     [@@deriving sexp_of]
 956
 957   val rust_parse :
 958     Full_fidelity_source_text.t ->
 959     Full_fidelity_parser_env.t ->
 960     unit * t * Full_fidelity_syntax_error.t list * Rust_pointer.t option
 961   val rust_parser_errors :
 962     Full_fidelity_source_text.t ->
 963     Rust_pointer.t ->
 964     ParserOptions.ffi_t ->
 965     Full_fidelity_syntax_error.t list
 966   val has_leading_trivia : TriviaKind.t -> Token.t -> bool
 967   val to_json : ?with_value:bool -> ?ignore_missing:bool -> t -> Hh_json.json
 968   val extract_text : t -> string option
 969   val is_in_body : t -> int -> bool
 970   val syntax_node_to_list : t -> t list
 971   val width : t -> int
 972   val full_width : t -> int
 973   val trailing_width : t -> int
 974   val leading_width : t -> int
 975   val leading_token : t -> Token.t option
 976   val children : t -> t list
 977   val syntax : t -> syntax
 978   val kind : t -> Full_fidelity_syntax_kind.t
 979   val value : t -> value
 980   val make_token : Token.t -> t
 981   val get_token : t -> Token.t option
 982   val all_tokens : t -> Token.t list
 983   val make_missing : Full_fidelity_source_text.t -> int -> t
 984   val make_list : Full_fidelity_source_text.t -> int -> t list -> t
 985   val is_namespace_prefix : t -> bool
 986   val syntax_list_fold : init:'a -> f:('a -> t -> 'a) -> t -> 'a
 987 CONSTRUCTOR_METHODS
 988
 989   val position : Relative_path.t -> t -> Pos.t option
 990   val offset : t -> int option
 991   val is_missing : t -> bool
 992   val is_list : t -> bool
 993 TYPE_TESTS
 994
 995   val is_specific_token : TokenKind.t -> t -> bool
 996   val is_loop_statement : t -> bool
 997   val is_external       : t -> bool
 998   val is_name           : t -> bool
 999   val is_construct      : t -> bool
1000   val is_static         : t -> bool
1001   val is_private        : t -> bool
1002   val is_public         : t -> bool
1003   val is_protected      : t -> bool
1004   val is_abstract       : t -> bool
1005   val is_final          : t -> bool
1006   val is_async          : t -> bool
1007   val is_void           : t -> bool
1008   val is_left_brace     : t -> bool
1009   val is_ellipsis       : t -> bool
1010   val is_comma          : t -> bool
1011   val is_ampersand      : t -> bool
1012   val is_inout          : t -> bool
1013
1014
1015 end
1016 "
1017
1018   let full_fidelity_syntax_sig =
1019     Full_fidelity_schema.make_template_file
1020       ~transformations:
1021         [
1022           { pattern = "SYNTAX"; func = to_syntax };
1023           { pattern = "CONSTRUCTOR_METHODS"; func = to_constructor_methods };
1024           { pattern = "TYPE_TESTS"; func = to_type_tests };
1025         ]
1026       ~filename:(full_fidelity_path_prefix ^ "syntax_sig.ml")
1027       ~template:full_fidelity_syntax_template
1028       ()
1029 end
1030
1031 (* GenerateFFSyntaxSig *)
1032
1033 module GenerateFFSmartConstructors = struct
1034   let full_fidelity_smart_constructors_template : string =
1035     make_header
1036       MLStyle
1037       "
1038  * This module contains a signature which can be used to describe smart
1039  * constructors.
1040   "
1041     ^ "
1042
1043 module ParserEnv = Full_fidelity_parser_env
1044
1045 module type SmartConstructors_S = sig
1046   module Token : Lexable_token_sig.LexableToken_S
1047
1048   type t (* state *) [@@deriving show, sexp_of]
1049
1050   type r (* smart constructor return type *) [@@deriving show]
1051
1052   val rust_parse :
1053     Full_fidelity_source_text.t ->
1054     ParserEnv.t ->
1055     t * r * Full_fidelity_syntax_error.t list * Rust_pointer.t option
1056
1057   val initial_state : ParserEnv.t -> t
1058 end
1059 "
1060
1061   let full_fidelity_smart_constructors =
1062     Full_fidelity_schema.make_template_file
1063       ~transformations:[]
1064       ~filename:
1065         (full_fidelity_path_prefix ^ "smart_constructors/smartConstructors.ml")
1066       ~template:full_fidelity_smart_constructors_template
1067       ()
1068 end
1069
1070 (* GenerateFFSmartConstructors *)
1071
1072 module GenerateFFRustSmartConstructors = struct
1073   let to_make_methods x =
1074     let fields =
1075       List.mapi x.fields ~f:(fun i _ ->
1076           "arg" ^ string_of_int i ^ ": Self::Output")
1077     in
1078     let stack = String.concat ~sep:", " fields in
1079     sprintf "    fn make_%s(&mut self, %s) -> Self::Output;\n" x.type_name stack
1080
1081   let full_fidelity_smart_constructors_template : string =
1082     make_header CStyle ""
1083     ^ "
1084 use parser_core_types::token_factory::TokenFactory;
1085 use parser_core_types::lexable_token::LexableToken;
1086
1087 pub type Token<S> = <<S as SmartConstructors>::Factory as TokenFactory>::Token;
1088 pub type Trivia<S> = <Token<S> as LexableToken>::Trivia;
1089
1090 pub trait SmartConstructors: Clone {
1091     type Factory: TokenFactory;
1092     type State;
1093     type Output;
1094
1095     fn state_mut(&mut self) -> &mut Self::State;
1096     fn into_state(self) -> Self::State;
1097     fn token_factory_mut(&mut self) -> &mut Self::Factory;
1098
1099     fn make_missing(&mut self, offset : usize) -> Self::Output;
1100     fn make_token(&mut self, arg0: Token<Self>) -> Self::Output;
1101     fn make_list(&mut self, arg0: Vec<Self::Output>, offset: usize) -> Self::Output;
1102
1103     fn begin_enumerator(&mut self) {}
1104     fn begin_enum_class_enumerator(&mut self) {}
1105     fn begin_constant_declarator(&mut self) {}
1106
1107 MAKE_METHODS
1108 }
1109 "
1110
1111   let full_fidelity_smart_constructors =
1112     Full_fidelity_schema.make_template_file
1113       ~transformations:[{ pattern = "MAKE_METHODS"; func = to_make_methods }]
1114       ~filename:(full_fidelity_path_prefix ^ "smart_constructors_generated.rs")
1115       ~template:full_fidelity_smart_constructors_template
1116       ()
1117 end
1118
1119 (* GenerateFFRustSmartConstructors *)
1120
1121 module GenerateFFRustPositionedSmartConstructors = struct
1122   let to_constructor_methods x =
1123     let args =
1124       List.mapi x.fields ~f:(fun i _ -> sprintf "arg%d: Self::Output" i)
1125     in
1126     let args = String.concat ~sep:", " args in
1127     let fwd_args = List.mapi x.fields ~f:(fun i _ -> sprintf "arg%d" i) in
1128     let fwd_args = String.concat ~sep:", " fwd_args in
1129     sprintf
1130       "    fn make_%s(&mut self, %s) -> Self::Output {
1131         <Self as SyntaxSmartConstructors<S, TF, St>>::make_%s(self, %s)
1132     }\n\n"
1133       x.type_name
1134       args
1135       x.type_name
1136       fwd_args
1137
1138   let positioned_smart_constructors_template : string =
1139     make_header CStyle ""
1140     ^ "
1141
1142
1143 use parser_core_types::{
1144     syntax::*,
1145     lexable_token::LexableToken,
1146     token_factory::TokenFactory,
1147 };
1148 use smart_constructors::SmartConstructors;
1149 use syntax_smart_constructors::{SyntaxSmartConstructors, StateType};
1150
1151 #[derive(Clone)]
1152 pub struct PositionedSmartConstructors<S, TF, St: StateType<S>> {
1153     pub state: St,
1154     token_factory: TF,
1155     phantom_s: std::marker::PhantomData<S>,
1156 }
1157
1158 impl<S, TF, St: StateType<S>> PositionedSmartConstructors<S, TF, St> {
1159     pub fn new(state: St, token_factory: TF) -> Self {
1160         Self { state, token_factory, phantom_s: std::marker::PhantomData }
1161     }
1162 }
1163
1164 impl<S, TF, St> SyntaxSmartConstructors<S, TF, St> for PositionedSmartConstructors<S, TF, St>
1165 where
1166     TF: TokenFactory<Token = S::Token>,
1167     St: StateType<S>,
1168     S: SyntaxType<St> + Clone,
1169     S::Token: LexableToken,
1170 {}
1171
1172 impl<S, TF, St> SmartConstructors for PositionedSmartConstructors<S, TF, St>
1173 where
1174     TF: TokenFactory<Token = S::Token>,
1175     S::Token: LexableToken,
1176     S: SyntaxType<St> + Clone,
1177     St: StateType<S>,
1178 {
1179     type Factory = TF;
1180     type State = St;
1181     type Output = S;
1182
1183     fn state_mut(&mut self) -> &mut St {
1184        &mut self.state
1185     }
1186
1187     fn into_state(self) -> St {
1188       self.state
1189     }
1190
1191     fn token_factory_mut(&mut self) -> &mut Self::Factory {
1192         &mut self.token_factory
1193     }
1194
1195     fn make_missing(&mut self, offset: usize) -> Self::Output {
1196         <Self as SyntaxSmartConstructors<S, TF, St>>::make_missing(self, offset)
1197     }
1198
1199     fn make_token(&mut self, offset: <Self::Factory as TokenFactory>::Token) -> Self::Output {
1200         <Self as SyntaxSmartConstructors<S, TF, St>>::make_token(self, offset)
1201     }
1202
1203     fn make_list(&mut self, lst: Vec<Self::Output>, offset: usize) -> Self::Output {
1204         <Self as SyntaxSmartConstructors<S, TF, St>>::make_list(self, lst, offset)
1205     }
1206 CONSTRUCTOR_METHODS}
1207 "
1208
1209   let positioned_smart_constructors =
1210     Full_fidelity_schema.make_template_file
1211       ~transformations:
1212         [{ pattern = "CONSTRUCTOR_METHODS"; func = to_constructor_methods }]
1213       ~filename:(full_fidelity_path_prefix ^ "positioned_smart_constructors.rs")
1214       ~template:positioned_smart_constructors_template
1215       ()
1216 end
1217
1218 module GenerateFFSyntaxSmartConstructors = struct
1219   let full_fidelity_syntax_smart_constructors_template : string =
1220     make_header
1221       MLStyle
1222       "
1223  * This module contains smart constructors implementation that can be used to
1224  * build AST.
1225  "
1226     ^ "
1227 open Sexplib.Std
1228
1229 module type SC_S = SmartConstructors.SmartConstructors_S
1230
1231 module ParserEnv = Full_fidelity_parser_env
1232
1233 module type State_S = sig
1234   type r [@@deriving show]
1235
1236   type t [@@deriving show, sexp_of]
1237
1238   val initial : ParserEnv.t -> t
1239
1240   val next : t -> r list -> t
1241 end
1242
1243 module type RustParser_S = sig
1244   type t
1245
1246   type r
1247
1248   val rust_parse :
1249     Full_fidelity_source_text.t ->
1250     ParserEnv.t ->
1251     t * r * Full_fidelity_syntax_error.t list * Rust_pointer.t option
1252 end
1253
1254 module WithSyntax (Syntax : Syntax_sig.Syntax_S) = struct
1255   module WithState (State : State_S with type r = Syntax.t) = struct
1256     module WithRustParser
1257         (RustParser : RustParser_S with type t = State.t with type r = Syntax.t) =
1258     struct
1259       module Token = Syntax.Token
1260
1261       type t = State.t [@@deriving show, sexp_of]
1262
1263       type r = Syntax.t [@@deriving show]
1264
1265       let rust_parse = RustParser.rust_parse
1266
1267       let initial_state = State.initial
1268     end
1269   end
1270
1271   include WithState (struct
1272     type r = Syntax.t [@@deriving show]
1273
1274     type t = unit [@@deriving show, sexp_of]
1275
1276     let initial _ = ()
1277
1278     let next () _ = ()
1279   end)
1280
1281   include WithRustParser (struct
1282     type r = Syntax.t
1283
1284     type t = unit
1285
1286     let rust_parse = Syntax.rust_parse
1287   end)
1288 end
1289 "
1290
1291   let full_fidelity_syntax_smart_constructors =
1292     Full_fidelity_schema.make_template_file
1293       ~transformations:[]
1294       ~filename:
1295         (full_fidelity_path_prefix
1296         ^ "smart_constructors/syntaxSmartConstructors.ml")
1297       ~template:full_fidelity_syntax_smart_constructors_template
1298       ()
1299 end
1300
1301 (* GenerateFFSyntaxSmartConstructors *)
1302
1303 module GenerateFFRustSyntaxSmartConstructors = struct
1304   let to_constructor_methods x =
1305     let params =
1306       List.mapi x.fields ~f:(fun i _ -> sprintf "arg%d : Self::Output" i)
1307     in
1308     let params = String.concat ~sep:", " params in
1309     let args = List.mapi x.fields ~f:(fun i _ -> sprintf "arg%d" i) in
1310     let args = String.concat ~sep:", " args in
1311     let next_args = List.mapi x.fields ~f:(fun i _ -> sprintf "&arg%d" i) in
1312     let next_args = String.concat ~sep:", " next_args in
1313     sprintf
1314       "    fn make_%s(&mut self, %s) -> Self::Output {
1315         self.state_mut().next(&[%s]);
1316         Self::Output::make_%s(self.state_mut(), %s)
1317     }\n\n"
1318       x.type_name
1319       params
1320       next_args
1321       x.type_name
1322       args
1323
1324   let full_fidelity_syntax_smart_constructors_template : string =
1325     make_header CStyle ""
1326     ^ "
1327 use parser_core_types::{
1328     syntax::*,
1329     token_factory::TokenFactory,
1330 };
1331 use smart_constructors::{NoState, SmartConstructors};
1332 use crate::StateType;
1333
1334 pub trait SyntaxSmartConstructors<S: SyntaxType<St>, TF: TokenFactory<Token = S::Token>, St = NoState>:
1335     SmartConstructors<State = St, Output=S, Factory = TF>
1336 where
1337     St: StateType<S>,
1338 {
1339     fn make_missing(&mut self, offset: usize) -> Self::Output {
1340         let r = Self::Output::make_missing(self.state_mut(), offset);
1341         self.state_mut().next(&[]);
1342         r
1343     }
1344
1345     fn make_token(&mut self, arg: <Self::Factory as TokenFactory>::Token) -> Self::Output {
1346         let r = Self::Output::make_token(self.state_mut(), arg);
1347         self.state_mut().next(&[]);
1348         r
1349     }
1350
1351     fn make_list(&mut self, items: Vec<Self::Output>, offset: usize) -> Self::Output {
1352         if items.is_empty() {
1353             <Self as SyntaxSmartConstructors<S, TF, St>>::make_missing(self, offset)
1354         } else {
1355             let item_refs: Vec<_> = items.iter().collect();
1356             self.state_mut().next(&item_refs);
1357             Self::Output::make_list(self.state_mut(), items, offset)
1358         }
1359     }
1360
1361 CONSTRUCTOR_METHODS}
1362 "
1363
1364   let full_fidelity_syntax_smart_constructors =
1365     Full_fidelity_schema.make_template_file
1366       ~transformations:
1367         [{ pattern = "CONSTRUCTOR_METHODS"; func = to_constructor_methods }]
1368       ~filename:
1369         (full_fidelity_path_prefix ^ "syntax_smart_constructors_generated.rs")
1370       ~template:full_fidelity_syntax_smart_constructors_template
1371       ()
1372 end
1373
1374 (* GenerateFFRustSyntaxSmartConstructors *)
1375
1376 module GenerateFFRustDeclModeSmartConstructors = struct
1377   let to_constructor_methods x =
1378     let args =
1379       List.mapi x.fields ~f:(fun i _ -> sprintf "arg%d: Self::Output" i)
1380     in
1381     let args = String.concat ~sep:", " args in
1382     let fwd_args = List.mapi x.fields ~f:(fun i _ -> sprintf "arg%d" i) in
1383     let fwd_args = String.concat ~sep:", " fwd_args in
1384     sprintf
1385       "    fn make_%s(&mut self, %s) -> Self::Output {
1386         <Self as SyntaxSmartConstructors<Self::Output, Self::Factory, State<'_, '_, Self::Output>>>::make_%s(self, %s)
1387     }\n\n"
1388       x.type_name
1389       args
1390       x.type_name
1391       fwd_args
1392
1393   let decl_mode_smart_constructors_template : string =
1394     make_header CStyle ""
1395     ^ "
1396 use parser_core_types::{
1397     lexable_token::LexableToken, syntax::SyntaxValueType, syntax_by_ref::syntax::Syntax,
1398     token_factory::TokenFactory,
1399 };
1400 use smart_constructors::SmartConstructors;
1401 use syntax_smart_constructors::SyntaxSmartConstructors;
1402 use crate::*;
1403
1404 impl<'s, 'a, Token, Value, TF> SmartConstructors
1405     for DeclModeSmartConstructors<'s, 'a, Syntax<'a, Token, Value>, Token, Value, TF>
1406 where
1407     TF: TokenFactory<Token = SyntaxToken<'s, 'a, Token, Value>>,
1408     Token: LexableToken + Copy,
1409     Value: SyntaxValueType<Token> + Clone,
1410 {
1411     type State = State<'s, 'a, Syntax<'a, Token, Value>>;
1412     type Factory = TF;
1413     type Output = Syntax<'a, Token, Value>;
1414
1415     fn state_mut(&mut self) -> &mut State<'s, 'a, Syntax<'a, Token, Value>> {
1416         &mut self.state
1417     }
1418
1419     fn into_state(self) -> State<'s, 'a, Syntax<'a, Token, Value>> {
1420         self.state
1421     }
1422
1423     fn token_factory_mut(&mut self) -> &mut Self::Factory {
1424         &mut self.token_factory
1425     }
1426
1427     fn make_missing(&mut self, o: usize) -> Self::Output {
1428         <Self as SyntaxSmartConstructors<Self::Output, Self::Factory, State<'_, '_, Self::Output>>>::make_missing(self, o)
1429     }
1430
1431     fn make_token(&mut self, token: <Self::Factory as TokenFactory>::Token) -> Self::Output {
1432         <Self as SyntaxSmartConstructors<Self::Output, Self::Factory, State<'_, '_, Self::Output>>>::make_token(self, token)
1433     }
1434
1435     fn make_list(&mut self, items: Vec<Self::Output>, offset: usize) -> Self::Output {
1436         <Self as SyntaxSmartConstructors<Self::Output, Self::Factory, State<'_, '_, Self::Output>>>::make_list(self, items, offset)
1437     }
1438
1439 CONSTRUCTOR_METHODS}
1440 "
1441
1442   let decl_mode_smart_constructors =
1443     Full_fidelity_schema.make_template_file
1444       ~transformations:
1445         [{ pattern = "CONSTRUCTOR_METHODS"; func = to_constructor_methods }]
1446       ~filename:
1447         (full_fidelity_path_prefix ^ "decl_mode_smart_constructors_generated.rs")
1448       ~template:decl_mode_smart_constructors_template
1449       ()
1450 end
1451
1452 (* GenerateFFRustDeclModeSmartConstructors *)
1453
1454 module GenerateRustFlattenSmartConstructors = struct
1455   let to_constructor_methods x =
1456     let args =
1457       List.mapi x.fields ~f:(fun i _ -> sprintf "arg%d: Self::Output" i)
1458     in
1459     let args = String.concat ~sep:", " args in
1460     let if_cond =
1461       List.mapi x.fields ~f:(fun i _ -> sprintf "Self::is_zero(&arg%d)" i)
1462     in
1463     let if_cond = String.concat ~sep:" && " if_cond in
1464     let flatten_args = List.mapi x.fields ~f:(fun i _ -> sprintf "arg%d" i) in
1465     let flatten_args = String.concat ~sep:", " flatten_args in
1466     sprintf
1467       "    fn make_%s(&mut self, %s) -> Self::Output {
1468         if %s {
1469           Self::zero(SyntaxKind::%s)
1470         } else {
1471           self.flatten(SyntaxKind::%s, vec!(%s))
1472         }
1473     }\n\n"
1474       x.type_name
1475       args
1476       if_cond
1477       x.kind_name
1478       x.kind_name
1479       flatten_args
1480
1481   let flatten_smart_constructors_template : string =
1482     make_header CStyle ""
1483     ^ "
1484 use smart_constructors::SmartConstructors;
1485 use parser_core_types::{
1486   lexable_token::LexableToken,
1487   syntax_kind::SyntaxKind,
1488   token_factory::TokenFactory,
1489 };
1490
1491 pub trait FlattenSmartConstructors: SmartConstructors
1492 {
1493     fn is_zero(s: &Self::Output) -> bool;
1494     fn zero(kind: SyntaxKind) -> Self::Output;
1495     fn flatten(&self, kind: SyntaxKind, lst: Vec<Self::Output>) -> Self::Output;
1496
1497     fn make_missing(&mut self, _: usize) -> Self::Output {
1498        Self::zero(SyntaxKind::Missing)
1499     }
1500
1501     fn make_token(&mut self, token: <Self::Factory as TokenFactory>::Token) -> Self::Output {
1502         Self::zero(SyntaxKind::Token(token.kind()))
1503     }
1504
1505     fn make_list(&mut self, _: Vec<Self::Output>, _: usize) -> Self::Output {
1506         Self::zero(SyntaxKind::SyntaxList)
1507     }
1508
1509     fn begin_enumerator(&mut self) {}
1510
1511     fn begin_enum_class_enumerator(&mut self) {}
1512
1513     fn begin_constant_declarator(&mut self) {}
1514
1515 CONSTRUCTOR_METHODS}
1516 "
1517
1518   let flatten_smart_constructors =
1519     Full_fidelity_schema.make_template_file
1520       ~transformations:
1521         [{ pattern = "CONSTRUCTOR_METHODS"; func = to_constructor_methods }]
1522       ~filename:(full_fidelity_path_prefix ^ "flatten_smart_constructors.rs")
1523       ~template:flatten_smart_constructors_template
1524       ()
1525 end
1526
1527 (* GenerateRustFlattenSmartConstructors *)
1528
1529 module GenerateRustDirectDeclSmartConstructors = struct
1530   let to_constructor_methods x =
1531     let args =
1532       List.map x.fields ~f:(fun (name, _) ->
1533           sprintf "%s: Self::Output" (escape_rust_keyword name))
1534     in
1535     let args = String.concat ~sep:", " args in
1536     let fwd_args =
1537       List.map x.fields ~f:(fun (name, _) -> escape_rust_keyword name)
1538     in
1539     let fwd_args = String.concat ~sep:", " fwd_args in
1540     sprintf
1541       "    fn make_%s(&mut self, %s) -> Self::Output {
1542         <Self as FlattenSmartConstructors>::make_%s(self, %s)
1543     }\n\n"
1544       x.type_name
1545       args
1546       x.type_name
1547       fwd_args
1548
1549   let direct_decl_smart_constructors_template : string =
1550     make_header CStyle ""
1551     ^ "
1552 use flatten_smart_constructors::*;
1553 use parser_core_types::compact_token::CompactToken;
1554 use parser_core_types::token_factory::SimpleTokenFactoryImpl;
1555 use smart_constructors::SmartConstructors;
1556
1557 use crate::{DirectDeclSmartConstructors, Node, SourceTextAllocator};
1558
1559 impl<'a, 'o, 't, S: SourceTextAllocator<'t, 'a>> SmartConstructors for DirectDeclSmartConstructors<'a, 'o, 't, S> {
1560     type State = Self;
1561     type Factory = SimpleTokenFactoryImpl<CompactToken>;
1562     type Output = Node<'a>;
1563
1564     fn state_mut(&mut self) -> &mut Self {
1565         self
1566     }
1567
1568     fn into_state(self) -> Self {
1569         self
1570     }
1571
1572     fn token_factory_mut(&mut self) -> &mut Self::Factory {
1573         &mut self.token_factory
1574     }
1575
1576     fn make_missing(&mut self, offset: usize) -> Self::Output {
1577         <Self as FlattenSmartConstructors>::make_missing(self, offset)
1578     }
1579
1580     fn make_token(&mut self, token: CompactToken) -> Self::Output {
1581         <Self as FlattenSmartConstructors>::make_token(self, token)
1582     }
1583
1584     fn make_list(&mut self, items: Vec<Self::Output>, offset: usize) -> Self::Output {
1585         <Self as FlattenSmartConstructors>::make_list(self, items, offset)
1586     }
1587
1588     fn begin_enumerator(&mut self) {
1589         <Self as FlattenSmartConstructors>::begin_enumerator(self)
1590     }
1591
1592     fn begin_enum_class_enumerator(&mut self) {
1593         <Self as FlattenSmartConstructors>::begin_enum_class_enumerator(self)
1594     }
1595
1596     fn begin_constant_declarator(&mut self) {
1597         <Self as FlattenSmartConstructors>::begin_constant_declarator(self)
1598     }
1599
1600
1601
1602 CONSTRUCTOR_METHODS}
1603 "
1604
1605   let direct_decl_smart_constructors =
1606     Full_fidelity_schema.make_template_file
1607       ~transformations:
1608         [{ pattern = "CONSTRUCTOR_METHODS"; func = to_constructor_methods }]
1609       ~filename:
1610         (full_fidelity_path_prefix
1611         ^ "../decl/direct_decl_smart_constructors_generated.rs")
1612       ~template:direct_decl_smart_constructors_template
1613       ()
1614 end
1615
1616 (* GenerateRustDirectDeclSmartConstructors *)
1617
1618 module GenerateRustPairSmartConstructors = struct
1619   let to_constructor_methods x =
1620     let args =
1621       List.map x.fields ~f:(fun (name, _) ->
1622           sprintf "%s: Self::Output" (escape_rust_keyword name))
1623     in
1624     let args = String.concat ~sep:", " args in
1625     let fwd_args =
1626       List.map x.fields ~f:(fun (name, _) -> escape_rust_keyword name)
1627     in
1628     let fwd_args idx =
1629       List.map fwd_args ~f:(fun name -> name ^ "." ^ idx)
1630       |> String.concat ~sep:", "
1631     in
1632     sprintf
1633       "    fn make_%s(&mut self, %s) -> Self::Output {
1634         Node(self.0.make_%s(%s), self.1.make_%s(%s))
1635     }\n\n"
1636       x.type_name
1637       args
1638       x.type_name
1639       (fwd_args "0")
1640       x.type_name
1641       (fwd_args "1")
1642
1643   let pair_smart_constructors_template : string =
1644     make_header CStyle ""
1645     ^ "
1646
1647 use parser_core_types::token_factory::TokenFactory;
1648 use smart_constructors::{NodeType, SmartConstructors};
1649
1650 use crate::{PairTokenFactory, Node};
1651
1652 #[derive(Clone)]
1653 pub struct PairSmartConstructors<SC0, SC1>(pub SC0, pub SC1, PairTokenFactory<SC0::Factory, SC1::Factory>)
1654 where
1655     SC0: SmartConstructors,
1656     SC0::Output: NodeType,
1657     SC1: SmartConstructors,
1658     SC1::Output: NodeType;
1659
1660 impl<SC0, SC1> PairSmartConstructors<SC0, SC1>
1661 where
1662     SC0: SmartConstructors,
1663     SC0::Output: NodeType,
1664     SC1: SmartConstructors,
1665     SC1::Output: NodeType,
1666 {
1667     pub fn new(mut sc0: SC0, mut sc1: SC1) -> Self {
1668         let tf0 = sc0.token_factory_mut().clone();
1669         let tf1 = sc1.token_factory_mut().clone();
1670         let tf = PairTokenFactory::new(tf0, tf1);
1671         Self(sc0, sc1, tf)
1672     }
1673 }
1674
1675 impl<SC0, SC1> SmartConstructors for PairSmartConstructors<SC0, SC1>
1676 where
1677     SC0: SmartConstructors,
1678     SC0::Output: NodeType,
1679     SC1: SmartConstructors,
1680     SC1::Output: NodeType,
1681 {
1682     type State = Self;
1683     type Factory = PairTokenFactory<SC0::Factory, SC1::Factory>;
1684     type Output = Node<SC0::Output, SC1::Output>;
1685
1686     fn state_mut(&mut self) -> &mut Self {
1687         self
1688     }
1689
1690     fn into_state(self) -> Self {
1691         self
1692     }
1693
1694     fn token_factory_mut(&mut self) -> &mut Self::Factory {
1695         &mut self.2
1696     }
1697
1698     fn make_missing(&mut self, offset: usize) -> Self::Output {
1699         Node(self.0.make_missing(offset), self.1.make_missing(offset))
1700     }
1701
1702     fn make_token(&mut self, token: <Self::Factory as TokenFactory>::Token) -> Self::Output {
1703         Node(self.0.make_token(token.0), self.1.make_token(token.1))
1704     }
1705
1706     fn make_list(&mut self, items: Vec<Self::Output>, offset: usize) -> Self::Output {
1707         let (items0, items1) = items.into_iter().map(|n| (n.0, n.1)).unzip();
1708         Node(self.0.make_list(items0, offset), self.1.make_list(items1, offset))
1709     }
1710
1711 CONSTRUCTOR_METHODS}
1712 "
1713
1714   let pair_smart_constructors =
1715     Full_fidelity_schema.make_template_file
1716       ~transformations:
1717         [{ pattern = "CONSTRUCTOR_METHODS"; func = to_constructor_methods }]
1718       ~filename:
1719         (full_fidelity_path_prefix ^ "pair_smart_constructors_generated.rs")
1720       ~template:pair_smart_constructors_template
1721       ()
1722 end
1723
1724 (* GenerateRustPairSmartConstructors *)
1725
1726 module GenerateFFSmartConstructorsWrappers = struct
1727   let full_fidelity_smart_constructors_wrappers_template : string =
1728     make_header
1729       MLStyle
1730       "
1731  * This module contains smart constructors implementation that can be used to
1732  * build AST.
1733  "
1734     ^ "
1735
1736 module type SC_S = SmartConstructors.SmartConstructors_S
1737
1738 module SK = Full_fidelity_syntax_kind
1739
1740 module type SyntaxKind_S = sig
1741   include SC_S
1742
1743   type original_sc_r [@@deriving show]
1744 end
1745
1746 module SyntaxKind (SC : SC_S) :
1747   SyntaxKind_S
1748     with module Token = SC.Token
1749      and type original_sc_r = SC.r
1750      and type t = SC.t = struct
1751   module Token = SC.Token
1752
1753   type original_sc_r = SC.r [@@deriving show]
1754
1755   type t = SC.t [@@deriving show, sexp_of]
1756
1757   type r = SK.t * SC.r [@@deriving show]
1758
1759   let compose : SK.t -> t * SC.r -> t * r =
1760    (fun kind (state, res) -> (state, (kind, res)))
1761
1762   let rust_parse text env =
1763     let (state, res, errors, pointer) = SC.rust_parse text env in
1764     let (state, res) = compose SK.Script (state, res) in
1765     (state, res, errors, pointer)
1766
1767   let initial_state = SC.initial_state
1768 end
1769 "
1770
1771   let full_fidelity_smart_constructors_wrappers =
1772     Full_fidelity_schema.make_template_file
1773       ~transformations:[]
1774       ~filename:
1775         (full_fidelity_path_prefix
1776         ^ "smart_constructors/smartConstructorsWrappers.ml")
1777       ~template:full_fidelity_smart_constructors_wrappers_template
1778       ()
1779 end
1780
1781 (* GenerateFFSmartConstructorsWrappers *)
1782
1783 module GenerateFFRustSmartConstructorsWrappers = struct
1784   let to_constructor_methods x =
1785     let params =
1786       List.mapi x.fields ~f:(fun i _ ->
1787           "arg" ^ string_of_int i ^ " : Self::Output")
1788     in
1789     let params = String.concat ~sep:", " params in
1790     let args = List.mapi x.fields ~f:(fun i _ -> sprintf "arg%d" i) in
1791     let raw_args = map_and_concat_separated ", " (fun x -> x ^ ".1") args in
1792     sprintf
1793       "    fn make_%s(&mut self, %s) -> Self::Output {
1794         compose(SyntaxKind::%s, self.s.make_%s(%s))
1795     }\n"
1796       x.type_name
1797       params
1798       x.kind_name
1799       x.type_name
1800       raw_args
1801
1802   let full_fidelity_smart_constructors_wrappers_template : string =
1803     make_header CStyle ""
1804     ^ "
1805  // This module contains smart constructors implementation that can be used to
1806  // build AST.
1807 "
1808     ^ "
1809 use parser_core_types::{
1810   lexable_token::LexableToken,
1811   syntax_kind::SyntaxKind,
1812   token_factory::TokenFactory,
1813 };
1814 use crate::SmartConstructors;
1815
1816 #[derive(Clone)]
1817 pub struct WithKind<S> {
1818     s: S,
1819 }
1820
1821 impl<S> WithKind<S> {
1822     pub fn new(s: S) -> Self {
1823         Self { s }
1824     }
1825 }
1826
1827 impl<S, St> SmartConstructors for WithKind<S>
1828 where S: SmartConstructors<State = St>,
1829 {
1830     type Factory = S::Factory;
1831     type State = St;
1832     type Output = (SyntaxKind, S::Output);
1833
1834     fn state_mut(&mut self) -> &mut St {
1835         self.s.state_mut()
1836     }
1837
1838     fn into_state(self) -> St {
1839       self.s.into_state()
1840     }
1841
1842     fn token_factory_mut(&mut self) -> &mut Self::Factory {
1843         self.s.token_factory_mut()
1844     }
1845
1846
1847     fn make_token(&mut self, token: <Self::Factory as TokenFactory>::Token) -> Self::Output {
1848         compose(SyntaxKind::Token(token.kind()), self.s.make_token(token))
1849     }
1850
1851     fn make_missing(&mut self, p: usize) -> Self::Output {
1852         compose(SyntaxKind::Missing, self.s.make_missing(p))
1853     }
1854
1855     fn make_list(&mut self, items: Vec<Self::Output>, p: usize) -> Self::Output {
1856         let kind = if items.is_empty() {
1857             SyntaxKind::Missing
1858         } else {
1859             SyntaxKind::SyntaxList
1860         };
1861         compose(kind, self.s.make_list(items.into_iter().map(|x| x.1).collect(), p))
1862     }
1863
1864 CONSTRUCTOR_METHODS
1865 }
1866
1867 #[inline(always)]
1868 fn compose<R>(kind: SyntaxKind, r: R) -> (SyntaxKind, R) {
1869     (kind, r)
1870 }
1871 "
1872
1873   let full_fidelity_smart_constructors_wrappers =
1874     Full_fidelity_schema.make_template_file
1875       ~transformations:
1876         [{ pattern = "CONSTRUCTOR_METHODS"; func = to_constructor_methods }]
1877       ~filename:(full_fidelity_path_prefix ^ "smart_constructors_wrappers.rs")
1878       ~template:full_fidelity_smart_constructors_wrappers_template
1879       ()
1880 end
1881
1882 (* GenerateFFRustSmartConstructorsWrappers *)
1883
1884 module GenerateFFSyntax = struct
1885   let to_to_kind x =
1886     sprintf
1887       ("      | " ^^ kind_name_fmt ^^ " _ -> SyntaxKind.%s\n")
1888       x.kind_name
1889       x.kind_name
1890
1891   let to_type_tests x =
1892     sprintf
1893       ("    let is_" ^^ type_name_fmt ^^ " = has_kind SyntaxKind.%s\n")
1894       x.type_name
1895       x.kind_name
1896
1897   let to_children x =
1898     let mapper (f, _) = sprintf "        %s_%s;\n" x.prefix f in
1899     let fields = map_and_concat mapper x.fields in
1900     sprintf
1901       "      | %s {\n%s      } -> [\n%s      ]\n"
1902       x.kind_name
1903       fields
1904       fields
1905
1906   let to_fold_from_syntax x =
1907     let mapper (f, _) = sprintf "        %s_%s;\n" x.prefix f in
1908     let fields = map_and_concat mapper x.fields in
1909     let mapper2 (f, _) =
1910       sprintf "         let acc = f acc %s_%s in\n" x.prefix f
1911     in
1912     let fields2 = map_and_concat mapper2 x.fields in
1913     sprintf
1914       "      | %s {\n%s      } ->\n%s         acc\n"
1915       x.kind_name
1916       fields
1917       fields2
1918
1919   let to_children_names x =
1920     let mapper1 (f, _) = sprintf "        %s_%s;\n" x.prefix f in
1921     let mapper2 (f, _) = sprintf "        \"%s_%s\";\n" x.prefix f in
1922     let fields1 = map_and_concat mapper1 x.fields in
1923     let fields2 = map_and_concat mapper2 x.fields in
1924     sprintf
1925       "      | %s {\n%s      } -> [\n%s      ]\n"
1926       x.kind_name
1927       fields1
1928       fields2
1929
1930   let to_syntax_from_children x =
1931     let mapper (f, _) = sprintf "          %s_%s;\n" x.prefix f in
1932     let fields = map_and_concat mapper x.fields in
1933     sprintf
1934       "      | (SyntaxKind.%s, [
1935 %s        ]) ->
1936         %s {
1937 %s        }
1938 "
1939       x.kind_name
1940       fields
1941       x.kind_name
1942       fields
1943
1944   let to_constructor_methods x =
1945     let mapper1 (f, _) = sprintf "        %s_%s\n" x.prefix f in
1946     let fields1 = map_and_concat mapper1 x.fields in
1947     let mapper2 (f, _) = sprintf "          %s_%s;\n" x.prefix f in
1948     let fields2 = map_and_concat mapper2 x.fields in
1949     sprintf
1950       "      let make_%s
1951 %s      =
1952         let syntax = %s {
1953 %s        } in
1954         let value = ValueBuilder.value_from_syntax syntax in
1955         make syntax value
1956
1957 "
1958       x.type_name
1959       fields1
1960       x.kind_name
1961       fields2
1962
1963   let to_from_methods x =
1964     if omit_syntax_record x then
1965       ""
1966     else
1967       let mapper (f, _) = sprintf "          %s_%s;\n" x.prefix f in
1968       let fields = map_and_concat mapper x.fields in
1969       sprintf
1970         "     let from_%s {
1971 %s       } = %s {
1972 %s       }
1973 "
1974         x.type_name
1975         fields
1976         x.kind_name
1977         fields
1978
1979   let to_get_methods x =
1980     if omit_syntax_record x then
1981       ""
1982     else
1983       let mapper (f, _) = sprintf "          %s_%s;\n" x.prefix f in
1984       let fields = map_and_concat mapper x.fields in
1985       sprintf
1986         "     let get_%s x =
1987         match x with
1988         | %s {\n%s            } -> {\n%s           }
1989         | _ -> failwith \"get_%s: not a %s\"
1990 "
1991         x.type_name
1992         x.kind_name
1993         fields
1994         fields
1995         x.type_name
1996         x.kind_name
1997
1998   let full_fidelity_syntax_template =
1999     make_header
2000       MLStyle
2001       "
2002  * With these factory methods, nodes can be built up from their child nodes. A
2003  * factory method must not just know all the children and the kind of node it is
2004  * constructing; it also must know how to construct the value that this node is
2005  * going to be tagged with. For that reason, an optional functor is provided.
2006  * This functor requires that methods be provided to construct the values
2007  * associated with a token or with any arbitrary node, given its children. If
2008  * this functor is used then the resulting module contains factory methods.
2009  *
2010  * This module also provides some useful helper functions, like an iterator,
2011  * a rewriting visitor, and so on."
2012     ^ "
2013
2014 open Hh_prelude
2015 open Full_fidelity_syntax_type
2016
2017 module SyntaxKind = Full_fidelity_syntax_kind
2018 module TokenKind = Full_fidelity_token_kind
2019 module Operator = Full_fidelity_operator
2020 [@@@warning \"-27\"] (* unused variable *)
2021
2022 module WithToken(Token: TokenType) = struct
2023   module WithSyntaxValue(SyntaxValue: SyntaxValueType) = struct
2024
2025     include MakeSyntaxType(Token)(SyntaxValue)
2026
2027     let make syntax value =
2028       { syntax; value }
2029
2030     let syntax node =
2031       node.syntax
2032
2033     let value node =
2034       node.value
2035
2036     let syntax_node_to_list node =
2037       match syntax node with
2038       | SyntaxList x -> x
2039       | Missing -> []
2040       | _ -> [node]
2041
2042     let to_kind syntax =
2043       match syntax with
2044       | Missing                             -> SyntaxKind.Missing
2045       | Token                             t -> SyntaxKind.Token (Token.kind t)
2046       | SyntaxList                        _ -> SyntaxKind.SyntaxList
2047 TO_KIND
2048
2049     let kind node =
2050       to_kind (syntax node)
2051
2052     let has_kind syntax_kind node =
2053       SyntaxKind.equal (kind node) syntax_kind
2054
2055     let is_missing node =
2056       match kind node with
2057       | SyntaxKind.Missing -> true
2058       | _ -> false
2059
2060     let is_list node =
2061       match kind node with
2062       | SyntaxKind.SyntaxList -> true
2063       | _ -> false
2064
2065 TYPE_TESTS
2066
2067     let is_loop_statement node =
2068       is_for_statement node ||
2069       is_foreach_statement node ||
2070       is_while_statement node ||
2071       is_do_statement node
2072
2073     let is_separable_prefix node =
2074       match syntax node with
2075       | Token t -> begin
2076         TokenKind.(match Token.kind t with
2077         | PlusPlus | MinusMinus -> false
2078         | _ -> true) end
2079       | _ -> true
2080
2081     let is_specific_token kind node =
2082       match syntax node with
2083       | Token t -> TokenKind.equal (Token.kind t) kind
2084       | _ -> false
2085
2086     let is_namespace_prefix node =
2087       match syntax node with
2088       | QualifiedName e ->
2089         begin match List.last (syntax_node_to_list e.qualified_name_parts) with
2090         | None -> false
2091         | Some p ->
2092           begin match syntax p with
2093           | ListItem p -> not (is_missing p.list_separator)
2094           | _ -> false
2095           end
2096         end
2097       | _ -> false
2098
2099     let has_leading_trivia kind token =
2100       List.exists (Token.leading token)
2101         ~f:(fun trivia ->
2102             Full_fidelity_trivia_kind.equal (Token.Trivia.kind trivia) kind)
2103
2104     let is_external e =
2105       is_specific_token TokenKind.Semicolon e || is_missing e
2106
2107     let is_name       = is_specific_token TokenKind.Name
2108     let is_construct  = is_specific_token TokenKind.Construct
2109     let is_static     = is_specific_token TokenKind.Static
2110     let is_private    = is_specific_token TokenKind.Private
2111     let is_public     = is_specific_token TokenKind.Public
2112     let is_protected  = is_specific_token TokenKind.Protected
2113     let is_abstract   = is_specific_token TokenKind.Abstract
2114     let is_final      = is_specific_token TokenKind.Final
2115     let is_async      = is_specific_token TokenKind.Async
2116     let is_void       = is_specific_token TokenKind.Void
2117     let is_left_brace = is_specific_token TokenKind.LeftBrace
2118     let is_ellipsis   = is_specific_token TokenKind.DotDotDot
2119     let is_comma      = is_specific_token TokenKind.Comma
2120     let is_ampersand  = is_specific_token TokenKind.Ampersand
2121     let is_inout      = is_specific_token TokenKind.Inout
2122
2123     let syntax_list_fold ~init ~f node =
2124       match syntax node with
2125       | SyntaxList sl ->
2126         List.fold_left
2127           ~init
2128           ~f:(fun init li -> match syntax li with
2129               | ListItem { list_item; _; }-> f init list_item
2130               | Missing -> init
2131               | _ -> f init li)
2132           sl
2133       | Missing -> init
2134       | _ -> f init node
2135
2136     let fold_over_children f acc syntax =
2137       match syntax with
2138       | Missing -> acc
2139       | Token _ -> acc
2140       | SyntaxList items ->
2141         List.fold_left ~f ~init:acc items
2142 FOLD_FROM_SYNTAX
2143
2144     (* The order that the children are returned in should match the order
2145        that they appear in the source text *)
2146     let children_from_syntax s =
2147       match s with
2148       | Missing -> []
2149       | Token _ -> []
2150       | SyntaxList x -> x
2151 CHILDREN
2152
2153     let children node =
2154       children_from_syntax node.syntax
2155
2156     let children_names node =
2157       match node.syntax with
2158       | Missing -> []
2159       | Token _ -> []
2160       | SyntaxList _ -> []
2161 CHILDREN_NAMES
2162
2163     let rec to_json_ ?(with_value = false) ?(ignore_missing = false) node =
2164       let open Hh_json in
2165       let ch = match node.syntax with
2166       | Token t -> [ \"token\", Token.to_json t ]
2167       | SyntaxList x -> [ (\"elements\",
2168         JSON_Array (List.filter_map ~f:(to_json_ ~with_value ~ignore_missing) x)) ]
2169       | _ ->
2170         let rec aux acc c n =
2171           match c, n with
2172           | ([], []) -> acc
2173           | ((hc :: tc), (hn :: tn)) ->
2174             let result = (to_json_ ~with_value ~ignore_missing) hc in
2175             (match result with
2176             | Some r -> aux ((hn, r):: acc) tc tn
2177             | None -> aux acc tc tn)
2178           | _ -> failwith \"mismatch between children and names\" in
2179         List.rev (aux [] (children node) (children_names node)) in
2180       let k = (\"kind\", JSON_String (SyntaxKind.to_string (kind node))) in
2181       let v = if with_value then
2182         (\"value\", SyntaxValue.to_json node.value) :: ch
2183         else ch in
2184       if ignore_missing && (List.is_empty ch) then None else Some(JSON_Object (k :: v))
2185
2186     let to_json ?(with_value = false) ?(ignore_missing = false) node =
2187       match to_json_ ~with_value ~ignore_missing node with
2188       | Some x -> x
2189       | None -> Hh_json.JSON_Object([])
2190
2191     let binary_operator_kind b =
2192       match syntax b with
2193       | Token token ->
2194         let kind = Token.kind token in
2195         if Operator.is_trailing_operator_token kind then
2196           Some (Operator.trailing_from_token kind)
2197         else
2198           None
2199       | _ -> None
2200
2201     let get_token node =
2202       match (syntax node) with
2203       | Token token -> Some token
2204       | _ -> None
2205
2206     let leading_token node =
2207       let rec aux nodes =
2208         match nodes with
2209         | [] -> None
2210         | h :: t ->
2211           let token = get_token h in
2212           if Option.is_none token then
2213             let result = aux (children h) in
2214             if Option.is_none result then aux t else result
2215           else
2216             token in
2217       aux [node]
2218
2219     let trailing_token node =
2220       let rec aux nodes =
2221         match nodes with
2222         | [] -> None
2223         | h :: t ->
2224           let token = get_token h in
2225           if Option.is_none token then
2226             let result = aux (List.rev (children h)) in
2227             if Option.is_none result then aux t else result
2228           else
2229             token in
2230       aux [node]
2231
2232     let syntax_from_children kind ts =
2233       match kind, ts with
2234 SYNTAX_FROM_CHILDREN      | (SyntaxKind.Missing, []) -> Missing
2235       | (SyntaxKind.SyntaxList, items) -> SyntaxList items
2236       | _ -> failwith
2237         \"syntax_from_children called with wrong number of children\"
2238
2239     let all_tokens node =
2240       let rec aux acc nodes =
2241         match nodes with
2242         | [] -> acc
2243         | h :: t ->
2244           begin
2245             match syntax h with
2246             | Token token -> aux (token :: acc) t
2247             | _ -> aux (aux acc (children h)) t
2248           end in
2249       List.rev (aux [] [node])
2250
2251     module type ValueBuilderType = sig
2252       val value_from_children:
2253         Full_fidelity_source_text.t ->
2254         int -> (* offset *)
2255         Full_fidelity_syntax_kind.t ->
2256         t list ->
2257         SyntaxValue.t
2258       val value_from_token: Token.t -> SyntaxValue.t
2259       val value_from_syntax: syntax -> SyntaxValue.t
2260     end
2261
2262     module WithValueBuilder(ValueBuilder: ValueBuilderType) = struct
2263       let from_children text offset kind ts =
2264         let syntax = syntax_from_children kind ts in
2265         let value = ValueBuilder.value_from_children text offset kind ts in
2266         make syntax value
2267
2268       let make_token token =
2269         let syntax = Token token in
2270         let value = ValueBuilder.value_from_token token in
2271         make syntax value
2272
2273       let make_missing text offset =
2274         from_children text offset SyntaxKind.Missing []
2275
2276       (* An empty list is represented by Missing; everything else is a
2277         SyntaxList, even if the list has only one item. *)
2278       let make_list text offset items =
2279         match items with
2280         | [] -> make_missing text offset
2281         | _ -> from_children text offset SyntaxKind.SyntaxList items
2282
2283 CONSTRUCTOR_METHODS
2284
2285 FROM_METHODS
2286
2287 GET_METHODS
2288
2289     end
2290   end
2291 end
2292 "
2293
2294   let full_fidelity_syntax =
2295     Full_fidelity_schema.make_template_file
2296       ~transformations:
2297         [
2298           { pattern = "TO_KIND"; func = to_to_kind };
2299           { pattern = "TYPE_TESTS"; func = to_type_tests };
2300           { pattern = "CHILDREN"; func = to_children };
2301           { pattern = "FOLD_FROM_SYNTAX"; func = to_fold_from_syntax };
2302           { pattern = "CHILDREN_NAMES"; func = to_children_names };
2303           { pattern = "SYNTAX_FROM_CHILDREN"; func = to_syntax_from_children };
2304           { pattern = "CONSTRUCTOR_METHODS"; func = to_constructor_methods };
2305           { pattern = "FROM_METHODS"; func = to_from_methods };
2306           { pattern = "GET_METHODS"; func = to_get_methods };
2307         ]
2308       ~filename:(full_fidelity_path_prefix ^ "full_fidelity_syntax.ml")
2309       ~template:full_fidelity_syntax_template
2310       ()
2311 end
2312
2313 module GenerateFFTriviaKind = struct
2314   let to_trivia { trivia_kind; trivia_text = _ } =
2315     sprintf "  | %s\n" trivia_kind
2316
2317   let to_to_string { trivia_kind; trivia_text } =
2318     sprintf
2319       ("  | " ^^ trivia_kind_fmt ^^ " -> \"%s\"\n")
2320       trivia_kind
2321       trivia_text
2322
2323   let full_fidelity_trivia_kind_template =
2324     make_header MLStyle ""
2325     ^ "
2326
2327 type t =
2328 TRIVIA
2329   [@@deriving show, enum, eq, sexp_of]
2330
2331 let to_string kind =
2332   match kind with
2333 TO_STRING"
2334
2335   let full_fidelity_trivia_kind =
2336     Full_fidelity_schema.make_template_file
2337       ~trivia_transformations:
2338         [
2339           { trivia_pattern = "TRIVIA"; trivia_func = map_and_concat to_trivia };
2340           {
2341             trivia_pattern = "TO_STRING";
2342             trivia_func = map_and_concat to_to_string;
2343           };
2344         ]
2345       ~filename:(full_fidelity_path_prefix ^ "/full_fidelity_trivia_kind.ml")
2346       ~template:full_fidelity_trivia_kind_template
2347       ()
2348 end
2349
2350 (* GenerateFFSyntaxKind *)
2351
2352 module GenerateFFRustTriviaKind = struct
2353   let ocaml_tag = ref (-1)
2354
2355   let to_trivia { trivia_kind; trivia_text = _ } =
2356     incr ocaml_tag;
2357     sprintf "    %s = %d,\n" trivia_kind !ocaml_tag
2358
2359   let to_to_string { trivia_kind; trivia_text } =
2360     sprintf "            TriviaKind::%s => \"%s\",\n" trivia_kind trivia_text
2361
2362   let full_fidelity_trivia_kind_template =
2363     make_header CStyle ""
2364     ^ "
2365
2366 use ocamlrep::{FromOcamlRep, ToOcamlRep};
2367
2368 #[derive(Debug, Copy, Clone, FromOcamlRep, ToOcamlRep, PartialEq)]
2369 #[repr(u8)]
2370 pub enum TriviaKind {
2371 TRIVIA}
2372
2373 impl TriviaKind {
2374     pub fn to_string(&self) -> &str {
2375         match self {
2376 TO_STRING        }
2377     }
2378
2379     pub const fn ocaml_tag(self) -> u8 {
2380         self as u8
2381     }
2382 }
2383 "
2384
2385   let full_fidelity_trivia_kind =
2386     Full_fidelity_schema.make_template_file
2387       ~trivia_transformations:
2388         [
2389           { trivia_pattern = "TRIVIA"; trivia_func = map_and_concat to_trivia };
2390           {
2391             trivia_pattern = "TO_STRING";
2392             trivia_func = map_and_concat to_to_string;
2393           };
2394         ]
2395       ~filename:(full_fidelity_path_prefix ^ "trivia_kind.rs")
2396       ~template:full_fidelity_trivia_kind_template
2397       ()
2398 end
2399
2400 (* GenerateFFRustTriviaKind *)
2401
2402 module GenerateFFSyntaxKind = struct
2403   let to_tokens x = sprintf "  | %s\n" x.kind_name
2404
2405   let to_to_string x =
2406     sprintf
2407       ("  | " ^^ kind_name_fmt ^^ " -> \"%s\"\n")
2408       x.kind_name
2409       x.description
2410
2411   let full_fidelity_syntax_kind_template =
2412     make_header MLStyle ""
2413     ^ "
2414
2415 type t =
2416   | Token of Full_fidelity_token_kind.t
2417   | Missing
2418   | SyntaxList
2419 TOKENS
2420   [@@deriving show, eq]
2421
2422 let to_string kind =
2423   match kind with
2424   | Token _                           -> \"token\"
2425   | Missing                           -> \"missing\"
2426   | SyntaxList                        -> \"list\"
2427 TO_STRING"
2428
2429   let full_fidelity_syntax_kind =
2430     Full_fidelity_schema.make_template_file
2431       ~transformations:
2432         [
2433           { pattern = "TOKENS"; func = to_tokens };
2434           { pattern = "TO_STRING"; func = to_to_string };
2435         ]
2436       ~filename:(full_fidelity_path_prefix ^ "full_fidelity_syntax_kind.ml")
2437       ~template:full_fidelity_syntax_kind_template
2438       ()
2439 end
2440
2441 (* GenerateFFTriviaKind *)
2442
2443 module GenerateFFRustSyntaxKind = struct
2444   let to_tokens x = sprintf "    %s,\n" x.kind_name
2445
2446   let to_to_string x =
2447     sprintf
2448       ("            SyntaxKind::" ^^ kind_name_fmt ^^ " => \"%s\",\n")
2449       x.kind_name
2450       x.description
2451
2452   let tag = ref 1
2453
2454   let to_ocaml_tag x =
2455     incr tag;
2456     sprintf "            SyntaxKind::%s => %d,\n" x.kind_name !tag
2457
2458   let full_fidelity_syntax_kind_template =
2459     make_header CStyle ""
2460     ^ "
2461
2462 use ocamlrep::{FromOcamlRep, ToOcamlRep};
2463
2464 use crate::token_kind::TokenKind;
2465
2466 #[derive(Debug, Copy, Clone, FromOcamlRep, ToOcamlRep, PartialEq)]
2467 pub enum SyntaxKind {
2468     Missing,
2469     Token(TokenKind),
2470     SyntaxList,
2471 TOKENS
2472 }
2473
2474 impl SyntaxKind {
2475     pub fn to_string(&self) -> &str {
2476         match self {
2477             SyntaxKind::SyntaxList => \"list\",
2478             SyntaxKind::Missing => \"missing\",
2479             SyntaxKind::Token(_) => \"token\",
2480 TO_STRING        }
2481     }
2482
2483     pub fn ocaml_tag(self) -> u8 {
2484         match self {
2485             SyntaxKind::Missing => 0,
2486             SyntaxKind::Token(_) => 0,
2487             SyntaxKind::SyntaxList => 1,
2488 OCAML_TAG        }
2489     }
2490 }
2491 "
2492
2493   let full_fidelity_syntax_kind =
2494     Full_fidelity_schema.make_template_file
2495       ~transformations:
2496         [
2497           { pattern = "TOKENS"; func = to_tokens };
2498           { pattern = "TO_STRING"; func = to_to_string };
2499           { pattern = "OCAML_TAG"; func = to_ocaml_tag };
2500         ]
2501       ~filename:(full_fidelity_path_prefix ^ "syntax_kind.rs")
2502       ~template:full_fidelity_syntax_kind_template
2503       ()
2504 end
2505
2506 (* GenerateFFRustSyntaxKind *)
2507
2508 module GenerateFFTokenKind = struct
2509   let given_text_width =
2510     let folder acc x = max acc (String.length x.token_text) in
2511     List.fold_left ~f:folder ~init:0 given_text_tokens
2512
2513   let to_kind_declaration x = sprintf "  | %s\n" x.token_kind
2514
2515   let add_guard_or_pad :
2516       cond:bool * string -> ?else_cond:bool * string -> string -> string =
2517    fun ~cond:(cond, guard) ?else_cond guards ->
2518     let pad str = String.make (String.length str) ' ' in
2519     let is_only_spaces str = String.equal str (pad str) in
2520     let make_same_length str1 str2 =
2521       let blanks n =
2522         try String.make n ' ' with
2523         | Invalid_argument _ -> ""
2524       in
2525       let (len1, len2) = (String.length str1, String.length str2) in
2526       let str1 = str1 ^ blanks (len2 - len1) in
2527       let str2 = str2 ^ blanks (len1 - len2) in
2528       (str1, str2)
2529     in
2530     let (else_cond, else_guard) =
2531       match else_cond with
2532       | Some (cond, guard) -> (cond, guard)
2533       | None -> (false, "")
2534     in
2535     let prefix =
2536       if cond || else_cond then
2537         if is_only_spaces guards then
2538           "when "
2539         else
2540           "&&   "
2541       else
2542         "     "
2543     in
2544     let (guard, else_guard) = make_same_length guard else_guard in
2545     let guard =
2546       if cond then
2547         guard
2548       else if else_cond then
2549         else_guard
2550       else
2551         pad guard
2552     in
2553     guards ^ prefix ^ guard ^ " "
2554
2555   let to_from_string x =
2556     let token_text = escape_token_text x.token_text in
2557     let spacer_width = given_text_width - String.length token_text in
2558     let spacer = String.make spacer_width ' ' in
2559     let guards =
2560       add_guard_or_pad "" ~cond:(x.allowed_as_identifier, "not only_reserved")
2561     in
2562     sprintf "  | \"%s\"%s %s-> Some %s\n" token_text spacer guards x.token_kind
2563
2564   let to_to_string x =
2565     let token_text = escape_token_text x.token_text in
2566     sprintf ("  | " ^^ token_kind_fmt ^^ " -> \"%s\"\n") x.token_kind token_text
2567
2568   let to_is_variable_text x = sprintf "  | %s -> true\n" x.token_kind
2569
2570   let full_fidelity_token_kind_template =
2571     make_header MLStyle ""
2572     ^ "
2573
2574 type t =
2575   (* No text tokens *)
2576 KIND_DECLARATIONS_NO_TEXT  (* Given text tokens *)
2577 KIND_DECLARATIONS_GIVEN_TEXT  (* Variable text tokens *)
2578 KIND_DECLARATIONS_VARIABLE_TEXT
2579   [@@deriving show, eq, sexp_of]
2580
2581 let from_string keyword ~only_reserved =
2582   match keyword with
2583   | \"true\"            when not only_reserved -> Some BooleanLiteral
2584   | \"false\"           when not only_reserved -> Some BooleanLiteral
2585 FROM_STRING_GIVEN_TEXT  | _              -> None
2586
2587 let to_string kind =
2588   match kind with
2589 (* No text tokens *)
2590 TO_STRING_NO_TEXT  (* Given text tokens *)
2591 TO_STRING_GIVEN_TEXT  (* Variable text tokens *)
2592 TO_STRING_VARIABLE_TEXT
2593
2594 let is_variable_text kind =
2595   match kind with
2596 IS_VARIABLE_TEXT_VARIABLE_TEXT  | _ -> false
2597 "
2598
2599   let full_fidelity_token_kind =
2600     Full_fidelity_schema.make_template_file
2601       ~token_no_text_transformations:
2602         [
2603           {
2604             token_pattern = "KIND_DECLARATIONS_NO_TEXT";
2605             token_func = map_and_concat to_kind_declaration;
2606           };
2607           {
2608             token_pattern = "TO_STRING_NO_TEXT";
2609             token_func = map_and_concat to_to_string;
2610           };
2611         ]
2612       ~token_given_text_transformations:
2613         [
2614           {
2615             token_pattern = "KIND_DECLARATIONS_GIVEN_TEXT";
2616             token_func = map_and_concat to_kind_declaration;
2617           };
2618           {
2619             token_pattern = "FROM_STRING_GIVEN_TEXT";
2620             token_func = map_and_concat to_from_string;
2621           };
2622           {
2623             token_pattern = "TO_STRING_GIVEN_TEXT";
2624             token_func = map_and_concat to_to_string;
2625           };
2626         ]
2627       ~token_variable_text_transformations:
2628         [
2629           {
2630             token_pattern = "KIND_DECLARATIONS_VARIABLE_TEXT";
2631             token_func = map_and_concat to_kind_declaration;
2632           };
2633           {
2634             token_pattern = "TO_STRING_VARIABLE_TEXT";
2635             token_func = map_and_concat to_to_string;
2636           };
2637           {
2638             token_pattern = "IS_VARIABLE_TEXT_VARIABLE_TEXT";
2639             token_func = map_and_concat to_is_variable_text;
2640           };
2641         ]
2642       ~filename:(full_fidelity_path_prefix ^ "full_fidelity_token_kind.ml")
2643       ~template:full_fidelity_token_kind_template
2644       ()
2645 end
2646
2647 (* GenerateFFTokenKind *)
2648
2649 module GenerateFFRustTokenKind = struct
2650   let token_kind x =
2651     match x.token_kind with
2652     | "Self" -> "SelfToken"
2653     | x -> x
2654
2655   let to_from_string x =
2656     let token_text = escape_token_text x.token_text in
2657     let guard =
2658       if x.allowed_as_identifier then
2659         "!only_reserved"
2660       else
2661         ""
2662     in
2663     let guard =
2664       if String.equal guard "" then
2665         ""
2666       else
2667         " if " ^ guard
2668     in
2669     sprintf
2670       "            b\"%s\"%s => Some(TokenKind::%s),\n"
2671       token_text
2672       guard
2673       (token_kind x)
2674
2675   let rust_tag = ref (-1)
2676
2677   let to_kind_declaration x =
2678     incr rust_tag;
2679     sprintf "    %s = %d,\n" (token_kind x) !rust_tag
2680
2681   let token_text x = escape_token_text x.token_text
2682
2683   let to_to_string x =
2684     sprintf
2685       "            TokenKind::%s => \"%s\",\n"
2686       (token_kind x)
2687       (token_text x)
2688
2689   let ocaml_tag = ref (-1)
2690
2691   let to_ocaml_tag x =
2692     incr ocaml_tag;
2693     sprintf "            TokenKind::%s => %d,\n" (token_kind x) !ocaml_tag
2694
2695   let from_u8_tag = ref (-1)
2696
2697   let to_try_from_u8 x =
2698     incr from_u8_tag;
2699     sprintf
2700       "            %d => Some(TokenKind::%s),\n"
2701       !from_u8_tag
2702       (token_kind x)
2703
2704   let to_width x =
2705     let len =
2706       if String.equal (token_kind x) "Backslash" then
2707         1
2708       else
2709         String.length (token_text x)
2710     in
2711     assert (len > 0);
2712     sprintf
2713       "            TokenKind::%s => Some(unsafe { NonZeroUsize::new_unchecked(%d) }),\n"
2714       (token_kind x)
2715       len
2716
2717   let full_fidelity_rust_token_kind_template =
2718     make_header CStyle ""
2719     ^ "
2720
2721 use std::num::NonZeroUsize;
2722
2723 use ocamlrep::{FromOcamlRep, ToOcamlRep};
2724
2725 #[allow(non_camel_case_types)] // allow Include_once and Require_once
2726 #[derive(Debug, Copy, Clone, PartialEq, Ord, Eq, PartialOrd, FromOcamlRep, ToOcamlRep)]
2727 #[repr(u8)]
2728 pub enum TokenKind {
2729     // No text tokens
2730 KIND_DECLARATIONS_NO_TEXT    // Given text tokens
2731 KIND_DECLARATIONS_GIVEN_TEXT    // Variable text tokens
2732 KIND_DECLARATIONS_VARIABLE_TEXT}
2733
2734 impl TokenKind {
2735     pub fn to_string(self) -> &'static str {
2736         match self {
2737             // No text tokens
2738 TO_STRING_NO_TEXT            // Given text tokens
2739 TO_STRING_GIVEN_TEXT            // Variable text tokes
2740 TO_STRING_VARIABLE_TEXT        }
2741     }
2742
2743     pub fn from_string(
2744         keyword: &[u8],
2745         only_reserved: bool,
2746     ) -> Option<Self> {
2747         match keyword {
2748             b\"true\" if !only_reserved => Some(TokenKind::BooleanLiteral),
2749             b\"false\" if !only_reserved => Some(TokenKind::BooleanLiteral),
2750 FROM_STRING_GIVEN_TEXT            _ => None,
2751         }
2752     }
2753
2754     pub fn ocaml_tag(self) -> u8 {
2755         match self {
2756 OCAML_TAG_NO_TEXTOCAML_TAG_GIVEN_TEXTOCAML_TAG_VARIABLE_TEXT        }
2757     }
2758
2759     pub fn try_from_u8(tag: u8) -> Option<Self> {
2760         match tag {
2761 FROM_U8_NO_TEXTFROM_U8_GIVEN_TEXTFROM_U8_VARIABLE_TEXT            _ => None,
2762         }
2763     }
2764
2765     pub fn fixed_width(self) -> Option<NonZeroUsize> {
2766         match self {
2767 WIDTH_GIVEN_TEXT            _ => None,
2768         }
2769     }
2770 }
2771 "
2772
2773   let full_fidelity_token_kind =
2774     Full_fidelity_schema.make_template_file
2775       ~token_no_text_transformations:
2776         [
2777           {
2778             token_pattern = "KIND_DECLARATIONS_NO_TEXT";
2779             token_func = map_and_concat to_kind_declaration;
2780           };
2781           {
2782             token_pattern = "TO_STRING_NO_TEXT";
2783             token_func = map_and_concat to_to_string;
2784           };
2785           {
2786             token_pattern = "OCAML_TAG_NO_TEXT";
2787             token_func = map_and_concat to_ocaml_tag;
2788           };
2789           {
2790             token_pattern = "FROM_U8_NO_TEXT";
2791             token_func = map_and_concat to_try_from_u8;
2792           };
2793         ]
2794       ~token_given_text_transformations:
2795         [
2796           {
2797             token_pattern = "KIND_DECLARATIONS_GIVEN_TEXT";
2798             token_func = map_and_concat to_kind_declaration;
2799           };
2800           {
2801             token_pattern = "FROM_STRING_GIVEN_TEXT";
2802             token_func = map_and_concat to_from_string;
2803           };
2804           {
2805             token_pattern = "TO_STRING_GIVEN_TEXT";
2806             token_func = map_and_concat to_to_string;
2807           };
2808           {
2809             token_pattern = "OCAML_TAG_GIVEN_TEXT";
2810             token_func = map_and_concat to_ocaml_tag;
2811           };
2812           {
2813             token_pattern = "FROM_U8_GIVEN_TEXT";
2814             token_func = map_and_concat to_try_from_u8;
2815           };
2816           {
2817             token_pattern = "WIDTH_GIVEN_TEXT";
2818             token_func = map_and_concat to_width;
2819           };
2820         ]
2821       ~token_variable_text_transformations:
2822         [
2823           {
2824             token_pattern = "KIND_DECLARATIONS_VARIABLE_TEXT";
2825             token_func = map_and_concat to_kind_declaration;
2826           };
2827           {
2828             token_pattern = "TO_STRING_VARIABLE_TEXT";
2829             token_func = map_and_concat to_to_string;
2830           };
2831           {
2832             token_pattern = "OCAML_TAG_VARIABLE_TEXT";
2833             token_func = map_and_concat to_ocaml_tag;
2834           };
2835           {
2836             token_pattern = "FROM_U8_VARIABLE_TEXT";
2837             token_func = map_and_concat to_try_from_u8;
2838           };
2839         ]
2840       ~filename:(full_fidelity_path_prefix ^ "token_kind.rs")
2841       ~template:full_fidelity_rust_token_kind_template
2842       ()
2843 end
2844
2845 (* GenerateFFTRustTokenKind *)
2846
2847 module GenerateFFOperatorRust = struct
2848   let template =
2849     make_header CStyle ""
2850     ^ "
2851
2852 use ocamlrep::{FromOcamlRep, ToOcamlRep};
2853
2854 #[derive(FromOcamlRep, ToOcamlRep)]
2855 pub enum Operator {
2856 OPERATORS}
2857 "
2858
2859   let full_fidelity_operators =
2860     Full_fidelity_schema.make_template_file
2861       ~operator_transformations:
2862         [
2863           {
2864             operator_pattern = "OPERATORS";
2865             operator_func =
2866               map_and_concat (fun op -> sprintf "    %sOperator,\n" op.name);
2867           };
2868         ]
2869       ~filename:(full_fidelity_path_prefix ^ "operator_generated.rs")
2870       ~template
2871       ()
2872 end
2873
2874 (* GenerateFFOperatorRust *)
2875
2876 module GenerateFFOperator = struct
2877   let template =
2878     make_header MLStyle ""
2879     ^ "
2880 module type Sig = sig
2881   type t =
2882 OPERATOR_DECL_SIGend
2883
2884 module Impl : Sig = struct
2885   type t =
2886 OPERATOR_DECL_IMPLend
2887 "
2888
2889   let op_pattern prefix op = sprintf "%s| %sOperator\n" prefix op.name
2890
2891   let transform pattern =
2892     {
2893       operator_pattern = pattern;
2894       operator_func = map_and_concat (op_pattern "  ");
2895     }
2896
2897   let full_fidelity_operator =
2898     Full_fidelity_schema.make_template_file
2899       ~operator_transformations:
2900         [transform "OPERATOR_DECL_SIG"; transform "OPERATOR_DECL_IMPL"]
2901       ~filename:
2902         (full_fidelity_path_prefix ^ "full_fidelity_operator_generated.ml")
2903       ~template
2904       ()
2905 end
2906
2907 module GenerateSchemaVersion = struct
2908   let template =
2909     make_header CStyle ""
2910     ^ sprintf
2911         "
2912 pub const VERSION: &str = \"%s\";
2913 "
2914         Full_fidelity_schema.full_fidelity_schema_version_number
2915
2916   let gen =
2917     Full_fidelity_schema.make_template_file
2918       ~filename:
2919         "hphp/hack/src/parser/schema/full_fidelity_schema_version_number.rs"
2920       ~template
2921       ()
2922 end
2923
2924 let templates =
2925   [
2926     GenerateFFOperatorRust.full_fidelity_operators;
2927     GenerateFFOperator.full_fidelity_operator;
2928     GenerateFFSyntaxType.full_fidelity_syntax_type;
2929     GenerateFFSyntaxSig.full_fidelity_syntax_sig;
2930     GenerateFFTriviaKind.full_fidelity_trivia_kind;
2931     GenerateFFRustTriviaKind.full_fidelity_trivia_kind;
2932     GenerateFFSyntax.full_fidelity_syntax;
2933     GenerateFFRustSyntax.full_fidelity_syntax;
2934     GenerateFFRustSyntaxType.full_fidelity_syntax;
2935     GenerateFFSyntaxKind.full_fidelity_syntax_kind;
2936     GenerateFFRustSyntaxKind.full_fidelity_syntax_kind;
2937     GenerateFFTokenKind.full_fidelity_token_kind;
2938     GenerateFFRustTokenKind.full_fidelity_token_kind;
2939     GenerateFFJSONSchema.full_fidelity_json_schema;
2940     GenerateFFSmartConstructors.full_fidelity_smart_constructors;
2941     GenerateFFRustSmartConstructors.full_fidelity_smart_constructors;
2942     GenerateFFRustPositionedSmartConstructors.positioned_smart_constructors;
2943     GenerateFFSyntaxSmartConstructors.full_fidelity_syntax_smart_constructors;
2944     GenerateFFRustSyntaxSmartConstructors
2945     .full_fidelity_syntax_smart_constructors;
2946     GenerateFFRustDeclModeSmartConstructors.decl_mode_smart_constructors;
2947     GenerateRustFlattenSmartConstructors.flatten_smart_constructors;
2948     GenerateRustDirectDeclSmartConstructors.direct_decl_smart_constructors;
2949     GenerateRustPairSmartConstructors.pair_smart_constructors;
2950     GenerateFFSmartConstructorsWrappers
2951     .full_fidelity_smart_constructors_wrappers;
2952     GenerateFFRustSmartConstructorsWrappers
2953     .full_fidelity_smart_constructors_wrappers;
2954     GenerateFFRustSyntaxVariantByRef.full_fidelity_syntax;
2955     GenerateSyntaxTypeImpl.full_fidelity_syntax;
2956     GenerateSyntaxChildrenIterator.full_fidelity_syntax;
2957     GenerateFFRustSyntaxImplByRef.full_fidelity_syntax;
2958     GenerateSyntaxSerialize.gen;
2959     GenerateSchemaVersion.gen;
2960   ]