hphp/hack/src/parser/rust_parser_ffi.rs

   1 // Copyright (c) 2019, Facebook, Inc.
   2 // All rights reserved.
   3 //
   4 // This source code is licensed under the MIT license found in the
   5 // LICENSE file in the "hack" directory of this source tree.
   6
   7 mod ocaml_coroutine_state;
   8 mod ocaml_syntax;
   9 mod ocaml_syntax_generated;
  10 pub mod rust_to_ocaml;
  11
  12 use parser_rust as parser;
  13
  14 use parser::lexer::Lexer;
  15 use parser::minimal_parser::MinimalSyntaxParser;
  16 use parser::minimal_syntax::MinimalSyntax;
  17 use parser::minimal_token::MinimalToken;
  18 use parser::minimal_trivia::MinimalTrivia;
  19 use parser::mode_parser::parse_mode;
  20 use parser::parser_env::ParserEnv;
  21 use parser::source_text::SourceText;
  22 use parser::stack_limit::StackLimit;
  23 use parser_core_types::syntax_tree::SyntaxTree;
  24
  25 use ocamlpool_rust::{caml_raise, ocamlvalue::Ocamlvalue, utils::*};
  26 use rust_to_ocaml::{to_list, SerializationContext, ToOcaml};
  27
  28 use parser::parser::Parser;
  29 use parser::positioned_smart_constructors::*;
  30 use parser::positioned_syntax::{PositionedSyntax, PositionedValue};
  31 use parser::positioned_token::PositionedToken;
  32 use parser::smart_constructors::NoState;
  33 use parser::smart_constructors_wrappers::WithKind;
  34
  35 use oxidized::relative_path::RelativePath;
  36
  37 type PositionedSyntaxParser<'a> = Parser<'a, WithKind<PositionedSmartConstructors>, NoState>;
  38
  39 use crate::ocaml_coroutine_state::OcamlCoroutineState;
  40 use crate::ocaml_syntax::OcamlSyntax;
  41 use parser::coroutine_smart_constructors::{CoroutineSmartConstructors, State as CoroutineState};
  42
  43 type CoroutineParser<'a> = Parser<
  44     'a,
  45     WithKind<
  46         CoroutineSmartConstructors<
  47             'a,
  48             OcamlSyntax<PositionedValue>,
  49             OcamlCoroutineState<'a, OcamlSyntax<PositionedValue>>,
  50         >,
  51     >,
  52     OcamlCoroutineState<'a, OcamlSyntax<PositionedValue>>,
  53 >;
  54
  55 type CoroutineParserLeakTree<'a> = Parser<
  56     'a,
  57     WithKind<
  58         CoroutineSmartConstructors<'a, PositionedSyntax, CoroutineState<'a, PositionedSyntax>>,
  59     >,
  60     CoroutineState<'a, PositionedSyntax>,
  61 >;
  62
  63 use parser::decl_mode_smart_constructors::DeclModeSmartConstructors;
  64 use parser::decl_mode_smart_constructors::State as DeclModeState;
  65
  66 type DeclModeParser<'a> = Parser<
  67     'a,
  68     WithKind<DeclModeSmartConstructors<'a, PositionedSyntax, PositionedToken, PositionedValue>>,
  69     DeclModeState<'a, PositionedSyntax>,
  70 >;
  71
  72 use parser::verify_smart_constructors::State as VerifyState;
  73 use parser::verify_smart_constructors::VerifySmartConstructors;
  74
  75 type VerifyParser<'a> = Parser<'a, WithKind<VerifySmartConstructors>, VerifyState>;
  76
  77 extern "C" {
  78     fn ocamlpool_enter();
  79     fn ocamlpool_leave();
  80 }
  81
  82 macro_rules! parse {
  83     ($name:ident, $parser:ident, $syntax:ty) => {
  84         caml_raise!($name, |ocaml_source_text, opts|, <l>, {
  85             let ocaml_source_text_value = ocaml_source_text.0;
  86
  87             let is_experimental_mode = bool_field(&opts, 0);
  88             let hhvm_compat_mode = bool_field(&opts, 1);
  89             let php5_compat_mode = bool_field(&opts, 2);
  90             let codegen = bool_field(&opts, 3);
  91             let allow_new_attribute_syntax = bool_field(&opts, 4);
  92             let leak_rust_tree = bool_field(&opts, 5);
  93             let disallow_func_ptrs_in_constants = bool_field(&opts, 6);
  94             let env = ParserEnv {
  95                 is_experimental_mode,
  96                 hhvm_compat_mode,
  97                 php5_compat_mode,
  98                 codegen,
  99                 allow_new_attribute_syntax,
 100                 disallow_func_ptrs_in_constants,
 101             };
 102
 103             // Note: Determining the current thread size cannot be done portably,
 104             // therefore assume the worst (running on non-main thread with min size, 2MiB)
 105             const KI: usize = 1024;
 106             const MI: usize = KI * KI;
 107             const MAX_STACK_SIZE: usize = 1024 * MI;
 108             let mut stack_size = 2 * MI;
 109             let mut default_stack_size_sufficient = true;
 110             parser::stack_limit::init();
 111             loop {
 112                 if stack_size > MAX_STACK_SIZE {
 113                     panic!("Rust FFI exceeded maximum allowed stack of {} KiB", MAX_STACK_SIZE / KI);
 114                 }
 115
 116                 // Avoid eagerly wasting of space that will not be used in practice (WWW),
 117                 // but only for degenerate test cases (/test/{slow,quick}), by starting off
 118                 // with small stack (default thread) then fall back to bigger ones (custom thread).
 119                 // Since we're doubling the stack the time is: t + 2*t + 4*t + ...
 120                 // where the total parse time with unbounded stack is T=k*t, which is
 121                 // bounded by 2*T (much less in practice due to superlinear parsing time).
 122                 let next_stack_size = if default_stack_size_sufficient {
 123                     13 * MI // assume we need much more if default stack size isn't enough
 124                 } else { // exponential backoff to limit parsing time to at most twice as long
 125                     2 * stack_size
 126                 };
 127                 // Note: detect almost full stack by setting "slack" of 60% for StackLimit because
 128                 // Syntax::to_ocaml is deeply & mutually recursive and uses nearly 2.5x of stack
 129                 // TODO: rewrite to_ocaml iteratively & reduce it to "stack_size - MB" as in HHVM
 130                 // (https://github.com/facebook/hhvm/blob/master/hphp/runtime/base/request-info.h)
 131                 let relative_stack_size = stack_size - stack_size*6/10;
 132                 let content = str_field(&ocaml_source_text, 2);
 133                 let relative_path_raw = block_field(&ocaml_source_text, 0);
 134
 135                 let env = env.clone();
 136                 let try_parse = move || {
 137                     let stack_limit = StackLimit::relative(relative_stack_size);
 138                     stack_limit.reset();
 139                     let stack_limit_ref = &stack_limit;
 140                     let relative_path = RelativePath::from_ocamlvalue(&relative_path_raw);
 141                     let file_path = relative_path.path_str().to_owned();
 142                     ocamlpool_enter();
 143                     let maybe_l = std::panic::catch_unwind(move || {
 144                         let source_text = SourceText::make_with_raw(
 145                             &relative_path,
 146                             &content.data(),
 147                             ocaml_source_text_value,
 148                         );
 149                         let mut parser = $parser::make(&source_text, env);
 150                         let root = parser.parse_script(Some(&stack_limit_ref));
 151                         let errors = parser.errors();
 152                         let state = parser.sc_state();
 153
 154                         // traversing the parsed syntax tree uses about 1/3 of the stack
 155                         let context = SerializationContext::new(ocaml_source_text_value);
 156                         let ocaml_root = root.to_ocaml(&context);
 157                         let ocaml_errors = errors.ocamlvalue();
 158                         let ocaml_state = state.to_ocaml(&context);
 159                         let tree = if leak_rust_tree {
 160                             let required_stack_size = if default_stack_size_sufficient {
 161                                 None
 162                             } else {
 163                                 Some(stack_size)
 164                             };
 165                             let mode = parse_mode(&source_text);
 166                             let tree = Box::new(SyntaxTree::build(&source_text, root, errors, mode, (), required_stack_size));
 167                             Some(Box::leak(tree) as *const SyntaxTree<$syntax, ()> as usize)
 168                         } else {
 169                             None
 170                         };
 171                         let ocaml_tree = tree.ocamlvalue();
 172
 173                         let res = caml_tuple(&[
 174                             ocaml_state,
 175                             ocaml_root,
 176                             ocaml_errors,
 177                             ocaml_tree,
 178                         ]);
 179                         let l = ocaml::Value::new(res);
 180                         l
 181                     });
 182                     ocamlpool_leave();  // note: must run even if a panic occurs
 183                     match maybe_l {
 184                         Ok(l) => Some(l),
 185                         Err(_) if stack_limit.exceeded() => {
 186                             // Not always printing warning here because this would fail some HHVM tests
 187                             let istty = libc::isatty(libc::STDERR_FILENO as i32) != 0;
 188                             if istty || std::env::var_os("HH_TEST_MODE").is_some() {
 189                                 eprintln!("[hrust] warning: parser exceeded stack of {} KiB on: {}",
 190                                           stack_limit.get() / KI,
 191                                           file_path,
 192                                 );
 193                             }
 194                             None
 195                         }
 196                         Err(msg) => panic!(msg),
 197                     }
 198                 };
 199                 stack_size = next_stack_size;
 200
 201                 let l_opt = if default_stack_size_sufficient {
 202                     try_parse()
 203                 } else {
 204                     std::thread::Builder::new().stack_size(stack_size).spawn(try_parse)
 205                         .expect("ERROR: thread::spawn")
 206                         .join().expect("ERROR: failed to wait on new thread")
 207                 };
 208
 209                 match l_opt {
 210                     Some(ocaml_result) => {
 211                         l = ocaml_result;
 212                         break;
 213                     },
 214                     _ => default_stack_size_sufficient = false,
 215                 }
 216             }
 217         } -> l);
 218     };
 219 }
 220
 221 parse!(parse_minimal, MinimalSyntaxParser, MinimalSyntax);
 222 parse!(parse_positioned, PositionedSyntaxParser, PositionedSyntax);
 223 parse!(
 224     parse_positioned_with_coroutine_sc,
 225     CoroutineParser,
 226     OcamlSyntax<PositionedValue>
 227 );
 228 parse!(
 229     parse_positioned_with_coroutine_sc_leak_tree,
 230     CoroutineParserLeakTree,
 231     PositionedSyntax
 232 );
 233 parse!(
 234     parse_positioned_with_decl_mode_sc,
 235     DeclModeParser,
 236     PositionedSyntax
 237 );
 238 parse!(
 239     parse_positioned_with_verify_sc,
 240     VerifyParser,
 241     PositionedSyntax
 242 );
 243
 244 caml_raise!(rust_parse_mode, |ocaml_source_text|, <l>, {
 245     let relative_path_raw = block_field(&ocaml_source_text, 0);
 246     let relative_path = RelativePath::from_ocamlvalue(&relative_path_raw);
 247     let content = str_field(&ocaml_source_text, 2);
 248     let source_text = SourceText::make(&relative_path, &content.data());
 249
 250     let mode = parse_mode(&source_text);
 251
 252     ocamlpool_enter();
 253     let ocaml_mode = mode.ocamlvalue();
 254     l = ocaml::Value::new(ocaml_mode);
 255     ocamlpool_leave();
 256 } -> l);
 257
 258 macro_rules! scan_trivia {
 259     ($name:ident) => {
 260         caml_raise!($name, |ocaml_source_text, offset|, <l>, {
 261             let relative_path_raw = block_field(&ocaml_source_text, 0);
 262             let relative_path = RelativePath::from_ocamlvalue(&relative_path_raw);
 263             let content = str_field(&ocaml_source_text, 2);
 264             let source_text = SourceText::make(&relative_path, &content.data());
 265
 266             let offset = offset.usize_val();
 267
 268             let is_experimental_mode = false;
 269
 270             let mut lexer : Lexer<MinimalToken> = Lexer::make_at(
 271                 &source_text,
 272                 is_experimental_mode,
 273                 offset,
 274             );
 275
 276             let res : Vec<MinimalTrivia> = lexer.$name();
 277
 278             ocamlpool_enter();
 279             let context = SerializationContext::new(ocaml_source_text.0);
 280             let trivia_list = to_list(&res, &context);
 281             l = ocaml::Value::new(trivia_list);
 282             ocamlpool_leave();
 283         } -> l);
 284     };
 285 }
 286
 287 scan_trivia!(scan_leading_xhp_trivia);
 288 scan_trivia!(scan_trailing_xhp_trivia);
 289 scan_trivia!(scan_leading_php_trivia);
 290 scan_trivia!(scan_trailing_php_trivia);