hphp/hack/src/parser/rust_parser_ffi.rs

   1 // Copyright (c) 2019, Facebook, Inc.
   2 // All rights reserved.
   3 //
   4 // This source code is licensed under the MIT license found in the
   5 // LICENSE file in the "hack" directory of this source tree.
   6
   7 use ocamlpool_rust::{caml_raise, ocamlvalue::Ocamlvalue, utils::*};
   8 use parser_rust::{
   9     lexer::Lexer, minimal_token::MinimalToken, minimal_trivia::MinimalTrivia,
  10     source_text::SourceText,
  11 };
  12 use rust_to_ocaml::{to_list, SerializationContext};
  13 use syntax_tree::mode_parser::parse_mode;
  14
  15 use oxidized::relative_path::RelativePath;
  16
  17 #[macro_export]
  18 macro_rules! parse {
  19     ($name:ident, $parser:ident, $syntax:ty) => {
  20         use ocamlpool_rust::caml_raise;
  21         use $name::{ocamlpool_enter, ocamlpool_leave};
  22         mod $name {
  23             use super::*;
  24             extern "C" {
  25                 pub fn ocamlpool_enter();
  26                 pub fn ocamlpool_leave();
  27             }
  28
  29             use ocaml::Value;
  30             use ocamlpool_rust::{caml_raise, ocamlvalue::Ocamlvalue, utils::*};
  31             use parser_rust::{
  32                 self as parser,
  33                 lexer::Lexer,
  34                 parser_env::ParserEnv,
  35                 source_text::SourceText,
  36                 stack_limit::StackLimit,
  37             };
  38             use rust_to_ocaml::{to_list, SerializationContext, ToOcaml};
  39             use syntax_tree::{mode_parser::parse_mode, SyntaxTree};
  40             use oxidized::relative_path::RelativePath;
  41
  42             pub unsafe fn parse(ocaml_source_text : Value, opts : Value, mut l : Value) -> Value {
  43                     let ocaml_source_text_value = ocaml_source_text.0;
  44
  45                     let is_experimental_mode = bool_field(&opts, 0);
  46                     let hhvm_compat_mode = bool_field(&opts, 1);
  47                     let php5_compat_mode = bool_field(&opts, 2);
  48                     let codegen = bool_field(&opts, 3);
  49                     let allow_new_attribute_syntax = bool_field(&opts, 4);
  50                     let leak_rust_tree = bool_field(&opts, 5);
  51                     let env = ParserEnv {
  52                         is_experimental_mode,
  53                         hhvm_compat_mode,
  54                         php5_compat_mode,
  55                         codegen,
  56                         allow_new_attribute_syntax,
  57                     };
  58
  59                     // Note: Determining the current thread size cannot be done portably,
  60                     // therefore assume the worst (running on non-main thread with min size, 2MiB)
  61                     const KI: usize = 1024;
  62                     const MI: usize = KI * KI;
  63                     const MAX_STACK_SIZE: usize = 1024 * MI;
  64                     let mut stack_size = 2 * MI;
  65                     let mut default_stack_size_sufficient = true;
  66                     parser::stack_limit::init();
  67                     loop {
  68                         if stack_size > MAX_STACK_SIZE {
  69                             panic!("Rust FFI exceeded maximum allowed stack of {} KiB", MAX_STACK_SIZE / KI);
  70                         }
  71
  72                         // Avoid eagerly wasting of space that will not be used in practice (WWW),
  73                         // but only for degenerate test cases (/test/{slow,quick}), by starting off
  74                         // with small stack (default thread) then fall back to bigger ones (custom thread).
  75                         // Since we're doubling the stack the time is: t + 2*t + 4*t + ...
  76                         // where the total parse time with unbounded stack is T=k*t, which is
  77                         // bounded by 2*T (much less in practice due to superlinear parsing time).
  78                         let next_stack_size = if default_stack_size_sufficient {
  79                             13 * MI // assume we need much more if default stack size isn't enough
  80                         } else { // exponential backoff to limit parsing time to at most twice as long
  81                             2 * stack_size
  82                         };
  83                         // Note: detect almost full stack by setting "slack" of 60% for StackLimit because
  84                         // Syntax::to_ocaml is deeply & mutually recursive and uses nearly 2.5x of stack
  85                         // TODO: rewrite to_ocaml iteratively & reduce it to "stack_size - MB" as in HHVM
  86                         // (https://github.com/facebook/hhvm/blob/master/hphp/runtime/base/request-info.h)
  87                         let relative_stack_size = stack_size - stack_size*6/10;
  88                         let content = str_field(&ocaml_source_text, 2);
  89                         let relative_path_raw = block_field(&ocaml_source_text, 0);
  90
  91                         let env = env.clone();
  92                         let try_parse = move || {
  93                             let stack_limit = StackLimit::relative(relative_stack_size);
  94                             stack_limit.reset();
  95                             let stack_limit_ref = &stack_limit;
  96                             let relative_path = RelativePath::from_ocamlvalue(&relative_path_raw);
  97                             let file_path = relative_path.path_str().to_owned();
  98                             ocamlpool_enter();
  99                             let maybe_l = std::panic::catch_unwind(move || {
 100                                 let source_text = SourceText::make_with_raw(
 101                                     &relative_path,
 102                                     &content.data(),
 103                                     ocaml_source_text_value,
 104                                 );
 105                                 let mut parser = $parser::make(&source_text, env);
 106                                 let root = parser.parse_script(Some(&stack_limit_ref));
 107                                 let errors = parser.errors();
 108                                 let state = parser.sc_state();
 109
 110                                 // traversing the parsed syntax tree uses about 1/3 of the stack
 111                                 let context = SerializationContext::new(ocaml_source_text_value);
 112                                 let ocaml_root = root.to_ocaml(&context);
 113                                 let ocaml_errors = errors.ocamlvalue();
 114                                 let ocaml_state = state.to_ocaml(&context);
 115                                 let tree = if leak_rust_tree {
 116                                     let required_stack_size = if default_stack_size_sufficient {
 117                                         None
 118                                     } else {
 119                                         Some(stack_size)
 120                                     };
 121                                     let mode = parse_mode(&source_text);
 122                                     let tree = Box::new(SyntaxTree::build(&source_text, root, errors, mode, (), required_stack_size));
 123                                     Some(Box::leak(tree) as *const SyntaxTree<$syntax, ()> as usize)
 124                                 } else {
 125                                     None
 126                                 };
 127                                 let ocaml_tree = tree.ocamlvalue();
 128
 129                                 let res = caml_tuple(&[
 130                                     ocaml_state,
 131                                     ocaml_root,
 132                                     ocaml_errors,
 133                                     ocaml_tree,
 134                                 ]);
 135                                 let l = ocaml::Value::new(res);
 136                                 l
 137                             });
 138                             ocamlpool_leave();  // note: must run even if a panic occurs
 139                             match maybe_l {
 140                                 Ok(l) => Some(l),
 141                                 Err(_) if stack_limit.exceeded() => {
 142                                     // Not always printing warning here because this would fail some HHVM tests
 143                                     let istty = libc::isatty(libc::STDERR_FILENO as i32) != 0;
 144                                     if istty || std::env::var_os("HH_TEST_MODE").is_some() {
 145                                         eprintln!("[hrust] warning: parser exceeded stack of {} KiB on: {}",
 146                                                   stack_limit.get() / KI,
 147                                                   file_path,
 148                                         );
 149                                     }
 150                                     None
 151                                 }
 152                                 Err(msg) => panic!(msg),
 153                             }
 154                         };
 155                         stack_size = next_stack_size;
 156
 157                         let l_opt = if default_stack_size_sufficient {
 158                             try_parse()
 159                         } else {
 160                             std::thread::Builder::new().stack_size(stack_size).spawn(try_parse)
 161                                 .expect("ERROR: thread::spawn")
 162                                 .join().expect("ERROR: failed to wait on new thread")
 163                         };
 164
 165                         match l_opt {
 166                             Some(ocaml_result) => {
 167                                 l = ocaml_result;
 168                                 break;
 169                             },
 170                             _ => default_stack_size_sufficient = false,
 171                         }
 172                     }
 173                     l
 174             }
 175         }
 176
 177         caml_raise!($name, |ocaml_source_text, opts|, <l>, {
 178             l = $name::parse(ocaml_source_text, opts, l)
 179         } -> l);
 180     }
 181 }
 182
 183 extern "C" {
 184     fn ocamlpool_enter();
 185     fn ocamlpool_leave();
 186 }
 187
 188 caml_raise!(rust_parse_mode, |ocaml_source_text|, <l>, {
 189     let relative_path_raw = block_field(&ocaml_source_text, 0);
 190     let relative_path = RelativePath::from_ocamlvalue(&relative_path_raw);
 191     let content = str_field(&ocaml_source_text, 2);
 192     let source_text = SourceText::make(&relative_path, &content.data());
 193
 194     let mode = parse_mode(&source_text);
 195
 196     ocamlpool_enter();
 197     let ocaml_mode = mode.ocamlvalue();
 198     l = ocaml::Value::new(ocaml_mode);
 199     ocamlpool_leave();
 200 } -> l);
 201
 202 macro_rules! scan_trivia {
 203     ($name:ident) => {
 204         caml_raise!($name, |ocaml_source_text, offset|, <l>, {
 205             let relative_path_raw = block_field(&ocaml_source_text, 0);
 206             let relative_path = RelativePath::from_ocamlvalue(&relative_path_raw);
 207             let content = str_field(&ocaml_source_text, 2);
 208             let source_text = SourceText::make(&relative_path, &content.data());
 209
 210             let offset = offset.usize_val();
 211
 212             let is_experimental_mode = false;
 213
 214             let mut lexer : Lexer<MinimalToken> = Lexer::make_at(
 215                 &source_text,
 216                 is_experimental_mode,
 217                 offset,
 218             );
 219
 220             let res : Vec<MinimalTrivia> = lexer.$name();
 221
 222             ocamlpool_enter();
 223             let context = SerializationContext::new(ocaml_source_text.0);
 224             let trivia_list = to_list(&res, &context);
 225             l = ocaml::Value::new(trivia_list);
 226             ocamlpool_leave();
 227         } -> l);
 228     };
 229 }
 230
 231 scan_trivia!(scan_leading_xhp_trivia);
 232 scan_trivia!(scan_trailing_xhp_trivia);
 233 scan_trivia!(scan_leading_php_trivia);
 234 scan_trivia!(scan_trailing_php_trivia);