hphp/hack/src/rupro/decl_folded_class_ffi.rs

   1 // Copyright (c) Meta Platforms, Inc. and affiliates.
   2 // This source code is licensed under the MIT license found in the
   3 // LICENSE file in the "hack" directory of this source tree.
   4
   5 use hackrs::decl_parser::DeclParser;
   6 use hackrs::folded_decl_provider::FoldedDeclProvider;
   7 use hackrs_test_utils::serde_store::{Compression, StoreOpts};
   8 use hackrs_test_utils::store::{make_shallow_decl_store, populate_shallow_decl_store};
   9 use indicatif::ParallelProgressIterator;
  10 use jwalk::WalkDir;
  11 use ocamlrep_ocamlpool::{ocaml_ffi_with_arena, Bump};
  12 use oxidized_by_ref::{decl_defs::DeclClassType, parser_options::ParserOptions};
  13 use pos::{Prefix, RelativePath, RelativePathCtx, ToOxidized, TypeName};
  14 use rayon::iter::{IntoParallelIterator, ParallelIterator};
  15 use std::cmp;
  16 use std::collections::BTreeMap;
  17 use std::path::{Path, PathBuf};
  18 use std::sync::Arc;
  19 use ty::decl::{folded::FoldedClass, shallow};
  20 use ty::reason::BReason;
  21
  22 fn find_hack_files(path: impl AsRef<Path>) -> impl Iterator<Item = PathBuf> {
  23     WalkDir::new(path)
  24         .into_iter()
  25         .filter_map(|e| e.ok())
  26         .filter(|e| !e.file_type().is_dir())
  27         .map(|e| e.path())
  28         .filter(|path| find_utils::is_hack(path))
  29 }
  30
  31 ocaml_ffi_with_arena! {
  32     fn fold_classes_in_files_ffi<'a>(
  33         arena: &'a Bump,
  34         root: &'a Path,
  35         opts: &'a ParserOptions<'a>,
  36         files: &'a [oxidized_by_ref::relative_path::RelativePath<'a>],
  37     ) -> Result<BTreeMap<RelativePath, Vec<DeclClassType<'a>>>, String> {
  38         let files: Vec<RelativePath> = files.iter().map(Into::into).collect();
  39         let path_ctx = Arc::new(RelativePathCtx {
  40             root: root.into(),
  41             ..Default::default()
  42         });
  43         let file_provider: Arc<dyn file_provider::FileProvider> =
  44             Arc::new(file_provider::PlainFileProvider::with_no_cache(path_ctx));
  45         let decl_parser = DeclParser::with_options(file_provider, opts);
  46         let shallow_decl_store = make_shallow_decl_store(StoreOpts::Unserialized);
  47
  48         let reverse_files = files.iter().copied().rev().collect::<Vec<_>>();
  49         for path in &reverse_files {
  50             let mut decls = decl_parser.parse(*path).unwrap();
  51             decls.reverse(); // To match OCaml behavior for name collisions
  52             shallow_decl_store.add_decls(decls).unwrap();
  53         };
  54
  55         let folded_decl_provider =
  56             hackrs_test_utils::decl_provider::make_folded_decl_provider(
  57                 StoreOpts::Unserialized,
  58                 None,
  59                 shallow_decl_store,
  60                 decl_parser.clone(),
  61             );
  62
  63         files.into_iter().map(|filename| {
  64             let classes: Vec<TypeName> = decl_parser
  65                 .parse(filename)
  66                 .map_err(|e| format!("Failed to parse {:?}: {:?}", filename, e))?
  67                 .into_iter()
  68                 .filter_map(|decl: ty::decl::shallow::Decl<BReason>| match decl {
  69                     shallow::Decl::Class(name, _) => Some(name),
  70                     _ => None,
  71                 })
  72                 .collect();
  73             // Declare the classes in the reverse of their order in the file, to
  74             // match the OCaml behavior. This should only matter when emitting
  75             // errors for cyclic definitions.
  76             for &name in classes.iter().rev() {
  77                 folded_decl_provider.get_class(name.into(), name)
  78                     .map_err(|e| format!("Failed to fold class {}: {:?}", name, e))?;
  79             }
  80             Ok((filename, classes.into_iter().map(|name| {
  81                 Ok(folded_decl_provider
  82                     .get_class(name.into(), name)
  83                     .map_err(|e| format!("Failed to fold class {}: {:?}", name, e))?
  84                     .ok_or_else(|| format!("Decl not found: class {}", name))?
  85                     .to_oxidized(arena))
  86             }).collect::<Result<_, String>>()?))
  87         })
  88         .collect()
  89     }
  90
  91     fn show_decl_class_type_ffi<'a>(arena: &'a Bump, decl: &'a DeclClassType<'a>) -> String {
  92         let decl = <FoldedClass<BReason>>::from(decl);
  93         format!("{:#?}", decl)
  94     }
  95
  96     // Due to memory constraints when folding over a large directory, it may be necessary to
  97     // fold over www in parts. This is achieved by finding the shallow decls of all of the files
  98     // in the directory but then only folding over a portion of the classes in those files
  99     // num_partitions controls the numbers of partitions (>= 1) the classes are divided into
 100     // and partition_index controls which partition is being folded
 101     // Returns a SMap from class_name to folded_decl
 102     fn partition_and_fold_dir_ffi<'a>(
 103         arena: &'a Bump,
 104         www_root: &'a Path,
 105         opts: &'a ParserOptions<'a>,
 106         num_partitions: &'a usize,
 107         partition_index: &'a usize,
 108     ) -> BTreeMap<String, DeclClassType<'a>> {
 109         // Collect hhi files
 110         let hhi_root = tempdir::TempDir::new("rupro_decl_repo_hhi").unwrap();
 111         hhi::write_hhi_files(hhi_root.path()).unwrap();
 112         let hhi_root_path: PathBuf = hhi_root.path().into();
 113         let mut filenames: Vec<RelativePath> = find_hack_files(&hhi_root_path)
 114             .map(|path| RelativePath::new(Prefix::Hhi, path.strip_prefix(&hhi_root_path).unwrap()))
 115             .collect();
 116         // Collect www files
 117         filenames.extend(
 118             find_hack_files(&www_root).map(|path| match path.strip_prefix(&www_root) {
 119                 Ok(suffix) => RelativePath::new(Prefix::Root, suffix),
 120                 Err(..) => RelativePath::new(Prefix::Dummy, &path),
 121             }),
 122         );
 123
 124         let path_ctx = Arc::new(RelativePathCtx {
 125             root: www_root.into(),
 126             hhi: hhi_root_path,
 127             ..Default::default()
 128         });
 129
 130         // Parse and gather shallow decls
 131         let file_provider: Arc<dyn file_provider::FileProvider> =
 132             Arc::new(file_provider::PlainFileProvider::with_no_cache(path_ctx));
 133         let decl_parser: DeclParser<BReason> = DeclParser::with_options(file_provider, opts);
 134         let shallow_decl_store = make_shallow_decl_store(StoreOpts::Serialized(Compression::default()));
 135         let mut classes =
 136             populate_shallow_decl_store(&shallow_decl_store, decl_parser.clone(), &filenames);
 137         classes.sort();
 138         let folded_decl_provider = hackrs_test_utils::decl_provider::make_folded_decl_provider(
 139             StoreOpts::Serialized(Compression::default()),
 140             None,
 141             shallow_decl_store,
 142             decl_parser,
 143         );
 144
 145         let len = classes.len();
 146         // Add 1 to size to ensure that we only have num_partitions
 147         let size_of_slices = (len / num_partitions) + 1;
 148         // Account for edge case where partition_index * size goes out of bounds of the array
 149         let start_index = cmp::min(len, partition_index * size_of_slices);
 150         // end_index is exclusive, so no need to say len - 1
 151         let end_index = cmp::min((partition_index + 1) * size_of_slices, len);
 152         // If start_index = end_index, this will be empty vec
 153         let s: Vec<(String, Arc<FoldedClass<BReason>>)> = (&classes[start_index..end_index])
 154             .into_par_iter()
 155             .progress_count(len.try_into().unwrap())
 156             .map(|class| {
 157                 let folded_decl = folded_decl_provider
 158                     .get_class((*class).into(), *class)
 159                     .expect("failed to fold class")
 160                     .expect("failed to look up class");
 161                 (class.as_str().into(), folded_decl)
 162             })
 163             .collect();
 164         s.into_iter()
 165             .map(|(name, fc)| (name, fc.to_oxidized(arena)))
 166             .collect()
 167     }
 168
 169     fn decls_equal_ffi<'a>(arena: &'a Bump, ocaml_decl: &'a DeclClassType<'a>, rust_decl: &'a DeclClassType<'a>) -> bool {
 170         let ocaml_decl = <FoldedClass<BReason>>::from(ocaml_decl);
 171         let rust_decl = <FoldedClass<BReason>>::from(rust_decl);
 172         ocaml_decl == rust_decl
 173     }
 174 }