hphp/hack/src/decl/decl_redecl_service.ml

   1 (*
   2  * Copyright (c) 2015, Facebook, Inc.
   3  * All rights reserved.
   4  *
   5  * This source code is licensed under the MIT license found in the
   6  * LICENSE file in the "hack" directory of this source tree.
   7  *
   8  *)
   9
  10 (** On the fly defs-declaration are called when the user modified a file
  11   and we are not at initilalization time anymore. Therefore, we have a bit more
  12   work to do. We need to calculate what must be re-checked. *)
  13
  14 open Hh_prelude
  15 open Reordered_argument_collections
  16 open Typing_deps
  17
  18 type get_classes_in_file = Relative_path.t -> SSet.t
  19
  20 type fanout = {
  21   changed: DepSet.t;
  22   to_redecl: DepSet.t;
  23   to_recheck: DepSet.t;
  24 }
  25
  26 type redo_type_decl_result = {
  27   errors: Errors.t;
  28   fanout: fanout;
  29   old_decl_missing_count: int;
  30 }
  31
  32 let lvl = Hh_logger.Level.Debug
  33
  34 let shallow_decl_enabled (ctx : Provider_context.t) =
  35   TypecheckerOptions.shallow_class_decl (Provider_context.get_tcopt ctx)
  36
  37 let force_shallow_decl_fanout_enabled (ctx : Provider_context.t) =
  38   TypecheckerOptions.force_shallow_decl_fanout (Provider_context.get_tcopt ctx)
  39
  40 let empty_fanout =
  41   let empty = DepSet.make () in
  42   { changed = empty; to_redecl = empty; to_recheck = empty }
  43
  44 let compute_deps_neutral = (empty_fanout, 0)
  45
  46 (** This is the place where we are going to put everything necessary for
  47   the redeclaration. We could "pass" the values directly to the workers,
  48   but it gives too much work to the master and slows things down,
  49   so what we do instead is pass the data through shared memory via
  50   OnTheFlyStore.
  51   I tried replicating the data to speed things up but it had no effect. *)
  52 module OnTheFlyStore = GlobalStorage.Make (struct
  53   type t = Naming_table.defs_per_file
  54 end)
  55
  56 (** Compute the decls in a file *)
  57 let decl_file ctx errors fn =
  58   let (decl_errors, ()) =
  59     Errors.do_with_context fn Errors.Decl (fun () ->
  60         Decl.make_env ~sh:SharedMem.Uses ctx fn)
  61   in
  62   Errors.merge decl_errors errors
  63
  64 (** Given a set of classes, compare the old and the new decl and deduce
  65   what must be rechecked accordingly. *)
  66 let compare_classes_and_get_fanout ctx old_classes new_classes acc classes =
  67   let { changed; to_redecl; to_recheck } = acc in
  68   let ((rc, rdd, rdc), old_classes_missing) =
  69     Decl_compare.get_classes_deps ~ctx old_classes new_classes classes
  70   in
  71   let changed = DepSet.union rc changed in
  72   let to_redecl = DepSet.union rdd to_redecl in
  73   let to_recheck = DepSet.union rdc to_recheck in
  74   ({ changed; to_redecl; to_recheck }, old_classes_missing)
  75
  76 (** Given a set of functions, compare the old and the new decl and deduce
  77   what must be rechecked accordingly. *)
  78 let compare_funs_and_get_fanout
  79     ctx old_funs { changed; to_redecl; to_recheck } funs =
  80   let ((rc, rdd, rdc), old_funs_missing) =
  81     Decl_compare.get_funs_deps ~ctx old_funs funs
  82   in
  83   let changed = DepSet.union rc changed in
  84   let to_redecl = DepSet.union rdd to_redecl in
  85   let to_recheck = DepSet.union rdc to_recheck in
  86   ({ changed; to_redecl; to_recheck }, old_funs_missing)
  87
  88 (** Given a set of typedefs, compare the old and the new decl and deduce
  89   what must be rechecked accordingly. *)
  90 let compare_types_and_get_fanout
  91     ctx old_types { changed; to_redecl; to_recheck } types =
  92   let ((rc, rdc), old_types_missing) =
  93     Decl_compare.get_types_deps ~ctx old_types types
  94   in
  95   let changed = DepSet.union rc changed in
  96   let to_recheck = DepSet.union rdc to_recheck in
  97   ({ changed; to_redecl; to_recheck }, old_types_missing)
  98
  99 (* Given a set of global constants, compare the old and the new decl and
 100   deduce what must be rechecked accordingly. *)
 101 let compare_gconsts_and_get_fanout
 102     ctx old_gconsts { changed; to_redecl; to_recheck } gconsts =
 103   let ((rc, rdd, rdc), old_gconsts_missing) =
 104     Decl_compare.get_gconsts_deps ~ctx old_gconsts gconsts
 105   in
 106   let changed = DepSet.union rc changed in
 107   let to_redecl = DepSet.union rdd to_redecl in
 108   let to_recheck = DepSet.union rdc to_recheck in
 109   ({ changed; to_redecl; to_recheck }, old_gconsts_missing)
 110
 111 (* Given a set of modules, compare the old and the new decl and
 112   deduce what must be rechecked accordingly. *)
 113 let compare_module_and_get_fanout
 114     ctx old_modules new_modules { changed; to_redecl; to_recheck } modules =
 115   let ((rc, rdd, rdc), old_modules_missing) =
 116     Decl_compare.get_module_deps ~ctx ~old_modules ~new_modules ~modules
 117   in
 118   let changed = DepSet.union rc changed in
 119   let to_redecl = DepSet.union rdd to_redecl in
 120   let to_recheck = DepSet.union rdc to_recheck in
 121   ({ changed; to_redecl; to_recheck }, old_modules_missing)
 122
 123 (*****************************************************************************)
 124 (* Redeclares a list of files
 125  * And then computes the files that must be redeclared/rechecked by looking
 126  * at what changed in the signatures of the classes/functions.
 127  *)
 128 (*****************************************************************************)
 129
 130 (** Compute decls in files. Return errors raised during decling. *)
 131 let redeclare_files ctx filel =
 132   List.fold_left filel ~f:(decl_file ctx) ~init:Errors.empty
 133
 134 (** Invalidate local caches and compute decls in files. Return errors raised during decling. *)
 135 let decl_files ctx filel =
 136   SharedMem.invalidate_local_caches ();
 137   redeclare_files ctx filel
 138
 139 let compare_decls_and_get_fanout
 140     ctx defs_per_file (filel : Relative_path.t list) : fanout * int =
 141   let defs_in_files =
 142     List.map filel ~f:(fun fn -> Relative_path.Map.find defs_per_file fn)
 143   in
 144   let all_defs =
 145     List.fold_left
 146       defs_in_files
 147       ~f:FileInfo.merge_names
 148       ~init:FileInfo.empty_names
 149   in
 150   let { FileInfo.n_classes; n_funs; n_types; n_consts; n_modules } = all_defs in
 151   let acc = empty_fanout in
 152   (* Fetching everything at once is faster *)
 153   let old_funs = Decl_heap.Funs.get_old_batch n_funs in
 154   let (acc, old_funs_missing) =
 155     compare_funs_and_get_fanout ctx old_funs acc n_funs
 156   in
 157   let old_types = Decl_heap.Typedefs.get_old_batch n_types in
 158   let (acc, old_types_missing) =
 159     compare_types_and_get_fanout ctx old_types acc n_types
 160   in
 161   let old_consts = Decl_heap.GConsts.get_old_batch n_consts in
 162   let (acc, old_gconsts_missing) =
 163     compare_gconsts_and_get_fanout ctx old_consts acc n_consts
 164   in
 165
 166   let (acc, old_classes_missing) =
 167     if shallow_decl_enabled ctx || force_shallow_decl_fanout_enabled ctx then
 168       (acc, 0)
 169     else
 170       let old_classes = Decl_heap.Classes.get_old_batch n_classes in
 171       let new_classes = Decl_heap.Classes.get_batch n_classes in
 172       compare_classes_and_get_fanout ctx old_classes new_classes acc n_classes
 173   in
 174
 175   let (acc, old_modules_missing) =
 176     let old_modules = Decl_heap.Modules.get_old_batch n_modules in
 177     let new_modules = Decl_heap.Modules.get_batch n_modules in
 178     compare_module_and_get_fanout ctx old_modules new_modules acc n_modules
 179   in
 180
 181   let old_decl_missing_count =
 182     old_funs_missing
 183     + old_types_missing
 184     + old_gconsts_missing
 185     + old_classes_missing
 186     + old_modules_missing
 187   in
 188   let { changed; to_redecl; to_recheck } = acc in
 189   ({ changed; to_redecl; to_recheck }, old_decl_missing_count)
 190
 191 (*****************************************************************************)
 192 (* Load the environment and then redeclare *)
 193 (*****************************************************************************)
 194
 195 (** Invalidate local caches and compute decls in files. Return the file count and errors raised during decling. *)
 196 let decl_files_job ctx _ filel =
 197   try (List.length filel, decl_files ctx filel) with
 198   | exn ->
 199     let e = Exception.wrap exn in
 200     Printf.printf "Error: %s\n" (Exception.get_ctor_string e);
 201     Out_channel.flush stdout;
 202     Exception.reraise e
 203
 204 let load_defs_compare_and_get_fanout ctx _acc (filel : Relative_path.t list) :
 205     (fanout * int) * int =
 206   try
 207     let defs_per_file = OnTheFlyStore.load () in
 208     let (fanout, old_decl_missing_count) =
 209       compare_decls_and_get_fanout ctx defs_per_file filel
 210     in
 211     ((fanout, List.length filel), old_decl_missing_count)
 212   with
 213   | exn ->
 214     let e = Exception.wrap exn in
 215     Printf.printf "Error: %s\n" (Exception.get_ctor_string e);
 216     Out_channel.flush stdout;
 217     Exception.reraise e
 218
 219 (*****************************************************************************)
 220 (* Merges the results coming back from the different workers *)
 221 (*****************************************************************************)
 222
 223 let merge_on_the_fly
 224     files_initial_count files_declared_count (count, errorl1) errorl2 =
 225   files_declared_count := !files_declared_count + count;
 226   ServerProgress.send_percentage_progress
 227     ~operation:"declaring"
 228     ~done_count:!files_declared_count
 229     ~total_count:files_initial_count
 230     ~unit:"files"
 231     ~extra:None;
 232
 233   Errors.merge errorl1 errorl2
 234
 235 let merge_compute_deps
 236     files_initial_count
 237     files_computed_count
 238     ((fanout1, computed_count), old_decl_missing_count1)
 239     (fanout2, old_decl_missing_count2) =
 240   files_computed_count := !files_computed_count + computed_count;
 241
 242   let { changed = changed1; to_redecl = to_redecl1; to_recheck = to_recheck1 } =
 243     fanout1
 244   in
 245   let { changed = changed2; to_redecl = to_redecl2; to_recheck = to_recheck2 } =
 246     fanout2
 247   in
 248   let fanout =
 249     {
 250       changed = DepSet.union changed1 changed2;
 251       to_redecl = DepSet.union to_redecl1 to_redecl2;
 252       to_recheck = DepSet.union to_recheck1 to_recheck2;
 253     }
 254   in
 255
 256   ServerProgress.send_percentage_progress
 257     ~operation:"computing dependencies of"
 258     ~done_count:!files_computed_count
 259     ~total_count:files_initial_count
 260     ~unit:"files"
 261     ~extra:None;
 262
 263   (fanout, old_decl_missing_count1 + old_decl_missing_count2)
 264
 265 (*****************************************************************************)
 266 (* The parallel worker *)
 267 (*****************************************************************************)
 268
 269 (** Invalidate local decl caches and recompute decls in files in parallel.
 270   Compare new and old decls and deduce the fanout.
 271   Return errors raised during decling, fanout and missing old decl count. *)
 272 let parallel_redecl_compare_and_get_fanout
 273     (ctx : Provider_context.t)
 274     (workers : MultiWorker.worker list option)
 275     (bucket_size : int)
 276     (defs_per_file : FileInfo.names Relative_path.Map.t)
 277     (fnl : Relative_path.t list) : (Errors.t * fanout) * int =
 278   try
 279     OnTheFlyStore.store defs_per_file;
 280     let files_initial_count = List.length fnl in
 281     let files_declared_count = ref 0 in
 282     let t = Unix.gettimeofday () in
 283     Hh_logger.log ~lvl "Declaring on-the-fly %d files" files_initial_count;
 284     ServerProgress.send_percentage_progress
 285       ~operation:"declaring"
 286       ~done_count:!files_declared_count
 287       ~total_count:files_initial_count
 288       ~unit:"files"
 289       ~extra:None;
 290     let errors =
 291       MultiWorker.call
 292         workers
 293         ~job:(decl_files_job ctx)
 294         ~neutral:Errors.empty
 295         ~merge:(merge_on_the_fly files_initial_count files_declared_count)
 296         ~next:(MultiWorker.next ~max_size:bucket_size workers fnl)
 297     in
 298     let t = Hh_logger.log_duration ~lvl "Finished declaring on-the-fly" t in
 299     Hh_logger.log ~lvl "Computing dependencies of %d files" files_initial_count;
 300     let files_computed_count = ref 0 in
 301     ServerProgress.send_percentage_progress
 302       ~operation:"computing dependencies of"
 303       ~done_count:!files_computed_count
 304       ~total_count:files_initial_count
 305       ~unit:"files"
 306       ~extra:None;
 307     let (fanout, old_decl_missing_count) =
 308       MultiWorker.call
 309         workers
 310         ~job:(load_defs_compare_and_get_fanout ctx)
 311         ~neutral:compute_deps_neutral
 312         ~merge:(merge_compute_deps files_initial_count files_computed_count)
 313         ~next:(MultiWorker.next ~max_size:bucket_size workers fnl)
 314     in
 315     let (_t : float) =
 316       Hh_logger.log_duration ~lvl "Finished computing dependencies" t
 317     in
 318     OnTheFlyStore.clear ();
 319     ((errors, fanout), old_decl_missing_count)
 320   with
 321   | exn ->
 322     let e = Exception.wrap exn in
 323     if SharedMem.SMTelemetry.is_heap_overflow () then
 324       Exit.exit Exit_status.Redecl_heap_overflow
 325     else
 326       Exception.reraise e
 327
 328 (*****************************************************************************)
 329 (* Code invalidating the heap *)
 330 (*****************************************************************************)
 331 let oldify_defs
 332     (ctx : Provider_context.t)
 333     { FileInfo.n_funs; n_classes; n_types; n_consts; n_modules }
 334     (elems : Decl_class_elements.t SMap.t)
 335     ~(collect_garbage : bool) : unit =
 336   Decl_heap.Funs.oldify_batch n_funs;
 337   Decl_class_elements.oldify_all elems;
 338   Decl_heap.Classes.oldify_batch n_classes;
 339   Shallow_classes_provider.oldify_batch ctx n_classes;
 340   Decl_heap.Typedefs.oldify_batch n_types;
 341   Decl_heap.GConsts.oldify_batch n_consts;
 342   Decl_heap.Modules.oldify_batch n_modules;
 343   if collect_garbage then SharedMem.GC.collect `gentle;
 344   ()
 345
 346 let remove_old_defs
 347     (ctx : Provider_context.t)
 348     { FileInfo.n_funs; n_classes; n_types; n_consts; n_modules }
 349     (elems : Decl_class_elements.t SMap.t) : unit =
 350   Decl_heap.Funs.remove_old_batch n_funs;
 351   Decl_class_elements.remove_old_all elems;
 352   Decl_heap.Classes.remove_old_batch n_classes;
 353   Shallow_classes_provider.remove_old_batch ctx n_classes;
 354   Decl_heap.Typedefs.remove_old_batch n_types;
 355   Decl_heap.GConsts.remove_old_batch n_consts;
 356   Decl_heap.Modules.remove_old_batch n_modules;
 357   SharedMem.GC.collect `gentle;
 358   ()
 359
 360 let remove_defs
 361     (ctx : Provider_context.t)
 362     { FileInfo.n_funs; n_classes; n_types; n_consts; n_modules }
 363     (elems : Decl_class_elements.t SMap.t)
 364     ~(collect_garbage : bool) : unit =
 365   Decl_heap.Funs.remove_batch n_funs;
 366   Decl_class_elements.remove_all elems;
 367   Decl_heap.Classes.remove_batch n_classes;
 368   Shallow_classes_provider.remove_batch ctx n_classes;
 369   Linearization_provider.remove_batch ctx n_classes;
 370   Decl_heap.Typedefs.remove_batch n_types;
 371   Decl_heap.GConsts.remove_batch n_consts;
 372   Decl_heap.Modules.remove_batch n_modules;
 373   if collect_garbage then SharedMem.GC.collect `gentle;
 374   ()
 375
 376 let is_dependent_class_of_any ctx classes (c : string) : bool =
 377   if SSet.mem classes c then
 378     true
 379   else if shallow_decl_enabled ctx then
 380     true
 381   else
 382     match Decl_heap.Classes.get c with
 383     | None -> false
 384     (* it might be a dependent class, but we are only doing this
 385      * check for the purpose of invalidating things from the heap
 386      * - if it's already not there, then we don't care. *)
 387     | Some c ->
 388       let intersection_nonempty s1 s2 = SSet.exists s1 ~f:(SSet.mem s2) in
 389       SMap.exists c.Decl_defs.dc_ancestors ~f:(fun c _ -> SSet.mem classes c)
 390       || intersection_nonempty c.Decl_defs.dc_extends classes
 391       || intersection_nonempty c.Decl_defs.dc_xhp_attr_deps classes
 392       || intersection_nonempty c.Decl_defs.dc_req_ancestors_extends classes
 393
 394 let get_maybe_dependent_classes
 395     (get_classes : Relative_path.t -> SSet.t)
 396     (classes : SSet.t)
 397     (files : Relative_path.Set.t) : string list =
 398   Relative_path.Set.fold files ~init:classes ~f:(fun x acc ->
 399       SSet.union acc @@ get_classes x)
 400   |> SSet.elements
 401
 402 let get_dependent_classes_files (ctx : Provider_context.t) (classes : SSet.t) :
 403     Relative_path.Set.t =
 404   let mode = Provider_context.get_deps_mode ctx in
 405   let visited = VisitedSet.make () in
 406   SSet.fold
 407     classes
 408     ~init:Typing_deps.(DepSet.make ())
 409     ~f:(fun c acc ->
 410       let source_class = Dep.make (Dep.Type c) in
 411       Typing_deps.get_extend_deps ~mode ~visited ~source_class ~acc)
 412   |> Naming_provider.get_files ctx
 413
 414 let filter_dependent_classes
 415     (ctx : Provider_context.t)
 416     (classes : SSet.t)
 417     (maybe_dependent_classes : string list) : string list =
 418   List.filter maybe_dependent_classes ~f:(is_dependent_class_of_any ctx classes)
 419
 420 module ClassSetStore = GlobalStorage.Make (struct
 421   type t = SSet.t
 422 end)
 423
 424 let load_and_filter_dependent_classes
 425     (ctx : Provider_context.t) (maybe_dependent_classes : string list) :
 426     string list * int =
 427   let classes = ClassSetStore.load () in
 428   ( filter_dependent_classes ctx classes maybe_dependent_classes,
 429     List.length maybe_dependent_classes )
 430
 431 let merge_dependent_classes
 432     classes_initial_count
 433     classes_filtered_count
 434     (dependent_classes, filtered)
 435     acc =
 436   classes_filtered_count := !classes_filtered_count + filtered;
 437   ServerProgress.send_percentage_progress
 438     ~operation:"filtering"
 439     ~done_count:!classes_filtered_count
 440     ~total_count:classes_initial_count
 441     ~unit:"classes"
 442     ~extra:None;
 443   dependent_classes @ acc
 444
 445 let filter_dependent_classes_parallel
 446     (ctx : Provider_context.t)
 447     (workers : MultiWorker.worker list option)
 448     ~(bucket_size : int)
 449     (classes : SSet.t)
 450     (maybe_dependent_classes : string list) : string list =
 451   if List.length maybe_dependent_classes < 10 then
 452     filter_dependent_classes ctx classes maybe_dependent_classes
 453   else (
 454     ClassSetStore.store classes;
 455     let classes_initial_count = List.length maybe_dependent_classes in
 456     let classes_filtered_count = ref 0 in
 457     let t = Unix.gettimeofday () in
 458     Hh_logger.log ~lvl "Filtering %d dependent classes" classes_initial_count;
 459     ServerProgress.send_percentage_progress
 460       ~operation:"filtering"
 461       ~done_count:!classes_filtered_count
 462       ~total_count:classes_initial_count
 463       ~unit:"classes"
 464       ~extra:None;
 465     let res =
 466       MultiWorker.call
 467         workers
 468         ~job:(fun _ c -> load_and_filter_dependent_classes ctx c)
 469         ~merge:
 470           (merge_dependent_classes classes_initial_count classes_filtered_count)
 471         ~neutral:[]
 472         ~next:
 473           (MultiWorker.next
 474              ~max_size:bucket_size
 475              workers
 476              maybe_dependent_classes)
 477     in
 478     let (_t : float) =
 479       Hh_logger.log_duration ~lvl "Finished filtering dependent classes" t
 480     in
 481     ClassSetStore.clear ();
 482     res
 483   )
 484
 485 let get_dependent_classes
 486     (ctx : Provider_context.t)
 487     (workers : MultiWorker.worker list option)
 488     ~(bucket_size : int)
 489     (get_classes : Relative_path.t -> SSet.t)
 490     (classes : SSet.t) : SSet.t =
 491   get_dependent_classes_files ctx classes
 492   |> get_maybe_dependent_classes get_classes classes
 493   |> filter_dependent_classes_parallel ctx workers ~bucket_size classes
 494   |> SSet.of_list
 495
 496 let merge_elements
 497     classes_initial_count classes_processed_count (elements, count) acc =
 498   classes_processed_count := !classes_processed_count + count;
 499
 500   let acc = SMap.union elements acc in
 501   ServerProgress.send_percentage_progress
 502     ~operation:"getting members of"
 503     ~done_count:!classes_processed_count
 504     ~total_count:classes_initial_count
 505     ~unit:"classes"
 506     ~extra:(Some (Printf.sprintf "%d elements" (SMap.cardinal acc)));
 507   acc
 508
 509 (**
 510  * Get the [Decl_class_elements.t]s corresponding to the classes contained in
 511  * [defs]. *)
 512 let get_elems
 513     (ctx : Provider_context.t)
 514     (workers : MultiWorker.worker list option)
 515     ~(bucket_size : int)
 516     ~(old : bool)
 517     (defs : FileInfo.names) : Decl_class_elements.t SMap.t =
 518   if shallow_decl_enabled ctx then
 519     SMap.empty
 520   else
 521     let classes = SSet.elements defs.FileInfo.n_classes in
 522     (* Getting the members of a class requires fetching the class from the heap.
 523      * Doing this for too many classes will cause a large amount of allocations
 524      * to be performed on the master process triggering the GC and slowing down
 525      * redeclaration. Using the workers prevents this from occurring
 526      *)
 527     let classes_initial_count = List.length classes in
 528     let t = Unix.gettimeofday () in
 529     Hh_logger.log ~lvl "Getting elements of %d classes" classes_initial_count;
 530     let elements =
 531       if classes_initial_count < 10 then
 532         Decl_class_elements.get_for_classes ~old classes
 533       else
 534         let classes_processed_count = ref 0 in
 535         ServerProgress.send_percentage_progress
 536           ~operation:"getting members of"
 537           ~done_count:!classes_processed_count
 538           ~total_count:classes_initial_count
 539           ~unit:"classes"
 540           ~extra:None;
 541         MultiWorker.call
 542           workers
 543           ~job:(fun _ c ->
 544             (Decl_class_elements.get_for_classes ~old c, List.length c))
 545           ~merge:(merge_elements classes_initial_count classes_processed_count)
 546           ~neutral:SMap.empty
 547           ~next:(MultiWorker.next ~max_size:bucket_size workers classes)
 548     in
 549
 550     let (_t : float) =
 551       Hh_logger.log_duration ~lvl "Finished getting elements" t
 552     in
 553     elements
 554
 555 let invalidate_folded_classes_for_shallow_fanout
 556     ctx workers ~bucket_size ~get_classes_in_file changed_classes =
 557   let invalidated =
 558     changed_classes
 559     |> Typing_deps.add_extend_deps (Provider_context.get_deps_mode ctx)
 560     |> Shallow_class_fanout.class_names_from_deps ~ctx ~get_classes_in_file
 561   in
 562   let get_elems n_classes =
 563     get_elems ctx workers ~bucket_size FileInfo.{ empty_names with n_classes }
 564   in
 565   Decl_class_elements.remove_old_all (get_elems invalidated ~old:true);
 566   Decl_class_elements.remove_all (get_elems invalidated ~old:false);
 567   Decl_heap.Classes.remove_old_batch invalidated;
 568   Decl_heap.Classes.remove_batch invalidated;
 569   SharedMem.GC.collect `gentle;
 570   ()
 571
 572 (*****************************************************************************)
 573 (* The main entry point *)
 574 (*****************************************************************************)
 575
 576 (** Oldify any defs in [defs] which aren't already in
 577 [previously_oldified_defs], then determines which symbols need to be
 578 re-typechecked as a result of comparing the current versions of the symbols
 579 to their old versions. *)
 580 let redo_type_decl
 581     (ctx : Provider_context.t)
 582     (workers : MultiWorker.worker list option)
 583     ~(bucket_size : int)
 584     (get_classes : Relative_path.t -> SSet.t)
 585     ~(previously_oldified_defs : FileInfo.names)
 586     ~(defs : FileInfo.names Relative_path.Map.t)
 587     ~(telemetry_label : string) : redo_type_decl_result =
 588   let all_defs =
 589     Relative_path.Map.fold defs ~init:FileInfo.empty_names ~f:(fun _ ->
 590         FileInfo.merge_names)
 591   in
 592   (* Some of the definitions are already in the old heap, left there by a
 593    * previous lazy check *)
 594   let (oldified_defs, current_defs) =
 595     Decl_utils.split_defs all_defs previously_oldified_defs
 596   in
 597   (* Oldify the remaining defs along with their elements *)
 598   let get_elems = get_elems ctx workers ~bucket_size in
 599   let current_elems = get_elems current_defs ~old:false in
 600   oldify_defs ctx current_defs current_elems ~collect_garbage:true;
 601
 602   (* Fetch the already oldified elements too so we can remove them later *)
 603   let oldified_elems = get_elems oldified_defs ~old:true in
 604   let all_elems = SMap.union current_elems oldified_elems in
 605   let fnl = Relative_path.Map.keys defs in
 606
 607   let ((errors, { changed; to_redecl; to_recheck }), old_decl_missing_count) =
 608     (* If there aren't enough files, let's do this ourselves ... it's faster! *)
 609     if List.length fnl < 10 then
 610       let errors = decl_files ctx fnl in
 611       let (fanout, old_decl_missing_count) =
 612         compare_decls_and_get_fanout ctx defs fnl
 613       in
 614       ((errors, fanout), old_decl_missing_count)
 615     else
 616       parallel_redecl_compare_and_get_fanout ctx workers bucket_size defs fnl
 617   in
 618   let (changed, to_recheck) =
 619     if shallow_decl_enabled ctx then (
 620       let AffectedDeps.{ changed = changed'; mro_invalidated; needs_recheck } =
 621         Shallow_decl_compare.compute_class_fanout
 622           ctx
 623           ~defs
 624           ~fetch_old_decls:
 625             (Remote_old_decl_client.fetch_old_decls ~ctx ~telemetry_label)
 626           fnl
 627       in
 628       let changed = DepSet.union changed changed' in
 629       let to_recheck = DepSet.union to_recheck needs_recheck in
 630       let mro_invalidated =
 631         mro_invalidated
 632         |> Naming_provider.get_files ctx
 633         |> Relative_path.Set.fold ~init:SSet.empty ~f:(fun path acc ->
 634                SSet.union acc (get_classes path))
 635       in
 636       Linearization_provider.remove_batch ctx mro_invalidated;
 637       (changed, to_recheck)
 638     ) else if force_shallow_decl_fanout_enabled ctx then (
 639       let AffectedDeps.
 640             { changed = changed'; mro_invalidated = _; needs_recheck } =
 641         Shallow_decl_compare.compute_class_fanout
 642           ctx
 643           ~defs
 644           ~fetch_old_decls:
 645             (Remote_old_decl_client.fetch_old_decls ~ctx ~telemetry_label)
 646           fnl
 647       in
 648
 649       invalidate_folded_classes_for_shallow_fanout
 650         ctx
 651         workers
 652         ~bucket_size
 653         ~get_classes_in_file:get_classes
 654         changed';
 655
 656       let changed = DepSet.union changed changed' in
 657       let to_recheck = DepSet.union to_recheck needs_recheck in
 658
 659       (changed, to_recheck)
 660     ) else
 661       (changed, to_recheck)
 662   in
 663   remove_old_defs ctx all_defs all_elems;
 664
 665   Hh_logger.log "Finished recomputing type declarations:";
 666   Hh_logger.log "  changed: %d" (DepSet.cardinal changed);
 667   Hh_logger.log "  to_redecl: %d" (DepSet.cardinal to_redecl);
 668   Hh_logger.log "  to_recheck: %d" (DepSet.cardinal to_recheck);
 669
 670   {
 671     errors;
 672     fanout = { changed; to_redecl; to_recheck };
 673     old_decl_missing_count;
 674   }
 675
 676 (** Mark all provided [defs] as old, as long as they were not previously
 677 oldified. All definitions in [previously_oldified_defs] are then removed from
 678 the decl heaps.
 679
 680 Typically, there are only any [previously_oldified_defs] during two-phase
 681 redeclaration. *)
 682 let oldify_type_decl
 683     (ctx : Provider_context.t)
 684     ?(collect_garbage = true)
 685     (workers : MultiWorker.worker list option)
 686     (get_classes : Relative_path.t -> SSet.t)
 687     ~(bucket_size : int)
 688     ~(previously_oldified_defs : FileInfo.names)
 689     ~(defs : FileInfo.names) : unit =
 690   (* Some defs are already oldified, waiting for their recheck *)
 691   let (oldified_defs, current_defs) =
 692     Decl_utils.split_defs defs previously_oldified_defs
 693   in
 694   let get_elems = get_elems ctx workers ~bucket_size in
 695   (* Oldify things that are not oldified yet *)
 696   let current_elems = get_elems current_defs ~old:false in
 697   oldify_defs ctx current_defs current_elems ~collect_garbage;
 698
 699   (* For the rest, just invalidate their current versions *)
 700   let oldified_elems = get_elems oldified_defs ~old:false in
 701   remove_defs ctx oldified_defs oldified_elems ~collect_garbage;
 702
 703   (* Oldifying/removing classes also affects their elements
 704    * (see Decl_class_elements), which might be shared with other classes. We
 705    * need to remove all of them too to avoid dangling references *)
 706   let all_classes = defs.FileInfo.n_classes in
 707   let dependent_classes =
 708     get_dependent_classes ctx workers get_classes ~bucket_size all_classes
 709   in
 710   let dependent_classes =
 711     FileInfo.
 712       { empty_names with n_classes = SSet.diff dependent_classes all_classes }
 713   in
 714   remove_defs ctx dependent_classes SMap.empty ~collect_garbage
 715
 716 let remove_old_defs
 717     (ctx : Provider_context.t)
 718     ~(bucket_size : int)
 719     (workers : MultiWorker.worker list option)
 720     (names : FileInfo.names) : unit =
 721   let elems = get_elems ctx workers ~bucket_size names ~old:true in
 722   remove_old_defs ctx names elems