Remove enable_disk_heap setting
[hiphop-php.git] / hphp / hack / src / server / serverApi.ml
blob02a435ef922b20e0c266c01044ea744596ddf2e9
1 (*
2 * Copyright (c) Facebook, Inc. and its affiliates.
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the "hack" directory of this source tree.
7 *)
9 open Hh_prelude
10 open Hh_prelude.Result.Monad_infix
11 open RemoteWorker
12 open Typing_service_types
14 let make_local_server_api
15 (naming_table : Naming_table.t)
16 ~(root : string)
17 ~(init_id : string)
18 ~(deps_mode : Typing_deps_mode.t) : (module LocalServerApi) =
19 (module struct
20 let send_progress (message : string) : unit =
21 ServerProgress.send_progress "%s" message
23 let update_state ~(state_filename : string) ~(check_id : string option) :
24 unit =
25 let check_id =
26 Option.value check_id ~default:(Random_id.short_string ())
28 HackEventLogger.with_id ~stage:`Recheck check_id @@ fun () ->
29 let start_t = Unix.gettimeofday () in
30 let edges = Typing_deps.load_discovered_edges deps_mode state_filename in
31 HackEventLogger.remote_scheduler_update_dependency_graph_end
32 ~edges
33 start_t;
34 let (_t : float) =
35 Hh_logger.log_duration
36 (Printf.sprintf "Updated dependency graph: added %d edges" edges)
37 start_t
41 let snapshot_naming_table_base ~destination_path : unit Future.t =
42 send_progress "Snapshotting the naming table for delegated type checking";
43 let start_t = Unix.gettimeofday () in
44 let future =
45 match Naming_table.get_forward_naming_fallback_path naming_table with
46 | Some source_path ->
47 Hh_logger.log
48 "Updating the existing table - moving %s to %s"
49 source_path
50 destination_path;
51 FileUtil.cp [source_path] destination_path;
52 let (_ : Naming_sqlite.save_result) =
53 Naming_table.save naming_table destination_path
55 Future.of_value ()
56 | None ->
57 Naming_table.save_async naming_table ~init_id ~root ~destination_path
59 Future.continue_with future @@ fun () ->
60 HackEventLogger.remote_scheduler_save_naming_end start_t;
61 let (start_t : float) =
62 Hh_logger.log_duration
63 (Printf.sprintf "Saved SQLite naming table to %s" destination_path)
64 start_t
66 send_progress
67 (Printf.sprintf "Snapshotted the naming table base: %f" start_t)
69 let snapshot_naming_table_diff ~(destination_path : string) : unit =
70 Hh_logger.log "snapshot_naming_table_diff: %s" destination_path;
71 Naming_table.save_changes_since_baseline naming_table ~destination_path
73 let begin_get_changed_files ~(mergebase : string option) :
74 string list Future.t =
75 let t = Unix.gettimeofday () in
76 match mergebase with
77 | Some mergebase ->
78 let hg_future = Hg.files_changed_since_rev (Hg.Hg_rev mergebase) root in
79 Future.continue_with hg_future @@ fun changed_files ->
80 let telemetry =
81 Telemetry.create ()
82 |> Telemetry.int_
83 ~key:"changed_files"
84 ~value:(List.length changed_files)
86 HackEventLogger.remote_scheduler_get_dirty_files_end telemetry t;
87 changed_files
88 | None -> Future.of_value []
90 let load_changed_files (changed_files : string list) :
91 (Relative_path.t * string option) list =
92 let changed_files_and_content =
93 List.map changed_files ~f:(fun changed_file ->
94 let changed_file = FilePath.make_absolute root changed_file in
95 let changed_file_path =
96 Relative_path.create Relative_path.Root changed_file
98 (changed_file_path, File_provider.get_contents changed_file_path))
100 changed_files_and_content
102 let write_changed_files
103 (changed_files : string list) ~(destination_path : string) : unit =
104 let changed_filepaths_and_content = load_changed_files changed_files in
105 let chan = Stdlib.open_out_bin destination_path in
106 Marshal.to_channel chan changed_filepaths_and_content [];
107 Stdlib.close_out chan
108 end : LocalServerApi)
110 let make_remote_server_api
111 (ctx : Provider_context.t)
112 (workers : MultiWorker.worker list option)
113 (root : Path.t) :
114 (module RemoteServerApi with type naming_table = Naming_table.t option) =
115 (module struct
116 type naming_table = Naming_table.t option
118 let load_naming_table_base ~(naming_table_base : Path.t option) :
119 (naming_table, string) result =
120 Hh_logger.log "Loading naming table base...";
122 match naming_table_base with
123 | None ->
124 Error
125 "Expected naming table base path to be set when loading naming table, but it was not"
126 | Some naming_table_base ->
128 (Some
129 (Naming_table.load_from_sqlite
131 (Path.to_string naming_table_base)))
134 There is a variety of state that the server accumulates after type
135 checking files. We want to make sure we remove such state before a
136 recheck. In order to do this cleaning, we need a list of files that
137 changed.
139 let clean_changed_files_state ctx naming_table changed_files ~t =
140 let (changed_names : FileInfo.names) =
141 List.fold changed_files ~init:FileInfo.empty_names ~f:(fun names file ->
142 match Naming_table.get_file_info naming_table file with
143 | Some (file_info : FileInfo.t) ->
144 FileInfo.merge_names names (FileInfo.simplify file_info)
145 | None -> names)
147 let t =
148 Hh_logger.log_duration "Got names changed since naming table baseline" t
150 let changed_files = Relative_path.set_of_list changed_files in
151 File_provider.remove_batch changed_files;
152 Ast_provider.remove_batch changed_files;
153 Fixme_provider.remove_batch changed_files;
154 Decl_redecl_service.remove_old_defs
156 ~bucket_size:1000
157 workers
158 changed_names;
159 Hh_logger.log_duration "Cleaned state associated with changed files" t
161 let load_naming_table_changes_since_baseline
162 (ctx : Provider_context.t)
163 ~(naming_table : Naming_table.t option)
164 ~(naming_table_diff : Naming_table.changes_since_baseline) :
165 (Naming_table.t option, string) result =
166 Hh_logger.log "Loading naming table changes since baseline...";
167 match naming_table with
168 | None -> Error "Expected naming table base"
169 | Some naming_table ->
170 begin
171 match Naming_table.get_forward_naming_fallback_path naming_table with
172 | None ->
173 Error "Expected naming table base path to be set, but it was not"
174 | Some naming_table_base ->
175 (try
176 let t = Unix.gettimeofday () in
177 let changed_files =
178 Naming_table.get_files_changed_since_baseline naming_table_diff
180 let t =
181 Hh_logger.log_duration
182 "Got files changed since naming table baseline"
185 let t =
186 clean_changed_files_state ctx naming_table changed_files ~t
188 Hh_logger.log "Prefetching naming dirty files...";
189 Vfs.prefetch changed_files;
190 let t =
191 Hh_logger.log_duration "Prefetched naming dirty files" t
193 let (naming_table : Naming_table.t) =
194 Naming_table.load_from_sqlite_with_changes_since_baseline
196 naming_table_diff
197 naming_table_base
199 HackEventLogger.remote_worker_load_naming_end t;
200 let _t : float =
201 Hh_logger.log_duration "Loaded naming table from SQLite" t
203 Ok (Some naming_table)
204 with
205 | e -> Error (Exn.to_string e))
208 let build_naming_table _ =
209 Hh_logger.log "Building naming table";
210 let indexer =
211 Find.make_next_files ~name:"root" ~filter:FindUtils.is_hack root
213 let get_next =
214 ServerUtils.make_next
215 ~hhi_filter:(fun _ -> true)
216 ~indexer
217 ~extra_roots:(ServerConfig.extra_paths ServerConfig.default_config)
219 Hh_logger.log "Building naming table - Parsing";
220 let defs_per_file =
221 Direct_decl_service.go
223 workers
224 ~ide_files:Relative_path.Set.empty
225 ~get_next
226 ~trace:false
227 ~cache_decls:true
229 Hh_logger.log "Building naming table - Naming";
230 let naming_table = Naming_table.create defs_per_file in
231 Naming_table.iter naming_table ~f:(fun k v ->
232 let _ = Naming_global.ndecl_file_error_if_already_bound ctx k v in
233 ());
234 Hh_logger.log "Building naming table - Done!";
237 let load_naming_and_dep_table
238 (saved_state_main_artifacts :
239 Saved_state_loader.Naming_and_dep_table_info.main_artifacts) :
240 (Naming_table.t * Path.t, string) result =
241 let {
242 Saved_state_loader.Naming_and_dep_table_info.naming_table_path = _;
243 dep_table_path;
244 naming_sqlite_table_path;
245 legacy_hot_decls_path = _;
246 shallow_hot_decls_path = _;
247 errors_path = _;
249 saved_state_main_artifacts
251 if not (Sys.file_exists (Path.to_string naming_sqlite_table_path)) then
252 Error
253 (Printf.sprintf
254 "Expected naming sqlite table at %s"
255 (Path.to_string naming_sqlite_table_path))
256 else
257 let naming_table =
258 Naming_table.load_from_sqlite
260 (Path.to_string naming_sqlite_table_path)
262 Ok (naming_table, dep_table_path)
264 let download_naming_and_dep_table
265 (manifold_api_key : string option)
266 (manifold_path : string)
267 ~(use_manifold_cython_client : bool) :
268 (Naming_table.t * Path.t, string) result =
269 let target_path = "/tmp/hh_server/" ^ Random_id.short_string () in
270 Disk.mkdir_p target_path;
272 let naming_table_future =
273 State_loader_futures.download_and_unpack_saved_state_from_manifold
274 ~env:
276 Saved_state_loader.log_saved_state_age_and_distance = false;
277 Saved_state_loader.saved_state_manifold_api_key = manifold_api_key;
278 Saved_state_loader.use_manifold_cython_client;
280 ~progress_callback:(fun _ -> ())
281 ~saved_state_type:
282 (Saved_state_loader.Naming_and_dep_table { naming_sqlite = true })
283 ~manifold_path
284 ~target_path:(Path.make target_path)
286 match Future.get ~timeout:60 naming_table_future with
287 | Error err ->
288 let err = Future.error_to_string err in
289 Hh_logger.error "Downloading dep table failed: %s" err;
290 Error err
291 | Ok download_result ->
292 begin
293 match download_result with
294 | Error (err, _telemetry) ->
295 Error (Saved_state_loader.debug_details_of_error err)
296 | Ok (main_artifacts, _telemetry) ->
297 let (_ : float) =
298 Hh_logger.log_duration
299 "Finished downloading dep table."
300 (Future.start_t naming_table_future)
302 let naming_table_path =
303 main_artifacts
304 .Saved_state_loader.Naming_and_dep_table_info
305 .naming_sqlite_table_path
307 Hh_logger.log
308 "Downloaded naming table to %s"
309 (Path.to_string naming_table_path);
310 load_naming_and_dep_table main_artifacts
313 let remove_decls naming_table fast_parsed =
314 Relative_path.Map.iter fast_parsed ~f:(fun fn _ ->
315 match Naming_table.get_file_info naming_table fn with
316 | None -> ()
317 | Some
319 FileInfo.funs;
320 classes;
321 typedefs;
322 consts;
323 modules;
324 file_mode = _;
325 comments = _;
326 hash = _;
327 } ->
328 (* we use [snd] to strip away positions *)
329 let snd (_, x, _) = x in
330 Naming_global.remove_decls
331 ~backend:(Provider_backend.get ())
332 ~funs:(List.map funs ~f:snd)
333 ~classes:(List.map classes ~f:snd)
334 ~typedefs:(List.map typedefs ~f:snd)
335 ~consts:(List.map consts ~f:snd)
336 ~modules:(List.map modules ~f:snd))
338 let update_naming_table
339 (naming_table : Naming_table.t)
340 (dep_table_path : Path.t)
341 (changed_files : Relative_path.t list option) : (string, string) result
343 match changed_files with
344 | None -> Error "No changed files uploaded for remote worker's payload"
345 | Some changed_files ->
346 Hh_logger.log "Cleaning naming table of changed files";
347 ignore
348 (clean_changed_files_state
350 naming_table
351 changed_files
352 ~t:(Unix.gettimeofday ()));
353 Ok (naming_table, changed_files, dep_table_path)
354 >>= fun (naming_table, changed_files, dep_table_path) ->
355 let changed_hack_files =
356 List.filter_map changed_files ~f:(fun file ->
357 if FindUtils.is_hack (Relative_path.suffix file) then
358 Some (Path.to_string root ^ "/" ^ Relative_path.suffix file)
359 else
360 None)
362 let indexer =
363 let state = ref changed_hack_files in
364 let max_files_per_batch = 1000 in
365 fun () ->
366 let (next, rest) = List.split_n !state max_files_per_batch in
367 state := rest;
368 next
370 let get_next =
371 ServerUtils.make_next
372 ~hhi_filter:(fun _ -> true)
373 ~indexer
374 ~extra_roots:(ServerConfig.extra_paths ServerConfig.default_config)
377 ( naming_table,
378 Direct_decl_service.go
380 workers
381 ~ide_files:Relative_path.Set.empty
382 ~get_next
383 ~trace:false
384 ~cache_decls:true,
385 dep_table_path )
386 >>= fun (naming_table, fast_parsed, dep_table_path) ->
387 Hh_logger.log "Built updated decls for naming table";
388 Hh_logger.log "Clearing old decls from naming table";
389 remove_decls naming_table fast_parsed;
390 Hh_logger.log "Updating naming table";
391 ignore (Naming_table.update_many naming_table fast_parsed);
392 Ok (fast_parsed, dep_table_path)
393 >>= fun (fast_parsed, dep_table_path) ->
394 Hh_logger.log "Updating naming global";
395 Naming_table.create fast_parsed
396 |> Naming_table.iter ~f:(fun k v ->
397 let _ =
398 Naming_global.ndecl_file_error_if_already_bound ctx k v
400 ());
401 Ok (Path.to_string dep_table_path)
403 let download_and_update_naming_table
404 ~(manifold_api_key : string option)
405 ~(use_manifold_cython_client : bool)
406 (saved_state_manifold_path : string option)
407 (changed_files : Relative_path.t list option) : string option =
408 match saved_state_manifold_path with
409 | None ->
410 Hh_logger.log
411 "[hulk lite] No saved_state_manifold_path, will fall back to building naming table locally";
412 build_naming_table ();
413 None
414 | Some path ->
415 let dep_table_path_result =
416 download_naming_and_dep_table
417 manifold_api_key
418 path
419 ~use_manifold_cython_client
420 >>= fun (naming_table, dep_table_path) ->
421 update_naming_table naming_table dep_table_path changed_files
423 (match dep_table_path_result with
424 | Ok dep_table_path -> Some dep_table_path
425 | Error err ->
426 Hh_logger.log "Could not build naming table from saved state: %s" err;
427 Hh_logger.log "Falling back to generating naming table";
428 build_naming_table ();
429 None)
431 let load_shallow_decls_saved_state
432 (saved_state_main_artifacts :
433 Saved_state_loader.Shallow_decls_info.main_artifacts) :
434 (string I64Map.t, string) result =
435 let { Saved_state_loader.Shallow_decls_info.shallow_decls_path } =
436 saved_state_main_artifacts
438 if not (Sys.file_exists (Path.to_string shallow_decls_path)) then
439 Error
440 (Printf.sprintf
441 "Expected shallow_decls_saved_state at %s"
442 (Path.to_string shallow_decls_path))
443 else
444 let chan = Stdlib.open_in_bin (Path.to_string shallow_decls_path) in
445 let contents = Marshal.from_channel chan in
446 Stdlib.close_in chan;
447 Ok contents
449 let download_shallow_decls_saved_state
450 (manifold_api_key : string option)
451 (manifold_path : string)
452 (use_manifold_cython_client : bool) : (string I64Map.t, string) result =
453 let target_path = "/tmp/hh_server/" ^ Random_id.short_string () in
454 Disk.mkdir_p target_path;
456 let shallow_decls_future =
457 State_loader_futures.download_and_unpack_saved_state_from_manifold
458 ~env:
460 Saved_state_loader.log_saved_state_age_and_distance = false;
461 Saved_state_loader.saved_state_manifold_api_key = manifold_api_key;
462 Saved_state_loader.use_manifold_cython_client;
464 ~progress_callback:(fun _ -> ())
465 ~saved_state_type:Saved_state_loader.Shallow_decls
466 ~manifold_path
467 ~target_path:(Path.make target_path)
469 match Future.get ~timeout:600 shallow_decls_future with
470 | Error err ->
471 let err = Future.error_to_string err in
472 Hh_logger.error "Downloading shallow_decls saved state failed: %s" err;
473 Error err
474 | Ok download_result ->
475 begin
476 match download_result with
477 | Error (err, _telemetry) ->
478 Error (Saved_state_loader.debug_details_of_error err)
479 | Ok (main_artifacts, _telemetry) ->
480 let (_ : float) =
481 Hh_logger.log_duration
482 "Finished downloading shallow_decls saved state."
483 (Future.start_t shallow_decls_future)
485 let shallow_decls_path =
486 main_artifacts
487 .Saved_state_loader.Shallow_decls_info.shallow_decls_path
489 Hh_logger.log
490 "Downloaded shallow_decls to %s"
491 (Path.to_string shallow_decls_path);
492 load_shallow_decls_saved_state main_artifacts
495 let unmarshal_decls_from_download_result
496 ~(ctx : Provider_context.t)
497 (shallow_decls : string I64Map.t)
498 (classnames : SSet.elt list) :
499 Shallow_decl_defs.shallow_class option SMap.t =
500 (* match shallow decls with classnames according to hash code *)
501 let db_path_opt = Remote_old_decl_client.Utils.db_path_of_ctx ~ctx in
502 match db_path_opt with
503 | None -> SMap.empty
504 | Some db_path ->
505 let decl_name_and_hashes =
506 List.filter_map
507 ~f:(fun name ->
508 match
509 Remote_old_decl_client.Utils.name_to_decl_hash_opt
510 ~name
511 ~db_path
512 with
513 | None -> None
514 | Some hash -> Some (name, Int64.of_string hash))
515 classnames
517 Hh_logger.log "constructed smap";
518 List.fold_left
519 ~init:SMap.empty
520 ~f:(fun acc (name, hash) ->
521 match I64Map.find_opt hash shallow_decls with
522 | None -> acc
523 | Some marshalled_decl ->
524 let decl = Marshal.from_string marshalled_decl 0 in
525 SMap.add name (Some decl) acc)
526 decl_name_and_hashes
528 let fetch_remote_decls_from_saved_state
529 manifold_api_key manifold_path use_manifold_cython_client classnames =
530 match
531 download_shallow_decls_saved_state
532 manifold_api_key
533 manifold_path
534 use_manifold_cython_client
535 with
536 | Ok shallow_decls_download_result ->
537 let _ =
538 Hh_logger.log
539 "loaded %d shallow decls from saved state"
540 (I64Map.cardinal shallow_decls_download_result)
542 let state_decls =
543 unmarshal_decls_from_download_result
544 ~ctx
545 shallow_decls_download_result
546 classnames
548 let _ =
549 Hh_logger.log
550 "extracted %d shallow decls from saved state"
551 (SMap.cardinal state_decls)
553 state_decls
554 | Error err -> failwith err
556 let fetch_remote_decls_from_remote_old_decl_service classnames =
557 let job (acc : 'a SMap.t) (classnames : string list) : 'a SMap.t =
558 Hh_logger.log
559 "Fecthing %d decls from the remote decl store"
560 (List.length classnames);
561 let remotely_fetched_decls =
562 Remote_old_decl_client.fetch_old_decls
563 ~telemetry_label:"hulk type check"
564 ~ctx
565 classnames
567 Hh_logger.log
568 "Fetched %d decls from the remote decl store"
569 (SMap.cardinal remotely_fetched_decls);
570 SMap.merge
571 (fun _key a b ->
572 if Option.is_some a then
574 else
577 remotely_fetched_decls
579 MultiWorker.call
580 workers
581 ~job
582 ~neutral:SMap.empty
583 ~merge:
584 (SMap.merge (fun _key a b ->
585 if Option.is_some a then
587 else
589 ~next:(MultiWorker.next ~max_size:50000 workers classnames)
591 let fetch_and_cache_remote_decls
592 ~ctx
593 naming_table
594 ~(from_saved_state : bool)
595 (manifold_api_key : string option)
596 (manifold_path : string)
597 (use_manifold_cython_client : bool) =
598 let start_t = Unix.gettimeofday () in
599 let fileinfos_from_naming_table =
600 naming_table
601 |> Naming_table.to_defs_per_file ~warn_on_naming_costly_iter:false
602 |> Relative_path.Map.elements
604 let classnames =
605 List.fold
606 fileinfos_from_naming_table
607 ~init:[]
608 ~f:(fun acc (_filename, fileinfo) ->
609 SSet.fold
610 (fun class_name acc -> class_name :: acc)
611 fileinfo.FileInfo.n_classes
612 acc)
614 let remotely_fetched_decls =
615 if from_saved_state then
616 fetch_remote_decls_from_saved_state
617 manifold_api_key
618 manifold_path
619 use_manifold_cython_client
620 classnames
621 else
622 fetch_remote_decls_from_remote_old_decl_service classnames
624 List.iter fileinfos_from_naming_table ~f:(fun (filename, fileinfo) ->
625 let class_names = fileinfo.FileInfo.n_classes in
626 let pfh_decls : (string * Shallow_decl_defs.decl * Int64.t) list =
627 SSet.fold
628 (fun name acc ->
629 let remotely_fetched_decl =
630 Option.join (SMap.find_opt name remotely_fetched_decls)
632 match
633 Option.Monad_infix.(
634 remotely_fetched_decl >>= fun decl ->
635 Remote_old_decl_client.Utils.db_path_of_ctx ~ctx
636 >>= fun db_path ->
637 Remote_old_decl_client.Utils.name_to_decl_hash_opt
638 ~name
639 ~db_path
640 >>= fun hash ->
641 Some
642 (name, Shallow_decl_defs.Class decl, Int64.of_string hash))
643 with
644 | Some pfh_decl -> pfh_decl :: acc
645 | None -> acc)
646 class_names
649 Direct_decl_utils.cache_decls ctx filename pfh_decls);
650 let (_ : float) =
651 Hh_logger.log_duration
652 "Fetched and cached decls from remote decl store"
653 start_t
657 let type_check
658 ctx ~init_id ~check_id files_to_check ~state_filename ~telemetry =
659 let t = Unix.gettimeofday () in
660 Hh_logger.log "Type checking a batch...";
661 let check_info =
663 init_id;
664 check_reason = "remote_server_api";
665 recheck_id = Some check_id;
666 use_max_typechecker_worker_memory_for_decl_deferral = false;
667 per_file_profiling = HackEventLogger.PerFileProfilingConfig.default;
668 memtrace_dir = None;
671 (* TODO: use the telemetry *)
672 let { Typing_check_service.errors; telemetry; _ } =
673 Typing_check_service.go
675 workers
676 Typing_service_delegate.default
677 telemetry
678 files_to_check
679 ~memory_cap:(Some 200000)
680 ~longlived_workers:false
681 ~mode:HulkStrategy.Legacy
682 ~check_info
684 HackEventLogger.remote_worker_type_check_end telemetry ~start_t:t;
685 let t = Hh_logger.log_duration "Type checked files in remote worker" t in
686 let dep_table_edges_added =
687 Typing_deps.save_discovered_edges
688 (Provider_context.get_deps_mode ctx)
689 ~dest:state_filename
690 ~reset_state_after_saving:true
692 let _t : float =
693 Hh_logger.log_duration
694 (Printf.sprintf
695 "Saved partial dependency graph (%d edges)"
696 dep_table_edges_added)
699 errors
700 end : RemoteServerApi
701 with type naming_table = Naming_table.t option)