2 * Copyright (c) Facebook, Inc. and its affiliates.
4 * This source code is licensed under the MIT license found in the
5 * LICENSE file in the "hack" directory of this source tree.
10 open Hh_prelude.Result.Monad_infix
12 open Typing_service_types
14 let make_local_server_api
15 (naming_table
: Naming_table.t
)
18 ~
(deps_mode
: Typing_deps_mode.t
) : (module LocalServerApi
) =
20 let send_progress (message
: string) : unit =
21 ServerProgress.send_progress "%s" message
23 let update_state ~
(state_filename
: string) ~
(check_id
: string option) :
26 Option.value check_id ~default
:(Random_id.short_string
())
28 HackEventLogger.with_id ~stage
:`Recheck
check_id @@ fun () ->
29 let start_t = Unix.gettimeofday
() in
30 let edges = Typing_deps.load_discovered_edges deps_mode state_filename
in
31 HackEventLogger.remote_scheduler_update_dependency_graph_end
35 Hh_logger.log_duration
36 (Printf.sprintf
"Updated dependency graph: added %d edges" edges)
41 let snapshot_naming_table_base ~destination_path
: unit Future.t
=
42 send_progress "Snapshotting the naming table for delegated type checking";
43 let start_t = Unix.gettimeofday
() in
45 match Naming_table.get_forward_naming_fallback_path naming_table
with
48 "Updating the existing table - moving %s to %s"
51 FileUtil.cp
[source_path
] destination_path
;
52 let (_
: Naming_sqlite.save_result
) =
53 Naming_table.save naming_table destination_path
57 Naming_table.save_async naming_table ~init_id ~root ~destination_path
59 Future.continue_with
future @@ fun () ->
60 HackEventLogger.remote_scheduler_save_naming_end
start_t;
61 let (start_t : float) =
62 Hh_logger.log_duration
63 (Printf.sprintf
"Saved SQLite naming table to %s" destination_path
)
67 (Printf.sprintf
"Snapshotted the naming table base: %f" start_t)
69 let snapshot_naming_table_diff ~
(destination_path
: string) : unit =
70 Hh_logger.log
"snapshot_naming_table_diff: %s" destination_path
;
71 Naming_table.save_changes_since_baseline naming_table ~destination_path
73 let begin_get_changed_files ~
(mergebase
: string option) :
74 string list
Future.t
=
75 let t = Unix.gettimeofday
() in
78 let hg_future = Hg.files_changed_since_rev
(Hg.Hg_rev mergebase
) root
in
79 Future.continue_with
hg_future @@ fun changed_files
->
84 ~
value:(List.length changed_files
)
86 HackEventLogger.remote_scheduler_get_dirty_files_end
telemetry t;
88 | None
-> Future.of_value
[]
90 let load_changed_files (changed_files
: string list
) :
91 (Relative_path.t * string option) list
=
92 let changed_files_and_content =
93 List.map changed_files ~f
:(fun changed_file
->
94 let changed_file = FilePath.make_absolute root
changed_file in
95 let changed_file_path =
96 Relative_path.create
Relative_path.Root
changed_file
98 (changed_file_path, File_provider.get_contents
changed_file_path))
100 changed_files_and_content
102 let write_changed_files
103 (changed_files
: string list
) ~
(destination_path
: string) : unit =
104 let changed_filepaths_and_content = load_changed_files changed_files
in
105 let chan = Stdlib.open_out_bin destination_path
in
106 Marshal.to_channel
chan changed_filepaths_and_content [];
107 Stdlib.close_out
chan
108 end : LocalServerApi
)
110 let make_remote_server_api
111 (ctx
: Provider_context.t)
112 (workers
: MultiWorker.worker list
option)
114 (module RemoteServerApi
with type naming_table
= Naming_table.t option) =
116 type naming_table
= Naming_table.t option
118 let load_naming_table_base ~
(naming_table_base
: Path.t option) :
119 (naming_table
, string) result
=
120 Hh_logger.log
"Loading naming table base...";
122 match naming_table_base
with
125 "Expected naming table base path to be set when loading naming table, but it was not"
126 | Some naming_table_base
->
129 (Naming_table.load_from_sqlite
131 (Path.to_string naming_table_base
)))
134 There is a variety of state that the server accumulates after type
135 checking files. We want to make sure we remove such state before a
136 recheck. In order to do this cleaning, we need a list of files that
139 let clean_changed_files_state ctx naming_table changed_files ~
t =
140 let (changed_names
: FileInfo.names
) =
141 List.fold changed_files ~init
:FileInfo.empty_names ~f
:(fun names file
->
142 match Naming_table.get_file_info naming_table file
with
143 | Some
(file_info
: FileInfo.t) ->
144 FileInfo.merge_names names
(FileInfo.simplify file_info
)
148 Hh_logger.log_duration
"Got names changed since naming table baseline" t
150 let changed_files = Relative_path.set_of_list
changed_files in
151 File_provider.remove_batch
changed_files;
152 Ast_provider.remove_batch
changed_files;
153 Fixme_provider.remove_batch
changed_files;
154 Decl_redecl_service.remove_old_defs
159 Hh_logger.log_duration
"Cleaned state associated with changed files" t
161 let load_naming_table_changes_since_baseline
162 (ctx
: Provider_context.t)
163 ~
(naming_table
: Naming_table.t option)
164 ~
(naming_table_diff
: Naming_table.changes_since_baseline
) :
165 (Naming_table.t option, string) result
=
166 Hh_logger.log
"Loading naming table changes since baseline...";
167 match naming_table
with
168 | None
-> Error
"Expected naming table base"
169 | Some naming_table
->
171 match Naming_table.get_forward_naming_fallback_path naming_table
with
173 Error
"Expected naming table base path to be set, but it was not"
174 | Some naming_table_base
->
176 let t = Unix.gettimeofday
() in
178 Naming_table.get_files_changed_since_baseline naming_table_diff
181 Hh_logger.log_duration
182 "Got files changed since naming table baseline"
186 clean_changed_files_state ctx naming_table
changed_files ~
t
188 Hh_logger.log
"Prefetching naming dirty files...";
189 Vfs.prefetch
changed_files;
191 Hh_logger.log_duration
"Prefetched naming dirty files" t
193 let (naming_table
: Naming_table.t) =
194 Naming_table.load_from_sqlite_with_changes_since_baseline
199 HackEventLogger.remote_worker_load_naming_end
t;
201 Hh_logger.log_duration
"Loaded naming table from SQLite" t
203 Ok
(Some naming_table
)
205 | e
-> Error
(Exn.to_string e
))
208 let build_naming_table _
=
209 Hh_logger.log
"Building naming table";
211 Find.make_next_files ~name
:"root" ~filter
:FindUtils.is_hack root
214 ServerUtils.make_next
215 ~hhi_filter
:(fun _
-> true)
217 ~extra_roots
:(ServerConfig.extra_paths
ServerConfig.default_config
)
219 Hh_logger.log
"Building naming table - Parsing";
221 Direct_decl_service.go
224 ~ide_files
:Relative_path.Set.empty
229 Hh_logger.log
"Building naming table - Naming";
230 let naming_table = Naming_table.create
defs_per_file in
231 Naming_table.iter
naming_table ~f
:(fun k v
->
232 let _ = Naming_global.ndecl_file_error_if_already_bound ctx k v
in
234 Hh_logger.log
"Building naming table - Done!";
237 let load_naming_and_dep_table
238 (saved_state_main_artifacts
:
239 Saved_state_loader.Naming_and_dep_table_info.main_artifacts
) :
240 (Naming_table.t * Path.t, string) result
=
242 Saved_state_loader.Naming_and_dep_table_info.naming_table_path
= _;
244 naming_sqlite_table_path
;
245 legacy_hot_decls_path
= _;
246 shallow_hot_decls_path
= _;
249 saved_state_main_artifacts
251 if not
(Sys.file_exists
(Path.to_string naming_sqlite_table_path
)) then
254 "Expected naming sqlite table at %s"
255 (Path.to_string naming_sqlite_table_path
))
258 Naming_table.load_from_sqlite
260 (Path.to_string naming_sqlite_table_path
)
262 Ok
(naming_table, dep_table_path
)
264 let download_naming_and_dep_table
265 (manifold_api_key
: string option)
266 (manifold_path
: string)
267 ~
(use_manifold_cython_client
: bool) :
268 (Naming_table.t * Path.t, string) result
=
269 let target_path = "/tmp/hh_server/" ^
Random_id.short_string
() in
270 Disk.mkdir_p
target_path;
272 let naming_table_future =
273 State_loader_futures.download_and_unpack_saved_state_from_manifold
276 Saved_state_loader.log_saved_state_age_and_distance
= false;
277 Saved_state_loader.saved_state_manifold_api_key
= manifold_api_key
;
278 Saved_state_loader.use_manifold_cython_client
;
280 ~progress_callback
:(fun _ -> ())
282 (Saved_state_loader.Naming_and_dep_table
{ naming_sqlite
= true })
284 ~
target_path:(Path.make
target_path)
286 match Future.get ~timeout
:60 naming_table_future with
288 let err = Future.error_to_string
err in
289 Hh_logger.error
"Downloading dep table failed: %s" err;
291 | Ok download_result
->
293 match download_result
with
294 | Error
(err, _telemetry
) ->
295 Error
(Saved_state_loader.debug_details_of_error
err)
296 | Ok
(main_artifacts
, _telemetry
) ->
298 Hh_logger.log_duration
299 "Finished downloading dep table."
300 (Future.start_t naming_table_future)
302 let naming_table_path =
304 .Saved_state_loader.Naming_and_dep_table_info
305 .naming_sqlite_table_path
308 "Downloaded naming table to %s"
309 (Path.to_string
naming_table_path);
310 load_naming_and_dep_table main_artifacts
313 let remove_decls naming_table fast_parsed
=
314 Relative_path.Map.iter fast_parsed ~f
:(fun fn
_ ->
315 match Naming_table.get_file_info
naming_table fn
with
328 (* we use [snd] to strip away positions *)
329 let snd (_, x
, _) = x
in
330 Naming_global.remove_decls
331 ~backend
:(Provider_backend.get
())
332 ~funs
:(List.map funs ~f
:snd)
333 ~classes
:(List.map classes ~f
:snd)
334 ~typedefs
:(List.map typedefs ~f
:snd)
335 ~consts
:(List.map consts ~f
:snd)
336 ~modules
:(List.map modules ~f
:snd))
338 let update_naming_table
339 (naming_table : Naming_table.t)
340 (dep_table_path
: Path.t)
341 (changed_files : Relative_path.t list
option) : (string, string) result
343 match changed_files with
344 | None
-> Error
"No changed files uploaded for remote worker's payload"
345 | Some
changed_files ->
346 Hh_logger.log
"Cleaning naming table of changed files";
348 (clean_changed_files_state
352 ~
t:(Unix.gettimeofday
()));
353 Ok
(naming_table, changed_files, dep_table_path
)
354 >>= fun (naming_table, changed_files, dep_table_path
) ->
355 let changed_hack_files =
356 List.filter_map
changed_files ~f
:(fun file
->
357 if FindUtils.is_hack
(Relative_path.suffix file
) then
358 Some
(Path.to_string root ^
"/" ^
Relative_path.suffix file
)
363 let state = ref changed_hack_files in
364 let max_files_per_batch = 1000 in
366 let (next
, rest
) = List.split_n
!state max_files_per_batch in
371 ServerUtils.make_next
372 ~hhi_filter
:(fun _ -> true)
374 ~extra_roots
:(ServerConfig.extra_paths
ServerConfig.default_config
)
378 Direct_decl_service.go
381 ~ide_files
:Relative_path.Set.empty
386 >>= fun (naming_table, fast_parsed
, dep_table_path
) ->
387 Hh_logger.log
"Built updated decls for naming table";
388 Hh_logger.log
"Clearing old decls from naming table";
389 remove_decls naming_table fast_parsed
;
390 Hh_logger.log
"Updating naming table";
391 ignore
(Naming_table.update_many
naming_table fast_parsed
);
392 Ok
(fast_parsed
, dep_table_path
)
393 >>= fun (fast_parsed
, dep_table_path
) ->
394 Hh_logger.log
"Updating naming global";
395 Naming_table.create fast_parsed
396 |> Naming_table.iter ~f
:(fun k v
->
398 Naming_global.ndecl_file_error_if_already_bound ctx k v
401 Ok
(Path.to_string dep_table_path
)
403 let download_and_update_naming_table
404 ~
(manifold_api_key
: string option)
405 ~
(use_manifold_cython_client
: bool)
406 (saved_state_manifold_path
: string option)
407 (changed_files : Relative_path.t list
option) : string option =
408 match saved_state_manifold_path
with
411 "[hulk lite] No saved_state_manifold_path, will fall back to building naming table locally";
412 build_naming_table ();
415 let dep_table_path_result =
416 download_naming_and_dep_table
419 ~use_manifold_cython_client
420 >>= fun (naming_table, dep_table_path
) ->
421 update_naming_table naming_table dep_table_path
changed_files
423 (match dep_table_path_result with
424 | Ok dep_table_path
-> Some dep_table_path
426 Hh_logger.log
"Could not build naming table from saved state: %s" err;
427 Hh_logger.log
"Falling back to generating naming table";
428 build_naming_table ();
431 let load_shallow_decls_saved_state
432 (saved_state_main_artifacts
:
433 Saved_state_loader.Shallow_decls_info.main_artifacts
) :
434 (string I64Map.t, string) result
=
435 let { Saved_state_loader.Shallow_decls_info.shallow_decls_path
} =
436 saved_state_main_artifacts
438 if not
(Sys.file_exists
(Path.to_string shallow_decls_path
)) then
441 "Expected shallow_decls_saved_state at %s"
442 (Path.to_string shallow_decls_path
))
444 let chan = Stdlib.open_in_bin
(Path.to_string shallow_decls_path
) in
445 let contents = Marshal.from_channel
chan in
446 Stdlib.close_in
chan;
449 let download_shallow_decls_saved_state
450 (manifold_api_key
: string option)
451 (manifold_path
: string)
452 (use_manifold_cython_client
: bool) : (string I64Map.t, string) result
=
453 let target_path = "/tmp/hh_server/" ^
Random_id.short_string
() in
454 Disk.mkdir_p
target_path;
456 let shallow_decls_future =
457 State_loader_futures.download_and_unpack_saved_state_from_manifold
460 Saved_state_loader.log_saved_state_age_and_distance
= false;
461 Saved_state_loader.saved_state_manifold_api_key
= manifold_api_key
;
462 Saved_state_loader.use_manifold_cython_client
;
464 ~progress_callback
:(fun _ -> ())
465 ~saved_state_type
:Saved_state_loader.Shallow_decls
467 ~
target_path:(Path.make
target_path)
469 match Future.get ~timeout
:600 shallow_decls_future with
471 let err = Future.error_to_string
err in
472 Hh_logger.error
"Downloading shallow_decls saved state failed: %s" err;
474 | Ok download_result
->
476 match download_result
with
477 | Error
(err, _telemetry
) ->
478 Error
(Saved_state_loader.debug_details_of_error
err)
479 | Ok
(main_artifacts
, _telemetry
) ->
481 Hh_logger.log_duration
482 "Finished downloading shallow_decls saved state."
483 (Future.start_t shallow_decls_future)
485 let shallow_decls_path =
487 .Saved_state_loader.Shallow_decls_info.shallow_decls_path
490 "Downloaded shallow_decls to %s"
491 (Path.to_string
shallow_decls_path);
492 load_shallow_decls_saved_state main_artifacts
495 let unmarshal_decls_from_download_result
496 ~
(ctx
: Provider_context.t)
497 (shallow_decls
: string I64Map.t)
498 (classnames
: SSet.elt list
) :
499 Shallow_decl_defs.shallow_class
option SMap.t =
500 (* match shallow decls with classnames according to hash code *)
501 let db_path_opt = Remote_old_decl_client.Utils.db_path_of_ctx ~ctx
in
502 match db_path_opt with
505 let decl_name_and_hashes =
509 Remote_old_decl_client.Utils.name_to_decl_hash_opt
514 | Some hash
-> Some
(name
, Int64.of_string hash
))
517 Hh_logger.log
"constructed smap";
520 ~f
:(fun acc
(name
, hash
) ->
521 match I64Map.find_opt hash shallow_decls
with
523 | Some marshalled_decl
->
524 let decl = Marshal.from_string marshalled_decl
0 in
525 SMap.add name
(Some
decl) acc
)
528 let fetch_remote_decls_from_saved_state
529 manifold_api_key manifold_path use_manifold_cython_client classnames
=
531 download_shallow_decls_saved_state
534 use_manifold_cython_client
536 | Ok shallow_decls_download_result
->
539 "loaded %d shallow decls from saved state"
540 (I64Map.cardinal shallow_decls_download_result
)
543 unmarshal_decls_from_download_result
545 shallow_decls_download_result
550 "extracted %d shallow decls from saved state"
551 (SMap.cardinal
state_decls)
554 | Error
err -> failwith
err
556 let fetch_remote_decls_from_remote_old_decl_service classnames
=
557 let job (acc
: 'a
SMap.t) (classnames
: string list
) : 'a
SMap.t =
559 "Fecthing %d decls from the remote decl store"
560 (List.length classnames
);
561 let remotely_fetched_decls =
562 Remote_old_decl_client.fetch_old_decls
563 ~telemetry_label
:"hulk type check"
568 "Fetched %d decls from the remote decl store"
569 (SMap.cardinal
remotely_fetched_decls);
572 if Option.is_some a
then
577 remotely_fetched_decls
584 (SMap.merge
(fun _key a b
->
585 if Option.is_some a
then
589 ~next
:(MultiWorker.next ~max_size
:50000 workers classnames
)
591 let fetch_and_cache_remote_decls
594 ~
(from_saved_state
: bool)
595 (manifold_api_key
: string option)
596 (manifold_path
: string)
597 (use_manifold_cython_client
: bool) =
598 let start_t = Unix.gettimeofday
() in
599 let fileinfos_from_naming_table =
601 |> Naming_table.to_defs_per_file ~warn_on_naming_costly_iter
:false
602 |> Relative_path.Map.elements
606 fileinfos_from_naming_table
608 ~f
:(fun acc
(_filename
, fileinfo
) ->
610 (fun class_name acc
-> class_name
:: acc
)
611 fileinfo
.FileInfo.n_classes
614 let remotely_fetched_decls =
615 if from_saved_state
then
616 fetch_remote_decls_from_saved_state
619 use_manifold_cython_client
622 fetch_remote_decls_from_remote_old_decl_service classnames
624 List.iter
fileinfos_from_naming_table ~f
:(fun (filename
, fileinfo
) ->
625 let class_names = fileinfo
.FileInfo.n_classes
in
626 let pfh_decls : (string * Shallow_decl_defs.decl * Int64.t) list
=
629 let remotely_fetched_decl =
630 Option.join
(SMap.find_opt name
remotely_fetched_decls)
634 remotely_fetched_decl >>= fun decl ->
635 Remote_old_decl_client.Utils.db_path_of_ctx ~ctx
637 Remote_old_decl_client.Utils.name_to_decl_hash_opt
642 (name
, Shallow_decl_defs.Class
decl, Int64.of_string hash
))
644 | Some pfh_decl
-> pfh_decl
:: acc
649 Direct_decl_utils.cache_decls ctx filename
pfh_decls);
651 Hh_logger.log_duration
652 "Fetched and cached decls from remote decl store"
658 ctx ~init_id ~
check_id files_to_check ~state_filename ~
telemetry =
659 let t = Unix.gettimeofday
() in
660 Hh_logger.log
"Type checking a batch...";
664 check_reason
= "remote_server_api";
665 recheck_id
= Some
check_id;
666 use_max_typechecker_worker_memory_for_decl_deferral
= false;
667 per_file_profiling
= HackEventLogger.PerFileProfilingConfig.default
;
671 (* TODO: use the telemetry *)
672 let { Typing_check_service.errors
; telemetry; _ } =
673 Typing_check_service.go
676 Typing_service_delegate.default
679 ~memory_cap
:(Some
200000)
680 ~longlived_workers
:false
681 ~mode
:HulkStrategy.Legacy
684 HackEventLogger.remote_worker_type_check_end
telemetry ~
start_t:t;
685 let t = Hh_logger.log_duration
"Type checked files in remote worker" t in
686 let dep_table_edges_added =
687 Typing_deps.save_discovered_edges
688 (Provider_context.get_deps_mode ctx
)
690 ~reset_state_after_saving
:true
693 Hh_logger.log_duration
695 "Saved partial dependency graph (%d edges)"
696 dep_table_edges_added)
700 end : RemoteServerApi
701 with type naming_table = Naming_table.t option)