From 6d38459ec8c25c59333b022f95f512aadd27a529 Mon Sep 17 00:00:00 2001 From: Lucian Wischik Date: Fri, 18 Oct 2019 10:13:03 -0700 Subject: [PATCH] ipc between worker and decl Summary: Workers and Decl-service have to communicate. The previous diffstacks left this part unimplemented. This diff provides the implementation. I decided to do the communication all in Rust, similar to what Young did: the ocaml client calls "ipc.write" via an FFI, and the rust service calls "ipc.read" natively. `decl_ipc.rs` is the Rust library, and `decl_ipc_ffi.rs` is the OCaml FFI wrapper. Difference with Young is that I'm trying to be parsimonious with what's sent back and forth -- I don't think there's need to invoke a serialization layer; the format of requests is simple, the format of responses is simple, so let's just pick our own wire format and have zero allocations. This diff also wires up the ocaml client side and the rust server side to get an end-to-end communication Reviewed By: 2BitSalute Differential Revision: D17969937 fbshipit-source-id: 09ec5536d7a034187bf6dc91e8452b6650b15449 --- hphp/hack/src/providers/decl_service_client.ml | 13 +- .../rearchitecture_proposal_1/hh_decl/decl_ipc.rs | 149 +++++++++++++++++++++ .../hh_decl/decl_ipc_ffi.rs | 30 +++++ .../hh_decl/decl_service.rs | 18 ++- 4 files changed, 204 insertions(+), 6 deletions(-) create mode 100644 hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_ipc.rs create mode 100644 hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_ipc_ffi.rs rewrite hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_service.rs (64%) diff --git a/hphp/hack/src/providers/decl_service_client.ml b/hphp/hack/src/providers/decl_service_client.ml index a6dd6665fa7..7c04d2b985c 100644 --- a/hphp/hack/src/providers/decl_service_client.ml +++ b/hphp/hack/src/providers/decl_service_client.ml @@ -6,14 +6,23 @@ * *) +open Core_kernel + type sharedmem_base_address type t = { rpc_get_gconst: string -> (string, Marshal_tools.error) result } +external get_gconst_ffi : + Unix.file_descr -> sharedmem_base_address -> string -> int = "get_gconst" + let rpc_get_gconst - (_fd : Unix.file_descr) (_base : sharedmem_base_address) (name : string) : + (fd : Unix.file_descr) (base : sharedmem_base_address) (name : string) : (string, Marshal_tools.error) result = - Error (Marshal_tools.Rpc_malformed ("not implemented", Utils.Callstack name)) + (* TODO: this is just a placeholder for now *) + Printf.printf "GET GCONST... %s\n%!" name; + let i = get_gconst_ffi fd base name in + Printf.printf "GOT GCONST... %s = %d\n%!" name i; + Ok (Printf.sprintf "%d" i) let init (decl_sock_file : string) (base : sharedmem_base_address) : (t, Marshal_tools.error) result = diff --git a/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_ipc.rs b/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_ipc.rs new file mode 100644 index 00000000000..ba292774e31 --- /dev/null +++ b/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_ipc.rs @@ -0,0 +1,149 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the "hack" directory of this source tree. + +// This IPC library is used for a worker to communicate with a decl service. +// It avoids all dynamic allocation where possible. The last remaining +// dynamic allocation is when we receive a request, and have to allocate a buffer for the string name. +// Mixed results on whether loopback TCP or unix domain sockets is faster +// https://stackoverflow.com/questions/14973942/tcp-loopback-connection-vs-unix-domain-socket-performance +// I haven't been able to find if it's faster to call "write" multiple times, or +// copy things into a single buffer and call "write" once. +// In any case it likely doesn't matter, since we'll want to switched to some form +// of communication involving shared-memory and busy-waits and maybe futexes. + +// Wrapper around libc::read which turns exit code into a Result +unsafe fn read( + fd: libc::c_int, + buf: *mut libc::c_void, + count: libc::size_t, +) -> std::io::Result> { + let r = libc::read(fd, buf, count); + if r > 0 { + // TODO: maybe fail on incomplete read here? + return Ok(Some(())); + } else if r < 0 { + return Err(std::io::Error::last_os_error()); + } else { + // note: if you specify count=0, it will return Ok(None) + return Ok(None); + } +} + +// Wrapper around libc::write which turns exit code into a Result +unsafe fn write( + fd: libc::c_int, + buf: *const libc::c_void, + count: libc::size_t, +) -> std::io::Result { + let r = libc::write(fd, buf, count); + if r > 0 { + // TODO: maybe fail on incomplete write here? + return Ok(r); + } else if r < 0 { + return Err(std::io::Error::last_os_error()); + } else if count == 0 { + return Ok(0); + } else { + return Err(std::io::Error::new( + std::io::ErrorKind::WriteZero, + "WriteZero", + )); + } +} + +// This function sends a request for symbol over FD. +// The wire format is (1) usize for length of symbol name, (2) u32 for kind, +// (3) symbol in UTF8 format +pub fn write_request(fd: std::os::unix::io::RawFd, symbol: &str, kind: u32) -> std::io::Result<()> { + unsafe { + // (1) write size of string + let len: usize = symbol.len(); + write( + fd, + (&len as *const usize) as *const libc::c_void, + std::mem::size_of::(), + )?; + // (2) write kind + write( + fd, + (&kind as *const u32) as *const libc::c_void, + std::mem::size_of::(), + )?; + // (3) write the string + write(fd, symbol.as_ptr() as *const libc::c_void, len)?; + return Ok(()); + } +} + +// This function receives a decl request. It returns Ok(None) upon EOF. +// See write_request for the wire format. +pub fn read_request(fd: std::os::unix::io::RawFd) -> std::io::Result> { + unsafe { + // (1) read size of string + let mut len = 0usize; + if read( + fd, + (&mut len as *mut usize) as *mut libc::c_void, + std::mem::size_of::(), + )? == None + { + return Ok(None); + } + // (2) read kind + let mut kind = 0u32; + if read( + fd, + (&mut kind as *mut u32) as *mut libc::c_void, + std::mem::size_of::(), + )? == None + { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "Unexpected EOF", + )); + }; + // (3) read string + let mut buf = Vec::with_capacity(len); + if read(fd, buf.as_mut_ptr() as *mut libc::c_void, len)? == None { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "Unexpected EOF", + )); + } + let ptr = buf.as_mut_ptr(); + std::mem::forget(buf); + let s = String::from_raw_parts(ptr, len, len); + return Ok(Some((kind, s))); + } +} + +// This function writes a decl response over the wire. +// The format is just a single usize, the offset from the start of sharedmem. +pub fn write_response(fd: std::os::unix::io::RawFd, offset: usize) -> std::io::Result<()> { + unsafe { + // (1) write usize offset + libc::write( + fd, + (&offset as *const usize) as *const libc::c_void, + std::mem::size_of::(), + ); + return Ok(()); + } +} + +// This function reads a decl response over the wire. +// See write_response for the wire format. +pub fn read_response(fd: std::os::unix::io::RawFd) -> std::io::Result { + unsafe { + // (1) read usize offset + let mut r = 0usize; + libc::read( + fd, + (&mut r as *mut usize) as *mut libc::c_void, + std::mem::size_of::(), + ); + return Ok(r); + } +} diff --git a/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_ipc_ffi.rs b/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_ipc_ffi.rs new file mode 100644 index 00000000000..f7b99a1ca42 --- /dev/null +++ b/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_ipc_ffi.rs @@ -0,0 +1,30 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the "hack" directory of this source tree. + +use ocaml::{caml, Str, Value}; + +// Unix.file_descr -> sharedmem_base_address -> string -> int = "get_gconst" +// Send an RPC request over file-descriptor to the decl service, +// block until it sends back an offset, and (not yet implemented) return +// the native ocaml value located at sharedmem+offset. +// For now it does the RPC but returns a dummy ocaml value. +// The ocaml signature is +// type addr (** opaque pointer for base address of sharedmem *) +// external get_gconst : Unix.file_descr -> addr -> string -> [dummy:int] +// The input string is the gconst symbol name we want to look up. +// TODO: figure out how we want to expose errors to ocaml... +// I'm eager that the common path should involve no allocations +caml!(get_gconst, |fd, base, name|, , { + let fd = fd.i32_val() as std::os::unix::io::RawFd; + let _base = base.ptr_val::<*const u8>(); + let name = Str::from(name); + let name = name.as_str(); + let kind = 1u32; + decl_ipc::write_request(fd, &name, kind).unwrap(); + let _offset = decl_ipc::read_response(fd).unwrap(); + // TODO: once cachelib has real ocaml data in its blobs, return a Ptr to that. + // Here's just a placeholder. + result = Value::i32(271); +} -> result); diff --git a/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_service.rs b/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_service.rs dissimilarity index 64% index 4bc352c1cbc..679b45edce0 100644 --- a/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_service.rs +++ b/hphp/hack/src/rearchitecture_proposal_1/hh_decl/decl_service.rs @@ -1,4 +1,14 @@ -pub fn handle(_stream: std::os::unix::net::UnixStream) { - println!("decl: received incoming connection"); - // TODO: handle the decl connection -} +pub fn handle(stream: std::os::unix::net::UnixStream) { + println!("decl: received incoming connection"); + let fd = std::os::unix::io::AsRawFd::as_raw_fd(&stream); + loop { + if let Some((kind, req)) = decl_ipc::read_request(fd).unwrap() { + println!("decl: received request: kind={} req={}", kind, req); + decl_ipc::write_response(fd, 0x4000).unwrap(); + println!("decl: sent response"); + } else { + break; + } + } + println!("decl: closing incoming connection"); +} -- 2.11.4.GIT