gfx/webrender_bindings/src/program_cache.rs

   1 /* This Source Code Form is subject to the terms of the Mozilla Public
   2  * License, v. 2.0. If a copy of the MPL was not distributed with this
   3  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   4
   5 use std::cell::RefCell;
   6 use std::ffi::OsString;
   7 use std::fs::{create_dir_all, read_dir, read_to_string, File};
   8 use std::io::{Error, ErrorKind};
   9 use std::io::{Read, Write};
  10 use std::path::{Path, PathBuf};
  11 use std::rc::Rc;
  12 use std::sync::Arc;
  13
  14 use nsstring::nsAString;
  15 use rayon::ThreadPool;
  16 use webrender::{ProgramBinary, ProgramCache, ProgramCacheObserver, ProgramSourceDigest};
  17
  18 const MAX_LOAD_TIME_MS: u64 = 400;
  19
  20 fn deserialize_program_binary(path: &Path) -> Result<Arc<ProgramBinary>, Error> {
  21     let mut buf = vec![];
  22     let mut file = File::open(path)?;
  23     file.read_to_end(&mut buf)?;
  24
  25     if buf.len() <= 8 + 4 {
  26         return Err(Error::new(ErrorKind::InvalidData, "File size is too small"));
  27     }
  28     let magic = &buf[0..4];
  29     let hash = &buf[4..8 + 4];
  30     let data = &buf[8 + 4..];
  31
  32     // Check if magic + version are correct.
  33     let mv: u32 = bincode::deserialize(&magic).unwrap();
  34     if mv != MAGIC_AND_VERSION {
  35         return Err(Error::new(
  36             ErrorKind::InvalidData,
  37             "File data is invalid (magic+version)",
  38         ));
  39     }
  40
  41     // Check if hash is correct
  42     let hash: u64 = bincode::deserialize(&hash).unwrap();
  43     let hash_data = fxhash::hash64(&data);
  44     if hash != hash_data {
  45         return Err(Error::new(ErrorKind::InvalidData, "File data is invalid (hash)"));
  46     }
  47
  48     // Deserialize ProgramBinary
  49     let binary = match bincode::deserialize(&data) {
  50         Ok(binary) => binary,
  51         Err(_) => {
  52             return Err(Error::new(
  53                 ErrorKind::InvalidData,
  54                 "Failed to deserialize ProgramBinary",
  55             ))
  56         },
  57     };
  58
  59     Ok(Arc::new(binary))
  60 }
  61
  62 #[cfg(target_os = "windows")]
  63 fn get_cache_path_from_prof_path(prof_path: &nsAString) -> Option<PathBuf> {
  64     if prof_path.is_empty() {
  65         // Empty means that we do not use disk cache.
  66         return None;
  67     }
  68
  69     use std::os::windows::prelude::*;
  70
  71     let prof_path = OsString::from_wide(prof_path.as_ref());
  72     let mut cache_path = PathBuf::from(&prof_path);
  73     cache_path.push("shader-cache");
  74
  75     Some(cache_path)
  76 }
  77
  78 #[cfg(not(target_os = "windows"))]
  79 fn get_cache_path_from_prof_path(prof_path: &nsAString) -> Option<PathBuf> {
  80     if prof_path.is_empty() {
  81         // Empty means that we do not use disk cache.
  82         return None;
  83     }
  84
  85     let utf8 = String::from_utf16(prof_path.as_ref()).unwrap();
  86     let prof_path = OsString::from(utf8);
  87     let mut cache_path = PathBuf::from(&prof_path);
  88     cache_path.push("shader-cache");
  89
  90     Some(cache_path)
  91 }
  92
  93 struct WrProgramBinaryDiskCache {
  94     cache_path: PathBuf,
  95     workers: Arc<ThreadPool>,
  96     cached_shader_filenames: Vec<OsString>,
  97 }
  98
  99 // Magic number + version. Increment the version when the binary format changes.
 100 const MAGIC: u32 = 0xB154AD30; // BI-SHADE + version.
 101 const VERSION: u32 = 2;
 102 const MAGIC_AND_VERSION: u32 = MAGIC + VERSION;
 103
 104 const WHITELIST_FILENAME: &str = "startup_shaders";
 105 const WHITELIST_SEPARATOR: &str = "\n";
 106
 107 /// Helper to convert a closure returning a `Result` to one that returns void.
 108 /// This allows the enclosed code to use the question-mark operator in a
 109 /// context where the calling function doesn't expect a `Result`.
 110 #[allow(unused_must_use)]
 111 fn result_to_void<F: FnOnce() -> Result<(), ()>>(f: F) {
 112     f();
 113 }
 114
 115 impl WrProgramBinaryDiskCache {
 116     #[allow(dead_code)]
 117     fn new(cache_path: PathBuf, workers: &Arc<ThreadPool>) -> Self {
 118         WrProgramBinaryDiskCache {
 119             cache_path,
 120             workers: Arc::clone(workers),
 121             cached_shader_filenames: Vec::new(),
 122         }
 123     }
 124
 125     /// Saves the specified binaries to the on-disk shader cache.
 126     fn save_shaders_to_disk(&mut self, entries: Vec<Arc<ProgramBinary>>) {
 127         info!("Saving binaries to on-disk shader cache");
 128
 129         // Write the entries to disk on a worker thread.
 130         for entry in entries {
 131             let file_name = entry.source_digest().to_string();
 132             let file_path = self.cache_path.join(&file_name);
 133
 134             self.workers.spawn(move || {
 135                 result_to_void(move || {
 136                     info!("Writing shader: {}", file_name);
 137
 138                     use std::time::Instant;
 139                     let start = Instant::now();
 140
 141                     let data: Vec<u8> =
 142                         bincode::serialize(&*entry).map_err(|e| error!("shader-cache: Failed to serialize: {}", e))?;
 143
 144                     let mut file =
 145                         File::create(&file_path).map_err(|e| error!("shader-cache: Failed to create file: {}", e))?;
 146
 147                     // Write magic + version.
 148                     let mv = MAGIC_AND_VERSION;
 149                     let mv = bincode::serialize(&mv).unwrap();
 150                     assert!(mv.len() == 4);
 151                     file.write_all(&mv)
 152                         .map_err(|e| error!("shader-cache: Failed to write magic + version: {}", e))?;
 153
 154                     // Write hash
 155                     let hash = fxhash::hash64(&data);
 156                     let hash = bincode::serialize(&hash).unwrap();
 157                     assert!(hash.len() == 8);
 158                     file.write_all(&hash)
 159                         .map_err(|e| error!("shader-cache: Failed to write hash: {}", e))?;
 160
 161                     // Write serialized data
 162                     file.write_all(&data)
 163                         .map_err(|e| error!("shader-cache: Failed to write program binary: {}", e))?;
 164
 165                     info!("Wrote shader {} in {:?}", file_name, start.elapsed());
 166                     Ok(())
 167                 })
 168             });
 169         }
 170     }
 171
 172     /// Writes the whitelist containing the set of startup shaders to disk.
 173     fn set_startup_shaders(&mut self, entries: Vec<Arc<ProgramBinary>>) {
 174         let whitelist = entries
 175             .iter()
 176             .map(|e| e.source_digest().to_string())
 177             .collect::<Vec<String>>()
 178             .join(WHITELIST_SEPARATOR);
 179
 180         let mut whitelist_path = self.cache_path.clone();
 181         whitelist_path.push(WHITELIST_FILENAME);
 182         self.workers.spawn(move || {
 183             result_to_void(move || {
 184                 info!("Writing startup shader whitelist");
 185                 File::create(&whitelist_path)
 186                     .and_then(|mut file| file.write_all(whitelist.as_bytes()))
 187                     .map_err(|e| error!("shader-cache: Failed to write startup whitelist: {}", e))?;
 188                 Ok(())
 189             })
 190         });
 191     }
 192
 193     pub fn try_load_shader_from_disk(&mut self, filename: &str, program_cache: &Rc<ProgramCache>) {
 194         if let Some(index) = self.cached_shader_filenames.iter().position(|e| e == filename) {
 195             let mut path = self.cache_path.clone();
 196             path.push(filename);
 197
 198             self.cached_shader_filenames.swap_remove(index);
 199
 200             info!("Loading shader: {}", filename);
 201
 202             match deserialize_program_binary(&path) {
 203                 Ok(program) => {
 204                     program_cache.load_program_binary(program);
 205                 },
 206                 Err(err) => {
 207                     error!("shader-cache: Failed to deserialize program binary: {}", err);
 208                 },
 209             };
 210         } else {
 211             info!("shader-cache: Program binary not found in disk cache");
 212         }
 213     }
 214
 215     pub fn try_load_startup_shaders_from_disk(&mut self, program_cache: &Rc<ProgramCache>) {
 216         use std::time::Instant;
 217         let start = Instant::now();
 218
 219         // Load and parse the whitelist if it exists
 220         let mut whitelist_path = self.cache_path.clone();
 221         whitelist_path.push(WHITELIST_FILENAME);
 222         let whitelist = match read_to_string(&whitelist_path) {
 223             Ok(whitelist) => whitelist
 224                 .split(WHITELIST_SEPARATOR)
 225                 .map(|s| s.to_string())
 226                 .collect::<Vec<String>>(),
 227             Err(err) => {
 228                 info!("shader-cache: Could not read startup whitelist: {}", err);
 229                 Vec::new()
 230             },
 231         };
 232         info!("Loaded startup shader whitelist in {:?}", start.elapsed());
 233
 234         self.cached_shader_filenames = read_dir(&self.cache_path)
 235             .map_err(|err| {
 236                 error!(
 237                     "shader-cache: Error reading directory whilst loading startup shaders: {}",
 238                     err
 239                 )
 240             })
 241             .into_iter()
 242             .flatten()
 243             .filter_map(Result::ok)
 244             .map(|entry| entry.file_name())
 245             .filter(|filename| filename != WHITELIST_FILENAME)
 246             .collect::<Vec<OsString>>();
 247
 248         // Load whitelisted program binaries if they exist
 249         for entry in &whitelist {
 250             self.try_load_shader_from_disk(&entry, program_cache);
 251
 252             let elapsed = start.elapsed();
 253             info!("Loaded shader in {:?}", elapsed);
 254             let elapsed_ms = (elapsed.as_secs() * 1_000) + elapsed.subsec_millis() as u64;
 255
 256             if elapsed_ms > MAX_LOAD_TIME_MS {
 257                 // Loading the startup shaders is taking too long, so bail out now.
 258                 // Additionally clear the list of remaining shaders cached on disk,
 259                 // so that we do not attempt to load any on demand during rendering.
 260                 error!("shader-cache: Timed out before finishing loads");
 261                 self.cached_shader_filenames.clear();
 262                 break;
 263             }
 264         }
 265     }
 266 }
 267
 268 pub struct WrProgramCacheObserver {
 269     disk_cache: Rc<RefCell<WrProgramBinaryDiskCache>>,
 270 }
 271
 272 impl WrProgramCacheObserver {
 273     #[allow(dead_code)]
 274     fn new(disk_cache: Rc<RefCell<WrProgramBinaryDiskCache>>) -> Self {
 275         WrProgramCacheObserver { disk_cache }
 276     }
 277 }
 278
 279 impl ProgramCacheObserver for WrProgramCacheObserver {
 280     fn save_shaders_to_disk(&self, entries: Vec<Arc<ProgramBinary>>) {
 281         self.disk_cache.borrow_mut().save_shaders_to_disk(entries);
 282     }
 283
 284     fn set_startup_shaders(&self, entries: Vec<Arc<ProgramBinary>>) {
 285         self.disk_cache.borrow_mut().set_startup_shaders(entries);
 286     }
 287
 288     fn try_load_shader_from_disk(&self, digest: &ProgramSourceDigest, program_cache: &Rc<ProgramCache>) {
 289         let filename = digest.to_string();
 290         self.disk_cache
 291             .borrow_mut()
 292             .try_load_shader_from_disk(&filename, program_cache);
 293     }
 294
 295     fn notify_program_binary_failed(&self, _program_binary: &Arc<ProgramBinary>) {
 296         error!("shader-cache: Failed program_binary");
 297     }
 298 }
 299
 300 pub struct WrProgramCache {
 301     pub program_cache: Rc<ProgramCache>,
 302     disk_cache: Option<Rc<RefCell<WrProgramBinaryDiskCache>>>,
 303 }
 304
 305 impl WrProgramCache {
 306     pub fn new(prof_path: &nsAString, workers: &Arc<ThreadPool>) -> Self {
 307         let cache_path = get_cache_path_from_prof_path(prof_path);
 308         let use_disk_cache = cache_path.as_ref().map_or(false, |p| create_dir_all(p).is_ok());
 309         let (disk_cache, program_cache_observer) = if use_disk_cache {
 310             let cache = Rc::new(RefCell::new(WrProgramBinaryDiskCache::new(
 311                 cache_path.unwrap(),
 312                 workers,
 313             )));
 314             let obs = Box::new(WrProgramCacheObserver::new(Rc::clone(&cache))) as Box<dyn ProgramCacheObserver>;
 315             (Some(cache), Some(obs))
 316         } else {
 317             (None, None)
 318         };
 319         let program_cache = ProgramCache::new(program_cache_observer);
 320
 321         WrProgramCache {
 322             program_cache,
 323             disk_cache,
 324         }
 325     }
 326
 327     pub fn rc_get(&self) -> &Rc<ProgramCache> {
 328         &self.program_cache
 329     }
 330
 331     pub fn try_load_startup_shaders_from_disk(&self) {
 332         if let Some(ref disk_cache) = self.disk_cache {
 333             disk_cache
 334                 .borrow_mut()
 335                 .try_load_startup_shaders_from_disk(&self.program_cache);
 336         } else {
 337             error!("shader-cache: Shader disk cache is not supported");
 338         }
 339     }
 340 }
 341
 342 pub fn remove_disk_cache(prof_path: &nsAString) -> Result<(), Error> {
 343     use std::time::Instant;
 344
 345     if let Some(cache_path) = get_cache_path_from_prof_path(prof_path) {
 346         if cache_path.exists() {
 347             let start = Instant::now();
 348             remove_dir_all::remove_dir_all(&cache_path)?;
 349             info!("removed all disk cache shaders in {:?}", start.elapsed());
 350         }
 351     }
 352     Ok(())
 353 }