1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 collections::{BTreeMap, BTreeSet},
12 use error_support::handle_error;
13 use once_cell::sync::OnceCell;
14 use parking_lot::Mutex;
15 use remote_settings::{
16 self, GetItemsOptions, RemoteSettingsConfig, RemoteSettingsRecord, SortOrder,
19 types::{FromSql, ToSqlOutput},
22 use serde::{de::DeserializeOwned, Deserialize, Serialize};
25 config::{SuggestGlobalConfig, SuggestProviderConfig},
27 ConnectionType, SuggestDao, SuggestDb, LAST_INGEST_META_UNPARSABLE,
28 UNPARSABLE_RECORDS_META_KEY,
31 provider::SuggestionProvider,
33 SuggestAttachment, SuggestRecord, SuggestRecordId, SuggestRecordType,
34 SuggestRemoteSettingsClient, DEFAULT_RECORDS_TYPES, REMOTE_SETTINGS_COLLECTION,
35 SUGGESTIONS_PER_ATTACHMENT,
38 Result, SuggestApiResult, Suggestion, SuggestionQuery,
41 /// The chunk size used to request unparsable records.
42 pub const UNPARSABLE_IDS_PER_REQUEST: usize = 150;
44 /// Builder for [SuggestStore]
46 /// Using a builder is preferred to calling the constructor directly since it's harder to confuse
47 /// the data_path and cache_path strings.
48 pub struct SuggestStoreBuilder(Mutex<SuggestStoreBuilderInner>);
51 struct SuggestStoreBuilderInner {
52 data_path: Option<String>,
53 remote_settings_config: Option<RemoteSettingsConfig>,
56 impl Default for SuggestStoreBuilder {
57 fn default() -> Self {
62 impl SuggestStoreBuilder {
63 pub fn new() -> SuggestStoreBuilder {
64 Self(Mutex::new(SuggestStoreBuilderInner::default()))
67 pub fn data_path(self: Arc<Self>, path: String) -> Arc<Self> {
68 self.0.lock().data_path = Some(path);
72 pub fn cache_path(self: Arc<Self>, _path: String) -> Arc<Self> {
73 // We used to use this, but we're not using it anymore, just ignore the call
77 pub fn remote_settings_config(self: Arc<Self>, config: RemoteSettingsConfig) -> Arc<Self> {
78 self.0.lock().remote_settings_config = Some(config);
82 #[handle_error(Error)]
83 pub fn build(&self) -> SuggestApiResult<Arc<SuggestStore>> {
84 let inner = self.0.lock();
88 .ok_or_else(|| Error::SuggestStoreBuilder("data_path not specified".to_owned()))?;
90 remote_settings::Client::new(inner.remote_settings_config.clone().unwrap_or_else(
91 || RemoteSettingsConfig {
94 collection_name: REMOTE_SETTINGS_COLLECTION.into(),
97 Ok(Arc::new(SuggestStore {
98 inner: SuggestStoreInner::new(data_path, settings_client),
103 /// The store is the entry point to the Suggest component. It incrementally
104 /// downloads suggestions from the Remote Settings service, stores them in a
105 /// local database, and returns them in response to user queries.
107 /// Your application should create a single store, and manage it as a singleton.
108 /// The store is thread-safe, and supports concurrent queries and ingests. We
109 /// expect that your application will call [`SuggestStore::query()`] to show
110 /// suggestions as the user types into the address bar, and periodically call
111 /// [`SuggestStore::ingest()`] in the background to update the database with
112 /// new suggestions from Remote Settings.
114 /// For responsiveness, we recommend always calling `query()` on a worker
115 /// thread. When the user types new input into the address bar, call
116 /// [`SuggestStore::interrupt()`] on the main thread to cancel the query
117 /// for the old input, and unblock the worker thread for the new query.
119 /// The store keeps track of the state needed to support incremental ingestion,
120 /// but doesn't schedule the ingestion work itself, or decide how many
121 /// suggestions to ingest at once. This is for two reasons:
123 /// 1. The primitives for scheduling background work vary between platforms, and
124 /// aren't available to the lower-level Rust layer. You might use an idle
125 /// timer on Desktop, `WorkManager` on Android, or `BGTaskScheduler` on iOS.
126 /// 2. Ingestion constraints can change, depending on the platform and the needs
127 /// of your application. A mobile device on a metered connection might want
128 /// to request a small subset of the Suggest data and download the rest
129 /// later, while a desktop on a fast link might download the entire dataset
130 /// on the first launch.
131 pub struct SuggestStore {
132 inner: SuggestStoreInner<remote_settings::Client>,
135 /// For records that aren't currently parsable,
136 /// the record ID and the schema version it's first seen in
137 /// is recorded in the meta table using `UNPARSABLE_RECORDS_META_KEY` as its key.
138 /// On the first ingest after an upgrade, re-request those records from Remote Settings,
139 /// and try to ingest them again.
140 #[derive(Deserialize, Serialize, Default, Debug)]
141 #[serde(transparent)]
142 pub(crate) struct UnparsableRecords(pub BTreeMap<String, UnparsableRecord>);
144 impl FromSql for UnparsableRecords {
145 fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult<Self> {
146 serde_json::from_str(value.as_str()?)
147 .map_err(|err| rusqlite::types::FromSqlError::Other(Box::new(err)))
151 impl ToSql for UnparsableRecords {
152 fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
153 Ok(ToSqlOutput::from(serde_json::to_string(self).map_err(
154 |err| rusqlite::Error::ToSqlConversionFailure(Box::new(err)),
159 #[derive(Deserialize, Serialize, Debug)]
160 pub(crate) struct UnparsableRecord {
161 #[serde(rename = "v")]
162 pub schema_version: u32,
166 /// Creates a Suggest store.
167 #[handle_error(Error)]
170 settings_config: Option<RemoteSettingsConfig>,
171 ) -> SuggestApiResult<Self> {
172 let settings_client = || -> Result<_> {
173 Ok(remote_settings::Client::new(
174 settings_config.unwrap_or_else(|| RemoteSettingsConfig {
177 collection_name: REMOTE_SETTINGS_COLLECTION.into(),
182 inner: SuggestStoreInner::new(path.to_owned(), settings_client),
186 /// Queries the database for suggestions.
187 #[handle_error(Error)]
188 pub fn query(&self, query: SuggestionQuery) -> SuggestApiResult<Vec<Suggestion>> {
189 self.inner.query(query)
192 /// Interrupts any ongoing queries.
194 /// This should be called when the user types new input into the address
195 /// bar, to ensure that they see fresh suggestions as they type. This
196 /// method does not interrupt any ongoing ingests.
197 pub fn interrupt(&self) {
198 self.inner.interrupt()
201 /// Ingests new suggestions from Remote Settings.
202 #[handle_error(Error)]
203 pub fn ingest(&self, constraints: SuggestIngestionConstraints) -> SuggestApiResult<()> {
204 self.inner.ingest(constraints)
207 /// Removes all content from the database.
208 #[handle_error(Error)]
209 pub fn clear(&self) -> SuggestApiResult<()> {
213 // Returns global Suggest configuration data.
214 #[handle_error(Error)]
215 pub fn fetch_global_config(&self) -> SuggestApiResult<SuggestGlobalConfig> {
216 self.inner.fetch_global_config()
219 // Returns per-provider Suggest configuration data.
220 #[handle_error(Error)]
221 pub fn fetch_provider_config(
223 provider: SuggestionProvider,
224 ) -> SuggestApiResult<Option<SuggestProviderConfig>> {
225 self.inner.fetch_provider_config(provider)
229 /// Constraints limit which suggestions to ingest from Remote Settings.
230 #[derive(Clone, Default, Debug)]
231 pub struct SuggestIngestionConstraints {
232 /// The approximate maximum number of suggestions to ingest. Set to [`None`]
235 /// Because of how suggestions are partitioned in Remote Settings, this is a
236 /// soft limit, and the store might ingest more than requested.
237 pub max_suggestions: Option<u64>,
238 pub providers: Option<Vec<SuggestionProvider>>,
241 /// The implementation of the store. This is generic over the Remote Settings
242 /// client, and is split out from the concrete [`SuggestStore`] for testing
243 /// with a mock client.
244 pub(crate) struct SuggestStoreInner<S> {
245 /// Path to the persistent SQL database.
247 /// This stores things that should persist when the user clears their cache.
248 /// It's not currently used because not all consumers pass this in yet.
251 dbs: OnceCell<SuggestStoreDbs>,
255 impl<S> SuggestStoreInner<S> {
256 fn new(data_path: impl Into<PathBuf>, settings_client: S) -> Self {
258 data_path: data_path.into(),
259 dbs: OnceCell::new(),
264 /// Returns this store's database connections, initializing them if
265 /// they're not already open.
266 fn dbs(&self) -> Result<&SuggestStoreDbs> {
268 .get_or_try_init(|| SuggestStoreDbs::open(&self.data_path))
271 fn query(&self, query: SuggestionQuery) -> Result<Vec<Suggestion>> {
272 if query.keyword.is_empty() || query.providers.is_empty() {
273 return Ok(Vec::new());
275 self.dbs()?.reader.read(|dao| dao.fetch_suggestions(&query))
278 fn interrupt(&self) {
279 if let Some(dbs) = self.dbs.get() {
280 // Only interrupt if the databases are already open.
281 dbs.reader.interrupt_handle.interrupt();
285 fn clear(&self) -> Result<()> {
286 self.dbs()?.writer.write(|dao| dao.clear())
289 pub fn fetch_global_config(&self) -> Result<SuggestGlobalConfig> {
290 self.dbs()?.reader.read(|dao| dao.get_global_config())
293 pub fn fetch_provider_config(
295 provider: SuggestionProvider,
296 ) -> Result<Option<SuggestProviderConfig>> {
299 .read(|dao| dao.get_provider_config(provider))
303 impl<S> SuggestStoreInner<S>
305 S: SuggestRemoteSettingsClient,
307 fn ingest(&self, constraints: SuggestIngestionConstraints) -> Result<()> {
308 let writer = &self.dbs()?.writer;
310 if let Some(unparsable_records) =
311 writer.read(|dao| dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY))?
313 let all_unparsable_ids = unparsable_records
316 .filter(|(_, unparsable_record)| unparsable_record.schema_version < VERSION)
317 .map(|(record_id, _)| record_id)
318 .collect::<Vec<_>>();
319 for unparsable_ids in all_unparsable_ids.chunks(UNPARSABLE_IDS_PER_REQUEST) {
320 let mut options = GetItemsOptions::new();
321 for unparsable_id in unparsable_ids {
322 options.eq("id", *unparsable_id);
324 let records_chunk = self
326 .get_records_with_options(&options)?
329 self.ingest_records(LAST_INGEST_META_UNPARSABLE, writer, &records_chunk)?;
333 // use std::collections::BTreeSet;
334 let ingest_record_types = if let Some(rt) = &constraints.providers {
336 .flat_map(|x| x.records_for_provider())
337 .collect::<BTreeSet<_>>()
341 DEFAULT_RECORDS_TYPES.to_vec()
344 for ingest_record_type in ingest_record_types {
345 self.ingest_records_by_type(ingest_record_type, writer, &constraints)?;
351 fn ingest_records_by_type(
353 ingest_record_type: SuggestRecordType,
355 constraints: &SuggestIngestionConstraints,
357 let mut options = GetItemsOptions::new();
359 // Remote Settings returns records in descending modification order
360 // (newest first), but we want them in ascending order (oldest first),
361 // so that we can eventually resume downloading where we left off.
362 options.sort("last_modified", SortOrder::Ascending);
364 options.eq("type", ingest_record_type.to_string());
366 // Get the last ingest value. This is the max of the last_ingest_keys
367 // that are in the database.
368 if let Some(last_ingest) = writer
369 .read(|dao| dao.get_meta::<u64>(ingest_record_type.last_ingest_meta_key().as_str()))?
371 // Only download changes since our last ingest. If our last ingest
372 // was interrupted, we'll pick up where we left off.
373 options.gt("last_modified", last_ingest.to_string());
376 if let Some(max_suggestions) = constraints.max_suggestions {
377 // Each record's attachment has 200 suggestions, so download enough
378 // records to cover the requested maximum.
379 let max_records = (max_suggestions.saturating_sub(1) / SUGGESTIONS_PER_ATTACHMENT) + 1;
380 options.limit(max_records);
385 .get_records_with_options(&options)?
387 self.ingest_records(&ingest_record_type.last_ingest_meta_key(), writer, &records)?;
393 last_ingest_key: &str,
395 records: &[RemoteSettingsRecord],
397 for record in records {
398 let record_id = SuggestRecordId::from(&record.id);
400 // If the entire record was deleted, drop all its suggestions
401 // and advance the last ingest time.
402 writer.write(|dao| dao.handle_deleted_record(last_ingest_key, record))?;
406 serde_json::from_value(serde_json::Value::Object(record.fields.clone()))
408 // We don't recognize this record's type, so we don't know how
409 // to ingest its suggestions. Skip processing this record.
410 writer.write(|dao| dao.handle_unparsable_record(record))?;
415 SuggestRecord::AmpWikipedia => {
416 self.ingest_attachment(
417 // TODO: Currently re-creating the last_ingest_key because using last_ingest_meta
418 // breaks the tests (particularly the unparsable functionality). So, keeping
419 // a direct reference until we remove the "unparsable" functionality.
420 &SuggestRecordType::AmpWikipedia.last_ingest_meta_key(),
423 |dao, record_id, suggestions| {
424 dao.insert_amp_wikipedia_suggestions(record_id, suggestions)
428 SuggestRecord::AmpMobile => {
429 self.ingest_attachment(
430 &SuggestRecordType::AmpMobile.last_ingest_meta_key(),
433 |dao, record_id, suggestions| {
434 dao.insert_amp_mobile_suggestions(record_id, suggestions)
438 SuggestRecord::Icon => {
439 let (Some(icon_id), Some(attachment)) =
440 (record_id.as_icon_id(), record.attachment.as_ref())
442 // An icon record should have an icon ID and an
443 // attachment. Icons that don't have these are
444 // malformed, so skip to the next record.
446 dao.put_last_ingest_if_newer(
447 &SuggestRecordType::Icon.last_ingest_meta_key(),
448 record.last_modified,
453 let data = self.settings_client.get_attachment(&attachment.location)?;
455 dao.put_icon(icon_id, &data, &attachment.mimetype)?;
456 dao.handle_ingested_record(
457 &SuggestRecordType::Icon.last_ingest_meta_key(),
462 SuggestRecord::Amo => {
463 self.ingest_attachment(
464 &SuggestRecordType::Amo.last_ingest_meta_key(),
467 |dao, record_id, suggestions| {
468 dao.insert_amo_suggestions(record_id, suggestions)
472 SuggestRecord::Pocket => {
473 self.ingest_attachment(
474 &SuggestRecordType::Pocket.last_ingest_meta_key(),
477 |dao, record_id, suggestions| {
478 dao.insert_pocket_suggestions(record_id, suggestions)
482 SuggestRecord::Yelp => {
483 self.ingest_attachment(
484 &SuggestRecordType::Yelp.last_ingest_meta_key(),
487 |dao, record_id, suggestions| match suggestions.first() {
488 Some(suggestion) => dao.insert_yelp_suggestions(record_id, suggestion),
493 SuggestRecord::Mdn => {
494 self.ingest_attachment(
495 &SuggestRecordType::Mdn.last_ingest_meta_key(),
498 |dao, record_id, suggestions| {
499 dao.insert_mdn_suggestions(record_id, suggestions)
503 SuggestRecord::Weather(data) => {
505 &SuggestRecordType::Weather.last_ingest_meta_key(),
508 |dao, record_id| dao.insert_weather_data(record_id, &data),
511 SuggestRecord::GlobalConfig(config) => {
513 &SuggestRecordType::GlobalConfig.last_ingest_meta_key(),
516 |dao, _| dao.put_global_config(&SuggestGlobalConfig::from(&config)),
526 last_ingest_key: &str,
528 record: &RemoteSettingsRecord,
529 ingestion_handler: impl FnOnce(&mut SuggestDao<'_>, &SuggestRecordId) -> Result<()>,
531 let record_id = SuggestRecordId::from(&record.id);
534 // Drop any data that we previously ingested from this record.
535 // Suggestions in particular don't have a stable identifier, and
536 // determining which suggestions in the record actually changed is
537 // more complicated than dropping and re-ingesting all of them.
538 dao.drop_suggestions(&record_id)?;
540 // Ingest (or re-ingest) all data in the record.
541 ingestion_handler(dao, &record_id)?;
543 dao.handle_ingested_record(last_ingest_key, record)
547 fn ingest_attachment<T>(
549 last_ingest_key: &str,
551 record: &RemoteSettingsRecord,
552 ingestion_handler: impl FnOnce(&mut SuggestDao<'_>, &SuggestRecordId, &[T]) -> Result<()>,
557 let Some(attachment) = record.attachment.as_ref() else {
558 // This method should be called only when a record is expected to
559 // have an attachment. If it doesn't have one, it's malformed, so
560 // skip to the next record.
562 .write(|dao| dao.put_last_ingest_if_newer(last_ingest_key, record.last_modified))?;
566 let attachment_data = self.settings_client.get_attachment(&attachment.location)?;
567 match serde_json::from_slice::<SuggestAttachment<T>>(&attachment_data) {
569 self.ingest_record(last_ingest_key, writer, record, |dao, record_id| {
570 ingestion_handler(dao, record_id, attachment.suggestions())
573 Err(_) => writer.write(|dao| dao.handle_unparsable_record(record)),
578 /// Holds a store's open connections to the Suggest database.
579 struct SuggestStoreDbs {
580 /// A read-write connection used to update the database with new data.
582 /// A read-only connection used to query the database.
586 impl SuggestStoreDbs {
587 fn open(path: &Path) -> Result<Self> {
588 // Order is important here: the writer must be opened first, so that it
589 // can set up the database and run any migrations.
590 let writer = SuggestDb::open(path, ConnectionType::ReadWrite)?;
591 let reader = SuggestDb::open(path, ConnectionType::ReadOnly)?;
592 Ok(Self { writer, reader })
600 use std::{cell::RefCell, collections::HashMap};
602 use anyhow::{anyhow, Context};
603 use expect_test::expect;
604 use parking_lot::Once;
606 use remote_settings::{RemoteSettingsRecord, RemoteSettingsResponse};
607 use serde_json::json;
608 use sql_support::ConnExt;
610 use crate::SuggestionProvider;
612 /// Creates a unique in-memory Suggest store.
613 fn unique_test_store<S>(settings_client: S) -> SuggestStoreInner<S>
615 S: SuggestRemoteSettingsClient,
617 let mut unique_suffix = [0u8; 8];
618 rand::fill(&mut unique_suffix).expect("Failed to generate unique suffix for test store");
619 // A store opens separate connections to the same database for reading
620 // and writing, so we must give our in-memory database a name, and open
621 // it in shared-cache mode so that both connections can access it.
622 SuggestStoreInner::new(
624 "file:test_store_data_{}?mode=memory&cache=shared",
625 hex::encode(unique_suffix),
631 /// A snapshot containing fake Remote Settings records and attachments for
632 /// the store to ingest. We use snapshots to test the store's behavior in a
635 records: Vec<RemoteSettingsRecord>,
636 attachments: HashMap<&'static str, Vec<u8>>,
640 /// Creates a snapshot from a JSON value that represents a collection of
641 /// Suggest Remote Settings records.
643 /// You can use the [`serde_json::json!`] macro to construct the JSON
644 /// value, then pass it to this function. It's easier to use the
645 /// `Snapshot::with_records(json!(...))` idiom than to construct the
647 fn with_records(value: serde_json::Value) -> anyhow::Result<Self> {
649 records: serde_json::from_value(value)
650 .context("Couldn't create snapshot with Remote Settings records")?,
651 attachments: HashMap::new(),
655 /// Adds a data attachment with one or more suggestions to the snapshot.
658 location: &'static str,
659 value: serde_json::Value,
660 ) -> anyhow::Result<Self> {
661 self.attachments.insert(
663 serde_json::to_vec(&value).context("Couldn't add data attachment to snapshot")?,
668 /// Adds an icon attachment to the snapshot.
669 fn with_icon(mut self, location: &'static str, bytes: Vec<u8>) -> Self {
670 self.attachments.insert(location, bytes);
675 /// A fake Remote Settings client that returns records and attachments from
677 struct SnapshotSettingsClient {
678 /// The current snapshot. You can modify it using
679 /// [`RefCell::borrow_mut()`] to simulate remote updates in tests.
680 snapshot: RefCell<Snapshot>,
682 /// The options passed to the last [`Self::get_records_with_options()`]
684 last_get_records_options: RefCell<Option<GetItemsOptions>>,
687 impl SnapshotSettingsClient {
688 /// Creates a client with an initial snapshot.
689 fn with_snapshot(snapshot: Snapshot) -> Self {
691 snapshot: RefCell::new(snapshot),
692 last_get_records_options: RefCell::default(),
696 /// Returns the most recent value of an option passed to
697 /// [`Self::get_records_with_options()`].
698 fn last_get_records_option(&self, option: &str) -> Option<String> {
699 self.last_get_records_options
702 .and_then(|options| {
705 .find(|(key, _)| key == option)
706 .map(|(_, value)| value.into())
711 impl SuggestRemoteSettingsClient for SnapshotSettingsClient {
712 fn get_records_with_options(
714 options: &GetItemsOptions,
715 ) -> Result<RemoteSettingsResponse> {
716 *self.last_get_records_options.borrow_mut() = Some(options.clone());
717 let records = self.snapshot.borrow().records.clone();
718 let last_modified = records
720 .map(|record| record.last_modified)
723 Ok(RemoteSettingsResponse {
729 fn get_attachment(&self, location: &str) -> Result<Vec<u8>> {
735 .unwrap_or_else(|| unreachable!("Unexpected request for attachment `{}`", location))
741 static ONCE: Once = Once::new();
747 /// Tests that `SuggestStore` is usable with UniFFI, which requires exposed
748 /// interfaces to be `Send` and `Sync`.
750 fn is_thread_safe() {
753 fn is_send_sync<T: Send + Sync>() {}
754 is_send_sync::<SuggestStore>();
757 /// Tests ingesting suggestions into an empty database.
759 fn ingest_suggestions() -> anyhow::Result<()> {
762 let snapshot = Snapshot::with_records(json!([{
767 "filename": "data-1.json",
768 "mimetype": "application/json",
769 "location": "data-1.json",
778 "advertiser": "Los Pollos Hermanos",
779 "iab_category": "8 - Food & Drink",
780 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
781 "title": "Los Pollos Hermanos - Albuquerque",
782 "url": "https://www.lph-nm.biz",
784 "impression_url": "https://example.com/impression_url",
785 "click_url": "https://example.com/click_url",
790 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
792 store.ingest(SuggestIngestionConstraints::default())?;
794 store.dbs()?.reader.read(|dao| {
797 SuggestRecordType::AmpWikipedia
798 .last_ingest_meta_key()
806 title: "Los Pollos Hermanos - Albuquerque",
807 url: "https://www.lph-nm.biz",
808 raw_url: "https://www.lph-nm.biz",
813 advertiser: "Los Pollos Hermanos",
814 iab_category: "8 - Food & Drink",
815 impression_url: "https://example.com/impression_url",
816 click_url: "https://example.com/click_url",
817 raw_click_url: "https://example.com/click_url",
822 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
823 keyword: "lo".into(),
824 providers: vec![SuggestionProvider::Amp],
834 /// Tests ingesting suggestions with icons.
836 fn ingest_icons() -> anyhow::Result<()> {
839 let snapshot = Snapshot::with_records(json!([{
844 "filename": "data-1.json",
845 "mimetype": "application/json",
846 "location": "data-1.json",
855 "filename": "icon-2.png",
856 "mimetype": "image/png",
857 "location": "icon-2.png",
866 "advertiser": "Good Place Eats",
867 "iab_category": "8 - Food & Drink",
868 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
869 "title": "Lasagna Come Out Tomorrow",
870 "url": "https://www.lasagna.restaurant",
872 "impression_url": "https://example.com/impression_url",
873 "click_url": "https://example.com/click_url"
876 "advertiser": "Good Place Eats",
877 "iab_category": "8 - Food & Drink",
878 "keywords": ["pe", "pen", "penne", "penne for your thoughts"],
879 "title": "Penne for Your Thoughts",
880 "url": "https://penne.biz",
882 "impression_url": "https://example.com/impression_url",
883 "click_url": "https://example.com/click_url",
887 .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into());
889 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
891 store.ingest(SuggestIngestionConstraints::default())?;
893 store.dbs()?.reader.read(|dao| {
897 title: "Lasagna Come Out Tomorrow",
898 url: "https://www.lasagna.restaurant",
899 raw_url: "https://www.lasagna.restaurant",
919 full_keyword: "lasagna",
921 advertiser: "Good Place Eats",
922 iab_category: "8 - Food & Drink",
923 impression_url: "https://example.com/impression_url",
924 click_url: "https://example.com/click_url",
925 raw_click_url: "https://example.com/click_url",
930 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
931 keyword: "la".into(),
932 providers: vec![SuggestionProvider::Amp],
938 title: "Penne for Your Thoughts",
939 url: "https://penne.biz",
940 raw_url: "https://penne.biz",
960 full_keyword: "penne",
962 advertiser: "Good Place Eats",
963 iab_category: "8 - Food & Drink",
964 impression_url: "https://example.com/impression_url",
965 click_url: "https://example.com/click_url",
966 raw_click_url: "https://example.com/click_url",
971 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
972 keyword: "pe".into(),
973 providers: vec![SuggestionProvider::Amp],
984 fn ingest_full_keywords() -> anyhow::Result<()> {
987 let snapshot = Snapshot::with_records(json!([{
992 "filename": "data-1.json",
993 "mimetype": "application/json",
994 "location": "data-1.json",
1001 "last_modified": 15,
1003 "filename": "data-2.json",
1004 "mimetype": "application/json",
1005 "location": "data-2.json",
1012 "last_modified": 15,
1014 "filename": "data-3.json",
1015 "mimetype": "application/json",
1016 "location": "data-3.json",
1022 "type": "amp-mobile-suggestions",
1023 "last_modified": 15,
1025 "filename": "data-4.json",
1026 "mimetype": "application/json",
1027 "location": "data-4.json",
1032 // AMP attachment with full keyword data
1037 "advertiser": "Los Pollos Hermanos",
1038 "iab_category": "8 - Food & Drink",
1039 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
1041 // Full keyword for the first 4 keywords
1043 // Full keyword for the next 2 keywords
1044 ("los pollos hermanos (restaurant)", 2),
1046 "title": "Los Pollos Hermanos - Albuquerque - 1",
1047 "url": "https://www.lph-nm.biz",
1049 "impression_url": "https://example.com/impression_url",
1050 "click_url": "https://example.com/click_url",
1054 // AMP attachment without a full keyword
1059 "advertiser": "Los Pollos Hermanos",
1060 "iab_category": "8 - Food & Drink",
1061 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
1062 "title": "Los Pollos Hermanos - Albuquerque - 2",
1063 "url": "https://www.lph-nm.biz",
1065 "impression_url": "https://example.com/impression_url",
1066 "click_url": "https://example.com/click_url",
1070 // Wikipedia attachment with full keyword data. We should ignore the full
1071 // keyword data for Wikipedia suggestions
1076 "advertiser": "Wikipedia",
1077 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
1078 "title": "Los Pollos Hermanos - Albuquerque - Wiki",
1080 ("Los Pollos Hermanos - Albuquerque", 6),
1082 "url": "https://www.lph-nm.biz",
1087 // Amp mobile suggestion, this is essentially the same as 1, except for the SuggestionProvider
1092 "advertiser": "Los Pollos Hermanos",
1093 "iab_category": "8 - Food & Drink",
1094 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
1096 // Full keyword for the first 4 keywords
1098 // Full keyword for the next 2 keywords
1099 ("los pollos hermanos (restaurant)", 2),
1101 "title": "Los Pollos Hermanos - Albuquerque - 4",
1102 "url": "https://www.lph-nm.biz",
1104 "impression_url": "https://example.com/impression_url",
1105 "click_url": "https://example.com/click_url",
1110 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
1112 store.ingest(SuggestIngestionConstraints::default())?;
1114 store.dbs()?.reader.read(|dao| {
1115 // This one should match the first full keyword for the first AMP item.
1119 title: "Los Pollos Hermanos - Albuquerque - 1",
1120 url: "https://www.lph-nm.biz",
1121 raw_url: "https://www.lph-nm.biz",
1123 icon_mimetype: None,
1124 full_keyword: "los pollos",
1126 advertiser: "Los Pollos Hermanos",
1127 iab_category: "8 - Food & Drink",
1128 impression_url: "https://example.com/impression_url",
1129 click_url: "https://example.com/click_url",
1130 raw_click_url: "https://example.com/click_url",
1134 title: "Los Pollos Hermanos - Albuquerque - 2",
1135 url: "https://www.lph-nm.biz",
1136 raw_url: "https://www.lph-nm.biz",
1138 icon_mimetype: None,
1139 full_keyword: "los",
1141 advertiser: "Los Pollos Hermanos",
1142 iab_category: "8 - Food & Drink",
1143 impression_url: "https://example.com/impression_url",
1144 click_url: "https://example.com/click_url",
1145 raw_click_url: "https://example.com/click_url",
1150 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1151 keyword: "lo".into(),
1152 providers: vec![SuggestionProvider::Amp],
1155 // This one should match the second full keyword for the first AMP item.
1159 title: "Los Pollos Hermanos - Albuquerque - 1",
1160 url: "https://www.lph-nm.biz",
1161 raw_url: "https://www.lph-nm.biz",
1163 icon_mimetype: None,
1164 full_keyword: "los pollos hermanos (restaurant)",
1166 advertiser: "Los Pollos Hermanos",
1167 iab_category: "8 - Food & Drink",
1168 impression_url: "https://example.com/impression_url",
1169 click_url: "https://example.com/click_url",
1170 raw_click_url: "https://example.com/click_url",
1174 title: "Los Pollos Hermanos - Albuquerque - 2",
1175 url: "https://www.lph-nm.biz",
1176 raw_url: "https://www.lph-nm.biz",
1178 icon_mimetype: None,
1179 full_keyword: "los pollos hermanos",
1181 advertiser: "Los Pollos Hermanos",
1182 iab_category: "8 - Food & Drink",
1183 impression_url: "https://example.com/impression_url",
1184 click_url: "https://example.com/click_url",
1185 raw_click_url: "https://example.com/click_url",
1190 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1191 keyword: "los pollos h".into(),
1192 providers: vec![SuggestionProvider::Amp],
1195 // This one matches a Wikipedia suggestion, so the full keyword should be ignored
1199 title: "Los Pollos Hermanos - Albuquerque - Wiki",
1200 url: "https://www.lph-nm.biz",
1202 icon_mimetype: None,
1203 full_keyword: "los",
1207 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1208 keyword: "los".into(),
1209 providers: vec![SuggestionProvider::Wikipedia],
1212 // This one matches a Wikipedia suggestion, so the full keyword should be ignored
1216 title: "Los Pollos Hermanos - Albuquerque - 4",
1217 url: "https://www.lph-nm.biz",
1218 raw_url: "https://www.lph-nm.biz",
1220 icon_mimetype: None,
1221 full_keyword: "los pollos hermanos (restaurant)",
1223 advertiser: "Los Pollos Hermanos",
1224 iab_category: "8 - Food & Drink",
1225 impression_url: "https://example.com/impression_url",
1226 click_url: "https://example.com/click_url",
1227 raw_click_url: "https://example.com/click_url",
1232 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1233 keyword: "los pollos h".into(),
1234 providers: vec![SuggestionProvider::AmpMobile],
1244 /// Tests ingesting a data attachment containing a single suggestion,
1245 /// instead of an array of suggestions.
1247 fn ingest_one_suggestion_in_data_attachment() -> anyhow::Result<()> {
1250 let snapshot = Snapshot::with_records(json!([{
1253 "last_modified": 15,
1255 "filename": "data-1.json",
1256 "mimetype": "application/json",
1257 "location": "data-1.json",
1266 "advertiser": "Good Place Eats",
1267 "iab_category": "8 - Food & Drink",
1268 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
1269 "title": "Lasagna Come Out Tomorrow",
1270 "url": "https://www.lasagna.restaurant",
1272 "impression_url": "https://example.com/impression_url",
1273 "click_url": "https://example.com/click_url",
1278 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
1280 store.ingest(SuggestIngestionConstraints::default())?;
1282 store.dbs()?.reader.read(|dao| {
1286 title: "Lasagna Come Out Tomorrow",
1287 url: "https://www.lasagna.restaurant",
1288 raw_url: "https://www.lasagna.restaurant",
1290 icon_mimetype: None,
1291 full_keyword: "lasagna",
1293 advertiser: "Good Place Eats",
1294 iab_category: "8 - Food & Drink",
1295 impression_url: "https://example.com/impression_url",
1296 click_url: "https://example.com/click_url",
1297 raw_click_url: "https://example.com/click_url",
1302 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1303 keyword: "la".into(),
1304 providers: vec![SuggestionProvider::Amp],
1314 /// Tests re-ingesting suggestions from an updated attachment.
1316 fn reingest_amp_suggestions() -> anyhow::Result<()> {
1319 // Ingest suggestions from the initial snapshot.
1320 let initial_snapshot = Snapshot::with_records(json!([{
1323 "last_modified": 15,
1325 "filename": "data-1.json",
1326 "mimetype": "application/json",
1327 "location": "data-1.json",
1336 "advertiser": "Good Place Eats",
1337 "iab_category": "8 - Food & Drink",
1338 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
1339 "title": "Lasagna Come Out Tomorrow",
1340 "url": "https://www.lasagna.restaurant",
1342 "impression_url": "https://example.com/impression_url",
1343 "click_url": "https://example.com/click_url",
1347 "advertiser": "Los Pollos Hermanos",
1348 "iab_category": "8 - Food & Drink",
1349 "keywords": ["lo", "los p", "los pollos h"],
1350 "title": "Los Pollos Hermanos - Albuquerque",
1351 "url": "https://www.lph-nm.biz",
1353 "impression_url": "https://example.com/impression_url",
1354 "click_url": "https://example.com/click_url",
1359 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(initial_snapshot));
1361 store.ingest(SuggestIngestionConstraints::default())?;
1363 store.dbs()?.reader.read(|dao| {
1366 SuggestRecordType::AmpWikipedia
1367 .last_ingest_meta_key()
1375 title: "Lasagna Come Out Tomorrow",
1376 url: "https://www.lasagna.restaurant",
1377 raw_url: "https://www.lasagna.restaurant",
1379 icon_mimetype: None,
1380 full_keyword: "lasagna",
1382 advertiser: "Good Place Eats",
1383 iab_category: "8 - Food & Drink",
1384 impression_url: "https://example.com/impression_url",
1385 click_url: "https://example.com/click_url",
1386 raw_click_url: "https://example.com/click_url",
1391 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1392 keyword: "la".into(),
1393 providers: vec![SuggestionProvider::Amp],
1399 // Update the snapshot with new suggestions: drop Lasagna, update Los
1400 // Pollos, and add Penne.
1401 *store.settings_client.snapshot.borrow_mut() = Snapshot::with_records(json!([{
1404 "last_modified": 30,
1406 "filename": "data-1-1.json",
1407 "mimetype": "application/json",
1408 "location": "data-1-1.json",
1417 "advertiser": "Los Pollos Hermanos",
1418 "iab_category": "8 - Food & Drink",
1419 "keywords": ["los ", "los pollos", "los pollos hermanos"],
1420 "title": "Los Pollos Hermanos - Now Serving at 14 Locations!",
1421 "url": "https://www.lph-nm.biz",
1423 "impression_url": "https://example.com/impression_url",
1424 "click_url": "https://example.com/click_url",
1428 "advertiser": "Good Place Eats",
1429 "iab_category": "8 - Food & Drink",
1430 "keywords": ["pe", "pen", "penne", "penne for your thoughts"],
1431 "title": "Penne for Your Thoughts",
1432 "url": "https://penne.biz",
1434 "impression_url": "https://example.com/impression_url",
1435 "click_url": "https://example.com/click_url",
1440 store.ingest(SuggestIngestionConstraints::default())?;
1442 store.dbs()?.reader.read(|dao: &SuggestDao<'_>| {
1445 SuggestRecordType::AmpWikipedia
1446 .last_ingest_meta_key()
1452 .fetch_suggestions(&SuggestionQuery {
1453 keyword: "la".into(),
1454 providers: vec![SuggestionProvider::Amp],
1461 title: "Los Pollos Hermanos - Now Serving at 14 Locations!",
1462 url: "https://www.lph-nm.biz",
1463 raw_url: "https://www.lph-nm.biz",
1465 icon_mimetype: None,
1466 full_keyword: "los pollos",
1468 advertiser: "Los Pollos Hermanos",
1469 iab_category: "8 - Food & Drink",
1470 impression_url: "https://example.com/impression_url",
1471 click_url: "https://example.com/click_url",
1472 raw_click_url: "https://example.com/click_url",
1477 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1478 keyword: "los ".into(),
1479 providers: vec![SuggestionProvider::Amp],
1485 title: "Penne for Your Thoughts",
1486 url: "https://penne.biz",
1487 raw_url: "https://penne.biz",
1489 icon_mimetype: None,
1490 full_keyword: "penne",
1492 advertiser: "Good Place Eats",
1493 iab_category: "8 - Food & Drink",
1494 impression_url: "https://example.com/impression_url",
1495 click_url: "https://example.com/click_url",
1496 raw_click_url: "https://example.com/click_url",
1501 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1502 keyword: "pe".into(),
1503 providers: vec![SuggestionProvider::Amp],
1512 /// Tests re-ingesting icons from an updated attachment.
1514 fn reingest_icons() -> anyhow::Result<()> {
1517 // Ingest suggestions and icons from the initial snapshot.
1518 let initial_snapshot = Snapshot::with_records(json!([{
1521 "last_modified": 15,
1523 "filename": "data-1.json",
1524 "mimetype": "application/json",
1525 "location": "data-1.json",
1532 "last_modified": 20,
1534 "filename": "icon-2.png",
1535 "mimetype": "image/png",
1536 "location": "icon-2.png",
1543 "last_modified": 25,
1545 "filename": "icon-3.png",
1546 "mimetype": "image/png",
1547 "location": "icon-3.png",
1556 "advertiser": "Good Place Eats",
1557 "iab_category": "8 - Food & Drink",
1558 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
1559 "title": "Lasagna Come Out Tomorrow",
1560 "url": "https://www.lasagna.restaurant",
1562 "impression_url": "https://example.com/impression_url",
1563 "click_url": "https://example.com/click_url",
1567 "advertiser": "Los Pollos Hermanos",
1568 "iab_category": "8 - Food & Drink",
1569 "keywords": ["lo", "los", "los pollos", "los pollos hermanos"],
1570 "title": "Los Pollos Hermanos - Albuquerque",
1571 "url": "https://www.lph-nm.biz",
1573 "impression_url": "https://example.com/impression_url",
1574 "click_url": "https://example.com/click_url",
1578 .with_icon("icon-2.png", "lasagna-icon".as_bytes().into())
1579 .with_icon("icon-3.png", "pollos-icon".as_bytes().into());
1581 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(initial_snapshot));
1583 store.ingest(SuggestIngestionConstraints::default())?;
1585 store.dbs()?.reader.read(|dao| {
1587 dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
1592 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
1595 assert_eq!(dao.conn.query_one::<i64>("SELECT count(*) FROM icons")?, 2);
1599 // Update the snapshot with new icons.
1600 *store.settings_client.snapshot.borrow_mut() = Snapshot::with_records(json!([{
1603 "last_modified": 30,
1605 "filename": "icon-2.png",
1606 "mimetype": "image/png",
1607 "location": "icon-2.png",
1614 "last_modified": 35,
1616 "filename": "icon-3.png",
1617 "mimetype": "image/png",
1618 "location": "icon-3.png",
1623 .with_icon("icon-2.png", "new-lasagna-icon".as_bytes().into())
1624 .with_icon("icon-3.png", "new-pollos-icon".as_bytes().into());
1626 store.ingest(SuggestIngestionConstraints::default())?;
1628 store.dbs()?.reader.read(|dao| {
1630 dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
1636 title: "Lasagna Come Out Tomorrow",
1637 url: "https://www.lasagna.restaurant",
1638 raw_url: "https://www.lasagna.restaurant",
1659 icon_mimetype: Some(
1662 full_keyword: "lasagna",
1664 advertiser: "Good Place Eats",
1665 iab_category: "8 - Food & Drink",
1666 impression_url: "https://example.com/impression_url",
1667 click_url: "https://example.com/click_url",
1668 raw_click_url: "https://example.com/click_url",
1673 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1674 keyword: "la".into(),
1675 providers: vec![SuggestionProvider::Amp],
1681 title: "Los Pollos Hermanos - Albuquerque",
1682 url: "https://www.lph-nm.biz",
1683 raw_url: "https://www.lph-nm.biz",
1703 icon_mimetype: Some(
1706 full_keyword: "los",
1708 advertiser: "Los Pollos Hermanos",
1709 iab_category: "8 - Food & Drink",
1710 impression_url: "https://example.com/impression_url",
1711 click_url: "https://example.com/click_url",
1712 raw_click_url: "https://example.com/click_url",
1717 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1718 keyword: "lo".into(),
1719 providers: vec![SuggestionProvider::Amp],
1728 /// Tests re-ingesting AMO suggestions from an updated attachment.
1730 fn reingest_amo_suggestions() -> anyhow::Result<()> {
1733 // Ingest suggestions from the initial snapshot.
1734 let initial_snapshot = Snapshot::with_records(json!([{
1736 "type": "amo-suggestions",
1737 "last_modified": 15,
1739 "filename": "data-1.json",
1740 "mimetype": "application/json",
1741 "location": "data-1.json",
1747 "type": "amo-suggestions",
1748 "last_modified": 15,
1750 "filename": "data-2.json",
1751 "mimetype": "application/json",
1752 "location": "data-2.json",
1760 "description": "First suggestion",
1761 "url": "https://example.org/amo-suggestion-1",
1762 "guid": "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
1763 "keywords": ["relay", "spam", "masking email", "alias"],
1764 "title": "AMO suggestion",
1765 "icon": "https://example.org/amo-suggestion-1/icon.png",
1767 "number_of_ratings": 800,
1774 "description": "Second suggestion",
1775 "url": "https://example.org/amo-suggestion-2",
1776 "guid": "{6d24e3b8-1400-4d37-9440-c798f9b79b1a}",
1777 "keywords": ["dark mode", "dark theme", "night mode"],
1778 "title": "Another AMO suggestion",
1779 "icon": "https://example.org/amo-suggestion-2/icon.png",
1781 "number_of_ratings": 750,
1784 "description": "Third suggestion",
1785 "url": "https://example.org/amo-suggestion-3",
1786 "guid": "{1e9d493b-0498-48bb-9b9a-8b45a44df146}",
1787 "keywords": ["grammar", "spelling", "edit"],
1788 "title": "Yet another AMO suggestion",
1789 "icon": "https://example.org/amo-suggestion-3/icon.png",
1791 "number_of_ratings": 900,
1796 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(initial_snapshot));
1798 store.ingest(SuggestIngestionConstraints::default())?;
1800 store.dbs()?.reader.read(|dao| {
1802 dao.get_meta(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?,
1809 title: "AMO suggestion",
1810 url: "https://example.org/amo-suggestion-1",
1811 icon_url: "https://example.org/amo-suggestion-1/icon.png",
1812 description: "First suggestion",
1816 number_of_ratings: 800,
1817 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
1822 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1823 keyword: "masking e".into(),
1824 providers: vec![SuggestionProvider::Amo],
1831 title: "Another AMO suggestion",
1832 url: "https://example.org/amo-suggestion-2",
1833 icon_url: "https://example.org/amo-suggestion-2/icon.png",
1834 description: "Second suggestion",
1838 number_of_ratings: 750,
1839 guid: "{6d24e3b8-1400-4d37-9440-c798f9b79b1a}",
1844 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1845 keyword: "night".into(),
1846 providers: vec![SuggestionProvider::Amo],
1853 // Update the snapshot with new suggestions: update the second, drop the
1854 // third, and add the fourth.
1855 *store.settings_client.snapshot.borrow_mut() = Snapshot::with_records(json!([{
1857 "type": "amo-suggestions",
1858 "last_modified": 30,
1860 "filename": "data-2-1.json",
1861 "mimetype": "application/json",
1862 "location": "data-2-1.json",
1870 "description": "Updated second suggestion",
1871 "url": "https://example.org/amo-suggestion-2",
1872 "guid": "{6d24e3b8-1400-4d37-9440-c798f9b79b1a}",
1873 "keywords": ["dark mode", "night mode"],
1874 "title": "Another AMO suggestion",
1875 "icon": "https://example.org/amo-suggestion-2/icon.png",
1877 "number_of_ratings": 775,
1880 "description": "Fourth suggestion",
1881 "url": "https://example.org/amo-suggestion-4",
1882 "guid": "{1ea82ebd-a1ba-4f57-b8bb-3824ead837bd}",
1883 "keywords": ["image search", "visual search"],
1884 "title": "New AMO suggestion",
1885 "icon": "https://example.org/amo-suggestion-4/icon.png",
1887 "number_of_ratings": 100,
1892 store.ingest(SuggestIngestionConstraints::default())?;
1894 store.dbs()?.reader.read(|dao| {
1896 dao.get_meta(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?,
1903 title: "AMO suggestion",
1904 url: "https://example.org/amo-suggestion-1",
1905 icon_url: "https://example.org/amo-suggestion-1/icon.png",
1906 description: "First suggestion",
1910 number_of_ratings: 800,
1911 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
1916 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1917 keyword: "masking e".into(),
1918 providers: vec![SuggestionProvider::Amo],
1925 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1926 keyword: "dark t".into(),
1927 providers: vec![SuggestionProvider::Amo],
1934 title: "Another AMO suggestion",
1935 url: "https://example.org/amo-suggestion-2",
1936 icon_url: "https://example.org/amo-suggestion-2/icon.png",
1937 description: "Updated second suggestion",
1941 number_of_ratings: 775,
1942 guid: "{6d24e3b8-1400-4d37-9440-c798f9b79b1a}",
1947 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1948 keyword: "night".into(),
1949 providers: vec![SuggestionProvider::Amo],
1956 title: "New AMO suggestion",
1957 url: "https://example.org/amo-suggestion-4",
1958 icon_url: "https://example.org/amo-suggestion-4/icon.png",
1959 description: "Fourth suggestion",
1963 number_of_ratings: 100,
1964 guid: "{1ea82ebd-a1ba-4f57-b8bb-3824ead837bd}",
1969 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1970 keyword: "image search".into(),
1971 providers: vec![SuggestionProvider::Amo],
1981 /// Tests ingesting tombstones for previously-ingested suggestions and
1984 fn ingest_tombstones() -> anyhow::Result<()> {
1987 // Ingest suggestions and icons from the initial snapshot.
1988 let initial_snapshot = Snapshot::with_records(json!([{
1991 "last_modified": 15,
1993 "filename": "data-1.json",
1994 "mimetype": "application/json",
1995 "location": "data-1.json",
2002 "last_modified": 20,
2004 "filename": "icon-2.png",
2005 "mimetype": "image/png",
2006 "location": "icon-2.png",
2015 "advertiser": "Good Place Eats",
2016 "iab_category": "8 - Food & Drink",
2017 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
2018 "title": "Lasagna Come Out Tomorrow",
2019 "url": "https://www.lasagna.restaurant",
2021 "impression_url": "https://example.com/impression_url",
2022 "click_url": "https://example.com/click_url",
2026 .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into());
2028 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(initial_snapshot));
2030 store.ingest(SuggestIngestionConstraints::default())?;
2032 store.dbs()?.reader.read(|dao| {
2035 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
2038 assert_eq!(dao.conn.query_one::<i64>("SELECT count(*) FROM icons")?, 1);
2041 SuggestRecordType::AmpWikipedia
2042 .last_ingest_meta_key()
2048 dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
2055 // Replace the records with tombstones. Ingesting these should remove
2056 // all their suggestions and icons.
2057 *store.settings_client.snapshot.borrow_mut() = Snapshot::with_records(json!([{
2059 "last_modified": 25,
2063 "last_modified": 30,
2067 store.ingest(SuggestIngestionConstraints::default())?;
2069 store.dbs()?.reader.read(|dao| {
2072 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
2075 assert_eq!(dao.conn.query_one::<i64>("SELECT count(*) FROM icons")?, 0);
2077 dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
2086 /// Tests ingesting suggestions with constraints.
2088 fn ingest_with_constraints() -> anyhow::Result<()> {
2091 let snapshot = Snapshot::with_records(json!([]))?;
2093 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
2095 store.ingest(SuggestIngestionConstraints::default())?;
2097 store.settings_client.last_get_records_option("_limit"),
2101 // 200 suggestions per record, so test with numbers around that
2112 for (max_suggestions, expected_limit) in table {
2113 store.ingest(SuggestIngestionConstraints {
2114 max_suggestions: Some(max_suggestions),
2115 providers: Some(vec![SuggestionProvider::Amp]),
2117 let actual_limit = store
2119 .last_get_records_option("_limit")
2121 anyhow!("Want limit = {} for {}", expected_limit, max_suggestions)
2124 actual_limit, expected_limit,
2125 "Want limit = {} for {}; got limit = {}",
2126 expected_limit, max_suggestions, actual_limit
2133 /// Tests clearing the store.
2135 fn clear() -> anyhow::Result<()> {
2138 let snapshot = Snapshot::with_records(json!([{
2141 "last_modified": 15,
2143 "filename": "data-1.json",
2144 "mimetype": "application/json",
2145 "location": "data-1.json",
2154 "advertiser": "Los Pollos Hermanos",
2155 "iab_category": "8 - Food & Drink",
2156 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
2157 "title": "Los Pollos Hermanos - Albuquerque",
2158 "url": "https://www.lph-nm.biz",
2160 "impression_url": "https://example.com",
2161 "click_url": "https://example.com",
2166 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
2168 store.ingest(SuggestIngestionConstraints::default())?;
2170 store.dbs()?.reader.read(|dao| {
2172 dao.get_meta::<u64>(
2173 SuggestRecordType::AmpWikipedia
2174 .last_ingest_meta_key()
2181 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
2185 dao.conn.query_one::<i64>("SELECT count(*) FROM keywords")?,
2194 store.dbs()?.reader.read(|dao| {
2196 dao.get_meta::<u64>(
2197 SuggestRecordType::AmpWikipedia
2198 .last_ingest_meta_key()
2205 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
2209 dao.conn.query_one::<i64>("SELECT count(*) FROM keywords")?,
2219 /// Tests querying suggestions.
2221 fn query() -> anyhow::Result<()> {
2224 let snapshot = Snapshot::with_records(json!([{
2227 "last_modified": 15,
2229 "filename": "data-1.json",
2230 "mimetype": "application/json",
2231 "location": "data-1.json",
2238 "type": "amo-suggestions",
2239 "last_modified": 15,
2241 "filename": "data-2.json",
2242 "mimetype": "application/json",
2243 "location": "data-2.json",
2249 "type": "pocket-suggestions",
2250 "last_modified": 15,
2252 "filename": "data-3.json",
2253 "mimetype": "application/json",
2254 "location": "data-3.json",
2260 "type": "yelp-suggestions",
2261 "last_modified": 15,
2263 "filename": "data-4.json",
2264 "mimetype": "application/json",
2265 "location": "data-4.json",
2271 "type": "mdn-suggestions",
2272 "last_modified": 15,
2274 "filename": "data-5.json",
2275 "mimetype": "application/json",
2276 "location": "data-5.json",
2283 "last_modified": 20,
2285 "filename": "icon-2.png",
2286 "mimetype": "image/png",
2287 "location": "icon-2.png",
2294 "last_modified": 25,
2296 "filename": "icon-3.png",
2297 "mimetype": "image/png",
2298 "location": "icon-3.png",
2303 "id": "icon-yelp-favicon",
2305 "last_modified": 25,
2307 "filename": "yelp-favicon.svg",
2308 "mimetype": "image/svg+xml",
2309 "location": "yelp-favicon.svg",
2318 "advertiser": "Good Place Eats",
2319 "iab_category": "8 - Food & Drink",
2320 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
2321 "title": "Lasagna Come Out Tomorrow",
2322 "url": "https://www.lasagna.restaurant",
2324 "impression_url": "https://example.com/impression_url",
2325 "click_url": "https://example.com/click_url",
2329 "advertiser": "Wikipedia",
2330 "iab_category": "5 - Education",
2331 "keywords": ["cal", "cali", "california"],
2332 "title": "California",
2333 "url": "https://wikipedia.org/California",
2337 "advertiser": "Wikipedia",
2338 "iab_category": "5 - Education",
2339 "keywords": ["cal", "cali", "california", "institute", "technology"],
2340 "title": "California Institute of Technology",
2341 "url": "https://wikipedia.org/California_Institute_of_Technology",
2345 "advertiser": "Wikipedia",
2346 "iab_category": "5 - Education",
2347 "keywords": ["multimatch"],
2348 "title": "Multimatch",
2349 "url": "https://wikipedia.org/Multimatch",
2357 "description": "amo suggestion",
2358 "url": "https://addons.mozilla.org/en-US/firefox/addon/example",
2359 "guid": "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2360 "keywords": ["relay", "spam", "masking email", "alias"],
2361 "title": "Firefox Relay",
2362 "icon": "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2364 "number_of_ratings": 888,
2368 "description": "amo suggestion multi-match",
2369 "url": "https://addons.mozilla.org/en-US/firefox/addon/multimatch",
2370 "guid": "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2371 "keywords": ["multimatch"],
2372 "title": "Firefox Multimatch",
2373 "icon": "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2375 "number_of_ratings": 888,
2384 "description": "pocket suggestion",
2385 "url": "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
2386 "lowConfidenceKeywords": ["soft life", "workaholism", "toxic work culture", "work-life balance"],
2387 "highConfidenceKeywords": ["burnout women", "grind culture", "women burnout"],
2388 "title": "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
2392 "description": "pocket suggestion multi-match",
2393 "url": "https://getpocket.com/collections/multimatch",
2394 "lowConfidenceKeywords": [],
2395 "highConfidenceKeywords": ["multimatch"],
2396 "title": "Multimatching",
2404 "subjects": ["ramen", "spicy ramen", "spicy random ramen", "rats", "raven", "raccoon", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789Z"],
2405 "preModifiers": ["best", "super best", "same_modifier"],
2406 "postModifiers": ["delivery", "super delivery", "same_modifier"],
2408 { "keyword": "in", "needLocation": true },
2409 { "keyword": "near", "needLocation": true },
2410 { "keyword": "near by", "needLocation": false },
2411 { "keyword": "near me", "needLocation": false },
2413 "yelpModifiers": ["yelp", "yelp keyword"],
2414 "icon": "yelp-favicon",
2422 "description": "Javascript Array",
2423 "url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
2424 "keywords": ["array javascript", "javascript array", "wildcard"],
2430 .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into())
2431 .with_icon("icon-3.png", "also-an-icon".as_bytes().into())
2432 .with_icon("yelp-favicon.svg", "yelp-icon".as_bytes().into());
2434 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
2436 store.ingest(SuggestIngestionConstraints::default())?;
2440 "empty keyword; all providers",
2442 keyword: String::new(),
2444 SuggestionProvider::Amp,
2445 SuggestionProvider::Wikipedia,
2446 SuggestionProvider::Amo,
2447 SuggestionProvider::Pocket,
2448 SuggestionProvider::Yelp,
2449 SuggestionProvider::Weather,
2458 "keyword = `la`; all providers",
2460 keyword: "la".into(),
2462 SuggestionProvider::Amp,
2463 SuggestionProvider::Wikipedia,
2464 SuggestionProvider::Amo,
2465 SuggestionProvider::Pocket,
2466 SuggestionProvider::Yelp,
2467 SuggestionProvider::Weather,
2474 title: "Lasagna Come Out Tomorrow",
2475 url: "https://www.lasagna.restaurant",
2476 raw_url: "https://www.lasagna.restaurant",
2493 icon_mimetype: Some(
2496 full_keyword: "lasagna",
2498 advertiser: "Good Place Eats",
2499 iab_category: "8 - Food & Drink",
2500 impression_url: "https://example.com/impression_url",
2501 click_url: "https://example.com/click_url",
2502 raw_click_url: "https://example.com/click_url",
2509 "multimatch; all providers",
2511 keyword: "multimatch".into(),
2513 SuggestionProvider::Amp,
2514 SuggestionProvider::Wikipedia,
2515 SuggestionProvider::Amo,
2516 SuggestionProvider::Pocket,
2523 title: "Multimatching",
2524 url: "https://getpocket.com/collections/multimatch",
2529 title: "Firefox Multimatch",
2530 url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch",
2531 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2532 description: "amo suggestion multi-match",
2536 number_of_ratings: 888,
2537 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2541 title: "Multimatch",
2542 url: "https://wikipedia.org/Multimatch",
2559 icon_mimetype: Some(
2562 full_keyword: "multimatch",
2568 "MultiMatch; all providers, mixed case",
2570 keyword: "MultiMatch".into(),
2572 SuggestionProvider::Amp,
2573 SuggestionProvider::Wikipedia,
2574 SuggestionProvider::Amo,
2575 SuggestionProvider::Pocket,
2582 title: "Multimatching",
2583 url: "https://getpocket.com/collections/multimatch",
2588 title: "Firefox Multimatch",
2589 url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch",
2590 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2591 description: "amo suggestion multi-match",
2595 number_of_ratings: 888,
2596 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2600 title: "Multimatch",
2601 url: "https://wikipedia.org/Multimatch",
2618 icon_mimetype: Some(
2621 full_keyword: "multimatch",
2627 "multimatch; all providers, limit 2",
2629 keyword: "multimatch".into(),
2631 SuggestionProvider::Amp,
2632 SuggestionProvider::Wikipedia,
2633 SuggestionProvider::Amo,
2634 SuggestionProvider::Pocket,
2641 title: "Multimatching",
2642 url: "https://getpocket.com/collections/multimatch",
2647 title: "Firefox Multimatch",
2648 url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch",
2649 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2650 description: "amo suggestion multi-match",
2654 number_of_ratings: 888,
2655 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2662 "keyword = `la`; AMP only",
2664 keyword: "la".into(),
2665 providers: vec![SuggestionProvider::Amp],
2671 title: "Lasagna Come Out Tomorrow",
2672 url: "https://www.lasagna.restaurant",
2673 raw_url: "https://www.lasagna.restaurant",
2690 icon_mimetype: Some(
2693 full_keyword: "lasagna",
2695 advertiser: "Good Place Eats",
2696 iab_category: "8 - Food & Drink",
2697 impression_url: "https://example.com/impression_url",
2698 click_url: "https://example.com/click_url",
2699 raw_click_url: "https://example.com/click_url",
2706 "keyword = `la`; Wikipedia, AMO, and Pocket",
2708 keyword: "la".into(),
2710 SuggestionProvider::Wikipedia,
2711 SuggestionProvider::Amo,
2712 SuggestionProvider::Pocket,
2721 "keyword = `la`; no providers",
2723 keyword: "la".into(),
2732 "keyword = `cal`; AMP, AMO, and Pocket",
2734 keyword: "cal".into(),
2736 SuggestionProvider::Amp,
2737 SuggestionProvider::Amo,
2738 SuggestionProvider::Pocket,
2747 "keyword = `cal`; Wikipedia only",
2749 keyword: "cal".into(),
2750 providers: vec![SuggestionProvider::Wikipedia],
2756 title: "California",
2757 url: "https://wikipedia.org/California",
2774 icon_mimetype: Some(
2777 full_keyword: "california",
2780 title: "California Institute of Technology",
2781 url: "https://wikipedia.org/California_Institute_of_Technology",
2798 icon_mimetype: Some(
2801 full_keyword: "california",
2807 "keyword = `cal`; Wikipedia with limit 1",
2809 keyword: "cal".into(),
2810 providers: vec![SuggestionProvider::Wikipedia],
2816 title: "California",
2817 url: "https://wikipedia.org/California",
2834 icon_mimetype: Some(
2837 full_keyword: "california",
2843 "keyword = `cal`; no providers",
2845 keyword: "cal".into(),
2854 "keyword = `spam`; AMO only",
2856 keyword: "spam".into(),
2857 providers: vec![SuggestionProvider::Amo],
2863 title: "Firefox Relay",
2864 url: "https://addons.mozilla.org/en-US/firefox/addon/example",
2865 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2866 description: "amo suggestion",
2870 number_of_ratings: 888,
2871 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2878 "keyword = `masking`; AMO only",
2880 keyword: "masking".into(),
2881 providers: vec![SuggestionProvider::Amo],
2887 title: "Firefox Relay",
2888 url: "https://addons.mozilla.org/en-US/firefox/addon/example",
2889 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2890 description: "amo suggestion",
2894 number_of_ratings: 888,
2895 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2902 "keyword = `masking e`; AMO only",
2904 keyword: "masking e".into(),
2905 providers: vec![SuggestionProvider::Amo],
2911 title: "Firefox Relay",
2912 url: "https://addons.mozilla.org/en-US/firefox/addon/example",
2913 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2914 description: "amo suggestion",
2918 number_of_ratings: 888,
2919 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2926 "keyword = `masking s`; AMO only",
2928 keyword: "masking s".into(),
2929 providers: vec![SuggestionProvider::Amo],
2937 "keyword = `soft`; AMP and Wikipedia",
2939 keyword: "soft".into(),
2940 providers: vec![SuggestionProvider::Amp, SuggestionProvider::Wikipedia],
2948 "keyword = `soft`; Pocket only",
2950 keyword: "soft".into(),
2951 providers: vec![SuggestionProvider::Pocket],
2957 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
2958 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
2966 "keyword = `soft l`; Pocket only",
2968 keyword: "soft l".into(),
2969 providers: vec![SuggestionProvider::Pocket],
2975 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
2976 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
2984 "keyword = `sof`; Pocket only",
2986 keyword: "sof".into(),
2987 providers: vec![SuggestionProvider::Pocket],
2995 "keyword = `burnout women`; Pocket only",
2997 keyword: "burnout women".into(),
2998 providers: vec![SuggestionProvider::Pocket],
3004 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
3005 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
3013 "keyword = `burnout person`; Pocket only",
3015 keyword: "burnout person".into(),
3016 providers: vec![SuggestionProvider::Pocket],
3024 "keyword = `best spicy ramen delivery in tokyo`; Yelp only",
3026 keyword: "best spicy ramen delivery in tokyo".into(),
3027 providers: vec![SuggestionProvider::Yelp],
3033 url: "https://www.yelp.com/search?find_desc=best+spicy+ramen+delivery&find_loc=tokyo",
3034 title: "best spicy ramen delivery in tokyo",
3048 icon_mimetype: Some(
3052 has_location_sign: true,
3053 subject_exact_match: true,
3054 location_param: "find_loc",
3060 "keyword = `BeSt SpIcY rAmEn DeLiVeRy In ToKyO`; Yelp only",
3062 keyword: "BeSt SpIcY rAmEn DeLiVeRy In ToKyO".into(),
3063 providers: vec![SuggestionProvider::Yelp],
3069 url: "https://www.yelp.com/search?find_desc=BeSt+SpIcY+rAmEn+DeLiVeRy&find_loc=ToKyO",
3070 title: "BeSt SpIcY rAmEn DeLiVeRy In ToKyO",
3084 icon_mimetype: Some(
3088 has_location_sign: true,
3089 subject_exact_match: true,
3090 location_param: "find_loc",
3096 "keyword = `best ramen delivery in tokyo`; Yelp only",
3098 keyword: "best ramen delivery in tokyo".into(),
3099 providers: vec![SuggestionProvider::Yelp],
3105 url: "https://www.yelp.com/search?find_desc=best+ramen+delivery&find_loc=tokyo",
3106 title: "best ramen delivery in tokyo",
3120 icon_mimetype: Some(
3124 has_location_sign: true,
3125 subject_exact_match: true,
3126 location_param: "find_loc",
3132 "keyword = `best invalid_ramen delivery in tokyo`; Yelp only",
3134 keyword: "best invalid_ramen delivery in tokyo".into(),
3135 providers: vec![SuggestionProvider::Yelp],
3143 "keyword = `best delivery in tokyo`; Yelp only",
3145 keyword: "best delivery in tokyo".into(),
3146 providers: vec![SuggestionProvider::Yelp],
3154 "keyword = `super best ramen delivery in tokyo`; Yelp only",
3156 keyword: "super best ramen delivery in tokyo".into(),
3157 providers: vec![SuggestionProvider::Yelp],
3163 url: "https://www.yelp.com/search?find_desc=super+best+ramen+delivery&find_loc=tokyo",
3164 title: "super best ramen delivery in tokyo",
3178 icon_mimetype: Some(
3182 has_location_sign: true,
3183 subject_exact_match: true,
3184 location_param: "find_loc",
3190 "keyword = `invalid_best ramen delivery in tokyo`; Yelp only",
3192 keyword: "invalid_best ramen delivery in tokyo".into(),
3193 providers: vec![SuggestionProvider::Yelp],
3201 "keyword = `ramen delivery in tokyo`; Yelp only",
3203 keyword: "ramen delivery in tokyo".into(),
3204 providers: vec![SuggestionProvider::Yelp],
3210 url: "https://www.yelp.com/search?find_desc=ramen+delivery&find_loc=tokyo",
3211 title: "ramen delivery in tokyo",
3225 icon_mimetype: Some(
3229 has_location_sign: true,
3230 subject_exact_match: true,
3231 location_param: "find_loc",
3237 "keyword = `ramen super delivery in tokyo`; Yelp only",
3239 keyword: "ramen super delivery in tokyo".into(),
3240 providers: vec![SuggestionProvider::Yelp],
3246 url: "https://www.yelp.com/search?find_desc=ramen+super+delivery&find_loc=tokyo",
3247 title: "ramen super delivery in tokyo",
3261 icon_mimetype: Some(
3265 has_location_sign: true,
3266 subject_exact_match: true,
3267 location_param: "find_loc",
3273 "keyword = `ramen invalid_delivery in tokyo`; Yelp only",
3275 keyword: "ramen invalid_delivery in tokyo".into(),
3276 providers: vec![SuggestionProvider::Yelp],
3284 "keyword = `ramen in tokyo`; Yelp only",
3286 keyword: "ramen in tokyo".into(),
3287 providers: vec![SuggestionProvider::Yelp],
3293 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo",
3294 title: "ramen in tokyo",
3308 icon_mimetype: Some(
3312 has_location_sign: true,
3313 subject_exact_match: true,
3314 location_param: "find_loc",
3320 "keyword = `ramen near tokyo`; Yelp only",
3322 keyword: "ramen near tokyo".into(),
3323 providers: vec![SuggestionProvider::Yelp],
3329 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo",
3330 title: "ramen near tokyo",
3344 icon_mimetype: Some(
3348 has_location_sign: true,
3349 subject_exact_match: true,
3350 location_param: "find_loc",
3356 "keyword = `ramen invalid_in tokyo`; Yelp only",
3358 keyword: "ramen invalid_in tokyo".into(),
3359 providers: vec![SuggestionProvider::Yelp],
3367 "keyword = `ramen in San Francisco`; Yelp only",
3369 keyword: "ramen in San Francisco".into(),
3370 providers: vec![SuggestionProvider::Yelp],
3376 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=San+Francisco",
3377 title: "ramen in San Francisco",
3391 icon_mimetype: Some(
3395 has_location_sign: true,
3396 subject_exact_match: true,
3397 location_param: "find_loc",
3403 "keyword = `ramen in`; Yelp only",
3405 keyword: "ramen in".into(),
3406 providers: vec![SuggestionProvider::Yelp],
3412 url: "https://www.yelp.com/search?find_desc=ramen",
3427 icon_mimetype: Some(
3431 has_location_sign: true,
3432 subject_exact_match: true,
3433 location_param: "find_loc",
3439 "keyword = `ramen near by`; Yelp only",
3441 keyword: "ramen near by".into(),
3442 providers: vec![SuggestionProvider::Yelp],
3448 url: "https://www.yelp.com/search?find_desc=ramen+near+by",
3449 title: "ramen near by",
3463 icon_mimetype: Some(
3467 has_location_sign: false,
3468 subject_exact_match: true,
3469 location_param: "find_loc",
3475 "keyword = `ramen near me`; Yelp only",
3477 keyword: "ramen near me".into(),
3478 providers: vec![SuggestionProvider::Yelp],
3484 url: "https://www.yelp.com/search?find_desc=ramen+near+me",
3485 title: "ramen near me",
3499 icon_mimetype: Some(
3503 has_location_sign: false,
3504 subject_exact_match: true,
3505 location_param: "find_loc",
3511 "keyword = `ramen near by tokyo`; Yelp only",
3513 keyword: "ramen near by tokyo".into(),
3514 providers: vec![SuggestionProvider::Yelp],
3522 "keyword = `ramen`; Yelp only",
3524 keyword: "ramen".into(),
3525 providers: vec![SuggestionProvider::Yelp],
3531 url: "https://www.yelp.com/search?find_desc=ramen",
3546 icon_mimetype: Some(
3550 has_location_sign: false,
3551 subject_exact_match: true,
3552 location_param: "find_loc",
3558 "keyword = maximum chars; Yelp only",
3560 keyword: "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789".into(),
3561 providers: vec![SuggestionProvider::Yelp],
3567 url: "https://www.yelp.com/search?find_desc=012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789",
3568 title: "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789",
3582 icon_mimetype: Some(
3586 has_location_sign: false,
3587 subject_exact_match: true,
3588 location_param: "find_loc",
3594 "keyword = over chars; Yelp only",
3596 keyword: "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789Z".into(),
3597 providers: vec![SuggestionProvider::Yelp],
3605 "keyword = `best delivery`; Yelp only",
3607 keyword: "best delivery".into(),
3608 providers: vec![SuggestionProvider::Yelp],
3616 "keyword = `same_modifier same_modifier`; Yelp only",
3618 keyword: "same_modifier same_modifier".into(),
3619 providers: vec![SuggestionProvider::Yelp],
3627 "keyword = `same_modifier `; Yelp only",
3629 keyword: "same_modifier ".into(),
3630 providers: vec![SuggestionProvider::Yelp],
3638 "keyword = `yelp ramen`; Yelp only",
3640 keyword: "yelp ramen".into(),
3641 providers: vec![SuggestionProvider::Yelp],
3647 url: "https://www.yelp.com/search?find_desc=ramen",
3662 icon_mimetype: Some(
3666 has_location_sign: false,
3667 subject_exact_match: true,
3668 location_param: "find_loc",
3674 "keyword = `yelp keyword ramen`; Yelp only",
3676 keyword: "yelp keyword ramen".into(),
3677 providers: vec![SuggestionProvider::Yelp],
3683 url: "https://www.yelp.com/search?find_desc=ramen",
3698 icon_mimetype: Some(
3702 has_location_sign: false,
3703 subject_exact_match: true,
3704 location_param: "find_loc",
3710 "keyword = `ramen in tokyo yelp`; Yelp only",
3712 keyword: "ramen in tokyo yelp".into(),
3713 providers: vec![SuggestionProvider::Yelp],
3719 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo",
3720 title: "ramen in tokyo",
3734 icon_mimetype: Some(
3738 has_location_sign: true,
3739 subject_exact_match: true,
3740 location_param: "find_loc",
3746 "keyword = `ramen in tokyo yelp keyword`; Yelp only",
3748 keyword: "ramen in tokyo yelp keyword".into(),
3749 providers: vec![SuggestionProvider::Yelp],
3755 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo",
3756 title: "ramen in tokyo",
3770 icon_mimetype: Some(
3774 has_location_sign: true,
3775 subject_exact_match: true,
3776 location_param: "find_loc",
3782 "keyword = `yelp ramen yelp`; Yelp only",
3784 keyword: "yelp ramen yelp".into(),
3785 providers: vec![SuggestionProvider::Yelp],
3791 url: "https://www.yelp.com/search?find_desc=ramen",
3806 icon_mimetype: Some(
3810 has_location_sign: false,
3811 subject_exact_match: true,
3812 location_param: "find_loc",
3818 "keyword = `best yelp ramen`; Yelp only",
3820 keyword: "best yelp ramen".into(),
3821 providers: vec![SuggestionProvider::Yelp],
3829 "keyword = `Spicy R`; Yelp only",
3831 keyword: "Spicy R".into(),
3832 providers: vec![SuggestionProvider::Yelp],
3838 url: "https://www.yelp.com/search?find_desc=Spicy+Ramen",
3839 title: "Spicy Ramen",
3853 icon_mimetype: Some(
3857 has_location_sign: false,
3858 subject_exact_match: false,
3859 location_param: "find_loc",
3865 "keyword = `BeSt Ramen`; Yelp only",
3867 keyword: "BeSt Ramen".into(),
3868 providers: vec![SuggestionProvider::Yelp],
3874 url: "https://www.yelp.com/search?find_desc=BeSt+Ramen",
3875 title: "BeSt Ramen",
3889 icon_mimetype: Some(
3893 has_location_sign: false,
3894 subject_exact_match: true,
3895 location_param: "find_loc",
3901 "keyword = `BeSt Spicy R`; Yelp only",
3903 keyword: "BeSt Spicy R".into(),
3904 providers: vec![SuggestionProvider::Yelp],
3910 url: "https://www.yelp.com/search?find_desc=BeSt+Spicy+Ramen",
3911 title: "BeSt Spicy Ramen",
3925 icon_mimetype: Some(
3929 has_location_sign: false,
3930 subject_exact_match: false,
3931 location_param: "find_loc",
3937 "keyword = `BeSt R`; Yelp only",
3939 keyword: "BeSt R".into(),
3940 providers: vec![SuggestionProvider::Yelp],
3948 "keyword = `r`; Yelp only",
3950 keyword: "r".into(),
3951 providers: vec![SuggestionProvider::Yelp],
3959 "keyword = `ra`; Yelp only",
3961 keyword: "ra".into(),
3962 providers: vec![SuggestionProvider::Yelp],
3968 url: "https://www.yelp.com/search?find_desc=rats",
3983 icon_mimetype: Some(
3987 has_location_sign: false,
3988 subject_exact_match: false,
3989 location_param: "find_loc",
3995 "keyword = `ram`; Yelp only",
3997 keyword: "ram".into(),
3998 providers: vec![SuggestionProvider::Yelp],
4004 url: "https://www.yelp.com/search?find_desc=ramen",
4019 icon_mimetype: Some(
4023 has_location_sign: false,
4024 subject_exact_match: false,
4025 location_param: "find_loc",
4031 "keyword = `rac`; Yelp only",
4033 keyword: "rac".into(),
4034 providers: vec![SuggestionProvider::Yelp],
4040 url: "https://www.yelp.com/search?find_desc=raccoon",
4055 icon_mimetype: Some(
4059 has_location_sign: false,
4060 subject_exact_match: false,
4061 location_param: "find_loc",
4067 "keyword = `best r`; Yelp only",
4069 keyword: "best r".into(),
4070 providers: vec![SuggestionProvider::Yelp],
4078 "keyword = `best ra`; Yelp only",
4080 keyword: "best ra".into(),
4081 providers: vec![SuggestionProvider::Yelp],
4087 url: "https://www.yelp.com/search?find_desc=best+rats",
4102 icon_mimetype: Some(
4106 has_location_sign: false,
4107 subject_exact_match: false,
4108 location_param: "find_loc",
4114 for (what, query, expect) in table {
4115 expect.assert_debug_eq(
4118 .with_context(|| format!("Couldn't query store for {}", what))?,
4125 // Tests querying amp wikipedia
4127 fn query_with_multiple_providers_and_diff_scores() -> anyhow::Result<()> {
4130 let snapshot = Snapshot::with_records(json!([{
4133 "last_modified": 15,
4135 "filename": "data-1.json",
4136 "mimetype": "application/json",
4137 "location": "data-1.json",
4143 "type": "pocket-suggestions",
4144 "last_modified": 15,
4146 "filename": "data-2.json",
4147 "mimetype": "application/json",
4148 "location": "data-2.json",
4155 "last_modified": 25,
4157 "filename": "icon-3.png",
4158 "mimetype": "image/png",
4159 "location": "icon-3.png",
4168 "advertiser": "Good Place Eats",
4169 "iab_category": "8 - Food & Drink",
4170 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow", "amp wiki match"],
4171 "title": "Lasagna Come Out Tomorrow",
4172 "url": "https://www.lasagna.restaurant",
4174 "impression_url": "https://example.com/impression_url",
4175 "click_url": "https://example.com/click_url",
4179 "advertiser": "Good Place Eats",
4180 "iab_category": "8 - Food & Drink",
4181 "keywords": ["pe", "pen", "penne", "penne for your thoughts", "amp wiki match"],
4182 "title": "Penne for Your Thoughts",
4183 "url": "https://penne.biz",
4185 "impression_url": "https://example.com/impression_url",
4186 "click_url": "https://example.com/click_url",
4190 "advertiser": "Wikipedia",
4191 "iab_category": "5 - Education",
4192 "keywords": ["amp wiki match", "pocket wiki match"],
4193 "title": "Multimatch",
4194 "url": "https://wikipedia.org/Multimatch",
4202 "description": "pocket suggestion",
4203 "url": "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4204 "lowConfidenceKeywords": ["soft life", "workaholism", "toxic work culture", "work-life balance", "pocket wiki match"],
4205 "highConfidenceKeywords": ["burnout women", "grind culture", "women burnout"],
4206 "title": "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4210 "description": "pocket suggestion multi-match",
4211 "url": "https://getpocket.com/collections/multimatch",
4212 "lowConfidenceKeywords": [],
4213 "highConfidenceKeywords": ["pocket wiki match"],
4214 "title": "Pocket wiki match",
4219 .with_icon("icon-3.png", "also-an-icon".as_bytes().into());
4221 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4223 store.ingest(SuggestIngestionConstraints::default())?;
4227 "keyword = `amp wiki match`; all providers",
4229 keyword: "amp wiki match".into(),
4231 SuggestionProvider::Amp,
4232 SuggestionProvider::Wikipedia,
4233 SuggestionProvider::Amo,
4234 SuggestionProvider::Pocket,
4235 SuggestionProvider::Yelp,
4242 title: "Lasagna Come Out Tomorrow",
4243 url: "https://www.lasagna.restaurant",
4244 raw_url: "https://www.lasagna.restaurant",
4246 icon_mimetype: None,
4247 full_keyword: "amp wiki match",
4249 advertiser: "Good Place Eats",
4250 iab_category: "8 - Food & Drink",
4251 impression_url: "https://example.com/impression_url",
4252 click_url: "https://example.com/click_url",
4253 raw_click_url: "https://example.com/click_url",
4257 title: "Multimatch",
4258 url: "https://wikipedia.org/Multimatch",
4275 icon_mimetype: Some(
4278 full_keyword: "amp wiki match",
4281 title: "Penne for Your Thoughts",
4282 url: "https://penne.biz",
4283 raw_url: "https://penne.biz",
4285 icon_mimetype: None,
4286 full_keyword: "amp wiki match",
4288 advertiser: "Good Place Eats",
4289 iab_category: "8 - Food & Drink",
4290 impression_url: "https://example.com/impression_url",
4291 click_url: "https://example.com/click_url",
4292 raw_click_url: "https://example.com/click_url",
4299 "keyword = `amp wiki match`; all providers, limit 2",
4301 keyword: "amp wiki match".into(),
4303 SuggestionProvider::Amp,
4304 SuggestionProvider::Wikipedia,
4305 SuggestionProvider::Amo,
4306 SuggestionProvider::Pocket,
4307 SuggestionProvider::Yelp,
4314 title: "Lasagna Come Out Tomorrow",
4315 url: "https://www.lasagna.restaurant",
4316 raw_url: "https://www.lasagna.restaurant",
4318 icon_mimetype: None,
4319 full_keyword: "amp wiki match",
4321 advertiser: "Good Place Eats",
4322 iab_category: "8 - Food & Drink",
4323 impression_url: "https://example.com/impression_url",
4324 click_url: "https://example.com/click_url",
4325 raw_click_url: "https://example.com/click_url",
4329 title: "Multimatch",
4330 url: "https://wikipedia.org/Multimatch",
4347 icon_mimetype: Some(
4350 full_keyword: "amp wiki match",
4356 "pocket wiki match; all providers",
4358 keyword: "pocket wiki match".into(),
4360 SuggestionProvider::Amp,
4361 SuggestionProvider::Wikipedia,
4362 SuggestionProvider::Amo,
4363 SuggestionProvider::Pocket,
4370 title: "Pocket wiki match",
4371 url: "https://getpocket.com/collections/multimatch",
4376 title: "Multimatch",
4377 url: "https://wikipedia.org/Multimatch",
4394 icon_mimetype: Some(
4397 full_keyword: "pocket wiki match",
4400 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4401 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4409 "pocket wiki match; all providers limit 1",
4411 keyword: "pocket wiki match".into(),
4413 SuggestionProvider::Amp,
4414 SuggestionProvider::Wikipedia,
4415 SuggestionProvider::Amo,
4416 SuggestionProvider::Pocket,
4423 title: "Pocket wiki match",
4424 url: "https://getpocket.com/collections/multimatch",
4432 "work-life balance; duplicate providers",
4434 keyword: "work-life balance".into(),
4435 providers: vec![SuggestionProvider::Pocket, SuggestionProvider::Pocket],
4441 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4442 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4450 for (what, query, expect) in table {
4451 expect.assert_debug_eq(
4454 .with_context(|| format!("Couldn't query store for {}", what))?,
4461 // Tests querying multiple suggestions with multiple keywords with same prefix keyword
4463 fn query_with_multiple_suggestions_with_same_prefix() -> anyhow::Result<()> {
4466 let snapshot = Snapshot::with_records(json!([{
4468 "type": "amo-suggestions",
4469 "last_modified": 15,
4471 "filename": "data-1.json",
4472 "mimetype": "application/json",
4473 "location": "data-1.json",
4479 "type": "pocket-suggestions",
4480 "last_modified": 15,
4482 "filename": "data-2.json",
4483 "mimetype": "application/json",
4484 "location": "data-2.json",
4491 "last_modified": 25,
4493 "filename": "icon-3.png",
4494 "mimetype": "image/png",
4495 "location": "icon-3.png",
4504 "description": "amo suggestion",
4505 "url": "https://addons.mozilla.org/en-US/firefox/addon/example",
4506 "guid": "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
4507 "keywords": ["relay", "spam", "masking email", "masking emails", "masking accounts", "alias" ],
4508 "title": "Firefox Relay",
4509 "icon": "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
4511 "number_of_ratings": 888,
4520 "description": "pocket suggestion",
4521 "url": "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4522 "lowConfidenceKeywords": ["soft life", "soft living", "soft work", "workaholism", "toxic work culture"],
4523 "highConfidenceKeywords": ["burnout women", "grind culture", "women burnout", "soft lives"],
4524 "title": "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4529 .with_icon("icon-3.png", "also-an-icon".as_bytes().into());
4531 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4533 store.ingest(SuggestIngestionConstraints::default())?;
4537 "keyword = `soft li`; pocket",
4539 keyword: "soft li".into(),
4540 providers: vec![SuggestionProvider::Pocket],
4546 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4547 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4555 "keyword = `soft lives`; pocket",
4557 keyword: "soft lives".into(),
4558 providers: vec![SuggestionProvider::Pocket],
4564 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4565 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4573 "keyword = `masking `; amo provider",
4575 keyword: "masking ".into(),
4576 providers: vec![SuggestionProvider::Amo],
4582 title: "Firefox Relay",
4583 url: "https://addons.mozilla.org/en-US/firefox/addon/example",
4584 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
4585 description: "amo suggestion",
4589 number_of_ratings: 888,
4590 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
4597 for (what, query, expect) in table {
4598 expect.assert_debug_eq(
4601 .with_context(|| format!("Couldn't query store for {}", what))?,
4608 // Tests querying multiple suggestions with multiple keywords with same prefix keyword
4610 fn query_with_amp_mobile_provider() -> anyhow::Result<()> {
4613 let snapshot = Snapshot::with_records(json!([{
4615 "type": "amp-mobile-suggestions",
4616 "last_modified": 15,
4618 "filename": "data-1.json",
4619 "mimetype": "application/json",
4620 "location": "data-1.json",
4627 "last_modified": 15,
4629 "filename": "data-2.json",
4630 "mimetype": "application/json",
4631 "location": "data-2.json",
4638 "last_modified": 25,
4640 "filename": "icon-3.png",
4641 "mimetype": "image/png",
4642 "location": "icon-3.png",
4652 "advertiser": "Good Place Eats",
4653 "iab_category": "8 - Food & Drink",
4654 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
4655 "title": "Mobile - Lasagna Come Out Tomorrow",
4656 "url": "https://www.lasagna.restaurant",
4658 "impression_url": "https://example.com/impression_url",
4659 "click_url": "https://example.com/click_url",
4669 "advertiser": "Good Place Eats",
4670 "iab_category": "8 - Food & Drink",
4671 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
4672 "title": "Desktop - Lasagna Come Out Tomorrow",
4673 "url": "https://www.lasagna.restaurant",
4675 "impression_url": "https://example.com/impression_url",
4676 "click_url": "https://example.com/click_url",
4681 .with_icon("icon-3.png", "also-an-icon".as_bytes().into());
4683 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4685 store.ingest(SuggestIngestionConstraints::default())?;
4689 "keyword = `las`; Amp Mobile",
4691 keyword: "las".into(),
4692 providers: vec![SuggestionProvider::AmpMobile],
4698 title: "Mobile - Lasagna Come Out Tomorrow",
4699 url: "https://www.lasagna.restaurant",
4700 raw_url: "https://www.lasagna.restaurant",
4717 icon_mimetype: Some(
4720 full_keyword: "lasagna",
4722 advertiser: "Good Place Eats",
4723 iab_category: "8 - Food & Drink",
4724 impression_url: "https://example.com/impression_url",
4725 click_url: "https://example.com/click_url",
4726 raw_click_url: "https://example.com/click_url",
4733 "keyword = `las`; Amp",
4735 keyword: "las".into(),
4736 providers: vec![SuggestionProvider::Amp],
4742 title: "Desktop - Lasagna Come Out Tomorrow",
4743 url: "https://www.lasagna.restaurant",
4744 raw_url: "https://www.lasagna.restaurant",
4761 icon_mimetype: Some(
4764 full_keyword: "lasagna",
4766 advertiser: "Good Place Eats",
4767 iab_category: "8 - Food & Drink",
4768 impression_url: "https://example.com/impression_url",
4769 click_url: "https://example.com/click_url",
4770 raw_click_url: "https://example.com/click_url",
4777 "keyword = `las `; amp and amp mobile",
4779 keyword: "las".into(),
4780 providers: vec![SuggestionProvider::Amp, SuggestionProvider::AmpMobile],
4786 title: "Mobile - Lasagna Come Out Tomorrow",
4787 url: "https://www.lasagna.restaurant",
4788 raw_url: "https://www.lasagna.restaurant",
4805 icon_mimetype: Some(
4808 full_keyword: "lasagna",
4810 advertiser: "Good Place Eats",
4811 iab_category: "8 - Food & Drink",
4812 impression_url: "https://example.com/impression_url",
4813 click_url: "https://example.com/click_url",
4814 raw_click_url: "https://example.com/click_url",
4818 title: "Desktop - Lasagna Come Out Tomorrow",
4819 url: "https://www.lasagna.restaurant",
4820 raw_url: "https://www.lasagna.restaurant",
4837 icon_mimetype: Some(
4840 full_keyword: "lasagna",
4842 advertiser: "Good Place Eats",
4843 iab_category: "8 - Food & Drink",
4844 impression_url: "https://example.com/impression_url",
4845 click_url: "https://example.com/click_url",
4846 raw_click_url: "https://example.com/click_url",
4853 for (what, query, expect) in table {
4854 expect.assert_debug_eq(
4857 .with_context(|| format!("Couldn't query store for {}", what))?,
4864 /// Tests ingesting malformed Remote Settings records that we understand,
4865 /// but that are missing fields, or aren't in the format we expect.
4867 fn ingest_malformed() -> anyhow::Result<()> {
4870 let snapshot = Snapshot::with_records(json!([{
4871 // Data record without an attachment.
4872 "id": "missing-data-attachment",
4874 "last_modified": 15,
4876 // Icon record without an attachment.
4877 "id": "missing-icon-attachment",
4879 "last_modified": 30,
4881 // Icon record with an ID that's not `icon-{id}`, so suggestions in
4882 // the data attachment won't be able to reference it.
4883 "id": "bad-icon-id",
4885 "last_modified": 45,
4887 "filename": "icon-1.png",
4888 "mimetype": "image/png",
4889 "location": "icon-1.png",
4894 .with_icon("icon-1.png", "i-am-an-icon".as_bytes().into());
4896 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4898 store.ingest(SuggestIngestionConstraints::default())?;
4900 store.dbs()?.reader.read(|dao| {
4902 dao.get_meta::<u64>(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
4907 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
4910 assert_eq!(dao.conn.query_one::<i64>("SELECT count(*) FROM icons")?, 0);
4918 /// Tests unparsable Remote Settings records, which we don't know how to
4921 fn ingest_unparsable() -> anyhow::Result<()> {
4924 let snapshot = Snapshot::with_records(json!([{
4925 "id": "fancy-new-suggestions-1",
4926 "type": "fancy-new-suggestions",
4927 "last_modified": 15,
4931 "last_modified": 30,
4934 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4936 store.ingest(SuggestIngestionConstraints::default())?;
4938 store.dbs()?.reader.read(|dao| {
4940 dao.get_meta("last_quicksuggest_ingest_unparsable")?,
4947 "clippy-2": UnparsableRecord {
4950 "fancy-new-suggestions-1": UnparsableRecord {
4957 .assert_debug_eq(&dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY)?);
4965 fn ingest_mixed_parsable_unparsable_records() -> anyhow::Result<()> {
4968 let snapshot = Snapshot::with_records(json!([{
4969 "id": "fancy-new-suggestions-1",
4970 "type": "fancy-new-suggestions",
4971 "last_modified": 15,
4976 "last_modified": 15,
4978 "filename": "data-1.json",
4979 "mimetype": "application/json",
4980 "location": "data-1.json",
4988 "last_modified": 30,
4994 "advertiser": "Los Pollos Hermanos",
4995 "iab_category": "8 - Food & Drink",
4996 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
4997 "title": "Los Pollos Hermanos - Albuquerque",
4998 "url": "https://www.lph-nm.biz",
5000 "impression_url": "https://example.com/impression_url",
5001 "click_url": "https://example.com/click_url",
5006 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5008 store.ingest(SuggestIngestionConstraints::default())?;
5010 store.dbs()?.reader.read(|dao| {
5012 dao.get_meta("last_quicksuggest_ingest_unparsable")?,
5019 "clippy-2": UnparsableRecord {
5022 "fancy-new-suggestions-1": UnparsableRecord {
5029 .assert_debug_eq(&dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY)?);
5036 /// Tests meta update field isn't updated for old unparsable Remote Settings
5039 fn ingest_unparsable_and_meta_update_stays_the_same() -> anyhow::Result<()> {
5042 let snapshot = Snapshot::with_records(json!([{
5043 "id": "fancy-new-suggestions-1",
5044 "type": "fancy-new-suggestions",
5045 "last_modified": 15,
5048 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5049 store.dbs()?.writer.write(|dao| {
5051 SuggestRecordType::AmpWikipedia
5052 .last_ingest_meta_key()
5058 store.ingest(SuggestIngestionConstraints::default())?;
5060 store.dbs()?.reader.read(|dao| {
5062 dao.get_meta::<u64>(
5063 SuggestRecordType::AmpWikipedia
5064 .last_ingest_meta_key()
5075 /// Tests that we only ingest providers that we're concerned with.
5077 fn ingest_constraints_provider() -> anyhow::Result<()> {
5080 let snapshot = Snapshot::with_records(json!([{
5083 "last_modified": 15,
5087 "last_modified": 30,
5090 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5091 store.dbs()?.writer.write(|dao| {
5092 // Check that existing data is updated properly.
5094 SuggestRecordType::AmpWikipedia
5095 .last_ingest_meta_key()
5102 let constraints = SuggestIngestionConstraints {
5103 max_suggestions: Some(100),
5104 providers: Some(vec![SuggestionProvider::Amp, SuggestionProvider::Pocket]),
5106 store.ingest(constraints)?;
5108 store.dbs()?.reader.read(|dao| {
5110 dao.get_meta::<u64>(
5111 SuggestRecordType::AmpWikipedia
5112 .last_ingest_meta_key()
5118 dao.get_meta::<u64>(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
5122 dao.get_meta::<u64>(SuggestRecordType::Pocket.last_ingest_meta_key().as_str())?,
5126 dao.get_meta::<u64>(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?,
5130 dao.get_meta::<u64>(SuggestRecordType::Yelp.last_ingest_meta_key().as_str())?,
5134 dao.get_meta::<u64>(SuggestRecordType::Mdn.last_ingest_meta_key().as_str())?,
5138 dao.get_meta::<u64>(SuggestRecordType::AmpMobile.last_ingest_meta_key().as_str())?,
5142 dao.get_meta::<u64>(
5143 SuggestRecordType::GlobalConfig
5144 .last_ingest_meta_key()
5156 fn remove_known_records_out_of_meta_table() -> anyhow::Result<()> {
5159 let snapshot = Snapshot::with_records(json!([{
5160 "id": "fancy-new-suggestions-1",
5161 "type": "fancy-new-suggestions",
5162 "last_modified": 15,
5167 "last_modified": 15,
5169 "filename": "data-1.json",
5170 "mimetype": "application/json",
5171 "location": "data-1.json",
5179 "last_modified": 15,
5185 "advertiser": "Los Pollos Hermanos",
5186 "iab_category": "8 - Food & Drink",
5187 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
5188 "title": "Los Pollos Hermanos - Albuquerque",
5189 "url": "https://www.lph-nm.biz",
5191 "impression_url": "https://example.com/impression_url",
5192 "click_url": "https://example.com/click_url",
5197 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5198 let mut initial_data = UnparsableRecords::default();
5201 .insert("data-1".to_string(), UnparsableRecord { schema_version: 1 });
5202 initial_data.0.insert(
5203 "clippy-2".to_string(),
5204 UnparsableRecord { schema_version: 1 },
5206 store.dbs()?.writer.write(|dao| {
5207 dao.put_meta(UNPARSABLE_RECORDS_META_KEY, initial_data)?;
5211 store.ingest(SuggestIngestionConstraints::default())?;
5213 store.dbs()?.reader.read(|dao| {
5218 "clippy-2": UnparsableRecord {
5221 "fancy-new-suggestions-1": UnparsableRecord {
5228 .assert_debug_eq(&dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY)?);
5235 /// Tests that records with invalid attachments are ignored and marked as unparsable.
5237 fn skip_over_invalid_records() -> anyhow::Result<()> {
5240 let snapshot = Snapshot::with_records(json!([
5242 "id": "invalid-attachment",
5244 "last_modified": 15,
5246 "filename": "data-2.json",
5247 "mimetype": "application/json",
5248 "location": "data-2.json",
5254 "id": "valid-record",
5256 "last_modified": 15,
5258 "filename": "data-1.json",
5259 "mimetype": "application/json",
5260 "location": "data-1.json",
5270 "advertiser": "Los Pollos Hermanos",
5271 "iab_category": "8 - Food & Drink",
5272 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
5273 "title": "Los Pollos Hermanos - Albuquerque",
5274 "url": "https://www.lph-nm.biz",
5276 "impression_url": "https://example.com/impression_url",
5277 "click_url": "https://example.com/click_url",
5281 // This attachment is missing the `keywords` field and is invalid
5286 "advertiser": "Los Pollos Hermanos",
5287 "iab_category": "8 - Food & Drink",
5288 "title": "Los Pollos Hermanos - Albuquerque",
5289 "url": "https://www.lph-nm.biz",
5291 "impression_url": "https://example.com/impression_url",
5292 "click_url": "https://example.com/click_url",
5297 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5299 store.ingest(SuggestIngestionConstraints::default())?;
5301 // Test that the invalid record marked as unparsable
5302 store.dbs()?.reader.read(|dao| {
5307 "invalid-attachment": UnparsableRecord {
5314 .assert_debug_eq(&dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY)?);
5318 // Test that the valid record was read
5319 store.dbs()?.reader.read(|dao| {
5322 SuggestRecordType::AmpWikipedia
5323 .last_ingest_meta_key()
5331 title: "Los Pollos Hermanos - Albuquerque",
5332 url: "https://www.lph-nm.biz",
5333 raw_url: "https://www.lph-nm.biz",
5335 icon_mimetype: None,
5336 full_keyword: "los",
5338 advertiser: "Los Pollos Hermanos",
5339 iab_category: "8 - Food & Drink",
5340 impression_url: "https://example.com/impression_url",
5341 click_url: "https://example.com/click_url",
5342 raw_click_url: "https://example.com/click_url",
5347 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
5348 keyword: "lo".into(),
5349 providers: vec![SuggestionProvider::Amp],
5360 fn unparsable_record_serialized_correctly() -> anyhow::Result<()> {
5361 let unparseable_record = UnparsableRecord { schema_version: 1 };
5362 assert_eq!(serde_json::to_value(unparseable_record)?, json!({ "v": 1 }),);
5367 fn query_mdn() -> anyhow::Result<()> {
5370 let snapshot = Snapshot::with_records(json!([{
5372 "type": "mdn-suggestions",
5373 "last_modified": 15,
5375 "filename": "data-1.json",
5376 "mimetype": "application/json",
5377 "location": "data-1.json",
5386 "description": "Javascript Array",
5387 "url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5388 "keywords": ["array javascript", "javascript array", "wildcard"],
5395 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5397 store.ingest(SuggestIngestionConstraints::default())?;
5401 "keyword = prefix; MDN only",
5403 keyword: "array".into(),
5404 providers: vec![SuggestionProvider::Mdn],
5411 url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5412 description: "Javascript Array",
5419 "keyword = prefix + partial suffix; MDN only",
5421 keyword: "array java".into(),
5422 providers: vec![SuggestionProvider::Mdn],
5429 url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5430 description: "Javascript Array",
5437 "keyword = prefix + entire suffix; MDN only",
5439 keyword: "javascript array".into(),
5440 providers: vec![SuggestionProvider::Mdn],
5447 url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5448 description: "Javascript Array",
5455 "keyword = `partial prefix word`; MDN only",
5457 keyword: "wild".into(),
5458 providers: vec![SuggestionProvider::Mdn],
5466 "keyword = single word; MDN only",
5468 keyword: "wildcard".into(),
5469 providers: vec![SuggestionProvider::Mdn],
5476 url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5477 description: "Javascript Array",
5485 for (what, query, expect) in table {
5486 expect.assert_debug_eq(
5489 .with_context(|| format!("Couldn't query store for {}", what))?,
5497 fn query_no_yelp_icon_data() -> anyhow::Result<()> {
5500 let snapshot = Snapshot::with_records(json!([{
5502 "type": "yelp-suggestions",
5503 "last_modified": 15,
5505 "filename": "data-1.json",
5506 "mimetype": "application/json",
5507 "location": "data-1.json",
5516 "subjects": ["ramen"],
5518 "postModifiers": [],
5519 "locationSigns": [],
5520 "yelpModifiers": [],
5521 "icon": "yelp-favicon",
5527 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5529 store.ingest(SuggestIngestionConstraints::default())?;
5532 "keyword = ramen; Yelp only",
5534 keyword: "ramen".into(),
5535 providers: vec![SuggestionProvider::Yelp],
5541 url: "https://www.yelp.com/search?find_desc=ramen",
5544 icon_mimetype: None,
5546 has_location_sign: false,
5547 subject_exact_match: true,
5548 location_param: "find_loc",
5554 for (what, query, expect) in table {
5555 expect.assert_debug_eq(
5558 .with_context(|| format!("Couldn't query store for {}", what))?,
5566 fn weather() -> anyhow::Result<()> {
5569 let snapshot = Snapshot::with_records(json!([{
5572 "last_modified": 15,
5574 "min_keyword_length": 3,
5575 "keywords": ["ab", "xyz", "weather"],
5580 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5581 store.ingest(SuggestIngestionConstraints::default())?;
5585 "keyword = 'ab'; Weather only, no match since query is too short",
5587 keyword: "ab".into(),
5588 providers: vec![SuggestionProvider::Weather],
5596 "keyword = 'xab'; Weather only, no matching keyword",
5598 keyword: "xab".into(),
5599 providers: vec![SuggestionProvider::Weather],
5607 "keyword = 'abx'; Weather only, no matching keyword",
5609 keyword: "abx".into(),
5610 providers: vec![SuggestionProvider::Weather],
5618 "keyword = 'xy'; Weather only, no match since query is too short",
5620 keyword: "xy".into(),
5621 providers: vec![SuggestionProvider::Weather],
5629 "keyword = 'xyz'; Weather only, match",
5631 keyword: "xyz".into(),
5632 providers: vec![SuggestionProvider::Weather],
5644 "keyword = 'xxyz'; Weather only, no matching keyword",
5646 keyword: "xxyz".into(),
5647 providers: vec![SuggestionProvider::Weather],
5655 "keyword = 'xyzx'; Weather only, no matching keyword",
5657 keyword: "xyzx".into(),
5658 providers: vec![SuggestionProvider::Weather],
5666 "keyword = 'we'; Weather only, no match since query is too short",
5668 keyword: "we".into(),
5669 providers: vec![SuggestionProvider::Weather],
5677 "keyword = 'wea'; Weather only, match",
5679 keyword: "wea".into(),
5680 providers: vec![SuggestionProvider::Weather],
5692 "keyword = 'weat'; Weather only, match",
5694 keyword: "weat".into(),
5695 providers: vec![SuggestionProvider::Weather],
5707 "keyword = 'weath'; Weather only, match",
5709 keyword: "weath".into(),
5710 providers: vec![SuggestionProvider::Weather],
5722 "keyword = 'weathe'; Weather only, match",
5724 keyword: "weathe".into(),
5725 providers: vec![SuggestionProvider::Weather],
5737 "keyword = 'weather'; Weather only, match",
5739 keyword: "weather".into(),
5740 providers: vec![SuggestionProvider::Weather],
5752 "keyword = 'weatherx'; Weather only, no matching keyword",
5754 keyword: "weatherx".into(),
5755 providers: vec![SuggestionProvider::Weather],
5763 "keyword = 'xweather'; Weather only, no matching keyword",
5765 keyword: "xweather".into(),
5766 providers: vec![SuggestionProvider::Weather],
5774 "keyword = 'xwea'; Weather only, no matching keyword",
5776 keyword: "xwea".into(),
5777 providers: vec![SuggestionProvider::Weather],
5785 "keyword = ' weather '; Weather only, match",
5787 keyword: " weather ".into(),
5788 providers: vec![SuggestionProvider::Weather],
5800 "keyword = 'x weather '; Weather only, no matching keyword",
5802 keyword: "x weather ".into(),
5803 providers: vec![SuggestionProvider::Weather],
5811 "keyword = ' weather x'; Weather only, no matching keyword",
5813 keyword: " weather x".into(),
5814 providers: vec![SuggestionProvider::Weather],
5823 for (what, query, expect) in table {
5824 expect.assert_debug_eq(
5827 .with_context(|| format!("Couldn't query store for {}", what))?,
5834 min_keyword_length: 3,
5840 .fetch_provider_config(SuggestionProvider::Weather)
5841 .with_context(|| "Couldn't fetch provider config")?,
5848 fn fetch_global_config() -> anyhow::Result<()> {
5851 let snapshot = Snapshot::with_records(json!([{
5853 "type": "configuration",
5854 "last_modified": 15,
5856 "show_less_frequently_cap": 3,
5860 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5861 store.ingest(SuggestIngestionConstraints::default())?;
5864 SuggestGlobalConfig {
5865 show_less_frequently_cap: 3,
5870 .fetch_global_config()
5871 .with_context(|| "fetch_global_config failed")?,
5878 fn fetch_global_config_default() -> anyhow::Result<()> {
5881 let snapshot = Snapshot::with_records(json!([]))?;
5882 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5883 store.ingest(SuggestIngestionConstraints::default())?;
5886 SuggestGlobalConfig {
5887 show_less_frequently_cap: 0,
5892 .fetch_global_config()
5893 .with_context(|| "fetch_global_config failed")?,
5900 fn fetch_provider_config_none() -> anyhow::Result<()> {
5903 let snapshot = Snapshot::with_records(json!([]))?;
5904 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5905 store.ingest(SuggestIngestionConstraints::default())?;
5912 .fetch_provider_config(SuggestionProvider::Amp)
5913 .with_context(|| "fetch_provider_config failed for Amp")?,
5921 .fetch_provider_config(SuggestionProvider::Weather)
5922 .with_context(|| "fetch_provider_config failed for Weather")?,
5929 fn fetch_provider_config_other() -> anyhow::Result<()> {
5932 // Add some weather config.
5933 let snapshot = Snapshot::with_records(json!([{
5936 "last_modified": 15,
5938 "min_keyword_length": 3,
5939 "keywords": ["weather"],
5944 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5945 store.ingest(SuggestIngestionConstraints::default())?;
5947 // Getting the config for a different provider should return None.
5953 .fetch_provider_config(SuggestionProvider::Amp)
5954 .with_context(|| "fetch_provider_config failed for Amp")?,