| blob | 4c925e5482d103129113812e383960152075dee7 |
1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
4 */
6 use std::{
7 collections::{BTreeMap, BTreeSet},
8 path::{Path, PathBuf},
9 sync::Arc,
10 };
12 use error_support::handle_error;
13 use once_cell::sync::OnceCell;
14 use parking_lot::Mutex;
15 use remote_settings::{
16 self, GetItemsOptions, RemoteSettingsConfig, RemoteSettingsRecord, SortOrder,
17 };
18 use rusqlite::{
19 types::{FromSql, ToSqlOutput},
20 ToSql,
21 };
22 use serde::{de::DeserializeOwned, Deserialize, Serialize};
24 use crate::{
25 config::{SuggestGlobalConfig, SuggestProviderConfig},
26 db::{
27 ConnectionType, SuggestDao, SuggestDb, LAST_INGEST_META_UNPARSABLE,
28 UNPARSABLE_RECORDS_META_KEY,
29 },
30 error::Error,
31 provider::SuggestionProvider,
32 rs::{
33 SuggestAttachment, SuggestRecord, SuggestRecordId, SuggestRecordType,
34 SuggestRemoteSettingsClient, DEFAULT_RECORDS_TYPES, REMOTE_SETTINGS_COLLECTION,
35 SUGGESTIONS_PER_ATTACHMENT,
36 },
37 schema::VERSION,
38 Result, SuggestApiResult, Suggestion, SuggestionQuery,
39 };
41 /// The chunk size used to request unparsable records.
42 pub const UNPARSABLE_IDS_PER_REQUEST: usize = 150;
44 /// Builder for [SuggestStore]
45 ///
46 /// Using a builder is preferred to calling the constructor directly since it's harder to confuse
47 /// the data_path and cache_path strings.
48 pub struct SuggestStoreBuilder(Mutex<SuggestStoreBuilderInner>);
50 #[derive(Default)]
51 struct SuggestStoreBuilderInner {
52 data_path: Option<String>,
53 remote_settings_config: Option<RemoteSettingsConfig>,
54 }
56 impl Default for SuggestStoreBuilder {
57 fn default() -> Self {
58 Self::new()
59 }
60 }
62 impl SuggestStoreBuilder {
63 pub fn new() -> SuggestStoreBuilder {
64 Self(Mutex::new(SuggestStoreBuilderInner::default()))
65 }
67 pub fn data_path(self: Arc<Self>, path: String) -> Arc<Self> {
68 self.0.lock().data_path = Some(path);
69 self
70 }
72 pub fn cache_path(self: Arc<Self>, _path: String) -> Arc<Self> {
73 // We used to use this, but we're not using it anymore, just ignore the call
74 self
75 }
77 pub fn remote_settings_config(self: Arc<Self>, config: RemoteSettingsConfig) -> Arc<Self> {
78 self.0.lock().remote_settings_config = Some(config);
79 self
80 }
82 #[handle_error(Error)]
83 pub fn build(&self) -> SuggestApiResult<Arc<SuggestStore>> {
84 let inner = self.0.lock();
85 let data_path = inner
86 .data_path
87 .clone()
88 .ok_or_else(|| Error::SuggestStoreBuilder("data_path not specified".to_owned()))?;
89 let settings_client =
90 remote_settings::Client::new(inner.remote_settings_config.clone().unwrap_or_else(
91 || RemoteSettingsConfig {
92 server_url: None,
93 bucket_name: None,
94 collection_name: REMOTE_SETTINGS_COLLECTION.into(),
95 },
96 ))?;
97 Ok(Arc::new(SuggestStore {
98 inner: SuggestStoreInner::new(data_path, settings_client),
99 }))
100 }
101 }
103 /// The store is the entry point to the Suggest component. It incrementally
104 /// downloads suggestions from the Remote Settings service, stores them in a
105 /// local database, and returns them in response to user queries.
106 ///
107 /// Your application should create a single store, and manage it as a singleton.
108 /// The store is thread-safe, and supports concurrent queries and ingests. We
109 /// expect that your application will call [`SuggestStore::query()`] to show
110 /// suggestions as the user types into the address bar, and periodically call
111 /// [`SuggestStore::ingest()`] in the background to update the database with
112 /// new suggestions from Remote Settings.
113 ///
114 /// For responsiveness, we recommend always calling `query()` on a worker
115 /// thread. When the user types new input into the address bar, call
116 /// [`SuggestStore::interrupt()`] on the main thread to cancel the query
117 /// for the old input, and unblock the worker thread for the new query.
118 ///
119 /// The store keeps track of the state needed to support incremental ingestion,
120 /// but doesn't schedule the ingestion work itself, or decide how many
121 /// suggestions to ingest at once. This is for two reasons:
122 ///
123 /// 1. The primitives for scheduling background work vary between platforms, and
124 /// aren't available to the lower-level Rust layer. You might use an idle
125 /// timer on Desktop, `WorkManager` on Android, or `BGTaskScheduler` on iOS.
126 /// 2. Ingestion constraints can change, depending on the platform and the needs
127 /// of your application. A mobile device on a metered connection might want
128 /// to request a small subset of the Suggest data and download the rest
129 /// later, while a desktop on a fast link might download the entire dataset
130 /// on the first launch.
131 pub struct SuggestStore {
132 inner: SuggestStoreInner<remote_settings::Client>,
133 }
135 /// For records that aren't currently parsable,
136 /// the record ID and the schema version it's first seen in
137 /// is recorded in the meta table using `UNPARSABLE_RECORDS_META_KEY` as its key.
138 /// On the first ingest after an upgrade, re-request those records from Remote Settings,
139 /// and try to ingest them again.
140 #[derive(Deserialize, Serialize, Default, Debug)]
141 #[serde(transparent)]
142 pub(crate) struct UnparsableRecords(pub BTreeMap<String, UnparsableRecord>);
144 impl FromSql for UnparsableRecords {
145 fn column_result(value: rusqlite::types::ValueRef<'_>) -> rusqlite::types::FromSqlResult<Self> {
146 serde_json::from_str(value.as_str()?)
147 .map_err(|err| rusqlite::types::FromSqlError::Other(Box::new(err)))
148 }
149 }
151 impl ToSql for UnparsableRecords {
152 fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput<'_>> {
153 Ok(ToSqlOutput::from(serde_json::to_string(self).map_err(
154 |err| rusqlite::Error::ToSqlConversionFailure(Box::new(err)),
155 )?))
156 }
157 }
159 #[derive(Deserialize, Serialize, Debug)]
160 pub(crate) struct UnparsableRecord {
161 #[serde(rename = "v")]
162 pub schema_version: u32,
163 }
165 impl SuggestStore {
166 /// Creates a Suggest store.
167 #[handle_error(Error)]
168 pub fn new(
169 path: &str,
170 settings_config: Option<RemoteSettingsConfig>,
171 ) -> SuggestApiResult<Self> {
172 let settings_client = || -> Result<_> {
173 Ok(remote_settings::Client::new(
174 settings_config.unwrap_or_else(|| RemoteSettingsConfig {
175 server_url: None,
176 bucket_name: None,
177 collection_name: REMOTE_SETTINGS_COLLECTION.into(),
178 }),
179 )?)
180 }()?;
181 Ok(Self {
182 inner: SuggestStoreInner::new(path.to_owned(), settings_client),
183 })
184 }
186 /// Queries the database for suggestions.
187 #[handle_error(Error)]
188 pub fn query(&self, query: SuggestionQuery) -> SuggestApiResult<Vec<Suggestion>> {
189 self.inner.query(query)
190 }
192 /// Interrupts any ongoing queries.
193 ///
194 /// This should be called when the user types new input into the address
195 /// bar, to ensure that they see fresh suggestions as they type. This
196 /// method does not interrupt any ongoing ingests.
197 pub fn interrupt(&self) {
198 self.inner.interrupt()
199 }
201 /// Ingests new suggestions from Remote Settings.
202 #[handle_error(Error)]
203 pub fn ingest(&self, constraints: SuggestIngestionConstraints) -> SuggestApiResult<()> {
204 self.inner.ingest(constraints)
205 }
207 /// Removes all content from the database.
208 #[handle_error(Error)]
209 pub fn clear(&self) -> SuggestApiResult<()> {
210 self.inner.clear()
211 }
213 // Returns global Suggest configuration data.
214 #[handle_error(Error)]
215 pub fn fetch_global_config(&self) -> SuggestApiResult<SuggestGlobalConfig> {
216 self.inner.fetch_global_config()
217 }
219 // Returns per-provider Suggest configuration data.
220 #[handle_error(Error)]
221 pub fn fetch_provider_config(
222 &self,
223 provider: SuggestionProvider,
224 ) -> SuggestApiResult<Option<SuggestProviderConfig>> {
225 self.inner.fetch_provider_config(provider)
226 }
227 }
229 /// Constraints limit which suggestions to ingest from Remote Settings.
230 #[derive(Clone, Default, Debug)]
231 pub struct SuggestIngestionConstraints {
232 /// The approximate maximum number of suggestions to ingest. Set to [`None`]
233 /// for "no limit".
234 ///
235 /// Because of how suggestions are partitioned in Remote Settings, this is a
236 /// soft limit, and the store might ingest more than requested.
237 pub max_suggestions: Option<u64>,
238 pub providers: Option<Vec<SuggestionProvider>>,
239 }
241 /// The implementation of the store. This is generic over the Remote Settings
242 /// client, and is split out from the concrete [`SuggestStore`] for testing
243 /// with a mock client.
244 pub(crate) struct SuggestStoreInner<S> {
245 /// Path to the persistent SQL database.
246 ///
247 /// This stores things that should persist when the user clears their cache.
248 /// It's not currently used because not all consumers pass this in yet.
249 #[allow(unused)]
250 data_path: PathBuf,
251 dbs: OnceCell<SuggestStoreDbs>,
252 settings_client: S,
253 }
255 impl<S> SuggestStoreInner<S> {
256 fn new(data_path: impl Into<PathBuf>, settings_client: S) -> Self {
257 Self {
258 data_path: data_path.into(),
259 dbs: OnceCell::new(),
260 settings_client,
261 }
262 }
264 /// Returns this store's database connections, initializing them if
265 /// they're not already open.
266 fn dbs(&self) -> Result<&SuggestStoreDbs> {
267 self.dbs
268 .get_or_try_init(|| SuggestStoreDbs::open(&self.data_path))
269 }
271 fn query(&self, query: SuggestionQuery) -> Result<Vec<Suggestion>> {
272 if query.keyword.is_empty() || query.providers.is_empty() {
273 return Ok(Vec::new());
274 }
275 self.dbs()?.reader.read(|dao| dao.fetch_suggestions(&query))
276 }
278 fn interrupt(&self) {
279 if let Some(dbs) = self.dbs.get() {
280 // Only interrupt if the databases are already open.
281 dbs.reader.interrupt_handle.interrupt();
282 }
283 }
285 fn clear(&self) -> Result<()> {
286 self.dbs()?.writer.write(|dao| dao.clear())
287 }
289 pub fn fetch_global_config(&self) -> Result<SuggestGlobalConfig> {
290 self.dbs()?.reader.read(|dao| dao.get_global_config())
291 }
293 pub fn fetch_provider_config(
294 &self,
295 provider: SuggestionProvider,
296 ) -> Result<Option<SuggestProviderConfig>> {
297 self.dbs()?
298 .reader
299 .read(|dao| dao.get_provider_config(provider))
300 }
301 }
303 impl<S> SuggestStoreInner<S>
304 where
305 S: SuggestRemoteSettingsClient,
306 {
307 fn ingest(&self, constraints: SuggestIngestionConstraints) -> Result<()> {
308 let writer = &self.dbs()?.writer;
310 if let Some(unparsable_records) =
311 writer.read(|dao| dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY))?
312 {
313 let all_unparsable_ids = unparsable_records
314 .0
315 .iter()
316 .filter(|(_, unparsable_record)| unparsable_record.schema_version < VERSION)
317 .map(|(record_id, _)| record_id)
318 .collect::<Vec<_>>();
319 for unparsable_ids in all_unparsable_ids.chunks(UNPARSABLE_IDS_PER_REQUEST) {
320 let mut options = GetItemsOptions::new();
321 for unparsable_id in unparsable_ids {
322 options.eq("id", *unparsable_id);
323 }
324 let records_chunk = self
325 .settings_client
326 .get_records_with_options(&options)?
327 .records;
329 self.ingest_records(LAST_INGEST_META_UNPARSABLE, writer, &records_chunk)?;
330 }
331 }
333 // use std::collections::BTreeSet;
334 let ingest_record_types = if let Some(rt) = &constraints.providers {
335 rt.iter()
336 .flat_map(|x| x.records_for_provider())
337 .collect::<BTreeSet<_>>()
338 .into_iter()
339 .collect()
340 } else {
341 DEFAULT_RECORDS_TYPES.to_vec()
342 };
344 for ingest_record_type in ingest_record_types {
345 self.ingest_records_by_type(ingest_record_type, writer, &constraints)?;
346 }
348 Ok(())
349 }
351 fn ingest_records_by_type(
352 &self,
353 ingest_record_type: SuggestRecordType,
354 writer: &SuggestDb,
355 constraints: &SuggestIngestionConstraints,
356 ) -> Result<()> {
357 let mut options = GetItemsOptions::new();
359 // Remote Settings returns records in descending modification order
360 // (newest first), but we want them in ascending order (oldest first),
361 // so that we can eventually resume downloading where we left off.
362 options.sort("last_modified", SortOrder::Ascending);
364 options.eq("type", ingest_record_type.to_string());
366 // Get the last ingest value. This is the max of the last_ingest_keys
367 // that are in the database.
368 if let Some(last_ingest) = writer
369 .read(|dao| dao.get_meta::<u64>(ingest_record_type.last_ingest_meta_key().as_str()))?
370 {
371 // Only download changes since our last ingest. If our last ingest
372 // was interrupted, we'll pick up where we left off.
373 options.gt("last_modified", last_ingest.to_string());
374 }
376 if let Some(max_suggestions) = constraints.max_suggestions {
377 // Each record's attachment has 200 suggestions, so download enough
378 // records to cover the requested maximum.
379 let max_records = (max_suggestions.saturating_sub(1) / SUGGESTIONS_PER_ATTACHMENT) + 1;
380 options.limit(max_records);
381 }
383 let records = self
384 .settings_client
385 .get_records_with_options(&options)?
386 .records;
387 self.ingest_records(&ingest_record_type.last_ingest_meta_key(), writer, &records)?;
388 Ok(())
389 }
391 fn ingest_records(
392 &self,
393 last_ingest_key: &str,
394 writer: &SuggestDb,
395 records: &[RemoteSettingsRecord],
396 ) -> Result<()> {
397 for record in records {
398 let record_id = SuggestRecordId::from(&record.id);
399 if record.deleted {
400 // If the entire record was deleted, drop all its suggestions
401 // and advance the last ingest time.
402 writer.write(|dao| dao.handle_deleted_record(last_ingest_key, record))?;
403 continue;
404 }
405 let Ok(fields) =
406 serde_json::from_value(serde_json::Value::Object(record.fields.clone()))
407 else {
408 // We don't recognize this record's type, so we don't know how
409 // to ingest its suggestions. Skip processing this record.
410 writer.write(|dao| dao.handle_unparsable_record(record))?;
411 continue;
412 };
414 match fields {
415 SuggestRecord::AmpWikipedia => {
416 self.ingest_attachment(
417 // TODO: Currently re-creating the last_ingest_key because using last_ingest_meta
418 // breaks the tests (particularly the unparsable functionality). So, keeping
419 // a direct reference until we remove the "unparsable" functionality.
420 &SuggestRecordType::AmpWikipedia.last_ingest_meta_key(),
421 writer,
422 record,
423 |dao, record_id, suggestions| {
424 dao.insert_amp_wikipedia_suggestions(record_id, suggestions)
425 },
426 )?;
427 }
428 SuggestRecord::AmpMobile => {
429 self.ingest_attachment(
430 &SuggestRecordType::AmpMobile.last_ingest_meta_key(),
431 writer,
432 record,
433 |dao, record_id, suggestions| {
434 dao.insert_amp_mobile_suggestions(record_id, suggestions)
435 },
436 )?;
437 }
438 SuggestRecord::Icon => {
439 let (Some(icon_id), Some(attachment)) =
440 (record_id.as_icon_id(), record.attachment.as_ref())
441 else {
442 // An icon record should have an icon ID and an
443 // attachment. Icons that don't have these are
444 // malformed, so skip to the next record.
445 writer.write(|dao| {
446 dao.put_last_ingest_if_newer(
447 &SuggestRecordType::Icon.last_ingest_meta_key(),
448 record.last_modified,
449 )
450 })?;
451 continue;
452 };
453 let data = self.settings_client.get_attachment(&attachment.location)?;
454 writer.write(|dao| {
455 dao.put_icon(icon_id, &data, &attachment.mimetype)?;
456 dao.handle_ingested_record(
457 &SuggestRecordType::Icon.last_ingest_meta_key(),
458 record,
459 )
460 })?;
461 }
462 SuggestRecord::Amo => {
463 self.ingest_attachment(
464 &SuggestRecordType::Amo.last_ingest_meta_key(),
465 writer,
466 record,
467 |dao, record_id, suggestions| {
468 dao.insert_amo_suggestions(record_id, suggestions)
469 },
470 )?;
471 }
472 SuggestRecord::Pocket => {
473 self.ingest_attachment(
474 &SuggestRecordType::Pocket.last_ingest_meta_key(),
475 writer,
476 record,
477 |dao, record_id, suggestions| {
478 dao.insert_pocket_suggestions(record_id, suggestions)
479 },
480 )?;
481 }
482 SuggestRecord::Yelp => {
483 self.ingest_attachment(
484 &SuggestRecordType::Yelp.last_ingest_meta_key(),
485 writer,
486 record,
487 |dao, record_id, suggestions| match suggestions.first() {
488 Some(suggestion) => dao.insert_yelp_suggestions(record_id, suggestion),
489 None => Ok(()),
490 },
491 )?;
492 }
493 SuggestRecord::Mdn => {
494 self.ingest_attachment(
495 &SuggestRecordType::Mdn.last_ingest_meta_key(),
496 writer,
497 record,
498 |dao, record_id, suggestions| {
499 dao.insert_mdn_suggestions(record_id, suggestions)
500 },
501 )?;
502 }
503 SuggestRecord::Weather(data) => {
504 self.ingest_record(
505 &SuggestRecordType::Weather.last_ingest_meta_key(),
506 writer,
507 record,
508 |dao, record_id| dao.insert_weather_data(record_id, &data),
509 )?;
510 }
511 SuggestRecord::GlobalConfig(config) => {
512 self.ingest_record(
513 &SuggestRecordType::GlobalConfig.last_ingest_meta_key(),
514 writer,
515 record,
516 |dao, _| dao.put_global_config(&SuggestGlobalConfig::from(&config)),
517 )?;
518 }
519 }
520 }
521 Ok(())
522 }
524 fn ingest_record(
525 &self,
526 last_ingest_key: &str,
527 writer: &SuggestDb,
528 record: &RemoteSettingsRecord,
529 ingestion_handler: impl FnOnce(&mut SuggestDao<'_>, &SuggestRecordId) -> Result<()>,
530 ) -> Result<()> {
531 let record_id = SuggestRecordId::from(&record.id);
533 writer.write(|dao| {
534 // Drop any data that we previously ingested from this record.
535 // Suggestions in particular don't have a stable identifier, and
536 // determining which suggestions in the record actually changed is
537 // more complicated than dropping and re-ingesting all of them.
538 dao.drop_suggestions(&record_id)?;
540 // Ingest (or re-ingest) all data in the record.
541 ingestion_handler(dao, &record_id)?;
543 dao.handle_ingested_record(last_ingest_key, record)
544 })
545 }
547 fn ingest_attachment<T>(
548 &self,
549 last_ingest_key: &str,
550 writer: &SuggestDb,
551 record: &RemoteSettingsRecord,
552 ingestion_handler: impl FnOnce(&mut SuggestDao<'_>, &SuggestRecordId, &[T]) -> Result<()>,
553 ) -> Result<()>
554 where
555 T: DeserializeOwned,
556 {
557 let Some(attachment) = record.attachment.as_ref() else {
558 // This method should be called only when a record is expected to
559 // have an attachment. If it doesn't have one, it's malformed, so
560 // skip to the next record.
561 writer
562 .write(|dao| dao.put_last_ingest_if_newer(last_ingest_key, record.last_modified))?;
563 return Ok(());
564 };
566 let attachment_data = self.settings_client.get_attachment(&attachment.location)?;
567 match serde_json::from_slice::<SuggestAttachment<T>>(&attachment_data) {
568 Ok(attachment) => {
569 self.ingest_record(last_ingest_key, writer, record, |dao, record_id| {
570 ingestion_handler(dao, record_id, attachment.suggestions())
571 })
572 }
573 Err(_) => writer.write(|dao| dao.handle_unparsable_record(record)),
574 }
575 }
576 }
578 /// Holds a store's open connections to the Suggest database.
579 struct SuggestStoreDbs {
580 /// A read-write connection used to update the database with new data.
581 writer: SuggestDb,
582 /// A read-only connection used to query the database.
583 reader: SuggestDb,
584 }
586 impl SuggestStoreDbs {
587 fn open(path: &Path) -> Result<Self> {
588 // Order is important here: the writer must be opened first, so that it
589 // can set up the database and run any migrations.
590 let writer = SuggestDb::open(path, ConnectionType::ReadWrite)?;
591 let reader = SuggestDb::open(path, ConnectionType::ReadOnly)?;
592 Ok(Self { writer, reader })
593 }
594 }
596 #[cfg(test)]
597 mod tests {
598 use super::*;
600 use std::{cell::RefCell, collections::HashMap};
602 use anyhow::{anyhow, Context};
603 use expect_test::expect;
604 use parking_lot::Once;
605 use rc_crypto::rand;
606 use remote_settings::{RemoteSettingsRecord, RemoteSettingsResponse};
607 use serde_json::json;
608 use sql_support::ConnExt;
610 use crate::SuggestionProvider;
612 /// Creates a unique in-memory Suggest store.
613 fn unique_test_store<S>(settings_client: S) -> SuggestStoreInner<S>
614 where
615 S: SuggestRemoteSettingsClient,
616 {
617 let mut unique_suffix = [0u8; 8];
618 rand::fill(&mut unique_suffix).expect("Failed to generate unique suffix for test store");
619 // A store opens separate connections to the same database for reading
620 // and writing, so we must give our in-memory database a name, and open
621 // it in shared-cache mode so that both connections can access it.
622 SuggestStoreInner::new(
623 format!(
624 "file:test_store_data_{}?mode=memory&cache=shared",
625 hex::encode(unique_suffix),
626 ),
627 settings_client,
628 )
629 }
631 /// A snapshot containing fake Remote Settings records and attachments for
632 /// the store to ingest. We use snapshots to test the store's behavior in a
633 /// data-driven way.
634 struct Snapshot {
635 records: Vec<RemoteSettingsRecord>,
636 attachments: HashMap<&'static str, Vec<u8>>,
637 }
639 impl Snapshot {
640 /// Creates a snapshot from a JSON value that represents a collection of
641 /// Suggest Remote Settings records.
642 ///
643 /// You can use the [`serde_json::json!`] macro to construct the JSON
644 /// value, then pass it to this function. It's easier to use the
645 /// `Snapshot::with_records(json!(...))` idiom than to construct the
646 /// records by hand.
647 fn with_records(value: serde_json::Value) -> anyhow::Result<Self> {
648 Ok(Self {
649 records: serde_json::from_value(value)
650 .context("Couldn't create snapshot with Remote Settings records")?,
651 attachments: HashMap::new(),
652 })
653 }
655 /// Adds a data attachment with one or more suggestions to the snapshot.
656 fn with_data(
657 mut self,
658 location: &'static str,
659 value: serde_json::Value,
660 ) -> anyhow::Result<Self> {
661 self.attachments.insert(
662 location,
663 serde_json::to_vec(&value).context("Couldn't add data attachment to snapshot")?,
664 );
665 Ok(self)
666 }
668 /// Adds an icon attachment to the snapshot.
669 fn with_icon(mut self, location: &'static str, bytes: Vec<u8>) -> Self {
670 self.attachments.insert(location, bytes);
671 self
672 }
673 }
675 /// A fake Remote Settings client that returns records and attachments from
676 /// a snapshot.
677 struct SnapshotSettingsClient {
678 /// The current snapshot. You can modify it using
679 /// [`RefCell::borrow_mut()`] to simulate remote updates in tests.
680 snapshot: RefCell<Snapshot>,
682 /// The options passed to the last [`Self::get_records_with_options()`]
683 /// call.
684 last_get_records_options: RefCell<Option<GetItemsOptions>>,
685 }
687 impl SnapshotSettingsClient {
688 /// Creates a client with an initial snapshot.
689 fn with_snapshot(snapshot: Snapshot) -> Self {
690 Self {
691 snapshot: RefCell::new(snapshot),
692 last_get_records_options: RefCell::default(),
693 }
694 }
696 /// Returns the most recent value of an option passed to
697 /// [`Self::get_records_with_options()`].
698 fn last_get_records_option(&self, option: &str) -> Option<String> {
699 self.last_get_records_options
700 .borrow()
701 .as_ref()
702 .and_then(|options| {
703 options
704 .iter_query_pairs()
705 .find(|(key, _)| key == option)
706 .map(|(_, value)| value.into())
707 })
708 }
709 }
711 impl SuggestRemoteSettingsClient for SnapshotSettingsClient {
712 fn get_records_with_options(
713 &self,
714 options: &GetItemsOptions,
715 ) -> Result<RemoteSettingsResponse> {
716 *self.last_get_records_options.borrow_mut() = Some(options.clone());
717 let records = self.snapshot.borrow().records.clone();
718 let last_modified = records
719 .iter()
720 .map(|record| record.last_modified)
721 .max()
722 .unwrap_or(0);
723 Ok(RemoteSettingsResponse {
724 records,
725 last_modified,
726 })
727 }
729 fn get_attachment(&self, location: &str) -> Result<Vec<u8>> {
730 Ok(self
731 .snapshot
732 .borrow()
733 .attachments
734 .get(location)
735 .unwrap_or_else(|| unreachable!("Unexpected request for attachment `{}`", location))
736 .clone())
737 }
738 }
740 fn before_each() {
741 static ONCE: Once = Once::new();
742 ONCE.call_once(|| {
743 env_logger::init();
744 });
745 }
747 /// Tests that `SuggestStore` is usable with UniFFI, which requires exposed
748 /// interfaces to be `Send` and `Sync`.
749 #[test]
750 fn is_thread_safe() {
751 before_each();
753 fn is_send_sync<T: Send + Sync>() {}
754 is_send_sync::<SuggestStore>();
755 }
757 /// Tests ingesting suggestions into an empty database.
758 #[test]
759 fn ingest_suggestions() -> anyhow::Result<()> {
760 before_each();
762 let snapshot = Snapshot::with_records(json!([{
763 "id": "1234",
764 "type": "data",
765 "last_modified": 15,
766 "attachment": {
767 "filename": "data-1.json",
768 "mimetype": "application/json",
769 "location": "data-1.json",
770 "hash": "",
771 "size": 0,
772 },
773 }]))?
774 .with_data(
775 "data-1.json",
776 json!([{
777 "id": 0,
778 "advertiser": "Los Pollos Hermanos",
779 "iab_category": "8 - Food & Drink",
780 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
781 "title": "Los Pollos Hermanos - Albuquerque",
782 "url": "https://www.lph-nm.biz",
783 "icon": "5678",
784 "impression_url": "https://example.com/impression_url",
785 "click_url": "https://example.com/click_url",
786 "score": 0.3
787 }]),
788 )?;
790 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
792 store.ingest(SuggestIngestionConstraints::default())?;
794 store.dbs()?.reader.read(|dao| {
795 assert_eq!(
796 dao.get_meta::<u64>(
797 SuggestRecordType::AmpWikipedia
798 .last_ingest_meta_key()
799 .as_str()
800 )?,
801 Some(15)
802 );
803 expect![[r#"
804 [
805 Amp {
806 title: "Los Pollos Hermanos - Albuquerque",
807 url: "https://www.lph-nm.biz",
808 raw_url: "https://www.lph-nm.biz",
809 icon: None,
810 icon_mimetype: None,
811 full_keyword: "los",
812 block_id: 0,
813 advertiser: "Los Pollos Hermanos",
814 iab_category: "8 - Food & Drink",
815 impression_url: "https://example.com/impression_url",
816 click_url: "https://example.com/click_url",
817 raw_click_url: "https://example.com/click_url",
818 score: 0.3,
819 },
820 ]
821 "#]]
822 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
823 keyword: "lo".into(),
824 providers: vec![SuggestionProvider::Amp],
825 limit: None,
826 })?);
828 Ok(())
829 })?;
831 Ok(())
832 }
834 /// Tests ingesting suggestions with icons.
835 #[test]
836 fn ingest_icons() -> anyhow::Result<()> {
837 before_each();
839 let snapshot = Snapshot::with_records(json!([{
840 "id": "data-1",
841 "type": "data",
842 "last_modified": 15,
843 "attachment": {
844 "filename": "data-1.json",
845 "mimetype": "application/json",
846 "location": "data-1.json",
847 "hash": "",
848 "size": 0,
849 },
850 }, {
851 "id": "icon-2",
852 "type": "icon",
853 "last_modified": 20,
854 "attachment": {
855 "filename": "icon-2.png",
856 "mimetype": "image/png",
857 "location": "icon-2.png",
858 "hash": "",
859 "size": 0,
860 },
861 }]))?
862 .with_data(
863 "data-1.json",
864 json!([{
865 "id": 0,
866 "advertiser": "Good Place Eats",
867 "iab_category": "8 - Food & Drink",
868 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
869 "title": "Lasagna Come Out Tomorrow",
870 "url": "https://www.lasagna.restaurant",
871 "icon": "2",
872 "impression_url": "https://example.com/impression_url",
873 "click_url": "https://example.com/click_url"
874 }, {
875 "id": 0,
876 "advertiser": "Good Place Eats",
877 "iab_category": "8 - Food & Drink",
878 "keywords": ["pe", "pen", "penne", "penne for your thoughts"],
879 "title": "Penne for Your Thoughts",
880 "url": "https://penne.biz",
881 "icon": "2",
882 "impression_url": "https://example.com/impression_url",
883 "click_url": "https://example.com/click_url",
884 "score": 0.3
885 }]),
886 )?
887 .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into());
889 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
891 store.ingest(SuggestIngestionConstraints::default())?;
893 store.dbs()?.reader.read(|dao| {
894 expect![[r#"
895 [
896 Amp {
897 title: "Lasagna Come Out Tomorrow",
898 url: "https://www.lasagna.restaurant",
899 raw_url: "https://www.lasagna.restaurant",
900 icon: Some(
901 [
902 105,
903 45,
904 97,
905 109,
906 45,
907 97,
908 110,
909 45,
910 105,
911 99,
912 111,
913 110,
914 ],
915 ),
916 icon_mimetype: Some(
917 "image/png",
918 ),
919 full_keyword: "lasagna",
920 block_id: 0,
921 advertiser: "Good Place Eats",
922 iab_category: "8 - Food & Drink",
923 impression_url: "https://example.com/impression_url",
924 click_url: "https://example.com/click_url",
925 raw_click_url: "https://example.com/click_url",
926 score: 0.2,
927 },
928 ]
929 "#]]
930 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
931 keyword: "la".into(),
932 providers: vec![SuggestionProvider::Amp],
933 limit: None,
934 })?);
935 expect![[r#"
936 [
937 Amp {
938 title: "Penne for Your Thoughts",
939 url: "https://penne.biz",
940 raw_url: "https://penne.biz",
941 icon: Some(
942 [
943 105,
944 45,
945 97,
946 109,
947 45,
948 97,
949 110,
950 45,
951 105,
952 99,
953 111,
954 110,
955 ],
956 ),
957 icon_mimetype: Some(
958 "image/png",
959 ),
960 full_keyword: "penne",
961 block_id: 0,
962 advertiser: "Good Place Eats",
963 iab_category: "8 - Food & Drink",
964 impression_url: "https://example.com/impression_url",
965 click_url: "https://example.com/click_url",
966 raw_click_url: "https://example.com/click_url",
967 score: 0.3,
968 },
969 ]
970 "#]]
971 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
972 keyword: "pe".into(),
973 providers: vec![SuggestionProvider::Amp],
974 limit: None,
975 })?);
977 Ok(())
978 })?;
980 Ok(())
981 }
983 #[test]
984 fn ingest_full_keywords() -> anyhow::Result<()> {
985 before_each();
987 let snapshot = Snapshot::with_records(json!([{
988 "id": "1",
989 "type": "data",
990 "last_modified": 15,
991 "attachment": {
992 "filename": "data-1.json",
993 "mimetype": "application/json",
994 "location": "data-1.json",
995 "hash": "",
996 "size": 0,
997 },
998 }, {
999 "id": "2",
1000 "type": "data",
1001 "last_modified": 15,
1002 "attachment": {
1003 "filename": "data-2.json",
1004 "mimetype": "application/json",
1005 "location": "data-2.json",
1006 "hash": "",
1007 "size": 0,
1008 },
1009 }, {
1010 "id": "3",
1011 "type": "data",
1012 "last_modified": 15,
1013 "attachment": {
1014 "filename": "data-3.json",
1015 "mimetype": "application/json",
1016 "location": "data-3.json",
1017 "hash": "",
1018 "size": 0,
1019 },
1020 }, {
1021 "id": "4",
1022 "type": "amp-mobile-suggestions",
1023 "last_modified": 15,
1024 "attachment": {
1025 "filename": "data-4.json",
1026 "mimetype": "application/json",
1027 "location": "data-4.json",
1028 "hash": "",
1029 "size": 0,
1030 },
1031 }]))?
1032 // AMP attachment with full keyword data
1033 .with_data(
1034 "data-1.json",
1035 json!([{
1036 "id": 0,
1037 "advertiser": "Los Pollos Hermanos",
1038 "iab_category": "8 - Food & Drink",
1039 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
1040 "full_keywords": [
1041 // Full keyword for the first 4 keywords
1042 ("los pollos", 4),
1043 // Full keyword for the next 2 keywords
1044 ("los pollos hermanos (restaurant)", 2),
1045 ],
1046 "title": "Los Pollos Hermanos - Albuquerque - 1",
1047 "url": "https://www.lph-nm.biz",
1048 "icon": "5678",
1049 "impression_url": "https://example.com/impression_url",
1050 "click_url": "https://example.com/click_url",
1051 "score": 0.3
1052 }]),
1053 )?
1054 // AMP attachment without a full keyword
1055 .with_data(
1056 "data-2.json",
1057 json!([{
1058 "id": 1,
1059 "advertiser": "Los Pollos Hermanos",
1060 "iab_category": "8 - Food & Drink",
1061 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
1062 "title": "Los Pollos Hermanos - Albuquerque - 2",
1063 "url": "https://www.lph-nm.biz",
1064 "icon": "5678",
1065 "impression_url": "https://example.com/impression_url",
1066 "click_url": "https://example.com/click_url",
1067 "score": 0.3
1068 }]),
1069 )?
1070 // Wikipedia attachment with full keyword data. We should ignore the full
1071 // keyword data for Wikipedia suggestions
1072 .with_data(
1073 "data-3.json",
1074 json!([{
1075 "id": 2,
1076 "advertiser": "Wikipedia",
1077 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
1078 "title": "Los Pollos Hermanos - Albuquerque - Wiki",
1079 "full_keywords": [
1080 ("Los Pollos Hermanos - Albuquerque", 6),
1081 ],
1082 "url": "https://www.lph-nm.biz",
1083 "icon": "5678",
1084 "score": 0.3,
1085 }]),
1086 )?
1087 // Amp mobile suggestion, this is essentially the same as 1, except for the SuggestionProvider
1088 .with_data(
1089 "data-4.json",
1090 json!([{
1091 "id": 0,
1092 "advertiser": "Los Pollos Hermanos",
1093 "iab_category": "8 - Food & Drink",
1094 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
1095 "full_keywords": [
1096 // Full keyword for the first 4 keywords
1097 ("los pollos", 4),
1098 // Full keyword for the next 2 keywords
1099 ("los pollos hermanos (restaurant)", 2),
1100 ],
1101 "title": "Los Pollos Hermanos - Albuquerque - 4",
1102 "url": "https://www.lph-nm.biz",
1103 "icon": "5678",
1104 "impression_url": "https://example.com/impression_url",
1105 "click_url": "https://example.com/click_url",
1106 "score": 0.3
1107 }]),
1108 )?;
1110 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
1112 store.ingest(SuggestIngestionConstraints::default())?;
1114 store.dbs()?.reader.read(|dao| {
1115 // This one should match the first full keyword for the first AMP item.
1116 expect![[r#"
1117 [
1118 Amp {
1119 title: "Los Pollos Hermanos - Albuquerque - 1",
1120 url: "https://www.lph-nm.biz",
1121 raw_url: "https://www.lph-nm.biz",
1122 icon: None,
1123 icon_mimetype: None,
1124 full_keyword: "los pollos",
1125 block_id: 0,
1126 advertiser: "Los Pollos Hermanos",
1127 iab_category: "8 - Food & Drink",
1128 impression_url: "https://example.com/impression_url",
1129 click_url: "https://example.com/click_url",
1130 raw_click_url: "https://example.com/click_url",
1131 score: 0.3,
1132 },
1133 Amp {
1134 title: "Los Pollos Hermanos - Albuquerque - 2",
1135 url: "https://www.lph-nm.biz",
1136 raw_url: "https://www.lph-nm.biz",
1137 icon: None,
1138 icon_mimetype: None,
1139 full_keyword: "los",
1140 block_id: 1,
1141 advertiser: "Los Pollos Hermanos",
1142 iab_category: "8 - Food & Drink",
1143 impression_url: "https://example.com/impression_url",
1144 click_url: "https://example.com/click_url",
1145 raw_click_url: "https://example.com/click_url",
1146 score: 0.3,
1147 },
1148 ]
1149 "#]]
1150 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1151 keyword: "lo".into(),
1152 providers: vec![SuggestionProvider::Amp],
1153 limit: None,
1154 })?);
1155 // This one should match the second full keyword for the first AMP item.
1156 expect![[r#"
1157 [
1158 Amp {
1159 title: "Los Pollos Hermanos - Albuquerque - 1",
1160 url: "https://www.lph-nm.biz",
1161 raw_url: "https://www.lph-nm.biz",
1162 icon: None,
1163 icon_mimetype: None,
1164 full_keyword: "los pollos hermanos (restaurant)",
1165 block_id: 0,
1166 advertiser: "Los Pollos Hermanos",
1167 iab_category: "8 - Food & Drink",
1168 impression_url: "https://example.com/impression_url",
1169 click_url: "https://example.com/click_url",
1170 raw_click_url: "https://example.com/click_url",
1171 score: 0.3,
1172 },
1173 Amp {
1174 title: "Los Pollos Hermanos - Albuquerque - 2",
1175 url: "https://www.lph-nm.biz",
1176 raw_url: "https://www.lph-nm.biz",
1177 icon: None,
1178 icon_mimetype: None,
1179 full_keyword: "los pollos hermanos",
1180 block_id: 1,
1181 advertiser: "Los Pollos Hermanos",
1182 iab_category: "8 - Food & Drink",
1183 impression_url: "https://example.com/impression_url",
1184 click_url: "https://example.com/click_url",
1185 raw_click_url: "https://example.com/click_url",
1186 score: 0.3,
1187 },
1188 ]
1189 "#]]
1190 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1191 keyword: "los pollos h".into(),
1192 providers: vec![SuggestionProvider::Amp],
1193 limit: None,
1194 })?);
1195 // This one matches a Wikipedia suggestion, so the full keyword should be ignored
1196 expect![[r#"
1197 [
1198 Wikipedia {
1199 title: "Los Pollos Hermanos - Albuquerque - Wiki",
1200 url: "https://www.lph-nm.biz",
1201 icon: None,
1202 icon_mimetype: None,
1203 full_keyword: "los",
1204 },
1205 ]
1206 "#]]
1207 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1208 keyword: "los".into(),
1209 providers: vec![SuggestionProvider::Wikipedia],
1210 limit: None,
1211 })?);
1212 // This one matches a Wikipedia suggestion, so the full keyword should be ignored
1213 expect![[r#"
1214 [
1215 Amp {
1216 title: "Los Pollos Hermanos - Albuquerque - 4",
1217 url: "https://www.lph-nm.biz",
1218 raw_url: "https://www.lph-nm.biz",
1219 icon: None,
1220 icon_mimetype: None,
1221 full_keyword: "los pollos hermanos (restaurant)",
1222 block_id: 0,
1223 advertiser: "Los Pollos Hermanos",
1224 iab_category: "8 - Food & Drink",
1225 impression_url: "https://example.com/impression_url",
1226 click_url: "https://example.com/click_url",
1227 raw_click_url: "https://example.com/click_url",
1228 score: 0.3,
1229 },
1230 ]
1231 "#]]
1232 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1233 keyword: "los pollos h".into(),
1234 providers: vec![SuggestionProvider::AmpMobile],
1235 limit: None,
1236 })?);
1238 Ok(())
1239 })?;
1241 Ok(())
1242 }
1244 /// Tests ingesting a data attachment containing a single suggestion,
1245 /// instead of an array of suggestions.
1246 #[test]
1247 fn ingest_one_suggestion_in_data_attachment() -> anyhow::Result<()> {
1248 before_each();
1250 let snapshot = Snapshot::with_records(json!([{
1251 "id": "data-1",
1252 "type": "data",
1253 "last_modified": 15,
1254 "attachment": {
1255 "filename": "data-1.json",
1256 "mimetype": "application/json",
1257 "location": "data-1.json",
1258 "hash": "",
1259 "size": 0,
1260 },
1261 }]))?
1262 .with_data(
1263 "data-1.json",
1264 json!({
1265 "id": 0,
1266 "advertiser": "Good Place Eats",
1267 "iab_category": "8 - Food & Drink",
1268 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
1269 "title": "Lasagna Come Out Tomorrow",
1270 "url": "https://www.lasagna.restaurant",
1271 "icon": "2",
1272 "impression_url": "https://example.com/impression_url",
1273 "click_url": "https://example.com/click_url",
1274 "score": 0.3
1275 }),
1276 )?;
1278 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
1280 store.ingest(SuggestIngestionConstraints::default())?;
1282 store.dbs()?.reader.read(|dao| {
1283 expect![[r#"
1284 [
1285 Amp {
1286 title: "Lasagna Come Out Tomorrow",
1287 url: "https://www.lasagna.restaurant",
1288 raw_url: "https://www.lasagna.restaurant",
1289 icon: None,
1290 icon_mimetype: None,
1291 full_keyword: "lasagna",
1292 block_id: 0,
1293 advertiser: "Good Place Eats",
1294 iab_category: "8 - Food & Drink",
1295 impression_url: "https://example.com/impression_url",
1296 click_url: "https://example.com/click_url",
1297 raw_click_url: "https://example.com/click_url",
1298 score: 0.3,
1299 },
1300 ]
1301 "#]]
1302 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1303 keyword: "la".into(),
1304 providers: vec![SuggestionProvider::Amp],
1305 limit: None,
1306 })?);
1308 Ok(())
1309 })?;
1311 Ok(())
1312 }
1314 /// Tests re-ingesting suggestions from an updated attachment.
1315 #[test]
1316 fn reingest_amp_suggestions() -> anyhow::Result<()> {
1317 before_each();
1319 // Ingest suggestions from the initial snapshot.
1320 let initial_snapshot = Snapshot::with_records(json!([{
1321 "id": "data-1",
1322 "type": "data",
1323 "last_modified": 15,
1324 "attachment": {
1325 "filename": "data-1.json",
1326 "mimetype": "application/json",
1327 "location": "data-1.json",
1328 "hash": "",
1329 "size": 0,
1330 },
1331 }]))?
1332 .with_data(
1333 "data-1.json",
1334 json!([{
1335 "id": 0,
1336 "advertiser": "Good Place Eats",
1337 "iab_category": "8 - Food & Drink",
1338 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
1339 "title": "Lasagna Come Out Tomorrow",
1340 "url": "https://www.lasagna.restaurant",
1341 "icon": "1",
1342 "impression_url": "https://example.com/impression_url",
1343 "click_url": "https://example.com/click_url",
1344 "score": 0.3
1345 }, {
1346 "id": 0,
1347 "advertiser": "Los Pollos Hermanos",
1348 "iab_category": "8 - Food & Drink",
1349 "keywords": ["lo", "los p", "los pollos h"],
1350 "title": "Los Pollos Hermanos - Albuquerque",
1351 "url": "https://www.lph-nm.biz",
1352 "icon": "2",
1353 "impression_url": "https://example.com/impression_url",
1354 "click_url": "https://example.com/click_url",
1355 "score": 0.3
1356 }]),
1357 )?;
1359 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(initial_snapshot));
1361 store.ingest(SuggestIngestionConstraints::default())?;
1363 store.dbs()?.reader.read(|dao| {
1364 assert_eq!(
1365 dao.get_meta(
1366 SuggestRecordType::AmpWikipedia
1367 .last_ingest_meta_key()
1368 .as_str()
1369 )?,
1370 Some(15u64)
1371 );
1372 expect![[r#"
1373 [
1374 Amp {
1375 title: "Lasagna Come Out Tomorrow",
1376 url: "https://www.lasagna.restaurant",
1377 raw_url: "https://www.lasagna.restaurant",
1378 icon: None,
1379 icon_mimetype: None,
1380 full_keyword: "lasagna",
1381 block_id: 0,
1382 advertiser: "Good Place Eats",
1383 iab_category: "8 - Food & Drink",
1384 impression_url: "https://example.com/impression_url",
1385 click_url: "https://example.com/click_url",
1386 raw_click_url: "https://example.com/click_url",
1387 score: 0.3,
1388 },
1389 ]
1390 "#]]
1391 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1392 keyword: "la".into(),
1393 providers: vec![SuggestionProvider::Amp],
1394 limit: None,
1395 })?);
1396 Ok(())
1397 })?;
1399 // Update the snapshot with new suggestions: drop Lasagna, update Los
1400 // Pollos, and add Penne.
1401 *store.settings_client.snapshot.borrow_mut() = Snapshot::with_records(json!([{
1402 "id": "data-1",
1403 "type": "data",
1404 "last_modified": 30,
1405 "attachment": {
1406 "filename": "data-1-1.json",
1407 "mimetype": "application/json",
1408 "location": "data-1-1.json",
1409 "hash": "",
1410 "size": 0,
1411 },
1412 }]))?
1413 .with_data(
1414 "data-1-1.json",
1415 json!([{
1416 "id": 0,
1417 "advertiser": "Los Pollos Hermanos",
1418 "iab_category": "8 - Food & Drink",
1419 "keywords": ["los ", "los pollos", "los pollos hermanos"],
1420 "title": "Los Pollos Hermanos - Now Serving at 14 Locations!",
1421 "url": "https://www.lph-nm.biz",
1422 "icon": "2",
1423 "impression_url": "https://example.com/impression_url",
1424 "click_url": "https://example.com/click_url",
1425 "score": 0.3
1426 }, {
1427 "id": 0,
1428 "advertiser": "Good Place Eats",
1429 "iab_category": "8 - Food & Drink",
1430 "keywords": ["pe", "pen", "penne", "penne for your thoughts"],
1431 "title": "Penne for Your Thoughts",
1432 "url": "https://penne.biz",
1433 "icon": "2",
1434 "impression_url": "https://example.com/impression_url",
1435 "click_url": "https://example.com/click_url",
1436 "score": 0.3
1437 }]),
1438 )?;
1440 store.ingest(SuggestIngestionConstraints::default())?;
1442 store.dbs()?.reader.read(|dao: &SuggestDao<'_>| {
1443 assert_eq!(
1444 dao.get_meta(
1445 SuggestRecordType::AmpWikipedia
1446 .last_ingest_meta_key()
1447 .as_str()
1448 )?,
1449 Some(30u64)
1450 );
1451 assert!(dao
1452 .fetch_suggestions(&SuggestionQuery {
1453 keyword: "la".into(),
1454 providers: vec![SuggestionProvider::Amp],
1455 limit: None,
1456 })?
1457 .is_empty());
1458 expect![[r#"
1459 [
1460 Amp {
1461 title: "Los Pollos Hermanos - Now Serving at 14 Locations!",
1462 url: "https://www.lph-nm.biz",
1463 raw_url: "https://www.lph-nm.biz",
1464 icon: None,
1465 icon_mimetype: None,
1466 full_keyword: "los pollos",
1467 block_id: 0,
1468 advertiser: "Los Pollos Hermanos",
1469 iab_category: "8 - Food & Drink",
1470 impression_url: "https://example.com/impression_url",
1471 click_url: "https://example.com/click_url",
1472 raw_click_url: "https://example.com/click_url",
1473 score: 0.3,
1474 },
1475 ]
1476 "#]]
1477 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1478 keyword: "los ".into(),
1479 providers: vec![SuggestionProvider::Amp],
1480 limit: None,
1481 })?);
1482 expect![[r#"
1483 [
1484 Amp {
1485 title: "Penne for Your Thoughts",
1486 url: "https://penne.biz",
1487 raw_url: "https://penne.biz",
1488 icon: None,
1489 icon_mimetype: None,
1490 full_keyword: "penne",
1491 block_id: 0,
1492 advertiser: "Good Place Eats",
1493 iab_category: "8 - Food & Drink",
1494 impression_url: "https://example.com/impression_url",
1495 click_url: "https://example.com/click_url",
1496 raw_click_url: "https://example.com/click_url",
1497 score: 0.3,
1498 },
1499 ]
1500 "#]]
1501 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1502 keyword: "pe".into(),
1503 providers: vec![SuggestionProvider::Amp],
1504 limit: None,
1505 })?);
1506 Ok(())
1507 })?;
1509 Ok(())
1510 }
1512 /// Tests re-ingesting icons from an updated attachment.
1513 #[test]
1514 fn reingest_icons() -> anyhow::Result<()> {
1515 before_each();
1517 // Ingest suggestions and icons from the initial snapshot.
1518 let initial_snapshot = Snapshot::with_records(json!([{
1519 "id": "data-1",
1520 "type": "data",
1521 "last_modified": 15,
1522 "attachment": {
1523 "filename": "data-1.json",
1524 "mimetype": "application/json",
1525 "location": "data-1.json",
1526 "hash": "",
1527 "size": 0,
1528 },
1529 }, {
1530 "id": "icon-2",
1531 "type": "icon",
1532 "last_modified": 20,
1533 "attachment": {
1534 "filename": "icon-2.png",
1535 "mimetype": "image/png",
1536 "location": "icon-2.png",
1537 "hash": "",
1538 "size": 0,
1539 },
1540 }, {
1541 "id": "icon-3",
1542 "type": "icon",
1543 "last_modified": 25,
1544 "attachment": {
1545 "filename": "icon-3.png",
1546 "mimetype": "image/png",
1547 "location": "icon-3.png",
1548 "hash": "",
1549 "size": 0,
1550 },
1551 }]))?
1552 .with_data(
1553 "data-1.json",
1554 json!([{
1555 "id": 0,
1556 "advertiser": "Good Place Eats",
1557 "iab_category": "8 - Food & Drink",
1558 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
1559 "title": "Lasagna Come Out Tomorrow",
1560 "url": "https://www.lasagna.restaurant",
1561 "icon": "2",
1562 "impression_url": "https://example.com/impression_url",
1563 "click_url": "https://example.com/click_url",
1564 "score": 0.3
1565 }, {
1566 "id": 0,
1567 "advertiser": "Los Pollos Hermanos",
1568 "iab_category": "8 - Food & Drink",
1569 "keywords": ["lo", "los", "los pollos", "los pollos hermanos"],
1570 "title": "Los Pollos Hermanos - Albuquerque",
1571 "url": "https://www.lph-nm.biz",
1572 "icon": "3",
1573 "impression_url": "https://example.com/impression_url",
1574 "click_url": "https://example.com/click_url",
1575 "score": 0.3
1576 }]),
1577 )?
1578 .with_icon("icon-2.png", "lasagna-icon".as_bytes().into())
1579 .with_icon("icon-3.png", "pollos-icon".as_bytes().into());
1581 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(initial_snapshot));
1583 store.ingest(SuggestIngestionConstraints::default())?;
1585 store.dbs()?.reader.read(|dao| {
1586 assert_eq!(
1587 dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
1588 Some(25u64)
1589 );
1590 assert_eq!(
1591 dao.conn
1592 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
1593 2
1594 );
1595 assert_eq!(dao.conn.query_one::<i64>("SELECT count(*) FROM icons")?, 2);
1596 Ok(())
1597 })?;
1599 // Update the snapshot with new icons.
1600 *store.settings_client.snapshot.borrow_mut() = Snapshot::with_records(json!([{
1601 "id": "icon-2",
1602 "type": "icon",
1603 "last_modified": 30,
1604 "attachment": {
1605 "filename": "icon-2.png",
1606 "mimetype": "image/png",
1607 "location": "icon-2.png",
1608 "hash": "",
1609 "size": 0,
1610 },
1611 }, {
1612 "id": "icon-3",
1613 "type": "icon",
1614 "last_modified": 35,
1615 "attachment": {
1616 "filename": "icon-3.png",
1617 "mimetype": "image/png",
1618 "location": "icon-3.png",
1619 "hash": "",
1620 "size": 0,
1621 }
1622 }]))?
1623 .with_icon("icon-2.png", "new-lasagna-icon".as_bytes().into())
1624 .with_icon("icon-3.png", "new-pollos-icon".as_bytes().into());
1626 store.ingest(SuggestIngestionConstraints::default())?;
1628 store.dbs()?.reader.read(|dao| {
1629 assert_eq!(
1630 dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
1631 Some(35u64)
1632 );
1633 expect![[r#"
1634 [
1635 Amp {
1636 title: "Lasagna Come Out Tomorrow",
1637 url: "https://www.lasagna.restaurant",
1638 raw_url: "https://www.lasagna.restaurant",
1639 icon: Some(
1640 [
1641 110,
1642 101,
1643 119,
1644 45,
1645 108,
1646 97,
1647 115,
1648 97,
1649 103,
1650 110,
1651 97,
1652 45,
1653 105,
1654 99,
1655 111,
1656 110,
1657 ],
1658 ),
1659 icon_mimetype: Some(
1660 "image/png",
1661 ),
1662 full_keyword: "lasagna",
1663 block_id: 0,
1664 advertiser: "Good Place Eats",
1665 iab_category: "8 - Food & Drink",
1666 impression_url: "https://example.com/impression_url",
1667 click_url: "https://example.com/click_url",
1668 raw_click_url: "https://example.com/click_url",
1669 score: 0.3,
1670 },
1671 ]
1672 "#]]
1673 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1674 keyword: "la".into(),
1675 providers: vec![SuggestionProvider::Amp],
1676 limit: None,
1677 })?);
1678 expect![[r#"
1679 [
1680 Amp {
1681 title: "Los Pollos Hermanos - Albuquerque",
1682 url: "https://www.lph-nm.biz",
1683 raw_url: "https://www.lph-nm.biz",
1684 icon: Some(
1685 [
1686 110,
1687 101,
1688 119,
1689 45,
1690 112,
1691 111,
1692 108,
1693 108,
1694 111,
1695 115,
1696 45,
1697 105,
1698 99,
1699 111,
1700 110,
1701 ],
1702 ),
1703 icon_mimetype: Some(
1704 "image/png",
1705 ),
1706 full_keyword: "los",
1707 block_id: 0,
1708 advertiser: "Los Pollos Hermanos",
1709 iab_category: "8 - Food & Drink",
1710 impression_url: "https://example.com/impression_url",
1711 click_url: "https://example.com/click_url",
1712 raw_click_url: "https://example.com/click_url",
1713 score: 0.3,
1714 },
1715 ]
1716 "#]]
1717 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1718 keyword: "lo".into(),
1719 providers: vec![SuggestionProvider::Amp],
1720 limit: None,
1721 })?);
1722 Ok(())
1723 })?;
1725 Ok(())
1726 }
1728 /// Tests re-ingesting AMO suggestions from an updated attachment.
1729 #[test]
1730 fn reingest_amo_suggestions() -> anyhow::Result<()> {
1731 before_each();
1733 // Ingest suggestions from the initial snapshot.
1734 let initial_snapshot = Snapshot::with_records(json!([{
1735 "id": "data-1",
1736 "type": "amo-suggestions",
1737 "last_modified": 15,
1738 "attachment": {
1739 "filename": "data-1.json",
1740 "mimetype": "application/json",
1741 "location": "data-1.json",
1742 "hash": "",
1743 "size": 0,
1744 },
1745 }, {
1746 "id": "data-2",
1747 "type": "amo-suggestions",
1748 "last_modified": 15,
1749 "attachment": {
1750 "filename": "data-2.json",
1751 "mimetype": "application/json",
1752 "location": "data-2.json",
1753 "hash": "",
1754 "size": 0,
1755 },
1756 }]))?
1757 .with_data(
1758 "data-1.json",
1759 json!({
1760 "description": "First suggestion",
1761 "url": "https://example.org/amo-suggestion-1",
1762 "guid": "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
1763 "keywords": ["relay", "spam", "masking email", "alias"],
1764 "title": "AMO suggestion",
1765 "icon": "https://example.org/amo-suggestion-1/icon.png",
1766 "rating": "4.9",
1767 "number_of_ratings": 800,
1768 "score": 0.25
1769 }),
1770 )?
1771 .with_data(
1772 "data-2.json",
1773 json!([{
1774 "description": "Second suggestion",
1775 "url": "https://example.org/amo-suggestion-2",
1776 "guid": "{6d24e3b8-1400-4d37-9440-c798f9b79b1a}",
1777 "keywords": ["dark mode", "dark theme", "night mode"],
1778 "title": "Another AMO suggestion",
1779 "icon": "https://example.org/amo-suggestion-2/icon.png",
1780 "rating": "4.6",
1781 "number_of_ratings": 750,
1782 "score": 0.25
1783 }, {
1784 "description": "Third suggestion",
1785 "url": "https://example.org/amo-suggestion-3",
1786 "guid": "{1e9d493b-0498-48bb-9b9a-8b45a44df146}",
1787 "keywords": ["grammar", "spelling", "edit"],
1788 "title": "Yet another AMO suggestion",
1789 "icon": "https://example.org/amo-suggestion-3/icon.png",
1790 "rating": "4.8",
1791 "number_of_ratings": 900,
1792 "score": 0.25
1793 }]),
1794 )?;
1796 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(initial_snapshot));
1798 store.ingest(SuggestIngestionConstraints::default())?;
1800 store.dbs()?.reader.read(|dao| {
1801 assert_eq!(
1802 dao.get_meta(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?,
1803 Some(15u64)
1804 );
1806 expect![[r#"
1807 [
1808 Amo {
1809 title: "AMO suggestion",
1810 url: "https://example.org/amo-suggestion-1",
1811 icon_url: "https://example.org/amo-suggestion-1/icon.png",
1812 description: "First suggestion",
1813 rating: Some(
1814 "4.9",
1815 ),
1816 number_of_ratings: 800,
1817 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
1818 score: 0.25,
1819 },
1820 ]
1821 "#]]
1822 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1823 keyword: "masking e".into(),
1824 providers: vec![SuggestionProvider::Amo],
1825 limit: None,
1826 })?);
1828 expect![[r#"
1829 [
1830 Amo {
1831 title: "Another AMO suggestion",
1832 url: "https://example.org/amo-suggestion-2",
1833 icon_url: "https://example.org/amo-suggestion-2/icon.png",
1834 description: "Second suggestion",
1835 rating: Some(
1836 "4.6",
1837 ),
1838 number_of_ratings: 750,
1839 guid: "{6d24e3b8-1400-4d37-9440-c798f9b79b1a}",
1840 score: 0.25,
1841 },
1842 ]
1843 "#]]
1844 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1845 keyword: "night".into(),
1846 providers: vec![SuggestionProvider::Amo],
1847 limit: None,
1848 })?);
1850 Ok(())
1851 })?;
1853 // Update the snapshot with new suggestions: update the second, drop the
1854 // third, and add the fourth.
1855 *store.settings_client.snapshot.borrow_mut() = Snapshot::with_records(json!([{
1856 "id": "data-2",
1857 "type": "amo-suggestions",
1858 "last_modified": 30,
1859 "attachment": {
1860 "filename": "data-2-1.json",
1861 "mimetype": "application/json",
1862 "location": "data-2-1.json",
1863 "hash": "",
1864 "size": 0,
1865 },
1866 }]))?
1867 .with_data(
1868 "data-2-1.json",
1869 json!([{
1870 "description": "Updated second suggestion",
1871 "url": "https://example.org/amo-suggestion-2",
1872 "guid": "{6d24e3b8-1400-4d37-9440-c798f9b79b1a}",
1873 "keywords": ["dark mode", "night mode"],
1874 "title": "Another AMO suggestion",
1875 "icon": "https://example.org/amo-suggestion-2/icon.png",
1876 "rating": "4.7",
1877 "number_of_ratings": 775,
1878 "score": 0.25
1879 }, {
1880 "description": "Fourth suggestion",
1881 "url": "https://example.org/amo-suggestion-4",
1882 "guid": "{1ea82ebd-a1ba-4f57-b8bb-3824ead837bd}",
1883 "keywords": ["image search", "visual search"],
1884 "title": "New AMO suggestion",
1885 "icon": "https://example.org/amo-suggestion-4/icon.png",
1886 "rating": "5.0",
1887 "number_of_ratings": 100,
1888 "score": 0.25
1889 }]),
1890 )?;
1892 store.ingest(SuggestIngestionConstraints::default())?;
1894 store.dbs()?.reader.read(|dao| {
1895 assert_eq!(
1896 dao.get_meta(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?,
1897 Some(30u64)
1898 );
1900 expect![[r#"
1901 [
1902 Amo {
1903 title: "AMO suggestion",
1904 url: "https://example.org/amo-suggestion-1",
1905 icon_url: "https://example.org/amo-suggestion-1/icon.png",
1906 description: "First suggestion",
1907 rating: Some(
1908 "4.9",
1909 ),
1910 number_of_ratings: 800,
1911 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
1912 score: 0.25,
1913 },
1914 ]
1915 "#]]
1916 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1917 keyword: "masking e".into(),
1918 providers: vec![SuggestionProvider::Amo],
1919 limit: None,
1920 })?);
1922 expect![[r#"
1923 []
1924 "#]]
1925 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1926 keyword: "dark t".into(),
1927 providers: vec![SuggestionProvider::Amo],
1928 limit: None,
1929 })?);
1931 expect![[r#"
1932 [
1933 Amo {
1934 title: "Another AMO suggestion",
1935 url: "https://example.org/amo-suggestion-2",
1936 icon_url: "https://example.org/amo-suggestion-2/icon.png",
1937 description: "Updated second suggestion",
1938 rating: Some(
1939 "4.7",
1940 ),
1941 number_of_ratings: 775,
1942 guid: "{6d24e3b8-1400-4d37-9440-c798f9b79b1a}",
1943 score: 0.25,
1944 },
1945 ]
1946 "#]]
1947 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1948 keyword: "night".into(),
1949 providers: vec![SuggestionProvider::Amo],
1950 limit: None,
1951 })?);
1953 expect![[r#"
1954 [
1955 Amo {
1956 title: "New AMO suggestion",
1957 url: "https://example.org/amo-suggestion-4",
1958 icon_url: "https://example.org/amo-suggestion-4/icon.png",
1959 description: "Fourth suggestion",
1960 rating: Some(
1961 "5.0",
1962 ),
1963 number_of_ratings: 100,
1964 guid: "{1ea82ebd-a1ba-4f57-b8bb-3824ead837bd}",
1965 score: 0.25,
1966 },
1967 ]
1968 "#]]
1969 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
1970 keyword: "image search".into(),
1971 providers: vec![SuggestionProvider::Amo],
1972 limit: None,
1973 })?);
1975 Ok(())
1976 })?;
1978 Ok(())
1979 }
1981 /// Tests ingesting tombstones for previously-ingested suggestions and
1982 /// icons.
1983 #[test]
1984 fn ingest_tombstones() -> anyhow::Result<()> {
1985 before_each();
1987 // Ingest suggestions and icons from the initial snapshot.
1988 let initial_snapshot = Snapshot::with_records(json!([{
1989 "id": "data-1",
1990 "type": "data",
1991 "last_modified": 15,
1992 "attachment": {
1993 "filename": "data-1.json",
1994 "mimetype": "application/json",
1995 "location": "data-1.json",
1996 "hash": "",
1997 "size": 0,
1998 },
1999 }, {
2000 "id": "icon-2",
2001 "type": "icon",
2002 "last_modified": 20,
2003 "attachment": {
2004 "filename": "icon-2.png",
2005 "mimetype": "image/png",
2006 "location": "icon-2.png",
2007 "hash": "",
2008 "size": 0,
2009 },
2010 }]))?
2011 .with_data(
2012 "data-1.json",
2013 json!([{
2014 "id": 0,
2015 "advertiser": "Good Place Eats",
2016 "iab_category": "8 - Food & Drink",
2017 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
2018 "title": "Lasagna Come Out Tomorrow",
2019 "url": "https://www.lasagna.restaurant",
2020 "icon": "2",
2021 "impression_url": "https://example.com/impression_url",
2022 "click_url": "https://example.com/click_url",
2023 "score": 0.3
2024 }]),
2025 )?
2026 .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into());
2028 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(initial_snapshot));
2030 store.ingest(SuggestIngestionConstraints::default())?;
2032 store.dbs()?.reader.read(|dao| {
2033 assert_eq!(
2034 dao.conn
2035 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
2036 1
2037 );
2038 assert_eq!(dao.conn.query_one::<i64>("SELECT count(*) FROM icons")?, 1);
2039 assert_eq!(
2040 dao.get_meta(
2041 SuggestRecordType::AmpWikipedia
2042 .last_ingest_meta_key()
2043 .as_str()
2044 )?,
2045 Some(15)
2046 );
2047 assert_eq!(
2048 dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
2049 Some(20)
2050 );
2052 Ok(())
2053 })?;
2055 // Replace the records with tombstones. Ingesting these should remove
2056 // all their suggestions and icons.
2057 *store.settings_client.snapshot.borrow_mut() = Snapshot::with_records(json!([{
2058 "id": "data-1",
2059 "last_modified": 25,
2060 "deleted": true,
2061 }, {
2062 "id": "icon-2",
2063 "last_modified": 30,
2064 "deleted": true,
2065 }]))?;
2067 store.ingest(SuggestIngestionConstraints::default())?;
2069 store.dbs()?.reader.read(|dao| {
2070 assert_eq!(
2071 dao.conn
2072 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
2073 0
2074 );
2075 assert_eq!(dao.conn.query_one::<i64>("SELECT count(*) FROM icons")?, 0);
2076 assert_eq!(
2077 dao.get_meta(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
2078 Some(30)
2079 );
2080 Ok(())
2081 })?;
2083 Ok(())
2084 }
2086 /// Tests ingesting suggestions with constraints.
2087 #[test]
2088 fn ingest_with_constraints() -> anyhow::Result<()> {
2089 before_each();
2091 let snapshot = Snapshot::with_records(json!([]))?;
2093 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
2095 store.ingest(SuggestIngestionConstraints::default())?;
2096 assert_eq!(
2097 store.settings_client.last_get_records_option("_limit"),
2098 None,
2099 );
2101 // 200 suggestions per record, so test with numbers around that
2102 // boundary.
2103 let table = [
2104 (0, "1"),
2105 (199, "1"),
2106 (200, "1"),
2107 (201, "2"),
2108 (300, "2"),
2109 (400, "2"),
2110 (401, "3"),
2111 ];
2112 for (max_suggestions, expected_limit) in table {
2113 store.ingest(SuggestIngestionConstraints {
2114 max_suggestions: Some(max_suggestions),
2115 providers: Some(vec![SuggestionProvider::Amp]),
2116 })?;
2117 let actual_limit = store
2118 .settings_client
2119 .last_get_records_option("_limit")
2120 .ok_or_else(|| {
2121 anyhow!("Want limit = {} for {}", expected_limit, max_suggestions)
2122 })?;
2123 assert_eq!(
2124 actual_limit, expected_limit,
2125 "Want limit = {} for {}; got limit = {}",
2126 expected_limit, max_suggestions, actual_limit
2127 );
2128 }
2130 Ok(())
2131 }
2133 /// Tests clearing the store.
2134 #[test]
2135 fn clear() -> anyhow::Result<()> {
2136 before_each();
2138 let snapshot = Snapshot::with_records(json!([{
2139 "id": "data-1",
2140 "type": "data",
2141 "last_modified": 15,
2142 "attachment": {
2143 "filename": "data-1.json",
2144 "mimetype": "application/json",
2145 "location": "data-1.json",
2146 "hash": "",
2147 "size": 0,
2148 },
2149 }]))?
2150 .with_data(
2151 "data-1.json",
2152 json!([{
2153 "id": 0,
2154 "advertiser": "Los Pollos Hermanos",
2155 "iab_category": "8 - Food & Drink",
2156 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
2157 "title": "Los Pollos Hermanos - Albuquerque",
2158 "url": "https://www.lph-nm.biz",
2159 "icon": "2",
2160 "impression_url": "https://example.com",
2161 "click_url": "https://example.com",
2162 "score": 0.3
2163 }]),
2164 )?;
2166 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
2168 store.ingest(SuggestIngestionConstraints::default())?;
2170 store.dbs()?.reader.read(|dao| {
2171 assert_eq!(
2172 dao.get_meta::<u64>(
2173 SuggestRecordType::AmpWikipedia
2174 .last_ingest_meta_key()
2175 .as_str()
2176 )?,
2177 Some(15)
2178 );
2179 assert_eq!(
2180 dao.conn
2181 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
2182 1
2183 );
2184 assert_eq!(
2185 dao.conn.query_one::<i64>("SELECT count(*) FROM keywords")?,
2186 6
2187 );
2189 Ok(())
2190 })?;
2192 store.clear()?;
2194 store.dbs()?.reader.read(|dao| {
2195 assert_eq!(
2196 dao.get_meta::<u64>(
2197 SuggestRecordType::AmpWikipedia
2198 .last_ingest_meta_key()
2199 .as_str()
2200 )?,
2201 None
2202 );
2203 assert_eq!(
2204 dao.conn
2205 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
2206 0
2207 );
2208 assert_eq!(
2209 dao.conn.query_one::<i64>("SELECT count(*) FROM keywords")?,
2210 0
2211 );
2213 Ok(())
2214 })?;
2216 Ok(())
2217 }
2219 /// Tests querying suggestions.
2220 #[test]
2221 fn query() -> anyhow::Result<()> {
2222 before_each();
2224 let snapshot = Snapshot::with_records(json!([{
2225 "id": "data-1",
2226 "type": "data",
2227 "last_modified": 15,
2228 "attachment": {
2229 "filename": "data-1.json",
2230 "mimetype": "application/json",
2231 "location": "data-1.json",
2232 "hash": "",
2233 "size": 0,
2234 },
2236 }, {
2237 "id": "data-2",
2238 "type": "amo-suggestions",
2239 "last_modified": 15,
2240 "attachment": {
2241 "filename": "data-2.json",
2242 "mimetype": "application/json",
2243 "location": "data-2.json",
2244 "hash": "",
2245 "size": 0,
2246 },
2247 }, {
2248 "id": "data-3",
2249 "type": "pocket-suggestions",
2250 "last_modified": 15,
2251 "attachment": {
2252 "filename": "data-3.json",
2253 "mimetype": "application/json",
2254 "location": "data-3.json",
2255 "hash": "",
2256 "size": 0,
2257 },
2258 }, {
2259 "id": "data-4",
2260 "type": "yelp-suggestions",
2261 "last_modified": 15,
2262 "attachment": {
2263 "filename": "data-4.json",
2264 "mimetype": "application/json",
2265 "location": "data-4.json",
2266 "hash": "",
2267 "size": 0,
2268 },
2269 }, {
2270 "id": "data-5",
2271 "type": "mdn-suggestions",
2272 "last_modified": 15,
2273 "attachment": {
2274 "filename": "data-5.json",
2275 "mimetype": "application/json",
2276 "location": "data-5.json",
2277 "hash": "",
2278 "size": 0,
2279 },
2280 }, {
2281 "id": "icon-2",
2282 "type": "icon",
2283 "last_modified": 20,
2284 "attachment": {
2285 "filename": "icon-2.png",
2286 "mimetype": "image/png",
2287 "location": "icon-2.png",
2288 "hash": "",
2289 "size": 0,
2290 },
2291 }, {
2292 "id": "icon-3",
2293 "type": "icon",
2294 "last_modified": 25,
2295 "attachment": {
2296 "filename": "icon-3.png",
2297 "mimetype": "image/png",
2298 "location": "icon-3.png",
2299 "hash": "",
2300 "size": 0,
2301 },
2302 }, {
2303 "id": "icon-yelp-favicon",
2304 "type": "icon",
2305 "last_modified": 25,
2306 "attachment": {
2307 "filename": "yelp-favicon.svg",
2308 "mimetype": "image/svg+xml",
2309 "location": "yelp-favicon.svg",
2310 "hash": "",
2311 "size": 0,
2312 },
2313 }]))?
2314 .with_data(
2315 "data-1.json",
2316 json!([{
2317 "id": 0,
2318 "advertiser": "Good Place Eats",
2319 "iab_category": "8 - Food & Drink",
2320 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
2321 "title": "Lasagna Come Out Tomorrow",
2322 "url": "https://www.lasagna.restaurant",
2323 "icon": "2",
2324 "impression_url": "https://example.com/impression_url",
2325 "click_url": "https://example.com/click_url",
2326 "score": 0.3
2327 }, {
2328 "id": 0,
2329 "advertiser": "Wikipedia",
2330 "iab_category": "5 - Education",
2331 "keywords": ["cal", "cali", "california"],
2332 "title": "California",
2333 "url": "https://wikipedia.org/California",
2334 "icon": "3"
2335 }, {
2336 "id": 0,
2337 "advertiser": "Wikipedia",
2338 "iab_category": "5 - Education",
2339 "keywords": ["cal", "cali", "california", "institute", "technology"],
2340 "title": "California Institute of Technology",
2341 "url": "https://wikipedia.org/California_Institute_of_Technology",
2342 "icon": "3"
2343 },{
2344 "id": 0,
2345 "advertiser": "Wikipedia",
2346 "iab_category": "5 - Education",
2347 "keywords": ["multimatch"],
2348 "title": "Multimatch",
2349 "url": "https://wikipedia.org/Multimatch",
2350 "icon": "3"
2351 }]),
2352 )?
2353 .with_data(
2354 "data-2.json",
2355 json!([
2356 {
2357 "description": "amo suggestion",
2358 "url": "https://addons.mozilla.org/en-US/firefox/addon/example",
2359 "guid": "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2360 "keywords": ["relay", "spam", "masking email", "alias"],
2361 "title": "Firefox Relay",
2362 "icon": "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2363 "rating": "4.9",
2364 "number_of_ratings": 888,
2365 "score": 0.25
2366 },
2367 {
2368 "description": "amo suggestion multi-match",
2369 "url": "https://addons.mozilla.org/en-US/firefox/addon/multimatch",
2370 "guid": "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2371 "keywords": ["multimatch"],
2372 "title": "Firefox Multimatch",
2373 "icon": "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2374 "rating": "4.9",
2375 "number_of_ratings": 888,
2376 "score": 0.25
2377 },
2378 ]),
2379 )?
2380 .with_data(
2381 "data-3.json",
2382 json!([
2383 {
2384 "description": "pocket suggestion",
2385 "url": "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
2386 "lowConfidenceKeywords": ["soft life", "workaholism", "toxic work culture", "work-life balance"],
2387 "highConfidenceKeywords": ["burnout women", "grind culture", "women burnout"],
2388 "title": "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
2389 "score": 0.25
2390 },
2391 {
2392 "description": "pocket suggestion multi-match",
2393 "url": "https://getpocket.com/collections/multimatch",
2394 "lowConfidenceKeywords": [],
2395 "highConfidenceKeywords": ["multimatch"],
2396 "title": "Multimatching",
2397 "score": 0.88
2398 },
2399 ]),
2400 )?
2401 .with_data(
2402 "data-4.json",
2403 json!({
2404 "subjects": ["ramen", "spicy ramen", "spicy random ramen", "rats", "raven", "raccoon", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789Z"],
2405 "preModifiers": ["best", "super best", "same_modifier"],
2406 "postModifiers": ["delivery", "super delivery", "same_modifier"],
2407 "locationSigns": [
2408 { "keyword": "in", "needLocation": true },
2409 { "keyword": "near", "needLocation": true },
2410 { "keyword": "near by", "needLocation": false },
2411 { "keyword": "near me", "needLocation": false },
2412 ],
2413 "yelpModifiers": ["yelp", "yelp keyword"],
2414 "icon": "yelp-favicon",
2415 "score": 0.5
2416 }),
2417 )?
2418 .with_data(
2419 "data-5.json",
2420 json!([
2421 {
2422 "description": "Javascript Array",
2423 "url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
2424 "keywords": ["array javascript", "javascript array", "wildcard"],
2425 "title": "Array",
2426 "score": 0.24
2427 },
2428 ]),
2429 )?
2430 .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into())
2431 .with_icon("icon-3.png", "also-an-icon".as_bytes().into())
2432 .with_icon("yelp-favicon.svg", "yelp-icon".as_bytes().into());
2434 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
2436 store.ingest(SuggestIngestionConstraints::default())?;
2438 let table = [
2439 (
2440 "empty keyword; all providers",
2441 SuggestionQuery {
2442 keyword: String::new(),
2443 providers: vec![
2444 SuggestionProvider::Amp,
2445 SuggestionProvider::Wikipedia,
2446 SuggestionProvider::Amo,
2447 SuggestionProvider::Pocket,
2448 SuggestionProvider::Yelp,
2449 SuggestionProvider::Weather,
2450 ],
2451 limit: None,
2452 },
2453 expect![[r#"
2454 []
2455 "#]],
2456 ),
2457 (
2458 "keyword = `la`; all providers",
2459 SuggestionQuery {
2460 keyword: "la".into(),
2461 providers: vec![
2462 SuggestionProvider::Amp,
2463 SuggestionProvider::Wikipedia,
2464 SuggestionProvider::Amo,
2465 SuggestionProvider::Pocket,
2466 SuggestionProvider::Yelp,
2467 SuggestionProvider::Weather,
2468 ],
2469 limit: None,
2470 },
2471 expect![[r#"
2472 [
2473 Amp {
2474 title: "Lasagna Come Out Tomorrow",
2475 url: "https://www.lasagna.restaurant",
2476 raw_url: "https://www.lasagna.restaurant",
2477 icon: Some(
2478 [
2479 105,
2480 45,
2481 97,
2482 109,
2483 45,
2484 97,
2485 110,
2486 45,
2487 105,
2488 99,
2489 111,
2490 110,
2491 ],
2492 ),
2493 icon_mimetype: Some(
2494 "image/png",
2495 ),
2496 full_keyword: "lasagna",
2497 block_id: 0,
2498 advertiser: "Good Place Eats",
2499 iab_category: "8 - Food & Drink",
2500 impression_url: "https://example.com/impression_url",
2501 click_url: "https://example.com/click_url",
2502 raw_click_url: "https://example.com/click_url",
2503 score: 0.3,
2504 },
2505 ]
2506 "#]],
2507 ),
2508 (
2509 "multimatch; all providers",
2510 SuggestionQuery {
2511 keyword: "multimatch".into(),
2512 providers: vec![
2513 SuggestionProvider::Amp,
2514 SuggestionProvider::Wikipedia,
2515 SuggestionProvider::Amo,
2516 SuggestionProvider::Pocket,
2517 ],
2518 limit: None,
2519 },
2520 expect![[r#"
2521 [
2522 Pocket {
2523 title: "Multimatching",
2524 url: "https://getpocket.com/collections/multimatch",
2525 score: 0.88,
2526 is_top_pick: true,
2527 },
2528 Amo {
2529 title: "Firefox Multimatch",
2530 url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch",
2531 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2532 description: "amo suggestion multi-match",
2533 rating: Some(
2534 "4.9",
2535 ),
2536 number_of_ratings: 888,
2537 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2538 score: 0.25,
2539 },
2540 Wikipedia {
2541 title: "Multimatch",
2542 url: "https://wikipedia.org/Multimatch",
2543 icon: Some(
2544 [
2545 97,
2546 108,
2547 115,
2548 111,
2549 45,
2550 97,
2551 110,
2552 45,
2553 105,
2554 99,
2555 111,
2556 110,
2557 ],
2558 ),
2559 icon_mimetype: Some(
2560 "image/png",
2561 ),
2562 full_keyword: "multimatch",
2563 },
2564 ]
2565 "#]],
2566 ),
2567 (
2568 "MultiMatch; all providers, mixed case",
2569 SuggestionQuery {
2570 keyword: "MultiMatch".into(),
2571 providers: vec![
2572 SuggestionProvider::Amp,
2573 SuggestionProvider::Wikipedia,
2574 SuggestionProvider::Amo,
2575 SuggestionProvider::Pocket,
2576 ],
2577 limit: None,
2578 },
2579 expect![[r#"
2580 [
2581 Pocket {
2582 title: "Multimatching",
2583 url: "https://getpocket.com/collections/multimatch",
2584 score: 0.88,
2585 is_top_pick: true,
2586 },
2587 Amo {
2588 title: "Firefox Multimatch",
2589 url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch",
2590 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2591 description: "amo suggestion multi-match",
2592 rating: Some(
2593 "4.9",
2594 ),
2595 number_of_ratings: 888,
2596 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2597 score: 0.25,
2598 },
2599 Wikipedia {
2600 title: "Multimatch",
2601 url: "https://wikipedia.org/Multimatch",
2602 icon: Some(
2603 [
2604 97,
2605 108,
2606 115,
2607 111,
2608 45,
2609 97,
2610 110,
2611 45,
2612 105,
2613 99,
2614 111,
2615 110,
2616 ],
2617 ),
2618 icon_mimetype: Some(
2619 "image/png",
2620 ),
2621 full_keyword: "multimatch",
2622 },
2623 ]
2624 "#]],
2625 ),
2626 (
2627 "multimatch; all providers, limit 2",
2628 SuggestionQuery {
2629 keyword: "multimatch".into(),
2630 providers: vec![
2631 SuggestionProvider::Amp,
2632 SuggestionProvider::Wikipedia,
2633 SuggestionProvider::Amo,
2634 SuggestionProvider::Pocket,
2635 ],
2636 limit: Some(2),
2637 },
2638 expect![[r#"
2639 [
2640 Pocket {
2641 title: "Multimatching",
2642 url: "https://getpocket.com/collections/multimatch",
2643 score: 0.88,
2644 is_top_pick: true,
2645 },
2646 Amo {
2647 title: "Firefox Multimatch",
2648 url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch",
2649 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2650 description: "amo suggestion multi-match",
2651 rating: Some(
2652 "4.9",
2653 ),
2654 number_of_ratings: 888,
2655 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2656 score: 0.25,
2657 },
2658 ]
2659 "#]],
2660 ),
2661 (
2662 "keyword = `la`; AMP only",
2663 SuggestionQuery {
2664 keyword: "la".into(),
2665 providers: vec![SuggestionProvider::Amp],
2666 limit: None,
2667 },
2668 expect![[r#"
2669 [
2670 Amp {
2671 title: "Lasagna Come Out Tomorrow",
2672 url: "https://www.lasagna.restaurant",
2673 raw_url: "https://www.lasagna.restaurant",
2674 icon: Some(
2675 [
2676 105,
2677 45,
2678 97,
2679 109,
2680 45,
2681 97,
2682 110,
2683 45,
2684 105,
2685 99,
2686 111,
2687 110,
2688 ],
2689 ),
2690 icon_mimetype: Some(
2691 "image/png",
2692 ),
2693 full_keyword: "lasagna",
2694 block_id: 0,
2695 advertiser: "Good Place Eats",
2696 iab_category: "8 - Food & Drink",
2697 impression_url: "https://example.com/impression_url",
2698 click_url: "https://example.com/click_url",
2699 raw_click_url: "https://example.com/click_url",
2700 score: 0.3,
2701 },
2702 ]
2703 "#]],
2704 ),
2705 (
2706 "keyword = `la`; Wikipedia, AMO, and Pocket",
2707 SuggestionQuery {
2708 keyword: "la".into(),
2709 providers: vec![
2710 SuggestionProvider::Wikipedia,
2711 SuggestionProvider::Amo,
2712 SuggestionProvider::Pocket,
2713 ],
2714 limit: None,
2715 },
2716 expect![[r#"
2717 []
2718 "#]],
2719 ),
2720 (
2721 "keyword = `la`; no providers",
2722 SuggestionQuery {
2723 keyword: "la".into(),
2724 providers: vec![],
2725 limit: None,
2726 },
2727 expect![[r#"
2728 []
2729 "#]],
2730 ),
2731 (
2732 "keyword = `cal`; AMP, AMO, and Pocket",
2733 SuggestionQuery {
2734 keyword: "cal".into(),
2735 providers: vec![
2736 SuggestionProvider::Amp,
2737 SuggestionProvider::Amo,
2738 SuggestionProvider::Pocket,
2739 ],
2740 limit: None,
2741 },
2742 expect![[r#"
2743 []
2744 "#]],
2745 ),
2746 (
2747 "keyword = `cal`; Wikipedia only",
2748 SuggestionQuery {
2749 keyword: "cal".into(),
2750 providers: vec![SuggestionProvider::Wikipedia],
2751 limit: None,
2752 },
2753 expect![[r#"
2754 [
2755 Wikipedia {
2756 title: "California",
2757 url: "https://wikipedia.org/California",
2758 icon: Some(
2759 [
2760 97,
2761 108,
2762 115,
2763 111,
2764 45,
2765 97,
2766 110,
2767 45,
2768 105,
2769 99,
2770 111,
2771 110,
2772 ],
2773 ),
2774 icon_mimetype: Some(
2775 "image/png",
2776 ),
2777 full_keyword: "california",
2778 },
2779 Wikipedia {
2780 title: "California Institute of Technology",
2781 url: "https://wikipedia.org/California_Institute_of_Technology",
2782 icon: Some(
2783 [
2784 97,
2785 108,
2786 115,
2787 111,
2788 45,
2789 97,
2790 110,
2791 45,
2792 105,
2793 99,
2794 111,
2795 110,
2796 ],
2797 ),
2798 icon_mimetype: Some(
2799 "image/png",
2800 ),
2801 full_keyword: "california",
2802 },
2803 ]
2804 "#]],
2805 ),
2806 (
2807 "keyword = `cal`; Wikipedia with limit 1",
2808 SuggestionQuery {
2809 keyword: "cal".into(),
2810 providers: vec![SuggestionProvider::Wikipedia],
2811 limit: Some(1),
2812 },
2813 expect![[r#"
2814 [
2815 Wikipedia {
2816 title: "California",
2817 url: "https://wikipedia.org/California",
2818 icon: Some(
2819 [
2820 97,
2821 108,
2822 115,
2823 111,
2824 45,
2825 97,
2826 110,
2827 45,
2828 105,
2829 99,
2830 111,
2831 110,
2832 ],
2833 ),
2834 icon_mimetype: Some(
2835 "image/png",
2836 ),
2837 full_keyword: "california",
2838 },
2839 ]
2840 "#]],
2841 ),
2842 (
2843 "keyword = `cal`; no providers",
2844 SuggestionQuery {
2845 keyword: "cal".into(),
2846 providers: vec![],
2847 limit: None,
2848 },
2849 expect![[r#"
2850 []
2851 "#]],
2852 ),
2853 (
2854 "keyword = `spam`; AMO only",
2855 SuggestionQuery {
2856 keyword: "spam".into(),
2857 providers: vec![SuggestionProvider::Amo],
2858 limit: None,
2859 },
2860 expect![[r#"
2861 [
2862 Amo {
2863 title: "Firefox Relay",
2864 url: "https://addons.mozilla.org/en-US/firefox/addon/example",
2865 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2866 description: "amo suggestion",
2867 rating: Some(
2868 "4.9",
2869 ),
2870 number_of_ratings: 888,
2871 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2872 score: 0.25,
2873 },
2874 ]
2875 "#]],
2876 ),
2877 (
2878 "keyword = `masking`; AMO only",
2879 SuggestionQuery {
2880 keyword: "masking".into(),
2881 providers: vec![SuggestionProvider::Amo],
2882 limit: None,
2883 },
2884 expect![[r#"
2885 [
2886 Amo {
2887 title: "Firefox Relay",
2888 url: "https://addons.mozilla.org/en-US/firefox/addon/example",
2889 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2890 description: "amo suggestion",
2891 rating: Some(
2892 "4.9",
2893 ),
2894 number_of_ratings: 888,
2895 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2896 score: 0.25,
2897 },
2898 ]
2899 "#]],
2900 ),
2901 (
2902 "keyword = `masking e`; AMO only",
2903 SuggestionQuery {
2904 keyword: "masking e".into(),
2905 providers: vec![SuggestionProvider::Amo],
2906 limit: None,
2907 },
2908 expect![[r#"
2909 [
2910 Amo {
2911 title: "Firefox Relay",
2912 url: "https://addons.mozilla.org/en-US/firefox/addon/example",
2913 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
2914 description: "amo suggestion",
2915 rating: Some(
2916 "4.9",
2917 ),
2918 number_of_ratings: 888,
2919 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
2920 score: 0.25,
2921 },
2922 ]
2923 "#]],
2924 ),
2925 (
2926 "keyword = `masking s`; AMO only",
2927 SuggestionQuery {
2928 keyword: "masking s".into(),
2929 providers: vec![SuggestionProvider::Amo],
2930 limit: None,
2931 },
2932 expect![[r#"
2933 []
2934 "#]],
2935 ),
2936 (
2937 "keyword = `soft`; AMP and Wikipedia",
2938 SuggestionQuery {
2939 keyword: "soft".into(),
2940 providers: vec![SuggestionProvider::Amp, SuggestionProvider::Wikipedia],
2941 limit: None,
2942 },
2943 expect![[r#"
2944 []
2945 "#]],
2946 ),
2947 (
2948 "keyword = `soft`; Pocket only",
2949 SuggestionQuery {
2950 keyword: "soft".into(),
2951 providers: vec![SuggestionProvider::Pocket],
2952 limit: None,
2953 },
2954 expect![[r#"
2955 [
2956 Pocket {
2957 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
2958 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
2959 score: 0.25,
2960 is_top_pick: false,
2961 },
2962 ]
2963 "#]],
2964 ),
2965 (
2966 "keyword = `soft l`; Pocket only",
2967 SuggestionQuery {
2968 keyword: "soft l".into(),
2969 providers: vec![SuggestionProvider::Pocket],
2970 limit: None,
2971 },
2972 expect![[r#"
2973 [
2974 Pocket {
2975 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
2976 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
2977 score: 0.25,
2978 is_top_pick: false,
2979 },
2980 ]
2981 "#]],
2982 ),
2983 (
2984 "keyword = `sof`; Pocket only",
2985 SuggestionQuery {
2986 keyword: "sof".into(),
2987 providers: vec![SuggestionProvider::Pocket],
2988 limit: None,
2989 },
2990 expect![[r#"
2991 []
2992 "#]],
2993 ),
2994 (
2995 "keyword = `burnout women`; Pocket only",
2996 SuggestionQuery {
2997 keyword: "burnout women".into(),
2998 providers: vec![SuggestionProvider::Pocket],
2999 limit: None,
3000 },
3001 expect![[r#"
3002 [
3003 Pocket {
3004 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
3005 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
3006 score: 0.25,
3007 is_top_pick: true,
3008 },
3009 ]
3010 "#]],
3011 ),
3012 (
3013 "keyword = `burnout person`; Pocket only",
3014 SuggestionQuery {
3015 keyword: "burnout person".into(),
3016 providers: vec![SuggestionProvider::Pocket],
3017 limit: None,
3018 },
3019 expect![[r#"
3020 []
3021 "#]],
3022 ),
3023 (
3024 "keyword = `best spicy ramen delivery in tokyo`; Yelp only",
3025 SuggestionQuery {
3026 keyword: "best spicy ramen delivery in tokyo".into(),
3027 providers: vec![SuggestionProvider::Yelp],
3028 limit: None,
3029 },
3030 expect![[r#"
3031 [
3032 Yelp {
3033 url: "https://www.yelp.com/search?find_desc=best+spicy+ramen+delivery&find_loc=tokyo",
3034 title: "best spicy ramen delivery in tokyo",
3035 icon: Some(
3036 [
3037 121,
3038 101,
3039 108,
3040 112,
3041 45,
3042 105,
3043 99,
3044 111,
3045 110,
3046 ],
3047 ),
3048 icon_mimetype: Some(
3049 "image/svg+xml",
3050 ),
3051 score: 0.5,
3052 has_location_sign: true,
3053 subject_exact_match: true,
3054 location_param: "find_loc",
3055 },
3056 ]
3057 "#]],
3058 ),
3059 (
3060 "keyword = `BeSt SpIcY rAmEn DeLiVeRy In ToKyO`; Yelp only",
3061 SuggestionQuery {
3062 keyword: "BeSt SpIcY rAmEn DeLiVeRy In ToKyO".into(),
3063 providers: vec![SuggestionProvider::Yelp],
3064 limit: None,
3065 },
3066 expect![[r#"
3067 [
3068 Yelp {
3069 url: "https://www.yelp.com/search?find_desc=BeSt+SpIcY+rAmEn+DeLiVeRy&find_loc=ToKyO",
3070 title: "BeSt SpIcY rAmEn DeLiVeRy In ToKyO",
3071 icon: Some(
3072 [
3073 121,
3074 101,
3075 108,
3076 112,
3077 45,
3078 105,
3079 99,
3080 111,
3081 110,
3082 ],
3083 ),
3084 icon_mimetype: Some(
3085 "image/svg+xml",
3086 ),
3087 score: 0.5,
3088 has_location_sign: true,
3089 subject_exact_match: true,
3090 location_param: "find_loc",
3091 },
3092 ]
3093 "#]],
3094 ),
3095 (
3096 "keyword = `best ramen delivery in tokyo`; Yelp only",
3097 SuggestionQuery {
3098 keyword: "best ramen delivery in tokyo".into(),
3099 providers: vec![SuggestionProvider::Yelp],
3100 limit: None,
3101 },
3102 expect![[r#"
3103 [
3104 Yelp {
3105 url: "https://www.yelp.com/search?find_desc=best+ramen+delivery&find_loc=tokyo",
3106 title: "best ramen delivery in tokyo",
3107 icon: Some(
3108 [
3109 121,
3110 101,
3111 108,
3112 112,
3113 45,
3114 105,
3115 99,
3116 111,
3117 110,
3118 ],
3119 ),
3120 icon_mimetype: Some(
3121 "image/svg+xml",
3122 ),
3123 score: 0.5,
3124 has_location_sign: true,
3125 subject_exact_match: true,
3126 location_param: "find_loc",
3127 },
3128 ]
3129 "#]],
3130 ),
3131 (
3132 "keyword = `best invalid_ramen delivery in tokyo`; Yelp only",
3133 SuggestionQuery {
3134 keyword: "best invalid_ramen delivery in tokyo".into(),
3135 providers: vec![SuggestionProvider::Yelp],
3136 limit: None,
3137 },
3138 expect![[r#"
3139 []
3140 "#]],
3141 ),
3142 (
3143 "keyword = `best delivery in tokyo`; Yelp only",
3144 SuggestionQuery {
3145 keyword: "best delivery in tokyo".into(),
3146 providers: vec![SuggestionProvider::Yelp],
3147 limit: None,
3148 },
3149 expect![[r#"
3150 []
3151 "#]],
3152 ),
3153 (
3154 "keyword = `super best ramen delivery in tokyo`; Yelp only",
3155 SuggestionQuery {
3156 keyword: "super best ramen delivery in tokyo".into(),
3157 providers: vec![SuggestionProvider::Yelp],
3158 limit: None,
3159 },
3160 expect![[r#"
3161 [
3162 Yelp {
3163 url: "https://www.yelp.com/search?find_desc=super+best+ramen+delivery&find_loc=tokyo",
3164 title: "super best ramen delivery in tokyo",
3165 icon: Some(
3166 [
3167 121,
3168 101,
3169 108,
3170 112,
3171 45,
3172 105,
3173 99,
3174 111,
3175 110,
3176 ],
3177 ),
3178 icon_mimetype: Some(
3179 "image/svg+xml",
3180 ),
3181 score: 0.5,
3182 has_location_sign: true,
3183 subject_exact_match: true,
3184 location_param: "find_loc",
3185 },
3186 ]
3187 "#]],
3188 ),
3189 (
3190 "keyword = `invalid_best ramen delivery in tokyo`; Yelp only",
3191 SuggestionQuery {
3192 keyword: "invalid_best ramen delivery in tokyo".into(),
3193 providers: vec![SuggestionProvider::Yelp],
3194 limit: None,
3195 },
3196 expect![[r#"
3197 []
3198 "#]],
3199 ),
3200 (
3201 "keyword = `ramen delivery in tokyo`; Yelp only",
3202 SuggestionQuery {
3203 keyword: "ramen delivery in tokyo".into(),
3204 providers: vec![SuggestionProvider::Yelp],
3205 limit: None,
3206 },
3207 expect![[r#"
3208 [
3209 Yelp {
3210 url: "https://www.yelp.com/search?find_desc=ramen+delivery&find_loc=tokyo",
3211 title: "ramen delivery in tokyo",
3212 icon: Some(
3213 [
3214 121,
3215 101,
3216 108,
3217 112,
3218 45,
3219 105,
3220 99,
3221 111,
3222 110,
3223 ],
3224 ),
3225 icon_mimetype: Some(
3226 "image/svg+xml",
3227 ),
3228 score: 0.5,
3229 has_location_sign: true,
3230 subject_exact_match: true,
3231 location_param: "find_loc",
3232 },
3233 ]
3234 "#]],
3235 ),
3236 (
3237 "keyword = `ramen super delivery in tokyo`; Yelp only",
3238 SuggestionQuery {
3239 keyword: "ramen super delivery in tokyo".into(),
3240 providers: vec![SuggestionProvider::Yelp],
3241 limit: None,
3242 },
3243 expect![[r#"
3244 [
3245 Yelp {
3246 url: "https://www.yelp.com/search?find_desc=ramen+super+delivery&find_loc=tokyo",
3247 title: "ramen super delivery in tokyo",
3248 icon: Some(
3249 [
3250 121,
3251 101,
3252 108,
3253 112,
3254 45,
3255 105,
3256 99,
3257 111,
3258 110,
3259 ],
3260 ),
3261 icon_mimetype: Some(
3262 "image/svg+xml",
3263 ),
3264 score: 0.5,
3265 has_location_sign: true,
3266 subject_exact_match: true,
3267 location_param: "find_loc",
3268 },
3269 ]
3270 "#]],
3271 ),
3272 (
3273 "keyword = `ramen invalid_delivery in tokyo`; Yelp only",
3274 SuggestionQuery {
3275 keyword: "ramen invalid_delivery in tokyo".into(),
3276 providers: vec![SuggestionProvider::Yelp],
3277 limit: None,
3278 },
3279 expect![[r#"
3280 []
3281 "#]],
3282 ),
3283 (
3284 "keyword = `ramen in tokyo`; Yelp only",
3285 SuggestionQuery {
3286 keyword: "ramen in tokyo".into(),
3287 providers: vec![SuggestionProvider::Yelp],
3288 limit: None,
3289 },
3290 expect![[r#"
3291 [
3292 Yelp {
3293 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo",
3294 title: "ramen in tokyo",
3295 icon: Some(
3296 [
3297 121,
3298 101,
3299 108,
3300 112,
3301 45,
3302 105,
3303 99,
3304 111,
3305 110,
3306 ],
3307 ),
3308 icon_mimetype: Some(
3309 "image/svg+xml",
3310 ),
3311 score: 0.5,
3312 has_location_sign: true,
3313 subject_exact_match: true,
3314 location_param: "find_loc",
3315 },
3316 ]
3317 "#]],
3318 ),
3319 (
3320 "keyword = `ramen near tokyo`; Yelp only",
3321 SuggestionQuery {
3322 keyword: "ramen near tokyo".into(),
3323 providers: vec![SuggestionProvider::Yelp],
3324 limit: None,
3325 },
3326 expect![[r#"
3327 [
3328 Yelp {
3329 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo",
3330 title: "ramen near tokyo",
3331 icon: Some(
3332 [
3333 121,
3334 101,
3335 108,
3336 112,
3337 45,
3338 105,
3339 99,
3340 111,
3341 110,
3342 ],
3343 ),
3344 icon_mimetype: Some(
3345 "image/svg+xml",
3346 ),
3347 score: 0.5,
3348 has_location_sign: true,
3349 subject_exact_match: true,
3350 location_param: "find_loc",
3351 },
3352 ]
3353 "#]],
3354 ),
3355 (
3356 "keyword = `ramen invalid_in tokyo`; Yelp only",
3357 SuggestionQuery {
3358 keyword: "ramen invalid_in tokyo".into(),
3359 providers: vec![SuggestionProvider::Yelp],
3360 limit: None,
3361 },
3362 expect![[r#"
3363 []
3364 "#]],
3365 ),
3366 (
3367 "keyword = `ramen in San Francisco`; Yelp only",
3368 SuggestionQuery {
3369 keyword: "ramen in San Francisco".into(),
3370 providers: vec![SuggestionProvider::Yelp],
3371 limit: None,
3372 },
3373 expect![[r#"
3374 [
3375 Yelp {
3376 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=San+Francisco",
3377 title: "ramen in San Francisco",
3378 icon: Some(
3379 [
3380 121,
3381 101,
3382 108,
3383 112,
3384 45,
3385 105,
3386 99,
3387 111,
3388 110,
3389 ],
3390 ),
3391 icon_mimetype: Some(
3392 "image/svg+xml",
3393 ),
3394 score: 0.5,
3395 has_location_sign: true,
3396 subject_exact_match: true,
3397 location_param: "find_loc",
3398 },
3399 ]
3400 "#]],
3401 ),
3402 (
3403 "keyword = `ramen in`; Yelp only",
3404 SuggestionQuery {
3405 keyword: "ramen in".into(),
3406 providers: vec![SuggestionProvider::Yelp],
3407 limit: None,
3408 },
3409 expect![[r#"
3410 [
3411 Yelp {
3412 url: "https://www.yelp.com/search?find_desc=ramen",
3413 title: "ramen in",
3414 icon: Some(
3415 [
3416 121,
3417 101,
3418 108,
3419 112,
3420 45,
3421 105,
3422 99,
3423 111,
3424 110,
3425 ],
3426 ),
3427 icon_mimetype: Some(
3428 "image/svg+xml",
3429 ),
3430 score: 0.5,
3431 has_location_sign: true,
3432 subject_exact_match: true,
3433 location_param: "find_loc",
3434 },
3435 ]
3436 "#]],
3437 ),
3438 (
3439 "keyword = `ramen near by`; Yelp only",
3440 SuggestionQuery {
3441 keyword: "ramen near by".into(),
3442 providers: vec![SuggestionProvider::Yelp],
3443 limit: None,
3444 },
3445 expect![[r#"
3446 [
3447 Yelp {
3448 url: "https://www.yelp.com/search?find_desc=ramen+near+by",
3449 title: "ramen near by",
3450 icon: Some(
3451 [
3452 121,
3453 101,
3454 108,
3455 112,
3456 45,
3457 105,
3458 99,
3459 111,
3460 110,
3461 ],
3462 ),
3463 icon_mimetype: Some(
3464 "image/svg+xml",
3465 ),
3466 score: 0.5,
3467 has_location_sign: false,
3468 subject_exact_match: true,
3469 location_param: "find_loc",
3470 },
3471 ]
3472 "#]],
3473 ),
3474 (
3475 "keyword = `ramen near me`; Yelp only",
3476 SuggestionQuery {
3477 keyword: "ramen near me".into(),
3478 providers: vec![SuggestionProvider::Yelp],
3479 limit: None,
3480 },
3481 expect![[r#"
3482 [
3483 Yelp {
3484 url: "https://www.yelp.com/search?find_desc=ramen+near+me",
3485 title: "ramen near me",
3486 icon: Some(
3487 [
3488 121,
3489 101,
3490 108,
3491 112,
3492 45,
3493 105,
3494 99,
3495 111,
3496 110,
3497 ],
3498 ),
3499 icon_mimetype: Some(
3500 "image/svg+xml",
3501 ),
3502 score: 0.5,
3503 has_location_sign: false,
3504 subject_exact_match: true,
3505 location_param: "find_loc",
3506 },
3507 ]
3508 "#]],
3509 ),
3510 (
3511 "keyword = `ramen near by tokyo`; Yelp only",
3512 SuggestionQuery {
3513 keyword: "ramen near by tokyo".into(),
3514 providers: vec![SuggestionProvider::Yelp],
3515 limit: None,
3516 },
3517 expect![[r#"
3518 []
3519 "#]],
3520 ),
3521 (
3522 "keyword = `ramen`; Yelp only",
3523 SuggestionQuery {
3524 keyword: "ramen".into(),
3525 providers: vec![SuggestionProvider::Yelp],
3526 limit: None,
3527 },
3528 expect![[r#"
3529 [
3530 Yelp {
3531 url: "https://www.yelp.com/search?find_desc=ramen",
3532 title: "ramen",
3533 icon: Some(
3534 [
3535 121,
3536 101,
3537 108,
3538 112,
3539 45,
3540 105,
3541 99,
3542 111,
3543 110,
3544 ],
3545 ),
3546 icon_mimetype: Some(
3547 "image/svg+xml",
3548 ),
3549 score: 0.5,
3550 has_location_sign: false,
3551 subject_exact_match: true,
3552 location_param: "find_loc",
3553 },
3554 ]
3555 "#]],
3556 ),
3557 (
3558 "keyword = maximum chars; Yelp only",
3559 SuggestionQuery {
3560 keyword: "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789".into(),
3561 providers: vec![SuggestionProvider::Yelp],
3562 limit: None,
3563 },
3564 expect![[r#"
3565 [
3566 Yelp {
3567 url: "https://www.yelp.com/search?find_desc=012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789",
3568 title: "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789",
3569 icon: Some(
3570 [
3571 121,
3572 101,
3573 108,
3574 112,
3575 45,
3576 105,
3577 99,
3578 111,
3579 110,
3580 ],
3581 ),
3582 icon_mimetype: Some(
3583 "image/svg+xml",
3584 ),
3585 score: 0.5,
3586 has_location_sign: false,
3587 subject_exact_match: true,
3588 location_param: "find_loc",
3589 },
3590 ]
3591 "#]],
3592 ),
3593 (
3594 "keyword = over chars; Yelp only",
3595 SuggestionQuery {
3596 keyword: "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789Z".into(),
3597 providers: vec![SuggestionProvider::Yelp],
3598 limit: None,
3599 },
3600 expect![[r#"
3601 []
3602 "#]],
3603 ),
3604 (
3605 "keyword = `best delivery`; Yelp only",
3606 SuggestionQuery {
3607 keyword: "best delivery".into(),
3608 providers: vec![SuggestionProvider::Yelp],
3609 limit: None,
3610 },
3611 expect![[r#"
3612 []
3613 "#]],
3614 ),
3615 (
3616 "keyword = `same_modifier same_modifier`; Yelp only",
3617 SuggestionQuery {
3618 keyword: "same_modifier same_modifier".into(),
3619 providers: vec![SuggestionProvider::Yelp],
3620 limit: None,
3621 },
3622 expect![[r#"
3623 []
3624 "#]],
3625 ),
3626 (
3627 "keyword = `same_modifier `; Yelp only",
3628 SuggestionQuery {
3629 keyword: "same_modifier ".into(),
3630 providers: vec![SuggestionProvider::Yelp],
3631 limit: None,
3632 },
3633 expect![[r#"
3634 []
3635 "#]],
3636 ),
3637 (
3638 "keyword = `yelp ramen`; Yelp only",
3639 SuggestionQuery {
3640 keyword: "yelp ramen".into(),
3641 providers: vec![SuggestionProvider::Yelp],
3642 limit: None,
3643 },
3644 expect![[r#"
3645 [
3646 Yelp {
3647 url: "https://www.yelp.com/search?find_desc=ramen",
3648 title: "ramen",
3649 icon: Some(
3650 [
3651 121,
3652 101,
3653 108,
3654 112,
3655 45,
3656 105,
3657 99,
3658 111,
3659 110,
3660 ],
3661 ),
3662 icon_mimetype: Some(
3663 "image/svg+xml",
3664 ),
3665 score: 0.5,
3666 has_location_sign: false,
3667 subject_exact_match: true,
3668 location_param: "find_loc",
3669 },
3670 ]
3671 "#]],
3672 ),
3673 (
3674 "keyword = `yelp keyword ramen`; Yelp only",
3675 SuggestionQuery {
3676 keyword: "yelp keyword ramen".into(),
3677 providers: vec![SuggestionProvider::Yelp],
3678 limit: None,
3679 },
3680 expect![[r#"
3681 [
3682 Yelp {
3683 url: "https://www.yelp.com/search?find_desc=ramen",
3684 title: "ramen",
3685 icon: Some(
3686 [
3687 121,
3688 101,
3689 108,
3690 112,
3691 45,
3692 105,
3693 99,
3694 111,
3695 110,
3696 ],
3697 ),
3698 icon_mimetype: Some(
3699 "image/svg+xml",
3700 ),
3701 score: 0.5,
3702 has_location_sign: false,
3703 subject_exact_match: true,
3704 location_param: "find_loc",
3705 },
3706 ]
3707 "#]],
3708 ),
3709 (
3710 "keyword = `ramen in tokyo yelp`; Yelp only",
3711 SuggestionQuery {
3712 keyword: "ramen in tokyo yelp".into(),
3713 providers: vec![SuggestionProvider::Yelp],
3714 limit: None,
3715 },
3716 expect![[r#"
3717 [
3718 Yelp {
3719 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo",
3720 title: "ramen in tokyo",
3721 icon: Some(
3722 [
3723 121,
3724 101,
3725 108,
3726 112,
3727 45,
3728 105,
3729 99,
3730 111,
3731 110,
3732 ],
3733 ),
3734 icon_mimetype: Some(
3735 "image/svg+xml",
3736 ),
3737 score: 0.5,
3738 has_location_sign: true,
3739 subject_exact_match: true,
3740 location_param: "find_loc",
3741 },
3742 ]
3743 "#]],
3744 ),
3745 (
3746 "keyword = `ramen in tokyo yelp keyword`; Yelp only",
3747 SuggestionQuery {
3748 keyword: "ramen in tokyo yelp keyword".into(),
3749 providers: vec![SuggestionProvider::Yelp],
3750 limit: None,
3751 },
3752 expect![[r#"
3753 [
3754 Yelp {
3755 url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo",
3756 title: "ramen in tokyo",
3757 icon: Some(
3758 [
3759 121,
3760 101,
3761 108,
3762 112,
3763 45,
3764 105,
3765 99,
3766 111,
3767 110,
3768 ],
3769 ),
3770 icon_mimetype: Some(
3771 "image/svg+xml",
3772 ),
3773 score: 0.5,
3774 has_location_sign: true,
3775 subject_exact_match: true,
3776 location_param: "find_loc",
3777 },
3778 ]
3779 "#]],
3780 ),
3781 (
3782 "keyword = `yelp ramen yelp`; Yelp only",
3783 SuggestionQuery {
3784 keyword: "yelp ramen yelp".into(),
3785 providers: vec![SuggestionProvider::Yelp],
3786 limit: None,
3787 },
3788 expect![[r#"
3789 [
3790 Yelp {
3791 url: "https://www.yelp.com/search?find_desc=ramen",
3792 title: "ramen",
3793 icon: Some(
3794 [
3795 121,
3796 101,
3797 108,
3798 112,
3799 45,
3800 105,
3801 99,
3802 111,
3803 110,
3804 ],
3805 ),
3806 icon_mimetype: Some(
3807 "image/svg+xml",
3808 ),
3809 score: 0.5,
3810 has_location_sign: false,
3811 subject_exact_match: true,
3812 location_param: "find_loc",
3813 },
3814 ]
3815 "#]],
3816 ),
3817 (
3818 "keyword = `best yelp ramen`; Yelp only",
3819 SuggestionQuery {
3820 keyword: "best yelp ramen".into(),
3821 providers: vec![SuggestionProvider::Yelp],
3822 limit: None,
3823 },
3824 expect![[r#"
3825 []
3826 "#]],
3827 ),
3828 (
3829 "keyword = `Spicy R`; Yelp only",
3830 SuggestionQuery {
3831 keyword: "Spicy R".into(),
3832 providers: vec![SuggestionProvider::Yelp],
3833 limit: None,
3834 },
3835 expect![[r#"
3836 [
3837 Yelp {
3838 url: "https://www.yelp.com/search?find_desc=Spicy+Ramen",
3839 title: "Spicy Ramen",
3840 icon: Some(
3841 [
3842 121,
3843 101,
3844 108,
3845 112,
3846 45,
3847 105,
3848 99,
3849 111,
3850 110,
3851 ],
3852 ),
3853 icon_mimetype: Some(
3854 "image/svg+xml",
3855 ),
3856 score: 0.5,
3857 has_location_sign: false,
3858 subject_exact_match: false,
3859 location_param: "find_loc",
3860 },
3861 ]
3862 "#]],
3863 ),
3864 (
3865 "keyword = `BeSt Ramen`; Yelp only",
3866 SuggestionQuery {
3867 keyword: "BeSt Ramen".into(),
3868 providers: vec![SuggestionProvider::Yelp],
3869 limit: None,
3870 },
3871 expect![[r#"
3872 [
3873 Yelp {
3874 url: "https://www.yelp.com/search?find_desc=BeSt+Ramen",
3875 title: "BeSt Ramen",
3876 icon: Some(
3877 [
3878 121,
3879 101,
3880 108,
3881 112,
3882 45,
3883 105,
3884 99,
3885 111,
3886 110,
3887 ],
3888 ),
3889 icon_mimetype: Some(
3890 "image/svg+xml",
3891 ),
3892 score: 0.5,
3893 has_location_sign: false,
3894 subject_exact_match: true,
3895 location_param: "find_loc",
3896 },
3897 ]
3898 "#]],
3899 ),
3900 (
3901 "keyword = `BeSt Spicy R`; Yelp only",
3902 SuggestionQuery {
3903 keyword: "BeSt Spicy R".into(),
3904 providers: vec![SuggestionProvider::Yelp],
3905 limit: None,
3906 },
3907 expect![[r#"
3908 [
3909 Yelp {
3910 url: "https://www.yelp.com/search?find_desc=BeSt+Spicy+Ramen",
3911 title: "BeSt Spicy Ramen",
3912 icon: Some(
3913 [
3914 121,
3915 101,
3916 108,
3917 112,
3918 45,
3919 105,
3920 99,
3921 111,
3922 110,
3923 ],
3924 ),
3925 icon_mimetype: Some(
3926 "image/svg+xml",
3927 ),
3928 score: 0.5,
3929 has_location_sign: false,
3930 subject_exact_match: false,
3931 location_param: "find_loc",
3932 },
3933 ]
3934 "#]],
3935 ),
3936 (
3937 "keyword = `BeSt R`; Yelp only",
3938 SuggestionQuery {
3939 keyword: "BeSt R".into(),
3940 providers: vec![SuggestionProvider::Yelp],
3941 limit: None,
3942 },
3943 expect![[r#"
3944 []
3945 "#]],
3946 ),
3947 (
3948 "keyword = `r`; Yelp only",
3949 SuggestionQuery {
3950 keyword: "r".into(),
3951 providers: vec![SuggestionProvider::Yelp],
3952 limit: None,
3953 },
3954 expect![[r#"
3955 []
3956 "#]],
3957 ),
3958 (
3959 "keyword = `ra`; Yelp only",
3960 SuggestionQuery {
3961 keyword: "ra".into(),
3962 providers: vec![SuggestionProvider::Yelp],
3963 limit: None,
3964 },
3965 expect![[r#"
3966 [
3967 Yelp {
3968 url: "https://www.yelp.com/search?find_desc=rats",
3969 title: "rats",
3970 icon: Some(
3971 [
3972 121,
3973 101,
3974 108,
3975 112,
3976 45,
3977 105,
3978 99,
3979 111,
3980 110,
3981 ],
3982 ),
3983 icon_mimetype: Some(
3984 "image/svg+xml",
3985 ),
3986 score: 0.5,
3987 has_location_sign: false,
3988 subject_exact_match: false,
3989 location_param: "find_loc",
3990 },
3991 ]
3992 "#]],
3993 ),
3994 (
3995 "keyword = `ram`; Yelp only",
3996 SuggestionQuery {
3997 keyword: "ram".into(),
3998 providers: vec![SuggestionProvider::Yelp],
3999 limit: None,
4000 },
4001 expect![[r#"
4002 [
4003 Yelp {
4004 url: "https://www.yelp.com/search?find_desc=ramen",
4005 title: "ramen",
4006 icon: Some(
4007 [
4008 121,
4009 101,
4010 108,
4011 112,
4012 45,
4013 105,
4014 99,
4015 111,
4016 110,
4017 ],
4018 ),
4019 icon_mimetype: Some(
4020 "image/svg+xml",
4021 ),
4022 score: 0.5,
4023 has_location_sign: false,
4024 subject_exact_match: false,
4025 location_param: "find_loc",
4026 },
4027 ]
4028 "#]],
4029 ),
4030 (
4031 "keyword = `rac`; Yelp only",
4032 SuggestionQuery {
4033 keyword: "rac".into(),
4034 providers: vec![SuggestionProvider::Yelp],
4035 limit: None,
4036 },
4037 expect![[r#"
4038 [
4039 Yelp {
4040 url: "https://www.yelp.com/search?find_desc=raccoon",
4041 title: "raccoon",
4042 icon: Some(
4043 [
4044 121,
4045 101,
4046 108,
4047 112,
4048 45,
4049 105,
4050 99,
4051 111,
4052 110,
4053 ],
4054 ),
4055 icon_mimetype: Some(
4056 "image/svg+xml",
4057 ),
4058 score: 0.5,
4059 has_location_sign: false,
4060 subject_exact_match: false,
4061 location_param: "find_loc",
4062 },
4063 ]
4064 "#]],
4065 ),
4066 (
4067 "keyword = `best r`; Yelp only",
4068 SuggestionQuery {
4069 keyword: "best r".into(),
4070 providers: vec![SuggestionProvider::Yelp],
4071 limit: None,
4072 },
4073 expect![[r#"
4074 []
4075 "#]],
4076 ),
4077 (
4078 "keyword = `best ra`; Yelp only",
4079 SuggestionQuery {
4080 keyword: "best ra".into(),
4081 providers: vec![SuggestionProvider::Yelp],
4082 limit: None,
4083 },
4084 expect![[r#"
4085 [
4086 Yelp {
4087 url: "https://www.yelp.com/search?find_desc=best+rats",
4088 title: "best rats",
4089 icon: Some(
4090 [
4091 121,
4092 101,
4093 108,
4094 112,
4095 45,
4096 105,
4097 99,
4098 111,
4099 110,
4100 ],
4101 ),
4102 icon_mimetype: Some(
4103 "image/svg+xml",
4104 ),
4105 score: 0.5,
4106 has_location_sign: false,
4107 subject_exact_match: false,
4108 location_param: "find_loc",
4109 },
4110 ]
4111 "#]],
4112 ),
4113 ];
4114 for (what, query, expect) in table {
4115 expect.assert_debug_eq(
4116 &store
4117 .query(query)
4118 .with_context(|| format!("Couldn't query store for {}", what))?,
4119 );
4120 }
4122 Ok(())
4123 }
4125 // Tests querying amp wikipedia
4126 #[test]
4127 fn query_with_multiple_providers_and_diff_scores() -> anyhow::Result<()> {
4128 before_each();
4130 let snapshot = Snapshot::with_records(json!([{
4131 "id": "data-1",
4132 "type": "data",
4133 "last_modified": 15,
4134 "attachment": {
4135 "filename": "data-1.json",
4136 "mimetype": "application/json",
4137 "location": "data-1.json",
4138 "hash": "",
4139 "size": 0,
4140 },
4141 }, {
4142 "id": "data-2",
4143 "type": "pocket-suggestions",
4144 "last_modified": 15,
4145 "attachment": {
4146 "filename": "data-2.json",
4147 "mimetype": "application/json",
4148 "location": "data-2.json",
4149 "hash": "",
4150 "size": 0,
4151 },
4152 }, {
4153 "id": "icon-3",
4154 "type": "icon",
4155 "last_modified": 25,
4156 "attachment": {
4157 "filename": "icon-3.png",
4158 "mimetype": "image/png",
4159 "location": "icon-3.png",
4160 "hash": "",
4161 "size": 0,
4162 },
4163 }]))?
4164 .with_data(
4165 "data-1.json",
4166 json!([{
4167 "id": 0,
4168 "advertiser": "Good Place Eats",
4169 "iab_category": "8 - Food & Drink",
4170 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow", "amp wiki match"],
4171 "title": "Lasagna Come Out Tomorrow",
4172 "url": "https://www.lasagna.restaurant",
4173 "icon": "2",
4174 "impression_url": "https://example.com/impression_url",
4175 "click_url": "https://example.com/click_url",
4176 "score": 0.3
4177 }, {
4178 "id": 0,
4179 "advertiser": "Good Place Eats",
4180 "iab_category": "8 - Food & Drink",
4181 "keywords": ["pe", "pen", "penne", "penne for your thoughts", "amp wiki match"],
4182 "title": "Penne for Your Thoughts",
4183 "url": "https://penne.biz",
4184 "icon": "2",
4185 "impression_url": "https://example.com/impression_url",
4186 "click_url": "https://example.com/click_url",
4187 "score": 0.1
4188 }, {
4189 "id": 0,
4190 "advertiser": "Wikipedia",
4191 "iab_category": "5 - Education",
4192 "keywords": ["amp wiki match", "pocket wiki match"],
4193 "title": "Multimatch",
4194 "url": "https://wikipedia.org/Multimatch",
4195 "icon": "3"
4196 }]),
4197 )?
4198 .with_data(
4199 "data-2.json",
4200 json!([
4201 {
4202 "description": "pocket suggestion",
4203 "url": "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4204 "lowConfidenceKeywords": ["soft life", "workaholism", "toxic work culture", "work-life balance", "pocket wiki match"],
4205 "highConfidenceKeywords": ["burnout women", "grind culture", "women burnout"],
4206 "title": "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4207 "score": 0.05
4208 },
4209 {
4210 "description": "pocket suggestion multi-match",
4211 "url": "https://getpocket.com/collections/multimatch",
4212 "lowConfidenceKeywords": [],
4213 "highConfidenceKeywords": ["pocket wiki match"],
4214 "title": "Pocket wiki match",
4215 "score": 0.88
4216 },
4217 ]),
4218 )?
4219 .with_icon("icon-3.png", "also-an-icon".as_bytes().into());
4221 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4223 store.ingest(SuggestIngestionConstraints::default())?;
4225 let table = [
4226 (
4227 "keyword = `amp wiki match`; all providers",
4228 SuggestionQuery {
4229 keyword: "amp wiki match".into(),
4230 providers: vec![
4231 SuggestionProvider::Amp,
4232 SuggestionProvider::Wikipedia,
4233 SuggestionProvider::Amo,
4234 SuggestionProvider::Pocket,
4235 SuggestionProvider::Yelp,
4236 ],
4237 limit: None,
4238 },
4239 expect![[r#"
4240 [
4241 Amp {
4242 title: "Lasagna Come Out Tomorrow",
4243 url: "https://www.lasagna.restaurant",
4244 raw_url: "https://www.lasagna.restaurant",
4245 icon: None,
4246 icon_mimetype: None,
4247 full_keyword: "amp wiki match",
4248 block_id: 0,
4249 advertiser: "Good Place Eats",
4250 iab_category: "8 - Food & Drink",
4251 impression_url: "https://example.com/impression_url",
4252 click_url: "https://example.com/click_url",
4253 raw_click_url: "https://example.com/click_url",
4254 score: 0.3,
4255 },
4256 Wikipedia {
4257 title: "Multimatch",
4258 url: "https://wikipedia.org/Multimatch",
4259 icon: Some(
4260 [
4261 97,
4262 108,
4263 115,
4264 111,
4265 45,
4266 97,
4267 110,
4268 45,
4269 105,
4270 99,
4271 111,
4272 110,
4273 ],
4274 ),
4275 icon_mimetype: Some(
4276 "image/png",
4277 ),
4278 full_keyword: "amp wiki match",
4279 },
4280 Amp {
4281 title: "Penne for Your Thoughts",
4282 url: "https://penne.biz",
4283 raw_url: "https://penne.biz",
4284 icon: None,
4285 icon_mimetype: None,
4286 full_keyword: "amp wiki match",
4287 block_id: 0,
4288 advertiser: "Good Place Eats",
4289 iab_category: "8 - Food & Drink",
4290 impression_url: "https://example.com/impression_url",
4291 click_url: "https://example.com/click_url",
4292 raw_click_url: "https://example.com/click_url",
4293 score: 0.1,
4294 },
4295 ]
4296 "#]],
4297 ),
4298 (
4299 "keyword = `amp wiki match`; all providers, limit 2",
4300 SuggestionQuery {
4301 keyword: "amp wiki match".into(),
4302 providers: vec![
4303 SuggestionProvider::Amp,
4304 SuggestionProvider::Wikipedia,
4305 SuggestionProvider::Amo,
4306 SuggestionProvider::Pocket,
4307 SuggestionProvider::Yelp,
4308 ],
4309 limit: Some(2),
4310 },
4311 expect![[r#"
4312 [
4313 Amp {
4314 title: "Lasagna Come Out Tomorrow",
4315 url: "https://www.lasagna.restaurant",
4316 raw_url: "https://www.lasagna.restaurant",
4317 icon: None,
4318 icon_mimetype: None,
4319 full_keyword: "amp wiki match",
4320 block_id: 0,
4321 advertiser: "Good Place Eats",
4322 iab_category: "8 - Food & Drink",
4323 impression_url: "https://example.com/impression_url",
4324 click_url: "https://example.com/click_url",
4325 raw_click_url: "https://example.com/click_url",
4326 score: 0.3,
4327 },
4328 Wikipedia {
4329 title: "Multimatch",
4330 url: "https://wikipedia.org/Multimatch",
4331 icon: Some(
4332 [
4333 97,
4334 108,
4335 115,
4336 111,
4337 45,
4338 97,
4339 110,
4340 45,
4341 105,
4342 99,
4343 111,
4344 110,
4345 ],
4346 ),
4347 icon_mimetype: Some(
4348 "image/png",
4349 ),
4350 full_keyword: "amp wiki match",
4351 },
4352 ]
4353 "#]],
4354 ),
4355 (
4356 "pocket wiki match; all providers",
4357 SuggestionQuery {
4358 keyword: "pocket wiki match".into(),
4359 providers: vec![
4360 SuggestionProvider::Amp,
4361 SuggestionProvider::Wikipedia,
4362 SuggestionProvider::Amo,
4363 SuggestionProvider::Pocket,
4364 ],
4365 limit: None,
4366 },
4367 expect![[r#"
4368 [
4369 Pocket {
4370 title: "Pocket wiki match",
4371 url: "https://getpocket.com/collections/multimatch",
4372 score: 0.88,
4373 is_top_pick: true,
4374 },
4375 Wikipedia {
4376 title: "Multimatch",
4377 url: "https://wikipedia.org/Multimatch",
4378 icon: Some(
4379 [
4380 97,
4381 108,
4382 115,
4383 111,
4384 45,
4385 97,
4386 110,
4387 45,
4388 105,
4389 99,
4390 111,
4391 110,
4392 ],
4393 ),
4394 icon_mimetype: Some(
4395 "image/png",
4396 ),
4397 full_keyword: "pocket wiki match",
4398 },
4399 Pocket {
4400 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4401 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4402 score: 0.05,
4403 is_top_pick: false,
4404 },
4405 ]
4406 "#]],
4407 ),
4408 (
4409 "pocket wiki match; all providers limit 1",
4410 SuggestionQuery {
4411 keyword: "pocket wiki match".into(),
4412 providers: vec![
4413 SuggestionProvider::Amp,
4414 SuggestionProvider::Wikipedia,
4415 SuggestionProvider::Amo,
4416 SuggestionProvider::Pocket,
4417 ],
4418 limit: Some(1),
4419 },
4420 expect![[r#"
4421 [
4422 Pocket {
4423 title: "Pocket wiki match",
4424 url: "https://getpocket.com/collections/multimatch",
4425 score: 0.88,
4426 is_top_pick: true,
4427 },
4428 ]
4429 "#]],
4430 ),
4431 (
4432 "work-life balance; duplicate providers",
4433 SuggestionQuery {
4434 keyword: "work-life balance".into(),
4435 providers: vec![SuggestionProvider::Pocket, SuggestionProvider::Pocket],
4436 limit: Some(-1),
4437 },
4438 expect![[r#"
4439 [
4440 Pocket {
4441 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4442 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4443 score: 0.05,
4444 is_top_pick: false,
4445 },
4446 ]
4447 "#]],
4448 ),
4449 ];
4450 for (what, query, expect) in table {
4451 expect.assert_debug_eq(
4452 &store
4453 .query(query)
4454 .with_context(|| format!("Couldn't query store for {}", what))?,
4455 );
4456 }
4458 Ok(())
4459 }
4461 // Tests querying multiple suggestions with multiple keywords with same prefix keyword
4462 #[test]
4463 fn query_with_multiple_suggestions_with_same_prefix() -> anyhow::Result<()> {
4464 before_each();
4466 let snapshot = Snapshot::with_records(json!([{
4467 "id": "data-1",
4468 "type": "amo-suggestions",
4469 "last_modified": 15,
4470 "attachment": {
4471 "filename": "data-1.json",
4472 "mimetype": "application/json",
4473 "location": "data-1.json",
4474 "hash": "",
4475 "size": 0,
4476 },
4477 }, {
4478 "id": "data-2",
4479 "type": "pocket-suggestions",
4480 "last_modified": 15,
4481 "attachment": {
4482 "filename": "data-2.json",
4483 "mimetype": "application/json",
4484 "location": "data-2.json",
4485 "hash": "",
4486 "size": 0,
4487 },
4488 }, {
4489 "id": "icon-3",
4490 "type": "icon",
4491 "last_modified": 25,
4492 "attachment": {
4493 "filename": "icon-3.png",
4494 "mimetype": "image/png",
4495 "location": "icon-3.png",
4496 "hash": "",
4497 "size": 0,
4498 },
4499 }]))?
4500 .with_data(
4501 "data-1.json",
4502 json!([
4503 {
4504 "description": "amo suggestion",
4505 "url": "https://addons.mozilla.org/en-US/firefox/addon/example",
4506 "guid": "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
4507 "keywords": ["relay", "spam", "masking email", "masking emails", "masking accounts", "alias" ],
4508 "title": "Firefox Relay",
4509 "icon": "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
4510 "rating": "4.9",
4511 "number_of_ratings": 888,
4512 "score": 0.25
4513 }
4514 ]),
4515 )?
4516 .with_data(
4517 "data-2.json",
4518 json!([
4519 {
4520 "description": "pocket suggestion",
4521 "url": "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4522 "lowConfidenceKeywords": ["soft life", "soft living", "soft work", "workaholism", "toxic work culture"],
4523 "highConfidenceKeywords": ["burnout women", "grind culture", "women burnout", "soft lives"],
4524 "title": "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4525 "score": 0.05
4526 }
4527 ]),
4528 )?
4529 .with_icon("icon-3.png", "also-an-icon".as_bytes().into());
4531 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4533 store.ingest(SuggestIngestionConstraints::default())?;
4535 let table = [
4536 (
4537 "keyword = `soft li`; pocket",
4538 SuggestionQuery {
4539 keyword: "soft li".into(),
4540 providers: vec![SuggestionProvider::Pocket],
4541 limit: None,
4542 },
4543 expect![[r#"
4544 [
4545 Pocket {
4546 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4547 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4548 score: 0.05,
4549 is_top_pick: false,
4550 },
4551 ]
4552 "#]],
4553 ),
4554 (
4555 "keyword = `soft lives`; pocket",
4556 SuggestionQuery {
4557 keyword: "soft lives".into(),
4558 providers: vec![SuggestionProvider::Pocket],
4559 limit: None,
4560 },
4561 expect![[r#"
4562 [
4563 Pocket {
4564 title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women",
4565 url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women",
4566 score: 0.05,
4567 is_top_pick: true,
4568 },
4569 ]
4570 "#]],
4571 ),
4572 (
4573 "keyword = `masking `; amo provider",
4574 SuggestionQuery {
4575 keyword: "masking ".into(),
4576 providers: vec![SuggestionProvider::Amo],
4577 limit: None,
4578 },
4579 expect![[r#"
4580 [
4581 Amo {
4582 title: "Firefox Relay",
4583 url: "https://addons.mozilla.org/en-US/firefox/addon/example",
4584 icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b",
4585 description: "amo suggestion",
4586 rating: Some(
4587 "4.9",
4588 ),
4589 number_of_ratings: 888,
4590 guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}",
4591 score: 0.25,
4592 },
4593 ]
4594 "#]],
4595 ),
4596 ];
4597 for (what, query, expect) in table {
4598 expect.assert_debug_eq(
4599 &store
4600 .query(query)
4601 .with_context(|| format!("Couldn't query store for {}", what))?,
4602 );
4603 }
4605 Ok(())
4606 }
4608 // Tests querying multiple suggestions with multiple keywords with same prefix keyword
4609 #[test]
4610 fn query_with_amp_mobile_provider() -> anyhow::Result<()> {
4611 before_each();
4613 let snapshot = Snapshot::with_records(json!([{
4614 "id": "data-1",
4615 "type": "amp-mobile-suggestions",
4616 "last_modified": 15,
4617 "attachment": {
4618 "filename": "data-1.json",
4619 "mimetype": "application/json",
4620 "location": "data-1.json",
4621 "hash": "",
4622 "size": 0,
4623 },
4624 }, {
4625 "id": "data-2",
4626 "type": "data",
4627 "last_modified": 15,
4628 "attachment": {
4629 "filename": "data-2.json",
4630 "mimetype": "application/json",
4631 "location": "data-2.json",
4632 "hash": "",
4633 "size": 0,
4634 },
4635 }, {
4636 "id": "icon-3",
4637 "type": "icon",
4638 "last_modified": 25,
4639 "attachment": {
4640 "filename": "icon-3.png",
4641 "mimetype": "image/png",
4642 "location": "icon-3.png",
4643 "hash": "",
4644 "size": 0,
4645 },
4646 }]))?
4647 .with_data(
4648 "data-1.json",
4649 json!([
4650 {
4651 "id": 0,
4652 "advertiser": "Good Place Eats",
4653 "iab_category": "8 - Food & Drink",
4654 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
4655 "title": "Mobile - Lasagna Come Out Tomorrow",
4656 "url": "https://www.lasagna.restaurant",
4657 "icon": "3",
4658 "impression_url": "https://example.com/impression_url",
4659 "click_url": "https://example.com/click_url",
4660 "score": 0.3
4661 }
4662 ]),
4663 )?
4664 .with_data(
4665 "data-2.json",
4666 json!([
4667 {
4668 "id": 0,
4669 "advertiser": "Good Place Eats",
4670 "iab_category": "8 - Food & Drink",
4671 "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow"],
4672 "title": "Desktop - Lasagna Come Out Tomorrow",
4673 "url": "https://www.lasagna.restaurant",
4674 "icon": "3",
4675 "impression_url": "https://example.com/impression_url",
4676 "click_url": "https://example.com/click_url",
4677 "score": 0.2
4678 }
4679 ]),
4680 )?
4681 .with_icon("icon-3.png", "also-an-icon".as_bytes().into());
4683 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4685 store.ingest(SuggestIngestionConstraints::default())?;
4687 let table = [
4688 (
4689 "keyword = `las`; Amp Mobile",
4690 SuggestionQuery {
4691 keyword: "las".into(),
4692 providers: vec![SuggestionProvider::AmpMobile],
4693 limit: None,
4694 },
4695 expect![[r#"
4696 [
4697 Amp {
4698 title: "Mobile - Lasagna Come Out Tomorrow",
4699 url: "https://www.lasagna.restaurant",
4700 raw_url: "https://www.lasagna.restaurant",
4701 icon: Some(
4702 [
4703 97,
4704 108,
4705 115,
4706 111,
4707 45,
4708 97,
4709 110,
4710 45,
4711 105,
4712 99,
4713 111,
4714 110,
4715 ],
4716 ),
4717 icon_mimetype: Some(
4718 "image/png",
4719 ),
4720 full_keyword: "lasagna",
4721 block_id: 0,
4722 advertiser: "Good Place Eats",
4723 iab_category: "8 - Food & Drink",
4724 impression_url: "https://example.com/impression_url",
4725 click_url: "https://example.com/click_url",
4726 raw_click_url: "https://example.com/click_url",
4727 score: 0.3,
4728 },
4729 ]
4730 "#]],
4731 ),
4732 (
4733 "keyword = `las`; Amp",
4734 SuggestionQuery {
4735 keyword: "las".into(),
4736 providers: vec![SuggestionProvider::Amp],
4737 limit: None,
4738 },
4739 expect![[r#"
4740 [
4741 Amp {
4742 title: "Desktop - Lasagna Come Out Tomorrow",
4743 url: "https://www.lasagna.restaurant",
4744 raw_url: "https://www.lasagna.restaurant",
4745 icon: Some(
4746 [
4747 97,
4748 108,
4749 115,
4750 111,
4751 45,
4752 97,
4753 110,
4754 45,
4755 105,
4756 99,
4757 111,
4758 110,
4759 ],
4760 ),
4761 icon_mimetype: Some(
4762 "image/png",
4763 ),
4764 full_keyword: "lasagna",
4765 block_id: 0,
4766 advertiser: "Good Place Eats",
4767 iab_category: "8 - Food & Drink",
4768 impression_url: "https://example.com/impression_url",
4769 click_url: "https://example.com/click_url",
4770 raw_click_url: "https://example.com/click_url",
4771 score: 0.2,
4772 },
4773 ]
4774 "#]],
4775 ),
4776 (
4777 "keyword = `las `; amp and amp mobile",
4778 SuggestionQuery {
4779 keyword: "las".into(),
4780 providers: vec![SuggestionProvider::Amp, SuggestionProvider::AmpMobile],
4781 limit: None,
4782 },
4783 expect![[r#"
4784 [
4785 Amp {
4786 title: "Mobile - Lasagna Come Out Tomorrow",
4787 url: "https://www.lasagna.restaurant",
4788 raw_url: "https://www.lasagna.restaurant",
4789 icon: Some(
4790 [
4791 97,
4792 108,
4793 115,
4794 111,
4795 45,
4796 97,
4797 110,
4798 45,
4799 105,
4800 99,
4801 111,
4802 110,
4803 ],
4804 ),
4805 icon_mimetype: Some(
4806 "image/png",
4807 ),
4808 full_keyword: "lasagna",
4809 block_id: 0,
4810 advertiser: "Good Place Eats",
4811 iab_category: "8 - Food & Drink",
4812 impression_url: "https://example.com/impression_url",
4813 click_url: "https://example.com/click_url",
4814 raw_click_url: "https://example.com/click_url",
4815 score: 0.3,
4816 },
4817 Amp {
4818 title: "Desktop - Lasagna Come Out Tomorrow",
4819 url: "https://www.lasagna.restaurant",
4820 raw_url: "https://www.lasagna.restaurant",
4821 icon: Some(
4822 [
4823 97,
4824 108,
4825 115,
4826 111,
4827 45,
4828 97,
4829 110,
4830 45,
4831 105,
4832 99,
4833 111,
4834 110,
4835 ],
4836 ),
4837 icon_mimetype: Some(
4838 "image/png",
4839 ),
4840 full_keyword: "lasagna",
4841 block_id: 0,
4842 advertiser: "Good Place Eats",
4843 iab_category: "8 - Food & Drink",
4844 impression_url: "https://example.com/impression_url",
4845 click_url: "https://example.com/click_url",
4846 raw_click_url: "https://example.com/click_url",
4847 score: 0.2,
4848 },
4849 ]
4850 "#]],
4851 ),
4852 ];
4853 for (what, query, expect) in table {
4854 expect.assert_debug_eq(
4855 &store
4856 .query(query)
4857 .with_context(|| format!("Couldn't query store for {}", what))?,
4858 );
4859 }
4861 Ok(())
4862 }
4864 /// Tests ingesting malformed Remote Settings records that we understand,
4865 /// but that are missing fields, or aren't in the format we expect.
4866 #[test]
4867 fn ingest_malformed() -> anyhow::Result<()> {
4868 before_each();
4870 let snapshot = Snapshot::with_records(json!([{
4871 // Data record without an attachment.
4872 "id": "missing-data-attachment",
4873 "type": "data",
4874 "last_modified": 15,
4875 }, {
4876 // Icon record without an attachment.
4877 "id": "missing-icon-attachment",
4878 "type": "icon",
4879 "last_modified": 30,
4880 }, {
4881 // Icon record with an ID that's not `icon-{id}`, so suggestions in
4882 // the data attachment won't be able to reference it.
4883 "id": "bad-icon-id",
4884 "type": "icon",
4885 "last_modified": 45,
4886 "attachment": {
4887 "filename": "icon-1.png",
4888 "mimetype": "image/png",
4889 "location": "icon-1.png",
4890 "hash": "",
4891 "size": 0,
4892 },
4893 }]))?
4894 .with_icon("icon-1.png", "i-am-an-icon".as_bytes().into());
4896 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4898 store.ingest(SuggestIngestionConstraints::default())?;
4900 store.dbs()?.reader.read(|dao| {
4901 assert_eq!(
4902 dao.get_meta::<u64>(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
4903 Some(45)
4904 );
4905 assert_eq!(
4906 dao.conn
4907 .query_one::<i64>("SELECT count(*) FROM suggestions")?,
4908 0
4909 );
4910 assert_eq!(dao.conn.query_one::<i64>("SELECT count(*) FROM icons")?, 0);
4912 Ok(())
4913 })?;
4915 Ok(())
4916 }
4918 /// Tests unparsable Remote Settings records, which we don't know how to
4919 /// ingest at all.
4920 #[test]
4921 fn ingest_unparsable() -> anyhow::Result<()> {
4922 before_each();
4924 let snapshot = Snapshot::with_records(json!([{
4925 "id": "fancy-new-suggestions-1",
4926 "type": "fancy-new-suggestions",
4927 "last_modified": 15,
4928 }, {
4929 "id": "clippy-2",
4930 "type": "clippy",
4931 "last_modified": 30,
4932 }]))?;
4934 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
4936 store.ingest(SuggestIngestionConstraints::default())?;
4938 store.dbs()?.reader.read(|dao| {
4939 assert_eq!(
4940 dao.get_meta("last_quicksuggest_ingest_unparsable")?,
4941 Some(30)
4942 );
4943 expect![[r#"
4944 Some(
4945 UnparsableRecords(
4946 {
4947 "clippy-2": UnparsableRecord {
4948 schema_version: 17,
4949 },
4950 "fancy-new-suggestions-1": UnparsableRecord {
4951 schema_version: 17,
4952 },
4953 },
4954 ),
4955 )
4956 "#]]
4957 .assert_debug_eq(&dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY)?);
4958 Ok(())
4959 })?;
4961 Ok(())
4962 }
4964 #[test]
4965 fn ingest_mixed_parsable_unparsable_records() -> anyhow::Result<()> {
4966 before_each();
4968 let snapshot = Snapshot::with_records(json!([{
4969 "id": "fancy-new-suggestions-1",
4970 "type": "fancy-new-suggestions",
4971 "last_modified": 15,
4972 },
4973 {
4974 "id": "data-1",
4975 "type": "data",
4976 "last_modified": 15,
4977 "attachment": {
4978 "filename": "data-1.json",
4979 "mimetype": "application/json",
4980 "location": "data-1.json",
4981 "hash": "",
4982 "size": 0,
4983 },
4984 },
4985 {
4986 "id": "clippy-2",
4987 "type": "clippy",
4988 "last_modified": 30,
4989 }]))?
4990 .with_data(
4991 "data-1.json",
4992 json!([{
4993 "id": 0,
4994 "advertiser": "Los Pollos Hermanos",
4995 "iab_category": "8 - Food & Drink",
4996 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
4997 "title": "Los Pollos Hermanos - Albuquerque",
4998 "url": "https://www.lph-nm.biz",
4999 "icon": "5678",
5000 "impression_url": "https://example.com/impression_url",
5001 "click_url": "https://example.com/click_url",
5002 "score": 0.3,
5003 }]),
5004 )?;
5006 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5008 store.ingest(SuggestIngestionConstraints::default())?;
5010 store.dbs()?.reader.read(|dao| {
5011 assert_eq!(
5012 dao.get_meta("last_quicksuggest_ingest_unparsable")?,
5013 Some(30)
5014 );
5015 expect![[r#"
5016 Some(
5017 UnparsableRecords(
5018 {
5019 "clippy-2": UnparsableRecord {
5020 schema_version: 17,
5021 },
5022 "fancy-new-suggestions-1": UnparsableRecord {
5023 schema_version: 17,
5024 },
5025 },
5026 ),
5027 )
5028 "#]]
5029 .assert_debug_eq(&dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY)?);
5030 Ok(())
5031 })?;
5033 Ok(())
5034 }
5036 /// Tests meta update field isn't updated for old unparsable Remote Settings
5037 /// records.
5038 #[test]
5039 fn ingest_unparsable_and_meta_update_stays_the_same() -> anyhow::Result<()> {
5040 before_each();
5042 let snapshot = Snapshot::with_records(json!([{
5043 "id": "fancy-new-suggestions-1",
5044 "type": "fancy-new-suggestions",
5045 "last_modified": 15,
5046 }]))?;
5048 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5049 store.dbs()?.writer.write(|dao| {
5050 dao.put_meta(
5051 SuggestRecordType::AmpWikipedia
5052 .last_ingest_meta_key()
5053 .as_str(),
5054 30,
5055 )?;
5056 Ok(())
5057 })?;
5058 store.ingest(SuggestIngestionConstraints::default())?;
5060 store.dbs()?.reader.read(|dao| {
5061 assert_eq!(
5062 dao.get_meta::<u64>(
5063 SuggestRecordType::AmpWikipedia
5064 .last_ingest_meta_key()
5065 .as_str()
5066 )?,
5067 Some(30)
5068 );
5069 Ok(())
5070 })?;
5072 Ok(())
5073 }
5075 /// Tests that we only ingest providers that we're concerned with.
5076 #[test]
5077 fn ingest_constraints_provider() -> anyhow::Result<()> {
5078 before_each();
5080 let snapshot = Snapshot::with_records(json!([{
5081 "id": "data-1",
5082 "type": "data",
5083 "last_modified": 15,
5084 }, {
5085 "id": "icon-1",
5086 "type": "icon",
5087 "last_modified": 30,
5088 }]))?;
5090 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5091 store.dbs()?.writer.write(|dao| {
5092 // Check that existing data is updated properly.
5093 dao.put_meta(
5094 SuggestRecordType::AmpWikipedia
5095 .last_ingest_meta_key()
5096 .as_str(),
5097 10,
5098 )?;
5099 Ok(())
5100 })?;
5102 let constraints = SuggestIngestionConstraints {
5103 max_suggestions: Some(100),
5104 providers: Some(vec![SuggestionProvider::Amp, SuggestionProvider::Pocket]),
5105 };
5106 store.ingest(constraints)?;
5108 store.dbs()?.reader.read(|dao| {
5109 assert_eq!(
5110 dao.get_meta::<u64>(
5111 SuggestRecordType::AmpWikipedia
5112 .last_ingest_meta_key()
5113 .as_str()
5114 )?,
5115 Some(15)
5116 );
5117 assert_eq!(
5118 dao.get_meta::<u64>(SuggestRecordType::Icon.last_ingest_meta_key().as_str())?,
5119 Some(30)
5120 );
5121 assert_eq!(
5122 dao.get_meta::<u64>(SuggestRecordType::Pocket.last_ingest_meta_key().as_str())?,
5123 None
5124 );
5125 assert_eq!(
5126 dao.get_meta::<u64>(SuggestRecordType::Amo.last_ingest_meta_key().as_str())?,
5127 None
5128 );
5129 assert_eq!(
5130 dao.get_meta::<u64>(SuggestRecordType::Yelp.last_ingest_meta_key().as_str())?,
5131 None
5132 );
5133 assert_eq!(
5134 dao.get_meta::<u64>(SuggestRecordType::Mdn.last_ingest_meta_key().as_str())?,
5135 None
5136 );
5137 assert_eq!(
5138 dao.get_meta::<u64>(SuggestRecordType::AmpMobile.last_ingest_meta_key().as_str())?,
5139 None
5140 );
5141 assert_eq!(
5142 dao.get_meta::<u64>(
5143 SuggestRecordType::GlobalConfig
5144 .last_ingest_meta_key()
5145 .as_str()
5146 )?,
5147 None
5148 );
5149 Ok(())
5150 })?;
5152 Ok(())
5153 }
5155 #[test]
5156 fn remove_known_records_out_of_meta_table() -> anyhow::Result<()> {
5157 before_each();
5159 let snapshot = Snapshot::with_records(json!([{
5160 "id": "fancy-new-suggestions-1",
5161 "type": "fancy-new-suggestions",
5162 "last_modified": 15,
5163 },
5164 {
5165 "id": "data-1",
5166 "type": "data",
5167 "last_modified": 15,
5168 "attachment": {
5169 "filename": "data-1.json",
5170 "mimetype": "application/json",
5171 "location": "data-1.json",
5172 "hash": "",
5173 "size": 0,
5174 },
5175 },
5176 {
5177 "id": "clippy-2",
5178 "type": "clippy",
5179 "last_modified": 15,
5180 }]))?
5181 .with_data(
5182 "data-1.json",
5183 json!([{
5184 "id": 0,
5185 "advertiser": "Los Pollos Hermanos",
5186 "iab_category": "8 - Food & Drink",
5187 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
5188 "title": "Los Pollos Hermanos - Albuquerque",
5189 "url": "https://www.lph-nm.biz",
5190 "icon": "5678",
5191 "impression_url": "https://example.com/impression_url",
5192 "click_url": "https://example.com/click_url",
5193 "score": 0.3
5194 }]),
5195 )?;
5197 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5198 let mut initial_data = UnparsableRecords::default();
5199 initial_data
5200 .0
5201 .insert("data-1".to_string(), UnparsableRecord { schema_version: 1 });
5202 initial_data.0.insert(
5203 "clippy-2".to_string(),
5204 UnparsableRecord { schema_version: 1 },
5205 );
5206 store.dbs()?.writer.write(|dao| {
5207 dao.put_meta(UNPARSABLE_RECORDS_META_KEY, initial_data)?;
5208 Ok(())
5209 })?;
5211 store.ingest(SuggestIngestionConstraints::default())?;
5213 store.dbs()?.reader.read(|dao| {
5214 expect![[r#"
5215 Some(
5216 UnparsableRecords(
5217 {
5218 "clippy-2": UnparsableRecord {
5219 schema_version: 17,
5220 },
5221 "fancy-new-suggestions-1": UnparsableRecord {
5222 schema_version: 17,
5223 },
5224 },
5225 ),
5226 )
5227 "#]]
5228 .assert_debug_eq(&dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY)?);
5229 Ok(())
5230 })?;
5232 Ok(())
5233 }
5235 /// Tests that records with invalid attachments are ignored and marked as unparsable.
5236 #[test]
5237 fn skip_over_invalid_records() -> anyhow::Result<()> {
5238 before_each();
5240 let snapshot = Snapshot::with_records(json!([
5241 {
5242 "id": "invalid-attachment",
5243 "type": "data",
5244 "last_modified": 15,
5245 "attachment": {
5246 "filename": "data-2.json",
5247 "mimetype": "application/json",
5248 "location": "data-2.json",
5249 "hash": "",
5250 "size": 0,
5251 },
5252 },
5253 {
5254 "id": "valid-record",
5255 "type": "data",
5256 "last_modified": 15,
5257 "attachment": {
5258 "filename": "data-1.json",
5259 "mimetype": "application/json",
5260 "location": "data-1.json",
5261 "hash": "",
5262 "size": 0,
5263 },
5264 },
5265 ]))?
5266 .with_data(
5267 "data-1.json",
5268 json!([{
5269 "id": 0,
5270 "advertiser": "Los Pollos Hermanos",
5271 "iab_category": "8 - Food & Drink",
5272 "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"],
5273 "title": "Los Pollos Hermanos - Albuquerque",
5274 "url": "https://www.lph-nm.biz",
5275 "icon": "5678",
5276 "impression_url": "https://example.com/impression_url",
5277 "click_url": "https://example.com/click_url",
5278 "score": 0.3
5279 }]),
5280 )?
5281 // This attachment is missing the `keywords` field and is invalid
5282 .with_data(
5283 "data-2.json",
5284 json!([{
5285 "id": 1,
5286 "advertiser": "Los Pollos Hermanos",
5287 "iab_category": "8 - Food & Drink",
5288 "title": "Los Pollos Hermanos - Albuquerque",
5289 "url": "https://www.lph-nm.biz",
5290 "icon": "5678",
5291 "impression_url": "https://example.com/impression_url",
5292 "click_url": "https://example.com/click_url",
5293 "score": 0.3
5294 }]),
5295 )?;
5297 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5299 store.ingest(SuggestIngestionConstraints::default())?;
5301 // Test that the invalid record marked as unparsable
5302 store.dbs()?.reader.read(|dao| {
5303 expect![[r#"
5304 Some(
5305 UnparsableRecords(
5306 {
5307 "invalid-attachment": UnparsableRecord {
5308 schema_version: 17,
5309 },
5310 },
5311 ),
5312 )
5313 "#]]
5314 .assert_debug_eq(&dao.get_meta::<UnparsableRecords>(UNPARSABLE_RECORDS_META_KEY)?);
5315 Ok(())
5316 })?;
5318 // Test that the valid record was read
5319 store.dbs()?.reader.read(|dao| {
5320 assert_eq!(
5321 dao.get_meta(
5322 SuggestRecordType::AmpWikipedia
5323 .last_ingest_meta_key()
5324 .as_str()
5325 )?,
5326 Some(15)
5327 );
5328 expect![[r#"
5329 [
5330 Amp {
5331 title: "Los Pollos Hermanos - Albuquerque",
5332 url: "https://www.lph-nm.biz",
5333 raw_url: "https://www.lph-nm.biz",
5334 icon: None,
5335 icon_mimetype: None,
5336 full_keyword: "los",
5337 block_id: 0,
5338 advertiser: "Los Pollos Hermanos",
5339 iab_category: "8 - Food & Drink",
5340 impression_url: "https://example.com/impression_url",
5341 click_url: "https://example.com/click_url",
5342 raw_click_url: "https://example.com/click_url",
5343 score: 0.3,
5344 },
5345 ]
5346 "#]]
5347 .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery {
5348 keyword: "lo".into(),
5349 providers: vec![SuggestionProvider::Amp],
5350 limit: None,
5351 })?);
5353 Ok(())
5354 })?;
5356 Ok(())
5357 }
5359 #[test]
5360 fn unparsable_record_serialized_correctly() -> anyhow::Result<()> {
5361 let unparseable_record = UnparsableRecord { schema_version: 1 };
5362 assert_eq!(serde_json::to_value(unparseable_record)?, json!({ "v": 1 }),);
5363 Ok(())
5364 }
5366 #[test]
5367 fn query_mdn() -> anyhow::Result<()> {
5368 before_each();
5370 let snapshot = Snapshot::with_records(json!([{
5371 "id": "data-1",
5372 "type": "mdn-suggestions",
5373 "last_modified": 15,
5374 "attachment": {
5375 "filename": "data-1.json",
5376 "mimetype": "application/json",
5377 "location": "data-1.json",
5378 "hash": "",
5379 "size": 0,
5380 },
5381 }]))?
5382 .with_data(
5383 "data-1.json",
5384 json!([
5385 {
5386 "description": "Javascript Array",
5387 "url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5388 "keywords": ["array javascript", "javascript array", "wildcard"],
5389 "title": "Array",
5390 "score": 0.24
5391 },
5392 ]),
5393 )?;
5395 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5397 store.ingest(SuggestIngestionConstraints::default())?;
5399 let table = [
5400 (
5401 "keyword = prefix; MDN only",
5402 SuggestionQuery {
5403 keyword: "array".into(),
5404 providers: vec![SuggestionProvider::Mdn],
5405 limit: None,
5406 },
5407 expect![[r#"
5408 [
5409 Mdn {
5410 title: "Array",
5411 url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5412 description: "Javascript Array",
5413 score: 0.24,
5414 },
5415 ]
5416 "#]],
5417 ),
5418 (
5419 "keyword = prefix + partial suffix; MDN only",
5420 SuggestionQuery {
5421 keyword: "array java".into(),
5422 providers: vec![SuggestionProvider::Mdn],
5423 limit: None,
5424 },
5425 expect![[r#"
5426 [
5427 Mdn {
5428 title: "Array",
5429 url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5430 description: "Javascript Array",
5431 score: 0.24,
5432 },
5433 ]
5434 "#]],
5435 ),
5436 (
5437 "keyword = prefix + entire suffix; MDN only",
5438 SuggestionQuery {
5439 keyword: "javascript array".into(),
5440 providers: vec![SuggestionProvider::Mdn],
5441 limit: None,
5442 },
5443 expect![[r#"
5444 [
5445 Mdn {
5446 title: "Array",
5447 url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5448 description: "Javascript Array",
5449 score: 0.24,
5450 },
5451 ]
5452 "#]],
5453 ),
5454 (
5455 "keyword = `partial prefix word`; MDN only",
5456 SuggestionQuery {
5457 keyword: "wild".into(),
5458 providers: vec![SuggestionProvider::Mdn],
5459 limit: None,
5460 },
5461 expect![[r#"
5462 []
5463 "#]],
5464 ),
5465 (
5466 "keyword = single word; MDN only",
5467 SuggestionQuery {
5468 keyword: "wildcard".into(),
5469 providers: vec![SuggestionProvider::Mdn],
5470 limit: None,
5471 },
5472 expect![[r#"
5473 [
5474 Mdn {
5475 title: "Array",
5476 url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array",
5477 description: "Javascript Array",
5478 score: 0.24,
5479 },
5480 ]
5481 "#]],
5482 ),
5483 ];
5485 for (what, query, expect) in table {
5486 expect.assert_debug_eq(
5487 &store
5488 .query(query)
5489 .with_context(|| format!("Couldn't query store for {}", what))?,
5490 );
5491 }
5493 Ok(())
5494 }
5496 #[test]
5497 fn query_no_yelp_icon_data() -> anyhow::Result<()> {
5498 before_each();
5500 let snapshot = Snapshot::with_records(json!([{
5501 "id": "data-1",
5502 "type": "yelp-suggestions",
5503 "last_modified": 15,
5504 "attachment": {
5505 "filename": "data-1.json",
5506 "mimetype": "application/json",
5507 "location": "data-1.json",
5508 "hash": "",
5509 "size": 0,
5510 },
5511 }]))?
5512 .with_data(
5513 "data-1.json",
5514 json!([
5515 {
5516 "subjects": ["ramen"],
5517 "preModifiers": [],
5518 "postModifiers": [],
5519 "locationSigns": [],
5520 "yelpModifiers": [],
5521 "icon": "yelp-favicon",
5522 "score": 0.5
5523 },
5524 ]),
5525 )?;
5527 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5529 store.ingest(SuggestIngestionConstraints::default())?;
5531 let table = [(
5532 "keyword = ramen; Yelp only",
5533 SuggestionQuery {
5534 keyword: "ramen".into(),
5535 providers: vec![SuggestionProvider::Yelp],
5536 limit: None,
5537 },
5538 expect![[r#"
5539 [
5540 Yelp {
5541 url: "https://www.yelp.com/search?find_desc=ramen",
5542 title: "ramen",
5543 icon: None,
5544 icon_mimetype: None,
5545 score: 0.5,
5546 has_location_sign: false,
5547 subject_exact_match: true,
5548 location_param: "find_loc",
5549 },
5550 ]
5551 "#]],
5552 )];
5554 for (what, query, expect) in table {
5555 expect.assert_debug_eq(
5556 &store
5557 .query(query)
5558 .with_context(|| format!("Couldn't query store for {}", what))?,
5559 );
5560 }
5562 Ok(())
5563 }
5565 #[test]
5566 fn weather() -> anyhow::Result<()> {
5567 before_each();
5569 let snapshot = Snapshot::with_records(json!([{
5570 "id": "data-1",
5571 "type": "weather",
5572 "last_modified": 15,
5573 "weather": {
5574 "min_keyword_length": 3,
5575 "keywords": ["ab", "xyz", "weather"],
5576 "score": "0.24"
5577 }
5578 }]))?;
5580 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5581 store.ingest(SuggestIngestionConstraints::default())?;
5583 let table = [
5584 (
5585 "keyword = 'ab'; Weather only, no match since query is too short",
5586 SuggestionQuery {
5587 keyword: "ab".into(),
5588 providers: vec![SuggestionProvider::Weather],
5589 limit: None,
5590 },
5591 expect![[r#"
5592 []
5593 "#]],
5594 ),
5595 (
5596 "keyword = 'xab'; Weather only, no matching keyword",
5597 SuggestionQuery {
5598 keyword: "xab".into(),
5599 providers: vec![SuggestionProvider::Weather],
5600 limit: None,
5601 },
5602 expect![[r#"
5603 []
5604 "#]],
5605 ),
5606 (
5607 "keyword = 'abx'; Weather only, no matching keyword",
5608 SuggestionQuery {
5609 keyword: "abx".into(),
5610 providers: vec![SuggestionProvider::Weather],
5611 limit: None,
5612 },
5613 expect![[r#"
5614 []
5615 "#]],
5616 ),
5617 (
5618 "keyword = 'xy'; Weather only, no match since query is too short",
5619 SuggestionQuery {
5620 keyword: "xy".into(),
5621 providers: vec![SuggestionProvider::Weather],
5622 limit: None,
5623 },
5624 expect![[r#"
5625 []
5626 "#]],
5627 ),
5628 (
5629 "keyword = 'xyz'; Weather only, match",
5630 SuggestionQuery {
5631 keyword: "xyz".into(),
5632 providers: vec![SuggestionProvider::Weather],
5633 limit: None,
5634 },
5635 expect![[r#"
5636 [
5637 Weather {
5638 score: 0.24,
5639 },
5640 ]
5641 "#]],
5642 ),
5643 (
5644 "keyword = 'xxyz'; Weather only, no matching keyword",
5645 SuggestionQuery {
5646 keyword: "xxyz".into(),
5647 providers: vec![SuggestionProvider::Weather],
5648 limit: None,
5649 },
5650 expect![[r#"
5651 []
5652 "#]],
5653 ),
5654 (
5655 "keyword = 'xyzx'; Weather only, no matching keyword",
5656 SuggestionQuery {
5657 keyword: "xyzx".into(),
5658 providers: vec![SuggestionProvider::Weather],
5659 limit: None,
5660 },
5661 expect![[r#"
5662 []
5663 "#]],
5664 ),
5665 (
5666 "keyword = 'we'; Weather only, no match since query is too short",
5667 SuggestionQuery {
5668 keyword: "we".into(),
5669 providers: vec![SuggestionProvider::Weather],
5670 limit: None,
5671 },
5672 expect![[r#"
5673 []
5674 "#]],
5675 ),
5676 (
5677 "keyword = 'wea'; Weather only, match",
5678 SuggestionQuery {
5679 keyword: "wea".into(),
5680 providers: vec![SuggestionProvider::Weather],
5681 limit: None,
5682 },
5683 expect![[r#"
5684 [
5685 Weather {
5686 score: 0.24,
5687 },
5688 ]
5689 "#]],
5690 ),
5691 (
5692 "keyword = 'weat'; Weather only, match",
5693 SuggestionQuery {
5694 keyword: "weat".into(),
5695 providers: vec![SuggestionProvider::Weather],
5696 limit: None,
5697 },
5698 expect![[r#"
5699 [
5700 Weather {
5701 score: 0.24,
5702 },
5703 ]
5704 "#]],
5705 ),
5706 (
5707 "keyword = 'weath'; Weather only, match",
5708 SuggestionQuery {
5709 keyword: "weath".into(),
5710 providers: vec![SuggestionProvider::Weather],
5711 limit: None,
5712 },
5713 expect![[r#"
5714 [
5715 Weather {
5716 score: 0.24,
5717 },
5718 ]
5719 "#]],
5720 ),
5721 (
5722 "keyword = 'weathe'; Weather only, match",
5723 SuggestionQuery {
5724 keyword: "weathe".into(),
5725 providers: vec![SuggestionProvider::Weather],
5726 limit: None,
5727 },
5728 expect![[r#"
5729 [
5730 Weather {
5731 score: 0.24,
5732 },
5733 ]
5734 "#]],
5735 ),
5736 (
5737 "keyword = 'weather'; Weather only, match",
5738 SuggestionQuery {
5739 keyword: "weather".into(),
5740 providers: vec![SuggestionProvider::Weather],
5741 limit: None,
5742 },
5743 expect![[r#"
5744 [
5745 Weather {
5746 score: 0.24,
5747 },
5748 ]
5749 "#]],
5750 ),
5751 (
5752 "keyword = 'weatherx'; Weather only, no matching keyword",
5753 SuggestionQuery {
5754 keyword: "weatherx".into(),
5755 providers: vec![SuggestionProvider::Weather],
5756 limit: None,
5757 },
5758 expect![[r#"
5759 []
5760 "#]],
5761 ),
5762 (
5763 "keyword = 'xweather'; Weather only, no matching keyword",
5764 SuggestionQuery {
5765 keyword: "xweather".into(),
5766 providers: vec![SuggestionProvider::Weather],
5767 limit: None,
5768 },
5769 expect![[r#"
5770 []
5771 "#]],
5772 ),
5773 (
5774 "keyword = 'xwea'; Weather only, no matching keyword",
5775 SuggestionQuery {
5776 keyword: "xwea".into(),
5777 providers: vec![SuggestionProvider::Weather],
5778 limit: None,
5779 },
5780 expect![[r#"
5781 []
5782 "#]],
5783 ),
5784 (
5785 "keyword = ' weather '; Weather only, match",
5786 SuggestionQuery {
5787 keyword: " weather ".into(),
5788 providers: vec![SuggestionProvider::Weather],
5789 limit: None,
5790 },
5791 expect![[r#"
5792 [
5793 Weather {
5794 score: 0.24,
5795 },
5796 ]
5797 "#]],
5798 ),
5799 (
5800 "keyword = 'x weather '; Weather only, no matching keyword",
5801 SuggestionQuery {
5802 keyword: "x weather ".into(),
5803 providers: vec![SuggestionProvider::Weather],
5804 limit: None,
5805 },
5806 expect![[r#"
5807 []
5808 "#]],
5809 ),
5810 (
5811 "keyword = ' weather x'; Weather only, no matching keyword",
5812 SuggestionQuery {
5813 keyword: " weather x".into(),
5814 providers: vec![SuggestionProvider::Weather],
5815 limit: None,
5816 },
5817 expect![[r#"
5818 []
5819 "#]],
5820 ),
5821 ];
5823 for (what, query, expect) in table {
5824 expect.assert_debug_eq(
5825 &store
5826 .query(query)
5827 .with_context(|| format!("Couldn't query store for {}", what))?,
5828 );
5829 }
5831 expect![[r#"
5832 Some(
5833 Weather {
5834 min_keyword_length: 3,
5835 },
5836 )
5837 "#]]
5838 .assert_debug_eq(
5839 &store
5840 .fetch_provider_config(SuggestionProvider::Weather)
5841 .with_context(|| "Couldn't fetch provider config")?,
5842 );
5844 Ok(())
5845 }
5847 #[test]
5848 fn fetch_global_config() -> anyhow::Result<()> {
5849 before_each();
5851 let snapshot = Snapshot::with_records(json!([{
5852 "id": "data-1",
5853 "type": "configuration",
5854 "last_modified": 15,
5855 "configuration": {
5856 "show_less_frequently_cap": 3,
5857 }
5858 }]))?;
5860 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5861 store.ingest(SuggestIngestionConstraints::default())?;
5863 expect![[r#"
5864 SuggestGlobalConfig {
5865 show_less_frequently_cap: 3,
5866 }
5867 "#]]
5868 .assert_debug_eq(
5869 &store
5870 .fetch_global_config()
5871 .with_context(|| "fetch_global_config failed")?,
5872 );
5874 Ok(())
5875 }
5877 #[test]
5878 fn fetch_global_config_default() -> anyhow::Result<()> {
5879 before_each();
5881 let snapshot = Snapshot::with_records(json!([]))?;
5882 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5883 store.ingest(SuggestIngestionConstraints::default())?;
5885 expect![[r#"
5886 SuggestGlobalConfig {
5887 show_less_frequently_cap: 0,
5888 }
5889 "#]]
5890 .assert_debug_eq(
5891 &store
5892 .fetch_global_config()
5893 .with_context(|| "fetch_global_config failed")?,
5894 );
5896 Ok(())
5897 }
5899 #[test]
5900 fn fetch_provider_config_none() -> anyhow::Result<()> {
5901 before_each();
5903 let snapshot = Snapshot::with_records(json!([]))?;
5904 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5905 store.ingest(SuggestIngestionConstraints::default())?;
5907 expect![[r#"
5908 None
5909 "#]]
5910 .assert_debug_eq(
5911 &store
5912 .fetch_provider_config(SuggestionProvider::Amp)
5913 .with_context(|| "fetch_provider_config failed for Amp")?,
5914 );
5916 expect![[r#"
5917 None
5918 "#]]
5919 .assert_debug_eq(
5920 &store
5921 .fetch_provider_config(SuggestionProvider::Weather)
5922 .with_context(|| "fetch_provider_config failed for Weather")?,
5923 );
5925 Ok(())
5926 }
5928 #[test]
5929 fn fetch_provider_config_other() -> anyhow::Result<()> {
5930 before_each();
5932 // Add some weather config.
5933 let snapshot = Snapshot::with_records(json!([{
5934 "id": "data-1",
5935 "type": "weather",
5936 "last_modified": 15,
5937 "weather": {
5938 "min_keyword_length": 3,
5939 "keywords": ["weather"],
5940 "score": "0.24"
5941 }
5942 }]))?;
5944 let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot));
5945 store.ingest(SuggestIngestionConstraints::default())?;
5947 // Getting the config for a different provider should return None.
5948 expect![[r#"
5949 None
5950 "#]]
5951 .assert_debug_eq(
5952 &store
5953 .fetch_provider_config(SuggestionProvider::Amp)
5954 .with_context(|| "fetch_provider_config failed for Amp")?,
5955 );
5957 Ok(())
5958 }
5959 }