1 /* This Source Code Form is subject to the terms of the Mozilla Public
2 * License, v. 2.0. If a copy of the MPL was not distributed with this
3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6 * This file implements a mirror and two-way merger for synced bookmarks. The
7 * mirror matches the complete tree stored on the Sync server, and stages new
8 * bookmarks changed on the server since the last sync. The merger walks the
9 * local tree in Places and the mirrored remote tree, produces a new merged
10 * tree, then updates the local tree to reflect the merged tree.
12 * Let's start with an overview of the different classes, and how they fit
15 * - `SyncedBookmarksMirror` sets up the database, validates and upserts new
16 * incoming records, attaches to Places, and applies the changed records.
17 * During application, we fetch the local and remote bookmark trees, merge
18 * them, and update Places to match. Merging and application happen in a
19 * single transaction, so applying the merged tree won't collide with local
20 * changes. A failure at this point aborts the merge and leaves Places
23 * - A `BookmarkTree` is a fully rooted tree that also notes deletions. A
24 * `BookmarkNode` represents a local item in Places, or a remote item in the
27 * - A `MergedBookmarkNode` holds a local node, a remote node, and a
28 * `MergeState` that indicates which node to prefer when updating Places and
29 * the server to match the merged tree.
31 * - `BookmarkObserverRecorder` records all changes made to Places during the
32 * merge, then dispatches `PlacesObservers` notifications. Places uses
33 * these notifications to update the UI and internal caches. We can't dispatch
34 * during the merge because observers won't see the changes until the merge
35 * transaction commits and the database is consistent again.
37 * - After application, we flag all applied incoming items as merged, create
38 * Sync records for the locally new and updated items in Places, and upload
39 * the records to the server. At this point, all outgoing items are flagged as
40 * changed in Places, so the next sync can resume cleanly if the upload is
41 * interrupted or fails.
43 * - Once upload succeeds, we update the mirror with the uploaded records, so
44 * that the mirror matches the server again. An interruption or error here
45 * will leave the uploaded items flagged as changed in Places, so we'll merge
46 * them again on the next sync. This is redundant work, but shouldn't cause
52 ChromeUtils.defineESModuleGetters(lazy, {
53 Async: "resource://services-common/async.sys.mjs",
54 Log: "resource://gre/modules/Log.sys.mjs",
55 PlacesSyncUtils: "resource://gre/modules/PlacesSyncUtils.sys.mjs",
56 PlacesUtils: "resource://gre/modules/PlacesUtils.sys.mjs",
59 ChromeUtils.defineLazyGetter(lazy, "MirrorLog", () =>
60 lazy.Log.repository.getLogger("Sync.Engine.Bookmarks.Mirror")
63 const SyncedBookmarksMerger = Components.Constructor(
64 "@mozilla.org/browser/synced-bookmarks-merger;1",
65 "mozISyncedBookmarksMerger"
68 // These can be removed once they're exposed in a central location (bug
70 const DB_URL_LENGTH_MAX = 65536;
71 const DB_TITLE_LENGTH_MAX = 4096;
73 // The current mirror database schema version. Bump for migrations, then add
74 // migration code to `migrateMirrorSchema`.
75 const MIRROR_SCHEMA_VERSION = 9;
77 // Use a shared jankYielder in these functions
78 ChromeUtils.defineLazyGetter(lazy, "yieldState", () => lazy.Async.yieldState());
80 /** Adapts a `Log.sys.mjs` logger to a `mozIServicesLogSink`. */
87 let level = this.log.level;
88 if (level <= lazy.Log.Level.All) {
89 return Ci.mozIServicesLogSink.LEVEL_TRACE;
91 if (level <= lazy.Log.Level.Info) {
92 return Ci.mozIServicesLogSink.LEVEL_DEBUG;
94 if (level <= lazy.Log.Level.Warn) {
95 return Ci.mozIServicesLogSink.LEVEL_WARN;
97 if (level <= lazy.Log.Level.Error) {
98 return Ci.mozIServicesLogSink.LEVEL_ERROR;
100 return Ci.mozIServicesLogSink.LEVEL_OFF;
104 this.log.trace(message);
108 this.log.debug(message);
112 this.log.warn(message);
116 this.log.error(message);
121 * A helper to track the progress of a merge for telemetry and shutdown hang
124 class ProgressTracker {
125 constructor(recordStepTelemetry) {
126 this.recordStepTelemetry = recordStepTelemetry;
131 * Records a merge step, updating the shutdown blocker state.
133 * @param {String} name A step name from `ProgressTracker.STEPS`. This is
134 * included in shutdown hang crash reports, along with the timestamp
135 * the step was recorded.
136 * @param {Number} [took] The time taken, in milliseconds.
137 * @param {Array} [counts] An array of additional counts to report in the
138 * shutdown blocker state.
140 step(name, took = -1, counts = null) {
141 let info = { step: name, at: Date.now() };
146 info.counts = counts;
148 this.steps.push(info);
152 * Records a merge step with timings and counts for telemetry.
154 * @param {String} name The step name.
155 * @param {Number} took The time taken, in milliseconds.
156 * @param {Array} [counts] An array of additional `{ name, count }` tuples to
157 * record in telemetry for this step.
159 stepWithTelemetry(name, took, counts = null) {
160 this.step(name, took, counts);
161 this.recordStepTelemetry(name, took, counts);
165 * Records a merge step with the time taken and item count.
167 * @param {String} name The step name.
168 * @param {Number} took The time taken, in milliseconds.
169 * @param {Number} count The number of items handled in this step.
171 stepWithItemCount(name, took, count) {
172 this.stepWithTelemetry(name, took, [{ name: "items", count }]);
176 * Clears all recorded merge steps.
183 * Returns the shutdown blocker state. This is included in shutdown hang
184 * crash reports, in the `AsyncShutdownTimeout` annotation.
186 * @see `fetchState` in `AsyncShutdown` for more details.
187 * @return {Object} A stringifiable object with the recorded steps.
190 return { steps: this.steps };
194 /** Merge steps for which we record progress. */
195 ProgressTracker.STEPS = {
196 FETCH_LOCAL_TREE: "fetchLocalTree",
197 FETCH_REMOTE_TREE: "fetchRemoteTree",
200 NOTIFY_OBSERVERS: "notifyObservers",
201 FETCH_LOCAL_CHANGE_RECORDS: "fetchLocalChangeRecords",
202 FINALIZE: "finalize",
206 * A mirror maintains a copy of the complete tree as stored on the Sync server.
209 * The mirror schema is a hybrid of how Sync and Places represent bookmarks.
210 * The `items` table contains item attributes (title, kind, URL, etc.), while
211 * the `structure` table stores parent-child relationships and position.
212 * This is similar to how iOS encodes "value" and "structure" state,
213 * though we handle these differently when merging. See `BookmarkMerger` for
216 * There's no guarantee that the remote state is consistent. We might be missing
217 * parents or children, or a bookmark and its parent might disagree about where
218 * it belongs. This means we need a strategy to handle missing parents and
221 * We treat the `children` of the last parent we see as canonical, and ignore
222 * the child's `parentid` entirely. We also ignore missing children, and
223 * temporarily reparent bookmarks with missing parents to "unfiled". When we
224 * eventually see the missing items, either during a later sync or as part of
225 * repair, we'll fill in the mirror's gaps and fix up the local tree.
227 * During merging, we won't intentionally try to fix inconsistencies on the
228 * server, and opt to build as complete a tree as we can from the remote state,
229 * even if we diverge from what's in the mirror. See bug 1433512 for context.
231 * If a sync is interrupted, we resume downloading from the server collection
232 * last modified time, or the server last modified time of the most recent
233 * record if newer. New incoming records always replace existing records in the
236 * We delete the mirror database on client reset, including when the sync ID
237 * changes on the server, and when the user is node reassigned, disables the
238 * bookmarks engine, or signs out.
240 export class SyncedBookmarksMirror {
246 recordValidationTelemetry,
247 finalizeAt = lazy.PlacesUtils.history.shutdownClient.jsclient,
251 this.wasCorrupt = wasCorrupt;
252 this.recordValidationTelemetry = recordValidationTelemetry;
254 this.merger = new SyncedBookmarksMerger();
255 this.merger.db = db.unsafeRawConnection.QueryInterface(
256 Ci.mozIStorageConnection
258 this.merger.logger = new LogAdapter(lazy.MirrorLog);
260 // Automatically close the database connection on shutdown. `progress`
261 // tracks state for shutdown hang reporting.
262 this.progress = new ProgressTracker(recordStepTelemetry);
263 this.finalizeController = new AbortController();
264 this.finalizeAt = finalizeAt;
265 this.finalizeBound = () => this.finalize({ alsoCleanup: false });
266 this.finalizeAt.addBlocker(
267 "SyncedBookmarksMirror: finalize",
269 { fetchState: () => this.progress }
274 * Sets up the mirror database connection and upgrades the mirror to the
275 * newest schema version. Automatically recreates the mirror if it's corrupt;
278 * @param {String} options.path
279 * The path to the mirror database file, either absolute or relative
280 * to the profile path.
281 * @param {Function} options.recordStepTelemetry
282 * A function with the signature `(name: String, took: Number,
283 * counts: Array?)`, where `name` is the name of the merge step,
284 * `took` is the time taken in milliseconds, and `counts` is an
285 * array of named counts (`{ name, count }` tuples) with additional
286 * counts for the step to record in the telemetry ping.
287 * @param {Function} options.recordValidationTelemetry
288 * A function with the signature `(took: Number, checked: Number,
289 * problems: Array)`, where `took` is the time taken to run
290 * validation in milliseconds, `checked` is the number of items
291 * checked, and `problems` is an array of named problem counts.
292 * @param {AsyncShutdown.Barrier} [options.finalizeAt]
293 * A shutdown phase, barrier, or barrier client that should
294 * automatically finalize the mirror when triggered. Exposed for
296 * @return {SyncedBookmarksMirror}
297 * A mirror ready for use.
299 static async open(options) {
300 let db = await lazy.PlacesUtils.promiseUnsafeWritableDBConnection();
302 throw new TypeError("Can't open mirror without Places connection");
305 if (PathUtils.isAbsolute(options.path)) {
308 path = PathUtils.join(PathUtils.profileDir, options.path);
310 let wasCorrupt = false;
312 await attachAndInitMirrorDatabase(db, path);
314 if (isDatabaseCorrupt(ex)) {
316 "Error attaching mirror to Places; removing and " +
321 await IOUtils.remove(path);
322 await attachAndInitMirrorDatabase(db, path);
324 lazy.MirrorLog.error(
325 "Unrecoverable error attaching mirror to Places",
331 return new SyncedBookmarksMirror(db, wasCorrupt, options);
335 * Returns the newer of the bookmarks collection last modified time, or the
336 * server modified time of the newest record. The bookmarks engine uses this
337 * timestamp as the "high water mark" for all downloaded records. Each sync
338 * downloads and stores records that are strictly newer than this time.
341 * The high water mark time, in seconds.
343 async getCollectionHighWaterMark() {
344 // The first case, where we have records with server modified times newer
345 // than the collection last modified time, occurs when a sync is interrupted
346 // before we call `setCollectionLastModified`. We subtract one second, the
347 // maximum time precision guaranteed by the server, so that we don't miss
348 // other records with the same time as the newest one we downloaded.
349 let rows = await this.db.executeCached(
352 IFNULL((SELECT MAX(serverModified) - 1000 FROM items), 0),
353 IFNULL((SELECT CAST(value AS INTEGER) FROM meta
354 WHERE key = :modifiedKey), 0)
356 { modifiedKey: SyncedBookmarksMirror.META_KEY.LAST_MODIFIED }
358 let highWaterMark = rows[0].getResultByName("highWaterMark");
359 return highWaterMark / 1000;
363 * Updates the bookmarks collection last modified time. Note that this may
364 * be newer than the modified time of the most recent record.
366 * @param {Number|String} lastModifiedSeconds
367 * The collection last modified time, in seconds.
369 async setCollectionLastModified(lastModifiedSeconds) {
370 let lastModified = Math.floor(lastModifiedSeconds * 1000);
371 if (!Number.isInteger(lastModified)) {
372 throw new TypeError("Invalid collection last modified time");
374 await this.db.executeBeforeShutdown(
375 "SyncedBookmarksMirror: setCollectionLastModified",
379 REPLACE INTO meta(key, value)
380 VALUES(:modifiedKey, :lastModified)`,
382 modifiedKey: SyncedBookmarksMirror.META_KEY.LAST_MODIFIED,
390 * Returns the bookmarks collection sync ID. This corresponds to
391 * `PlacesSyncUtils.bookmarks.getSyncId`.
394 * The sync ID, or `""` if one isn't set.
397 let rows = await this.db.executeCached(
399 SELECT value FROM meta WHERE key = :syncIdKey`,
400 { syncIdKey: SyncedBookmarksMirror.META_KEY.SYNC_ID }
402 return rows.length ? rows[0].getResultByName("value") : "";
406 * Ensures that the sync ID in the mirror is up-to-date with the server and
407 * Places, and discards the mirror on mismatch.
409 * The bookmarks engine store the same sync ID in Places and the mirror to
410 * "tie" the two together. This allows Sync to do the right thing if the
411 * database files are copied between profiles connected to different accounts.
413 * See `PlacesSyncUtils.bookmarks.ensureCurrentSyncId` for an explanation of
414 * how Places handles sync ID mismatches.
416 * @param {String} newSyncId
417 * The server's sync ID.
419 async ensureCurrentSyncId(newSyncId) {
420 if (!newSyncId || typeof newSyncId != "string") {
421 throw new TypeError("Invalid new bookmarks sync ID");
423 let existingSyncId = await this.getSyncId();
424 if (existingSyncId == newSyncId) {
425 lazy.MirrorLog.trace("Sync ID up-to-date in mirror", { existingSyncId });
429 "Sync ID changed from ${existingSyncId} to " +
430 "${newSyncId}; resetting mirror",
431 { existingSyncId, newSyncId }
433 await this.db.executeBeforeShutdown(
434 "SyncedBookmarksMirror: ensureCurrentSyncId",
436 db.executeTransaction(async function () {
437 await resetMirror(db);
440 REPLACE INTO meta(key, value)
441 VALUES(:syncIdKey, :newSyncId)`,
442 { syncIdKey: SyncedBookmarksMirror.META_KEY.SYNC_ID, newSyncId }
449 * Stores incoming or uploaded Sync records in the mirror. Rejects if any
450 * records are invalid.
452 * @param {PlacesItem[]} records
453 * Sync records to store in the mirror.
454 * @param {Boolean} [options.needsMerge]
455 * Indicates if the records were changed remotely since the last sync,
456 * and should be merged into the local tree. This option is set to
457 * `true` for incoming records, and `false` for successfully uploaded
458 * records. Tests can also pass `false` to set up an existing mirror.
459 * @param {AbortSignal} [options.signal]
460 * An abort signal that can be used to interrupt the operation. If
461 * omitted, storing incoming items can still be interrupted when the
462 * mirror is finalized.
464 async store(records, { needsMerge = true, signal = null } = {}) {
467 signal: anyAborted(this.finalizeController.signal, signal),
469 await this.db.executeBeforeShutdown("SyncedBookmarksMirror: store", db =>
470 db.executeTransaction(async () => {
471 for (let record of records) {
472 if (options.signal.aborted) {
473 throw new SyncedBookmarksMirror.InterruptedError(
474 "Interrupted while storing incoming items"
477 let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
478 if (guid == lazy.PlacesUtils.bookmarks.rootGuid) {
479 // The engine should hard DELETE Places roots from the server.
480 throw new TypeError("Can't store Places root");
482 if (lazy.MirrorLog.level <= lazy.Log.Level.Trace) {
483 lazy.MirrorLog.trace(
484 `Storing in mirror: ${record.cleartextToString()}`
487 switch (record.type) {
489 await this.storeRemoteBookmark(record, options);
493 await this.storeRemoteQuery(record, options);
497 await this.storeRemoteFolder(record, options);
501 await this.storeRemoteLivemark(record, options);
505 await this.storeRemoteSeparator(record, options);
509 if (record.deleted) {
510 await this.storeRemoteTombstone(record, options);
514 lazy.MirrorLog.warn("Ignoring record with unknown type", record.type);
521 * Builds a complete merged tree from the local and remote trees, resolves
522 * value and structure conflicts, dedupes local items, applies the merged
523 * tree back to Places, and notifies observers about the changes.
525 * Merging and application happen in a transaction, meaning code that uses the
526 * main Places connection, including the UI, will fail to write to the
527 * database until the transaction commits. Asynchronous consumers will retry
528 * on `SQLITE_BUSY`; synchronous consumers will fail after waiting for 100ms.
529 * See bug 1305563, comment 122 for details.
531 * @param {Number} [options.localTimeSeconds]
532 * The current local time, in seconds.
533 * @param {Number} [options.remoteTimeSeconds]
534 * The current server time, in seconds.
535 * @param {Boolean} [options.notifyInStableOrder]
536 * If `true`, fire observer notifications for items in the same folder
537 * in a stable order. This is disabled by default, to avoid the cost
538 * of sorting the notifications, but enabled in some tests to simplify
540 * @param {AbortSignal} [options.signal]
541 * An abort signal that can be used to interrupt a merge when its
542 * associated `AbortController` is aborted. If omitted, the merge can
543 * still be interrupted when the mirror is finalized.
544 * @return {Object.<String, BookmarkChangeRecord>}
545 * A changeset containing locally changed and reconciled records to
546 * upload to the server, and to store in the mirror once upload
555 // We intentionally don't use `executeBeforeShutdown` in this function,
556 // since merging can take a while for large trees, and we don't want to
557 // block shutdown. Since all new items are in the mirror, we'll just try
558 // to merge again on the next sync.
560 let finalizeOrInterruptSignal = anyAborted(
561 this.finalizeController.signal,
567 changeRecords = await this.tryApply(
568 finalizeOrInterruptSignal,
574 this.progress.reset();
577 return changeRecords;
584 notifyInStableOrder = false
586 let wasMerged = await withTiming("Merging bookmarks in Rust", () =>
587 this.merge(signal, localTimeSeconds, remoteTimeSeconds)
591 lazy.MirrorLog.debug("No changes detected in both mirror and Places");
595 // At this point, the database is consistent, so we can notify observers and
596 // inflate records for outgoing items.
598 let observersToNotify = new BookmarkObserverRecorder(this.db, {
604 "Notifying Places observers",
607 // Note that we don't use a transaction when fetching info for
608 // observers, so it's possible we might notify with stale info if the
609 // main connection changes Places between the time we finish merging,
610 // and the time we notify observers.
611 await observersToNotify.notifyAll();
613 // Places relies on observer notifications to update internal caches.
614 // If notifying observers failed, these caches may be inconsistent,
615 // so we invalidate them just in case.
616 await lazy.PlacesUtils.keywords.invalidateCachedKeywords();
617 lazy.MirrorLog.warn("Error notifying Places observers", ex);
619 await this.db.executeTransaction(async () => {
620 await this.db.execute(`DELETE FROM itemsAdded`);
621 await this.db.execute(`DELETE FROM guidsChanged`);
622 await this.db.execute(`DELETE FROM itemsChanged`);
623 await this.db.execute(`DELETE FROM itemsRemoved`);
624 await this.db.execute(`DELETE FROM itemsMoved`);
629 this.progress.stepWithTelemetry(
630 ProgressTracker.STEPS.NOTIFY_OBSERVERS,
635 let { changeRecords } = await withTiming(
636 "Fetching records for local items to upload",
639 let result = await this.fetchLocalChangeRecords(signal);
642 await this.db.execute(`DELETE FROM itemsToUpload`);
646 this.progress.stepWithItemCount(
647 ProgressTracker.STEPS.FETCH_LOCAL_CHANGE_RECORDS,
653 return changeRecords;
656 merge(signal, localTimeSeconds = Date.now() / 1000, remoteTimeSeconds = 0) {
657 return new Promise((resolve, reject) => {
660 signal.removeEventListener("abort", onAbort);
664 QueryInterface: ChromeUtils.generateQI([
665 "mozISyncedBookmarksMirrorProgressListener",
666 "mozISyncedBookmarksMirrorCallback",
668 // `mozISyncedBookmarksMirrorProgressListener` methods.
669 onFetchLocalTree: (took, itemCount, deleteCount) => {
680 this.progress.stepWithTelemetry(
681 ProgressTracker.STEPS.FETCH_LOCAL_TREE,
685 // We don't record local tree problems in validation telemetry.
687 onFetchRemoteTree: (took, itemCount, deleteCount, problemsBag) => {
698 this.progress.stepWithTelemetry(
699 ProgressTracker.STEPS.FETCH_REMOTE_TREE,
703 // Record validation telemetry for problems in the remote tree.
704 let problems = bagToNamedCounts(problemsBag, [
709 "parentChildDisagreements",
712 let checked = itemCount + deleteCount;
713 this.recordValidationTelemetry(took, checked, problems);
715 onMerge: (took, countsBag) => {
716 let counts = bagToNamedCounts(countsBag, [
724 this.progress.stepWithTelemetry(
725 ProgressTracker.STEPS.MERGE,
731 this.progress.stepWithTelemetry(ProgressTracker.STEPS.APPLY, took);
733 // `mozISyncedBookmarksMirrorCallback` methods.
734 handleSuccess(result) {
735 signal.removeEventListener("abort", onAbort);
738 handleError(code, message) {
739 signal.removeEventListener("abort", onAbort);
741 case Cr.NS_ERROR_STORAGE_BUSY:
742 reject(new SyncedBookmarksMirror.MergeConflictError(message));
745 case Cr.NS_ERROR_ABORT:
746 reject(new SyncedBookmarksMirror.InterruptedError(message));
750 reject(new SyncedBookmarksMirror.MergeError(message));
754 op = this.merger.merge(localTimeSeconds, remoteTimeSeconds, callback);
755 if (signal.aborted) {
758 signal.addEventListener("abort", onAbort);
764 * Discards the mirror contents. This is called when the user is node
765 * reassigned, disables the bookmarks engine, or signs out.
768 await this.db.executeBeforeShutdown("SyncedBookmarksMirror: reset", db =>
769 db.executeTransaction(() => resetMirror(db))
774 * Fetches the GUIDs of all items in the remote tree that need to be merged
775 * into the local tree.
778 * Remotely changed GUIDs that need to be merged into Places.
780 async fetchUnmergedGuids() {
781 let rows = await this.db.execute(`
782 SELECT guid FROM items
785 return rows.map(row => row.getResultByName("guid"));
788 async storeRemoteBookmark(record, { needsMerge, signal }) {
789 let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
791 let url = validateURL(record.bmkUri);
793 await this.maybeStoreRemoteURL(url);
796 let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(
799 let serverModified = determineServerModified(record);
800 let dateAdded = determineDateAdded(record);
801 let title = validateTitle(record.title);
802 let keyword = validateKeyword(record.keyword);
804 ? Ci.mozISyncedBookmarksMerger.VALIDITY_VALID
805 : Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE;
807 let unknownFields = lazy.PlacesSyncUtils.extractUnknownFields(
815 ...COMMON_UNKNOWN_FIELDS,
818 await this.db.executeCached(
820 REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
821 dateAdded, title, keyword, validity, unknownFields,
823 VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
824 :dateAdded, NULLIF(:title, ''), :keyword, :validity, :unknownFields,
826 WHERE hash = hash(:url) AND
833 kind: Ci.mozISyncedBookmarksMerger.KIND_BOOKMARK,
837 url: url ? url.href : null,
843 let tags = record.tags;
844 if (tags && Array.isArray(tags)) {
845 for (let rawTag of tags) {
846 if (signal.aborted) {
847 throw new SyncedBookmarksMirror.InterruptedError(
848 "Interrupted while storing tags for incoming bookmark"
851 let tag = validateTag(rawTag);
855 await this.db.executeCached(
857 INSERT INTO tags(itemId, tag)
858 SELECT id, :tag FROM items
866 async storeRemoteQuery(record, { needsMerge }) {
867 let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
869 let validity = Ci.mozISyncedBookmarksMerger.VALIDITY_VALID;
871 let url = validateURL(record.bmkUri);
873 // The query has a valid URL. Determine if we need to rewrite and reupload
875 let params = new URLSearchParams(url.href.slice(url.protocol.length));
876 let type = +params.get("type");
877 if (type == Ci.nsINavHistoryQueryOptions.RESULTS_AS_TAG_CONTENTS) {
878 // Legacy tag queries with this type use a `place:` URL with a `folder`
879 // param that points to the tag folder ID. Rewrite the query to directly
880 // reference the tag stored in its `folderName`, then flag the rewritten
881 // query for reupload.
882 let tagFolderName = validateTag(record.folderName);
885 url.href = `place:tag=${tagFolderName}`;
886 validity = Ci.mozISyncedBookmarksMerger.VALIDITY_REUPLOAD;
888 // The tag folder name isn't URL-encoded (bug 1449939), so we might
889 // produce an invalid URL. However, invalid URLs are already likely
890 // to cause other issues, and it's better to replace or delete the
891 // query than break syncing or the Firefox UI.
895 // The tag folder name is invalid, so replace or delete the remote
900 let folder = params.get("folder");
901 if (folder && !params.has("excludeItems")) {
902 // We don't sync enough information to rewrite other queries with a
903 // `folder` param (bug 1377175). Referencing a nonexistent folder ID
904 // causes the query to return all items in the database, so we add
905 // `excludeItems=1` to stop it from doing so. We also flag the
906 // rewritten query for reupload.
908 url.href = `${url.href}&excludeItems=1`;
909 validity = Ci.mozISyncedBookmarksMerger.VALIDITY_REUPLOAD;
916 // Other queries are implicitly valid, and don't need to be reuploaded
921 await this.maybeStoreRemoteURL(url);
923 // If the query doesn't have a valid URL, we must replace the remote copy
924 // with either a valid local copy, or a tombstone if the query doesn't
926 validity = Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE;
929 let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(
932 let serverModified = determineServerModified(record);
933 let dateAdded = determineDateAdded(record);
934 let title = validateTitle(record.title);
936 let unknownFields = lazy.PlacesSyncUtils.extractUnknownFields(
946 ...COMMON_UNKNOWN_FIELDS,
950 await this.db.executeCached(
952 REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
955 validity, unknownFields)
956 VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
957 :dateAdded, NULLIF(:title, ''),
959 WHERE hash = hash(:url) AND
961 :validity, :unknownFields)`,
967 kind: Ci.mozISyncedBookmarksMerger.KIND_QUERY,
970 url: url ? url.href : null,
977 async storeRemoteFolder(record, { needsMerge, signal }) {
978 let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
979 let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(
982 let serverModified = determineServerModified(record);
983 let dateAdded = determineDateAdded(record);
984 let title = validateTitle(record.title);
985 let unknownFields = lazy.PlacesSyncUtils.extractUnknownFields(
987 ["children", "description", "title", ...COMMON_UNKNOWN_FIELDS]
989 await this.db.executeCached(
991 REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
992 dateAdded, title, unknownFields)
993 VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
994 :dateAdded, NULLIF(:title, ''), :unknownFields)`,
1000 kind: Ci.mozISyncedBookmarksMerger.KIND_FOLDER,
1007 let children = record.children;
1008 if (children && Array.isArray(children)) {
1010 for (let chunk of lazy.PlacesUtils.chunkArray(
1012 this.db.variableLimit - 1
1014 if (signal.aborted) {
1015 throw new SyncedBookmarksMirror.InterruptedError(
1016 "Interrupted while storing children for incoming folder"
1019 // Builds a fragment like `(?2, ?1, 0), (?3, ?1, 1), ...`, where ?1 is
1020 // the folder's GUID, [?2, ?3] are the first and second child GUIDs
1021 // (SQLite binding parameters index from 1), and [0, 1] are the
1022 // positions. This lets us store the folder's children using as few
1023 // statements as possible.
1024 let valuesFragment = Array.from(
1025 { length: chunk.length },
1026 (_, index) => `(?${index + 2}, ?1, ${offset + index})`
1028 await this.db.execute(
1030 INSERT INTO structure(guid, parentGuid, position)
1031 VALUES ${valuesFragment}`,
1032 [guid, ...chunk.map(lazy.PlacesSyncUtils.bookmarks.recordIdToGuid)]
1034 offset += chunk.length;
1039 async storeRemoteLivemark(record, { needsMerge }) {
1040 let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
1041 let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(
1044 let serverModified = determineServerModified(record);
1045 let feedURL = validateURL(record.feedUri);
1046 let dateAdded = determineDateAdded(record);
1047 let title = validateTitle(record.title);
1048 let siteURL = validateURL(record.siteUri);
1050 let validity = feedURL
1051 ? Ci.mozISyncedBookmarksMerger.VALIDITY_VALID
1052 : Ci.mozISyncedBookmarksMerger.VALIDITY_REPLACE;
1054 let unknownFields = lazy.PlacesSyncUtils.extractUnknownFields(
1062 ...COMMON_UNKNOWN_FIELDS,
1066 await this.db.executeCached(
1068 REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
1069 dateAdded, title, feedURL, siteURL, validity, unknownFields)
1070 VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
1071 :dateAdded, NULLIF(:title, ''), :feedURL, :siteURL, :validity, :unknownFields)`,
1077 kind: Ci.mozISyncedBookmarksMerger.KIND_LIVEMARK,
1080 feedURL: feedURL ? feedURL.href : null,
1081 siteURL: siteURL ? siteURL.href : null,
1088 async storeRemoteSeparator(record, { needsMerge }) {
1089 let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
1090 let parentGuid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(
1093 let serverModified = determineServerModified(record);
1094 let dateAdded = determineDateAdded(record);
1095 let unknownFields = lazy.PlacesSyncUtils.extractUnknownFields(
1097 ["pos", ...COMMON_UNKNOWN_FIELDS]
1100 await this.db.executeCached(
1102 REPLACE INTO items(guid, parentGuid, serverModified, needsMerge, kind,
1103 dateAdded, unknownFields)
1104 VALUES(:guid, :parentGuid, :serverModified, :needsMerge, :kind,
1105 :dateAdded, :unknownFields)`,
1111 kind: Ci.mozISyncedBookmarksMerger.KIND_SEPARATOR,
1118 async storeRemoteTombstone(record, { needsMerge }) {
1119 let guid = lazy.PlacesSyncUtils.bookmarks.recordIdToGuid(record.id);
1120 let serverModified = determineServerModified(record);
1122 await this.db.executeCached(
1124 REPLACE INTO items(guid, serverModified, needsMerge, isDeleted)
1125 VALUES(:guid, :serverModified, :needsMerge, 1)`,
1126 { guid, serverModified, needsMerge }
1130 async maybeStoreRemoteURL(url) {
1131 await this.db.executeCached(
1133 INSERT OR IGNORE INTO urls(guid, url, hash, revHost)
1134 VALUES(IFNULL((SELECT guid FROM urls
1135 WHERE hash = hash(:url) AND
1137 GENERATE_GUID()), :url, hash(:url), :revHost)`,
1138 { url: url.href, revHost: lazy.PlacesUtils.getReversedHost(url) }
1143 * Inflates Sync records for all staged outgoing items.
1145 * @param {AbortSignal} signal
1146 * Stops fetching records when the associated `AbortController`
1149 * A `{ changeRecords, count }` tuple, where `changeRecords` is a
1150 * changeset containing Sync record cleartexts for outgoing items and
1151 * tombstones, keyed by their Sync record IDs, and `count` is the
1152 * number of records.
1154 async fetchLocalChangeRecords(signal) {
1155 let changeRecords = {};
1156 let childRecordIdsByLocalParentId = new Map();
1157 let tagsByLocalId = new Map();
1159 let childGuidRows = [];
1160 await this.db.execute(
1161 `SELECT parentId, guid FROM structureToUpload
1162 ORDER BY parentId, position`,
1165 if (signal.aborted) {
1168 // `Sqlite.sys.mjs` callbacks swallow exceptions (bug 1387775), so we
1169 // accumulate all rows in an array, and process them after.
1170 childGuidRows.push(row);
1175 await lazy.Async.yieldingForEach(
1178 if (signal.aborted) {
1179 throw new SyncedBookmarksMirror.InterruptedError(
1180 "Interrupted while fetching structure to upload"
1183 let localParentId = row.getResultByName("parentId");
1184 let childRecordId = lazy.PlacesSyncUtils.bookmarks.guidToRecordId(
1185 row.getResultByName("guid")
1187 let childRecordIds = childRecordIdsByLocalParentId.get(localParentId);
1188 if (childRecordIds) {
1189 childRecordIds.push(childRecordId);
1191 childRecordIdsByLocalParentId.set(localParentId, [childRecordId]);
1198 await this.db.execute(
1199 `SELECT id, tag FROM tagsToUpload`,
1202 if (signal.aborted) {
1210 await lazy.Async.yieldingForEach(
1213 if (signal.aborted) {
1214 throw new SyncedBookmarksMirror.InterruptedError(
1215 "Interrupted while fetching tags to upload"
1218 let localId = row.getResultByName("id");
1219 let tag = row.getResultByName("tag");
1220 let tags = tagsByLocalId.get(localId);
1224 tagsByLocalId.set(localId, [tag]);
1231 await this.db.execute(
1232 `SELECT id, syncChangeCounter, guid, isDeleted, type, isQuery,
1233 tagFolderName, keyword, url, IFNULL(title, '') AS title,
1234 position, parentGuid, unknownFields,
1235 IFNULL(parentTitle, '') AS parentTitle, dateAdded
1236 FROM itemsToUpload`,
1239 if (signal.interrupted) {
1247 await lazy.Async.yieldingForEach(
1250 if (signal.aborted) {
1251 throw new SyncedBookmarksMirror.InterruptedError(
1252 "Interrupted while fetching items to upload"
1255 let syncChangeCounter = row.getResultByName("syncChangeCounter");
1257 let guid = row.getResultByName("guid");
1258 let recordId = lazy.PlacesSyncUtils.bookmarks.guidToRecordId(guid);
1260 // Tombstones don't carry additional properties.
1261 let isDeleted = row.getResultByName("isDeleted");
1263 changeRecords[recordId] = new BookmarkChangeRecord(
1273 let parentGuid = row.getResultByName("parentGuid");
1274 let parentRecordId =
1275 lazy.PlacesSyncUtils.bookmarks.guidToRecordId(parentGuid);
1277 let unknownFieldsRow = row.getResultByName("unknownFields");
1278 let unknownFields = unknownFieldsRow
1279 ? JSON.parse(unknownFieldsRow)
1281 let type = row.getResultByName("type");
1283 case lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK: {
1284 let isQuery = row.getResultByName("isQuery");
1286 let queryCleartext = {
1289 // We ignore `parentid` and use the parent's `children`, but older
1290 // Desktops and Android use `parentid` as the canonical parent.
1291 // iOS is stricter and requires both `children` and `parentid` to
1293 parentid: parentRecordId,
1294 // Older Desktops use `hasDupe` (along with `parentName` for
1295 // deduping), if hasDupe is true, then they won't attempt deduping
1296 // (since they believe that a duplicate for this record should
1297 // exist). We set it to true to prevent them from applying their
1300 parentName: row.getResultByName("parentTitle"),
1301 // Omit `dateAdded` from the record if it's not set locally.
1302 dateAdded: row.getResultByName("dateAdded") || undefined,
1303 bmkUri: row.getResultByName("url"),
1304 title: row.getResultByName("title"),
1305 // folderName should never be an empty string or null
1306 folderName: row.getResultByName("tagFolderName") || undefined,
1309 changeRecords[recordId] = new BookmarkChangeRecord(
1316 let bookmarkCleartext = {
1319 parentid: parentRecordId,
1321 parentName: row.getResultByName("parentTitle"),
1322 dateAdded: row.getResultByName("dateAdded") || undefined,
1323 bmkUri: row.getResultByName("url"),
1324 title: row.getResultByName("title"),
1327 let keyword = row.getResultByName("keyword");
1329 bookmarkCleartext.keyword = keyword;
1331 let localId = row.getResultByName("id");
1332 let tags = tagsByLocalId.get(localId);
1334 bookmarkCleartext.tags = tags;
1336 changeRecords[recordId] = new BookmarkChangeRecord(
1343 case lazy.PlacesUtils.bookmarks.TYPE_FOLDER: {
1344 let folderCleartext = {
1347 parentid: parentRecordId,
1349 parentName: row.getResultByName("parentTitle"),
1350 dateAdded: row.getResultByName("dateAdded") || undefined,
1351 title: row.getResultByName("title"),
1354 let localId = row.getResultByName("id");
1355 let childRecordIds = childRecordIdsByLocalParentId.get(localId);
1356 folderCleartext.children = childRecordIds || [];
1357 changeRecords[recordId] = new BookmarkChangeRecord(
1364 case lazy.PlacesUtils.bookmarks.TYPE_SEPARATOR: {
1365 let separatorCleartext = {
1368 parentid: parentRecordId,
1370 parentName: row.getResultByName("parentTitle"),
1371 dateAdded: row.getResultByName("dateAdded") || undefined,
1372 // Older Desktops use `pos` for deduping.
1373 pos: row.getResultByName("position"),
1376 changeRecords[recordId] = new BookmarkChangeRecord(
1384 throw new TypeError("Can't create record for unknown Places item");
1390 return { changeRecords, count: itemRows.length };
1394 * Closes the mirror database connection. This is called automatically on
1395 * shutdown, but may also be called explicitly when the mirror is no longer
1398 * @param {Boolean} [options.alsoCleanup]
1399 * If specified, drop all temp tables, views, and triggers,
1400 * and detach from the mirror database before closing the
1401 * connection. Defaults to `true`.
1403 finalize({ alsoCleanup = true } = {}) {
1404 if (!this.finalizePromise) {
1405 this.finalizePromise = (async () => {
1406 this.progress.step(ProgressTracker.STEPS.FINALIZE);
1407 this.finalizeController.abort();
1408 this.merger.reset();
1410 // If the mirror is finalized explicitly, clean up temp entities and
1411 // detach from the mirror database. We can skip this for automatic
1412 // finalization, since the Places connection is already shutting
1414 await cleanupMirrorDatabase(this.db);
1416 await this.db.execute(`PRAGMA mirror.optimize(0x02)`);
1417 await this.db.execute(`DETACH mirror`);
1418 this.finalizeAt.removeBlocker(this.finalizeBound);
1421 return this.finalizePromise;
1425 /** Key names for the key-value `meta` table. */
1426 SyncedBookmarksMirror.META_KEY = {
1427 LAST_MODIFIED: "collection/lastModified",
1428 SYNC_ID: "collection/syncId",
1432 * An error thrown when the merge was interrupted.
1434 class InterruptedError extends Error {
1435 constructor(message) {
1437 this.name = "InterruptedError";
1440 SyncedBookmarksMirror.InterruptedError = InterruptedError;
1443 * An error thrown when the merge failed for an unexpected reason.
1445 class MergeError extends Error {
1446 constructor(message) {
1448 this.name = "MergeError";
1451 SyncedBookmarksMirror.MergeError = MergeError;
1454 * An error thrown when the merge can't proceed because the local tree
1455 * changed during the merge.
1457 class MergeConflictError extends Error {
1458 constructor(message) {
1460 this.name = "MergeConflictError";
1463 SyncedBookmarksMirror.MergeConflictError = MergeConflictError;
1466 * An error thrown when the mirror database is corrupt, or can't be migrated to
1467 * the latest schema version, and must be replaced.
1469 class DatabaseCorruptError extends Error {
1470 constructor(message) {
1472 this.name = "DatabaseCorruptError";
1476 // Indicates if the mirror should be replaced because the database file is
1478 function isDatabaseCorrupt(error) {
1479 if (error instanceof DatabaseCorruptError) {
1483 return error.errors.some(
1485 error instanceof Ci.mozIStorageError &&
1486 (error.result == Ci.mozIStorageError.CORRUPT ||
1487 error.result == Ci.mozIStorageError.NOTADB)
1494 * Attaches a cloned Places database connection to the mirror database,
1495 * migrates the mirror schema to the latest version, and creates temporary
1496 * tables, views, and triggers.
1498 * @param {Sqlite.OpenedConnection} db
1499 * The Places database connection.
1500 * @param {String} path
1501 * The full path to the mirror database file.
1503 async function attachAndInitMirrorDatabase(db, path) {
1504 await db.execute(`ATTACH :path AS mirror`, { path });
1506 await db.executeTransaction(async function () {
1507 let currentSchemaVersion = await db.getSchemaVersion("mirror");
1508 if (currentSchemaVersion > 0) {
1509 if (currentSchemaVersion < MIRROR_SCHEMA_VERSION) {
1510 await migrateMirrorSchema(db, currentSchemaVersion);
1513 await initializeMirrorDatabase(db);
1515 // Downgrading from a newer profile to an older profile rolls back the
1516 // schema version, but leaves all new columns in place. We'll run the
1517 // migration logic again on the next upgrade.
1518 await db.setSchemaVersion(MIRROR_SCHEMA_VERSION, "mirror");
1519 await initializeTempMirrorEntities(db);
1522 await db.execute(`DETACH mirror`);
1528 * Migrates the mirror database schema to the latest version.
1530 * @param {Sqlite.OpenedConnection} db
1531 * The mirror database connection.
1532 * @param {Number} currentSchemaVersion
1533 * The current mirror database schema version.
1535 async function migrateMirrorSchema(db, currentSchemaVersion) {
1536 if (currentSchemaVersion < 5) {
1537 // The mirror was pref'd off by default for schema versions 1-4.
1538 throw new DatabaseCorruptError(
1539 `Can't migrate from schema version ${currentSchemaVersion}; too old`
1542 if (currentSchemaVersion < 6) {
1543 await db.execute(`CREATE INDEX IF NOT EXISTS mirror.itemURLs ON
1545 await db.execute(`CREATE INDEX IF NOT EXISTS mirror.itemKeywords ON
1546 items(keyword) WHERE keyword NOT NULL`);
1548 if (currentSchemaVersion < 7) {
1549 await db.execute(`CREATE INDEX IF NOT EXISTS mirror.structurePositions ON
1550 structure(parentGuid, position)`);
1552 if (currentSchemaVersion < 8) {
1553 // Not really a "schema" update, but addresses the defect from bug 1635859.
1554 // In short, every bookmark with a corresponding entry in the mirror should
1555 // have syncStatus = NORMAL.
1556 await db.execute(`UPDATE moz_bookmarks AS b
1557 SET syncStatus = ${lazy.PlacesUtils.bookmarks.SYNC_STATUS.NORMAL}
1558 WHERE EXISTS (SELECT 1 FROM mirror.items
1559 WHERE guid = b.guid)`);
1561 if (currentSchemaVersion < 9) {
1562 // Adding unknownFields to the mirror table, which allows us to
1563 // keep fields we may not yet understand from other clients and roundtrip
1564 // them during the sync process
1565 let columns = await db.execute(`PRAGMA table_info(items)`);
1566 // migration needs to be idempotent, so we check if the column exists first
1567 let exists = columns.find(
1568 row => row.getResultByName("name") === "unknownFields"
1571 await db.execute(`ALTER TABLE items ADD COLUMN unknownFields TEXT`);
1577 * Initializes a new mirror database, creating persistent tables, indexes, and
1580 * @param {Sqlite.OpenedConnection} db
1581 * The mirror database connection.
1583 async function initializeMirrorDatabase(db) {
1584 // Key-value metadata table. Stores the server collection last modified time
1586 await db.execute(`CREATE TABLE mirror.meta(
1587 key TEXT PRIMARY KEY,
1591 // Note: description and loadInSidebar are not used as of Firefox 63, but
1592 // remain to avoid rebuilding the database if the user happens to downgrade.
1593 await db.execute(`CREATE TABLE mirror.items(
1594 id INTEGER PRIMARY KEY,
1595 guid TEXT UNIQUE NOT NULL,
1596 /* The "parentid" from the record. */
1598 /* The server modified time, in milliseconds. */
1599 serverModified INTEGER NOT NULL DEFAULT 0,
1600 needsMerge BOOLEAN NOT NULL DEFAULT 0,
1601 validity INTEGER NOT NULL DEFAULT ${Ci.mozISyncedBookmarksMerger.VALIDITY_VALID},
1602 isDeleted BOOLEAN NOT NULL DEFAULT 0,
1603 kind INTEGER NOT NULL DEFAULT -1,
1604 /* The creation date, in milliseconds. */
1605 dateAdded INTEGER NOT NULL DEFAULT 0,
1607 urlId INTEGER REFERENCES urls(id)
1611 loadInSidebar BOOLEAN,
1612 smartBookmarkName TEXT,
1618 await db.execute(`CREATE TABLE mirror.structure(
1620 parentGuid TEXT REFERENCES items(guid)
1622 position INTEGER NOT NULL,
1623 PRIMARY KEY(parentGuid, guid)
1626 await db.execute(`CREATE TABLE mirror.urls(
1627 id INTEGER PRIMARY KEY,
1630 hash INTEGER NOT NULL,
1631 revHost TEXT NOT NULL
1634 await db.execute(`CREATE TABLE mirror.tags(
1635 itemId INTEGER NOT NULL REFERENCES items(id)
1641 `CREATE INDEX mirror.structurePositions ON structure(parentGuid, position)`
1644 await db.execute(`CREATE INDEX mirror.urlHashes ON urls(hash)`);
1646 await db.execute(`CREATE INDEX mirror.itemURLs ON items(urlId)`);
1648 await db.execute(`CREATE INDEX mirror.itemKeywords ON items(keyword)
1649 WHERE keyword NOT NULL`);
1651 await createMirrorRoots(db);
1655 * Drops all temp tables, views, and triggers used for merging, and detaches
1656 * from the mirror database.
1658 * @param {Sqlite.OpenedConnection} db
1659 * The mirror database connection.
1661 async function cleanupMirrorDatabase(db) {
1662 await db.executeTransaction(async function () {
1663 await db.execute(`DROP TABLE changeGuidOps`);
1664 await db.execute(`DROP TABLE itemsToApply`);
1665 await db.execute(`DROP TABLE applyNewLocalStructureOps`);
1666 await db.execute(`DROP VIEW localTags`);
1667 await db.execute(`DROP TABLE itemsAdded`);
1668 await db.execute(`DROP TABLE guidsChanged`);
1669 await db.execute(`DROP TABLE itemsChanged`);
1670 await db.execute(`DROP TABLE itemsMoved`);
1671 await db.execute(`DROP TABLE itemsRemoved`);
1672 await db.execute(`DROP TABLE itemsToUpload`);
1673 await db.execute(`DROP TABLE structureToUpload`);
1674 await db.execute(`DROP TABLE tagsToUpload`);
1679 * Sets up the syncable roots. All items in the mirror we apply will descend
1680 * from these roots - however, malformed records from the server which create
1681 * a different root *will* be created in the mirror - just not applied.
1684 * @param {Sqlite.OpenedConnection} db
1685 * The mirror database connection.
1687 async function createMirrorRoots(db) {
1688 const syncableRoots = [
1690 guid: lazy.PlacesUtils.bookmarks.rootGuid,
1691 // The Places root is its own parent, to satisfy the foreign key and
1692 // `NOT NULL` constraints on `structure`.
1693 parentGuid: lazy.PlacesUtils.bookmarks.rootGuid,
1697 ...lazy.PlacesUtils.bookmarks.userContentRoots.map((guid, position) => {
1700 parentGuid: lazy.PlacesUtils.bookmarks.rootGuid,
1707 for (let { guid, parentGuid, position, needsMerge } of syncableRoots) {
1708 await db.executeCached(
1710 INSERT INTO items(guid, parentGuid, kind, needsMerge)
1711 VALUES(:guid, :parentGuid, :kind, :needsMerge)`,
1715 kind: Ci.mozISyncedBookmarksMerger.KIND_FOLDER,
1720 await db.executeCached(
1722 INSERT INTO structure(guid, parentGuid, position)
1723 VALUES(:guid, :parentGuid, :position)`,
1724 { guid, parentGuid, position }
1730 * Creates temporary tables, views, and triggers to apply the mirror to Places.
1732 * @param {Sqlite.OpenedConnection} db
1733 * The mirror database connection.
1735 async function initializeTempMirrorEntities(db) {
1736 await db.execute(`CREATE TEMP TABLE changeGuidOps(
1737 localGuid TEXT PRIMARY KEY,
1738 mergedGuid TEXT UNIQUE NOT NULL,
1740 level INTEGER NOT NULL,
1741 lastModifiedMicroseconds INTEGER NOT NULL
1745 CREATE TEMP TRIGGER changeGuids
1746 AFTER DELETE ON changeGuidOps
1748 /* Record item changed notifications for the updated GUIDs. */
1749 INSERT INTO guidsChanged(itemId, oldGuid, level)
1750 SELECT b.id, OLD.localGuid, OLD.level
1751 FROM moz_bookmarks b
1752 WHERE b.guid = OLD.localGuid;
1754 UPDATE moz_bookmarks SET
1755 guid = OLD.mergedGuid,
1756 lastModified = OLD.lastModifiedMicroseconds,
1757 syncStatus = IFNULL(OLD.syncStatus, syncStatus)
1758 WHERE guid = OLD.localGuid;
1761 await db.execute(`CREATE TEMP TABLE itemsToApply(
1762 mergedGuid TEXT PRIMARY KEY,
1763 localId INTEGER UNIQUE,
1764 remoteId INTEGER UNIQUE NOT NULL,
1765 remoteGuid TEXT UNIQUE NOT NULL,
1766 newLevel INTEGER NOT NULL,
1767 newType INTEGER NOT NULL,
1768 localDateAddedMicroseconds INTEGER,
1769 remoteDateAddedMicroseconds INTEGER NOT NULL,
1770 lastModifiedMicroseconds INTEGER NOT NULL,
1778 await db.execute(`CREATE INDEX existingItems ON itemsToApply(localId)
1779 WHERE localId NOT NULL`);
1781 await db.execute(`CREATE INDEX oldPlaceIds ON itemsToApply(oldPlaceId)
1782 WHERE oldPlaceId NOT NULL`);
1784 await db.execute(`CREATE INDEX newPlaceIds ON itemsToApply(newPlaceId)
1785 WHERE newPlaceId NOT NULL`);
1787 await db.execute(`CREATE INDEX newKeywords ON itemsToApply(newKeyword)
1788 WHERE newKeyword NOT NULL`);
1790 await db.execute(`CREATE TEMP TABLE applyNewLocalStructureOps(
1791 mergedGuid TEXT PRIMARY KEY,
1792 mergedParentGuid TEXT NOT NULL,
1793 position INTEGER NOT NULL,
1794 level INTEGER NOT NULL,
1795 lastModifiedMicroseconds INTEGER NOT NULL
1799 CREATE TEMP TRIGGER applyNewLocalStructure
1800 AFTER DELETE ON applyNewLocalStructureOps
1802 INSERT INTO itemsMoved(itemId, oldParentId, oldParentGuid, oldPosition,
1804 SELECT b.id, p.id, p.guid, b.position, OLD.level
1805 FROM moz_bookmarks b
1806 JOIN moz_bookmarks p ON p.id = b.parent
1807 WHERE b.guid = OLD.mergedGuid;
1809 UPDATE moz_bookmarks SET
1810 parent = (SELECT id FROM moz_bookmarks
1811 WHERE guid = OLD.mergedParentGuid),
1812 position = OLD.position,
1813 lastModified = OLD.lastModifiedMicroseconds
1814 WHERE guid = OLD.mergedGuid;
1817 // A view of local bookmark tags. Tags, like keywords, are associated with
1818 // URLs, so two bookmarks with the same URL should have the same tags. Unlike
1819 // keywords, one tag may be associated with many different URLs. Tags are also
1820 // different because they're implemented as bookmarks under the hood. Each tag
1821 // is stored as a folder under the tags root, and tagged URLs are stored as
1822 // untitled bookmarks under these folders. This complexity can be removed once
1823 // bug 424160 lands.
1825 CREATE TEMP VIEW localTags(tagEntryId, tagEntryGuid, tagFolderId,
1826 tagFolderGuid, tagEntryPosition, tagEntryType,
1827 tag, placeId, lastModifiedMicroseconds) AS
1828 SELECT b.id, b.guid, p.id, p.guid, b.position, b.type,
1829 p.title, b.fk, b.lastModified
1830 FROM moz_bookmarks b
1831 JOIN moz_bookmarks p ON p.id = b.parent
1832 WHERE b.type = ${lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK} AND
1833 p.parent = (SELECT id FROM moz_bookmarks
1834 WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}')`);
1836 // Untags a URL by removing its tag entry.
1838 CREATE TEMP TRIGGER untagLocalPlace
1839 INSTEAD OF DELETE ON localTags
1841 /* Record an item removed notification for the tag entry. */
1842 INSERT INTO itemsRemoved(itemId, parentId, position, type, placeId, guid,
1843 parentGuid, title, isUntagging)
1844 VALUES(OLD.tagEntryId, OLD.tagFolderId, OLD.tagEntryPosition,
1845 OLD.tagEntryType, OLD.placeId, OLD.tagEntryGuid,
1846 OLD.tagFolderGuid, OLD.tag, 1);
1848 DELETE FROM moz_bookmarks WHERE id = OLD.tagEntryId;
1850 /* Fix the positions of the sibling tag entries. */
1851 UPDATE moz_bookmarks SET
1852 position = position - 1
1853 WHERE parent = OLD.tagFolderId AND
1854 position > OLD.tagEntryPosition;
1857 // Tags a URL by creating a tag folder if it doesn't exist, then inserting a
1858 // tag entry for the URL into the tag folder. `NEW.placeId` can be NULL, in
1859 // which case we'll just create the tag folder.
1861 CREATE TEMP TRIGGER tagLocalPlace
1862 INSTEAD OF INSERT ON localTags
1864 /* Ensure the tag folder exists. */
1865 INSERT OR IGNORE INTO moz_bookmarks(guid, parent, position, type, title,
1866 dateAdded, lastModified)
1867 VALUES(IFNULL((SELECT b.guid FROM moz_bookmarks b
1868 JOIN moz_bookmarks p ON p.id = b.parent
1869 WHERE b.title = NEW.tag AND
1870 p.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'),
1872 (SELECT id FROM moz_bookmarks
1873 WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'),
1874 (SELECT COUNT(*) FROM moz_bookmarks b
1875 JOIN moz_bookmarks p ON p.id = b.parent
1876 WHERE p.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'),
1877 ${lazy.PlacesUtils.bookmarks.TYPE_FOLDER}, NEW.tag,
1878 NEW.lastModifiedMicroseconds,
1879 NEW.lastModifiedMicroseconds);
1881 /* Record an item added notification if we created a tag folder.
1882 "CHANGES()" returns the number of rows affected by the INSERT above:
1883 1 if we created the folder, or 0 if the folder already existed. */
1884 INSERT INTO itemsAdded(guid, isTagging)
1886 FROM moz_bookmarks b
1887 JOIN moz_bookmarks p ON p.id = b.parent
1888 WHERE CHANGES() > 0 AND
1889 b.title = NEW.tag AND
1890 p.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}';
1892 /* Add a tag entry for the URL under the tag folder. Omitting the place
1893 ID creates a tag folder without tagging the URL. */
1894 INSERT OR IGNORE INTO moz_bookmarks(guid, parent, position, type, fk,
1895 dateAdded, lastModified)
1896 SELECT IFNULL((SELECT b.guid FROM moz_bookmarks b
1897 JOIN moz_bookmarks p ON p.id = b.parent
1898 WHERE b.fk = NEW.placeId AND
1899 p.title = NEW.tag AND
1900 p.parent = (SELECT id FROM moz_bookmarks
1901 WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}')),
1903 (SELECT b.id FROM moz_bookmarks b
1904 JOIN moz_bookmarks p ON p.id = b.parent
1905 WHERE p.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}' AND
1907 (SELECT COUNT(*) FROM moz_bookmarks b
1908 JOIN moz_bookmarks p ON p.id = b.parent
1909 WHERE p.title = NEW.tag AND
1910 p.parent = (SELECT id FROM moz_bookmarks
1911 WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}')),
1912 ${lazy.PlacesUtils.bookmarks.TYPE_BOOKMARK}, NEW.placeId,
1913 NEW.lastModifiedMicroseconds,
1914 NEW.lastModifiedMicroseconds
1915 WHERE NEW.placeId NOT NULL;
1917 /* Record an item added notification for the tag entry. */
1918 INSERT INTO itemsAdded(guid, isTagging)
1920 FROM moz_bookmarks b
1921 JOIN moz_bookmarks p ON p.id = b.parent
1922 WHERE CHANGES() > 0 AND
1923 b.fk = NEW.placeId AND
1924 p.title = NEW.tag AND
1925 p.parent = (SELECT id FROM moz_bookmarks
1926 WHERE guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}');
1929 // Stores properties to pass to `onItem{Added, Changed, Moved, Removed}`
1930 // bookmark observers for new, updated, moved, and deleted items.
1931 await db.execute(`CREATE TEMP TABLE itemsAdded(
1932 guid TEXT PRIMARY KEY,
1933 isTagging BOOLEAN NOT NULL DEFAULT 0,
1934 keywordChanged BOOLEAN NOT NULL DEFAULT 0,
1935 level INTEGER NOT NULL DEFAULT -1
1938 await db.execute(`CREATE INDEX addedItemLevels ON itemsAdded(level)`);
1940 await db.execute(`CREATE TEMP TABLE guidsChanged(
1941 itemId INTEGER PRIMARY KEY,
1942 oldGuid TEXT NOT NULL,
1943 level INTEGER NOT NULL DEFAULT -1
1946 await db.execute(`CREATE INDEX changedGuidLevels ON guidsChanged(level)`);
1948 await db.execute(`CREATE TEMP TABLE itemsChanged(
1949 itemId INTEGER PRIMARY KEY,
1952 keywordChanged BOOLEAN NOT NULL DEFAULT 0,
1953 level INTEGER NOT NULL DEFAULT -1
1956 await db.execute(`CREATE INDEX changedItemLevels ON itemsChanged(level)`);
1958 await db.execute(`CREATE TEMP TABLE itemsMoved(
1959 itemId INTEGER PRIMARY KEY,
1960 oldParentId INTEGER NOT NULL,
1961 oldParentGuid TEXT NOT NULL,
1962 oldPosition INTEGER NOT NULL,
1963 level INTEGER NOT NULL DEFAULT -1
1966 await db.execute(`CREATE INDEX movedItemLevels ON itemsMoved(level)`);
1968 await db.execute(`CREATE TEMP TABLE itemsRemoved(
1969 itemId INTEGER PRIMARY KEY,
1971 parentId INTEGER NOT NULL,
1972 position INTEGER NOT NULL,
1973 type INTEGER NOT NULL,
1974 title TEXT NOT NULL,
1976 parentGuid TEXT NOT NULL,
1977 /* We record the original level of the removed item in the tree so that we
1978 can notify children before parents. */
1979 level INTEGER NOT NULL DEFAULT -1,
1980 isUntagging BOOLEAN NOT NULL DEFAULT 0,
1981 keywordRemoved BOOLEAN NOT NULL DEFAULT 0
1985 `CREATE INDEX removedItemLevels ON itemsRemoved(level DESC)`
1988 // Stores locally changed items staged for upload.
1989 await db.execute(`CREATE TEMP TABLE itemsToUpload(
1990 id INTEGER PRIMARY KEY,
1991 guid TEXT UNIQUE NOT NULL,
1992 syncChangeCounter INTEGER NOT NULL,
1993 isDeleted BOOLEAN NOT NULL DEFAULT 0,
1996 dateAdded INTEGER, /* In milliseconds. */
2000 isQuery BOOLEAN NOT NULL DEFAULT 0,
2008 await db.execute(`CREATE TEMP TABLE structureToUpload(
2009 guid TEXT PRIMARY KEY,
2010 parentId INTEGER NOT NULL REFERENCES itemsToUpload(id)
2012 position INTEGER NOT NULL
2016 `CREATE INDEX parentsToUpload ON structureToUpload(parentId, position)`
2019 await db.execute(`CREATE TEMP TABLE tagsToUpload(
2020 id INTEGER REFERENCES itemsToUpload(id)
2023 PRIMARY KEY(id, tag)
2027 async function resetMirror(db) {
2028 await db.execute(`DELETE FROM meta`);
2029 await db.execute(`DELETE FROM structure`);
2030 await db.execute(`DELETE FROM items`);
2031 await db.execute(`DELETE FROM urls`);
2033 // Since we need to reset the modified times and merge flags for the syncable
2034 // roots, we simply delete and recreate them.
2035 await createMirrorRoots(db);
2038 // Converts a Sync record's last modified time to milliseconds.
2039 function determineServerModified(record) {
2040 return Math.max(record.modified * 1000, 0) || 0;
2043 // Determines a Sync record's creation date.
2044 function determineDateAdded(record) {
2045 let serverModified = determineServerModified(record);
2046 return lazy.PlacesSyncUtils.bookmarks.ratchetTimestampBackwards(
2052 function validateTitle(rawTitle) {
2053 if (typeof rawTitle != "string" || !rawTitle) {
2056 return rawTitle.slice(0, DB_TITLE_LENGTH_MAX);
2059 function validateURL(rawURL) {
2060 if (typeof rawURL != "string" || rawURL.length > DB_URL_LENGTH_MAX) {
2063 return URL.parse(rawURL);
2066 function validateKeyword(rawKeyword) {
2067 if (typeof rawKeyword != "string") {
2070 let keyword = rawKeyword.trim();
2071 // Drop empty keywords.
2072 return keyword ? keyword.toLowerCase() : null;
2075 function validateTag(rawTag) {
2076 if (typeof rawTag != "string") {
2079 let tag = rawTag.trim();
2080 if (!tag || tag.length > lazy.PlacesUtils.bookmarks.MAX_TAG_LENGTH) {
2081 // Drop empty and oversized tags.
2088 * Measures and logs the time taken to execute a function, using a monotonic
2091 * @param {String} name
2092 * The name of the operation, used for logging.
2093 * @param {Function} func
2094 * The function to time.
2095 * @param {Function} [recordTiming]
2096 * An optional function with the signature `(time: Number)`, where
2097 * `time` is the measured time.
2098 * @return The return value of the timed function.
2100 async function withTiming(name, func, recordTiming) {
2101 lazy.MirrorLog.debug(name);
2103 let startTime = Cu.now();
2104 let result = await func();
2105 let elapsedTime = Cu.now() - startTime;
2107 lazy.MirrorLog.debug(`${name} took ${elapsedTime.toFixed(3)}ms`);
2108 if (typeof recordTiming == "function") {
2109 recordTiming(elapsedTime, result);
2116 * Fires bookmark and keyword observer notifications for all changes made during
2119 class BookmarkObserverRecorder {
2120 constructor(db, { notifyInStableOrder, signal }) {
2122 this.notifyInStableOrder = notifyInStableOrder;
2123 this.signal = signal;
2124 this.placesEvents = [];
2125 this.shouldInvalidateKeywords = false;
2129 * Fires observer notifications for all changed items, invalidates the
2130 * livemark cache if necessary, and recalculates frecencies for changed
2131 * URLs. This is called outside the merge transaction.
2134 await this.noteAllChanges();
2135 if (this.shouldInvalidateKeywords) {
2136 await lazy.PlacesUtils.keywords.invalidateCachedKeywords();
2138 this.notifyBookmarkObservers();
2139 if (this.signal.aborted) {
2140 throw new SyncedBookmarksMirror.InterruptedError(
2141 "Interrupted before recalculating frecencies for new URLs"
2146 orderBy(level, parent, position) {
2148 this.notifyInStableOrder ? `${level}, ${parent}, ${position}` : level
2153 * Records Places observer notifications for removed, added, moved, and
2156 async noteAllChanges() {
2157 lazy.MirrorLog.trace("Recording observer notifications for removed items");
2158 // `ORDER BY v.level DESC` sorts deleted children before parents, to ensure
2159 // that we update caches in the correct order (bug 1297941).
2160 await this.db.execute(
2161 `SELECT v.itemId AS id, v.parentId, v.parentGuid, v.position, v.type,
2162 (SELECT h.url FROM moz_places h WHERE h.id = v.placeId) AS url,
2163 v.title, v.guid, v.isUntagging, v.keywordRemoved
2165 ${this.orderBy("v.level", "v.parentId", "v.position")}`,
2168 if (this.signal.aborted) {
2173 id: row.getResultByName("id"),
2174 parentId: row.getResultByName("parentId"),
2175 position: row.getResultByName("position"),
2176 type: row.getResultByName("type"),
2177 urlHref: row.getResultByName("url"),
2178 title: row.getResultByName("title"),
2179 guid: row.getResultByName("guid"),
2180 parentGuid: row.getResultByName("parentGuid"),
2181 isUntagging: row.getResultByName("isUntagging"),
2183 this.noteItemRemoved(info);
2184 if (row.getResultByName("keywordRemoved")) {
2185 this.shouldInvalidateKeywords = true;
2189 if (this.signal.aborted) {
2190 throw new SyncedBookmarksMirror.InterruptedError(
2191 "Interrupted while recording observer notifications for removed items"
2195 lazy.MirrorLog.trace("Recording observer notifications for changed GUIDs");
2196 await this.db.execute(
2197 `SELECT b.id, b.lastModified, b.type, b.guid AS newGuid,
2198 p.guid AS parentGuid, gp.guid AS grandParentGuid
2200 JOIN moz_bookmarks b ON b.id = c.itemId
2201 JOIN moz_bookmarks p ON p.id = b.parent
2202 LEFT JOIN moz_bookmarks gp ON gp.id = p.parent
2203 ${this.orderBy("c.level", "b.parent", "b.position")}`,
2206 if (this.signal.aborted) {
2211 id: row.getResultByName("id"),
2212 lastModified: row.getResultByName("lastModified"),
2213 type: row.getResultByName("type"),
2214 newGuid: row.getResultByName("newGuid"),
2215 parentGuid: row.getResultByName("parentGuid"),
2216 grandParentGuid: row.getResultByName("grandParentGuid"),
2218 this.noteGuidChanged(info);
2221 if (this.signal.aborted) {
2222 throw new SyncedBookmarksMirror.InterruptedError(
2223 "Interrupted while recording observer notifications for changed GUIDs"
2227 lazy.MirrorLog.trace("Recording observer notifications for new items");
2228 await this.db.execute(
2229 `SELECT b.id, p.id AS parentId, b.position, b.type,
2230 IFNULL(b.title, '') AS title, b.dateAdded, b.guid,
2231 p.guid AS parentGuid, n.isTagging, n.keywordChanged,
2232 h.url AS url, IFNULL(h.frecency, 0) AS frecency,
2233 IFNULL(h.hidden, 0) AS hidden,
2234 IFNULL(h.visit_count, 0) AS visit_count,
2236 (SELECT group_concat(pp.title ORDER BY pp.title)
2237 FROM moz_bookmarks bb
2238 JOIN moz_bookmarks pp ON pp.id = bb.parent
2239 JOIN moz_bookmarks gg ON gg.id = pp.parent
2241 AND gg.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'
2243 t.guid AS tGuid, t.id AS tId, t.title AS tTitle
2245 JOIN moz_bookmarks b ON b.guid = n.guid
2246 JOIN moz_bookmarks p ON p.id = b.parent
2247 LEFT JOIN moz_places h ON h.id = b.fk
2248 LEFT JOIN moz_bookmarks t ON t.guid = target_folder_guid(url)
2249 ${this.orderBy("n.level", "b.parent", "b.position")}`,
2252 if (this.signal.aborted) {
2257 let lastVisitDate = row.getResultByName("last_visit_date");
2260 id: row.getResultByName("id"),
2261 parentId: row.getResultByName("parentId"),
2262 position: row.getResultByName("position"),
2263 type: row.getResultByName("type"),
2264 urlHref: row.getResultByName("url"),
2265 title: row.getResultByName("title"),
2266 dateAdded: row.getResultByName("dateAdded"),
2267 guid: row.getResultByName("guid"),
2268 parentGuid: row.getResultByName("parentGuid"),
2269 isTagging: row.getResultByName("isTagging"),
2270 frecency: row.getResultByName("frecency"),
2271 hidden: row.getResultByName("hidden"),
2272 visitCount: row.getResultByName("visit_count"),
2273 lastVisitDate: lastVisitDate
2274 ? lazy.PlacesUtils.toDate(lastVisitDate).getTime()
2276 tags: row.getResultByName("tags"),
2277 targetFolderGuid: row.getResultByName("tGuid"),
2278 targetFolderItemId: row.getResultByName("tId"),
2279 targetFolderTitle: row.getResultByName("tTitle"),
2282 this.noteItemAdded(info);
2283 if (row.getResultByName("keywordChanged")) {
2284 this.shouldInvalidateKeywords = true;
2288 if (this.signal.aborted) {
2289 throw new SyncedBookmarksMirror.InterruptedError(
2290 "Interrupted while recording observer notifications for new items"
2294 lazy.MirrorLog.trace("Recording observer notifications for moved items");
2295 await this.db.execute(
2296 `SELECT b.id, b.guid, b.type, p.guid AS newParentGuid, c.oldParentGuid,
2297 b.position AS newPosition, c.oldPosition,
2298 gp.guid AS grandParentGuid,
2299 h.url AS url, IFNULL(b.title, '') AS title,
2300 IFNULL(h.frecency, 0) AS frecency, IFNULL(h.hidden, 0) AS hidden,
2301 IFNULL(h.visit_count, 0) AS visit_count,
2302 b.dateAdded, h.last_visit_date,
2303 (SELECT group_concat(pp.title ORDER BY pp.title)
2304 FROM moz_bookmarks bb
2305 JOIN moz_bookmarks pp ON pp.id = bb.parent
2306 JOIN moz_bookmarks gg ON gg.id = pp.parent
2308 AND gg.guid = '${lazy.PlacesUtils.bookmarks.tagsGuid}'
2311 JOIN moz_bookmarks b ON b.id = c.itemId
2312 JOIN moz_bookmarks p ON p.id = b.parent
2313 LEFT JOIN moz_bookmarks gp ON gp.id = p.parent
2314 LEFT JOIN moz_places h ON h.id = b.fk
2315 ${this.orderBy("c.level", "b.parent", "b.position")}`,
2318 if (this.signal.aborted) {
2322 let lastVisitDate = row.getResultByName("last_visit_date");
2324 id: row.getResultByName("id"),
2325 guid: row.getResultByName("guid"),
2326 type: row.getResultByName("type"),
2327 newParentGuid: row.getResultByName("newParentGuid"),
2328 oldParentGuid: row.getResultByName("oldParentGuid"),
2329 newPosition: row.getResultByName("newPosition"),
2330 oldPosition: row.getResultByName("oldPosition"),
2331 urlHref: row.getResultByName("url"),
2332 grandParentGuid: row.getResultByName("grandParentGuid"),
2333 title: row.getResultByName("title"),
2334 frecency: row.getResultByName("frecency"),
2335 hidden: row.getResultByName("hidden"),
2336 visitCount: row.getResultByName("visit_count"),
2337 dateAdded: lazy.PlacesUtils.toDate(
2338 row.getResultByName("dateAdded")
2340 lastVisitDate: lastVisitDate
2341 ? lazy.PlacesUtils.toDate(lastVisitDate).getTime()
2343 tags: row.getResultByName("tags"),
2345 this.noteItemMoved(info);
2348 if (this.signal.aborted) {
2349 throw new SyncedBookmarksMirror.InterruptedError(
2350 "Interrupted while recording observer notifications for moved items"
2354 lazy.MirrorLog.trace("Recording observer notifications for changed items");
2355 await this.db.execute(
2356 `SELECT b.id, b.guid, b.lastModified, b.type,
2357 IFNULL(b.title, '') AS newTitle,
2358 IFNULL(c.oldTitle, '') AS oldTitle,
2359 (SELECT h.url FROM moz_places h
2360 WHERE h.id = b.fk) AS newURL,
2361 (SELECT h.url FROM moz_places h
2362 WHERE h.id = c.oldPlaceId) AS oldURL,
2363 p.id AS parentId, p.guid AS parentGuid,
2365 gp.guid AS grandParentGuid,
2366 (SELECT h.url FROM moz_places h WHERE h.id = b.fk) AS url
2368 JOIN moz_bookmarks b ON b.id = c.itemId
2369 JOIN moz_bookmarks p ON p.id = b.parent
2370 LEFT JOIN moz_bookmarks gp ON gp.id = p.parent
2371 ${this.orderBy("c.level", "b.parent", "b.position")}`,
2374 if (this.signal.aborted) {
2379 id: row.getResultByName("id"),
2380 guid: row.getResultByName("guid"),
2381 lastModified: row.getResultByName("lastModified"),
2382 type: row.getResultByName("type"),
2383 newTitle: row.getResultByName("newTitle"),
2384 oldTitle: row.getResultByName("oldTitle"),
2385 newURLHref: row.getResultByName("newURL"),
2386 oldURLHref: row.getResultByName("oldURL"),
2387 parentId: row.getResultByName("parentId"),
2388 parentGuid: row.getResultByName("parentGuid"),
2389 grandParentGuid: row.getResultByName("grandParentGuid"),
2391 this.noteItemChanged(info);
2392 if (row.getResultByName("keywordChanged")) {
2393 this.shouldInvalidateKeywords = true;
2397 if (this.signal.aborted) {
2398 throw new SyncedBookmarksMirror.InterruptedError(
2399 "Interrupted while recording observer notifications for changed items"
2404 noteItemAdded(info) {
2405 this.placesEvents.push(
2406 new PlacesBookmarkAddition({
2408 parentId: info.parentId,
2409 index: info.position,
2410 url: info.urlHref || "",
2412 // Note that both the database and the legacy `onItem{Moved, Removed,
2413 // Changed}` notifications use microsecond timestamps, but
2414 // `PlacesBookmarkAddition` uses milliseconds.
2415 dateAdded: info.dateAdded / 1000,
2417 parentGuid: info.parentGuid,
2418 source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC,
2419 itemType: info.type,
2420 isTagging: info.isTagging,
2422 frecency: info.frecency,
2423 hidden: info.hidden,
2424 visitCount: info.visitCount,
2425 lastVisitDate: info.lastVisitDate,
2426 targetFolderGuid: info.targetFolderGuid,
2427 targetFolderItemId: info.targetFolderItemId,
2428 targetFolderTitle: info.targetFolderTitle,
2433 noteGuidChanged(info) {
2434 this.placesEvents.push(
2435 new PlacesBookmarkGuid({
2437 itemType: info.type,
2440 parentGuid: info.parentGuid,
2441 lastModified: info.lastModified,
2442 source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC,
2444 info.parentGuid === lazy.PlacesUtils.bookmarks.tagsGuid ||
2445 info.grandParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid,
2450 noteItemMoved(info) {
2451 this.placesEvents.push(
2452 new PlacesBookmarkMoved({
2454 itemType: info.type,
2458 parentGuid: info.newParentGuid,
2459 source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC,
2460 index: info.newPosition,
2461 oldParentGuid: info.oldParentGuid,
2462 oldIndex: info.oldPosition,
2464 info.newParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid ||
2465 info.grandParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid,
2467 frecency: info.frecency,
2468 hidden: info.hidden,
2469 visitCount: info.visitCount,
2470 dateAdded: info.dateAdded,
2471 lastVisitDate: info.lastVisitDate,
2476 noteItemChanged(info) {
2477 if (info.oldTitle != info.newTitle) {
2478 this.placesEvents.push(
2479 new PlacesBookmarkTitle({
2481 itemType: info.type,
2484 parentGuid: info.parentGuid,
2485 title: info.newTitle,
2486 lastModified: info.lastModified,
2487 source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC,
2489 info.parentGuid === lazy.PlacesUtils.bookmarks.tagsGuid ||
2490 info.grandParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid,
2494 if (info.oldURLHref != info.newURLHref) {
2495 this.placesEvents.push(
2496 new PlacesBookmarkUrl({
2498 itemType: info.type,
2499 url: info.newURLHref,
2501 parentGuid: info.parentGuid,
2502 lastModified: info.lastModified,
2503 source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC,
2505 info.parentGuid === lazy.PlacesUtils.bookmarks.tagsGuid ||
2506 info.grandParentGuid === lazy.PlacesUtils.bookmarks.tagsGuid,
2512 noteItemRemoved(info) {
2513 this.placesEvents.push(
2514 new PlacesBookmarkRemoved({
2516 parentId: info.parentId,
2517 index: info.position,
2518 url: info.urlHref || "",
2521 parentGuid: info.parentGuid,
2522 source: lazy.PlacesUtils.bookmarks.SOURCES.SYNC,
2523 itemType: info.type,
2524 isTagging: info.isUntagging,
2525 isDescendantRemoval: false,
2530 notifyBookmarkObservers() {
2531 lazy.MirrorLog.trace("Notifying bookmark observers");
2533 if (this.placesEvents.length) {
2534 PlacesObservers.notifyListeners(this.placesEvents);
2537 lazy.MirrorLog.trace("Notified bookmark observers");
2542 * Holds Sync metadata and the cleartext for a locally changed record. The
2543 * bookmarks engine inflates a Sync record from the cleartext, and updates the
2544 * `synced` property for successfully uploaded items.
2546 * At the end of the sync, the engine writes the uploaded cleartext back to the
2547 * mirror, and passes the updated change record as part of the changeset to
2548 * `PlacesSyncUtils.bookmarks.pushChanges`.
2550 class BookmarkChangeRecord {
2551 constructor(syncChangeCounter, cleartext) {
2552 this.tombstone = cleartext.deleted === true;
2553 this.counter = syncChangeCounter;
2554 this.cleartext = cleartext;
2555 this.synced = false;
2559 function bagToNamedCounts(bag, names) {
2561 for (let name of names) {
2562 let count = bag.getProperty(name);
2564 counts.push({ name, count });
2571 * Returns an `AbortSignal` that aborts if either `finalizeSignal` or
2572 * `interruptSignal` aborts. This is like `Promise.race`, but for
2575 * @param {AbortSignal} finalizeSignal
2576 * @param {AbortSignal?} signal
2577 * @return {AbortSignal}
2579 function anyAborted(finalizeSignal, interruptSignal = null) {
2580 if (finalizeSignal.aborted || !interruptSignal) {
2581 // If the mirror was already finalized, or we don't have an interrupt
2582 // signal for this merge, just use the finalize signal.
2583 return finalizeSignal;
2585 if (interruptSignal.aborted) {
2586 // If the merge was interrupted, return its already-aborted signal.
2587 return interruptSignal;
2589 // Otherwise, we return a new signal that aborts if either the mirror is
2590 // finalized, or the merge is interrupted, whichever happens first.
2591 let controller = new AbortController();
2592 function onAbort() {
2593 finalizeSignal.removeEventListener("abort", onAbort);
2594 interruptSignal.removeEventListener("abort", onAbort);
2597 finalizeSignal.addEventListener("abort", onAbort);
2598 interruptSignal.addEventListener("abort", onAbort);
2599 return controller.signal;
2602 // Common unknown fields for places items
2603 const COMMON_UNKNOWN_FIELDS = [
2613 // In conclusion, this is why bookmark syncing is hard.