src/Imap/Tasks/ObtainSynchronizedMailboxTask.cpp

   1 /* Copyright (C) 2006 - 2012 Jan Kundrát <jkt@flaska.net>
   2
   3    This file is part of the Trojita Qt IMAP e-mail client,
   4    http://trojita.flaska.net/
   5
   6    This program is free software; you can redistribute it and/or
   7    modify it under the terms of the GNU General Public License as
   8    published by the Free Software Foundation; either version 2 of
   9    the License or (at your option) version 3 or any later version
  10    accepted by the membership of KDE e.V. (or its successor approved
  11    by the membership of KDE e.V.), which shall act as a proxy
  12    defined in Section 14 of version 3 of the license.
  13
  14    This program is distributed in the hope that it will be useful,
  15    but WITHOUT ANY WARRANTY; without even the implied warranty of
  16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17    GNU General Public License for more details.
  18
  19    You should have received a copy of the GNU General Public License
  20    along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21 */
  22
  23 #include "ObtainSynchronizedMailboxTask.h"
  24 #include <algorithm>
  25 #include <sstream>
  26 #include <QTimer>
  27 #include "ItemRoles.h"
  28 #include "KeepMailboxOpenTask.h"
  29 #include "MailboxTree.h"
  30 #include "Model.h"
  31 #include "UnSelectTask.h"
  32
  33 namespace Imap
  34 {
  35 namespace Mailbox
  36 {
  37
  38 ObtainSynchronizedMailboxTask::ObtainSynchronizedMailboxTask(Model *model, const QModelIndex &mailboxIndex, ImapTask *parentTask,
  39         KeepMailboxOpenTask *keepTask):
  40     ImapTask(model), conn(parentTask), mailboxIndex(mailboxIndex), status(STATE_WAIT_FOR_CONN), uidSyncingMode(UID_SYNC_ALL),
  41     firstUnknownUidOffset(0), m_usingQresync(false), unSelectTask(0), keepTaskChild(keepTask)
  42 {
  43     // The Parser* is not provided by our parent task, but instead through the keepTaskChild.  The reason is simple, the parent
  44     // task might not even exist, but there's always an KeepMailboxOpenTask in the game.
  45     parser = keepTaskChild->parser;
  46     Q_ASSERT(parser);
  47     if (conn) {
  48         conn->addDependentTask(this);
  49     }
  50     CHECK_TASK_TREE
  51     addDependentTask(keepTaskChild);
  52     CHECK_TASK_TREE
  53 }
  54
  55 void ObtainSynchronizedMailboxTask::addDependentTask(ImapTask *task)
  56 {
  57     if (!dependentTasks.isEmpty()) {
  58         throw CantHappen("Attempted to add another dependent task to an ObtainSynchronizedMailboxTask");
  59     }
  60     ImapTask::addDependentTask(task);
  61 }
  62
  63 void ObtainSynchronizedMailboxTask::perform()
  64 {
  65     CHECK_TASK_TREE
  66     markAsActiveTask();
  67
  68     if (_dead || _aborted) {
  69         // We're at the very start, so let's try to abort in a sane way
  70         _failed("Asked to abort or die");
  71         die();
  72         return;
  73     }
  74
  75     if (! mailboxIndex.isValid()) {
  76         // FIXME: proper error handling
  77         log("The mailbox went missing, sorry", Common::LOG_MAILBOX_SYNC);
  78         _completed();
  79         return;
  80     }
  81
  82     TreeItemMailbox *mailbox = dynamic_cast<TreeItemMailbox *>(static_cast<TreeItem *>(mailboxIndex.internalPointer()));
  83     Q_ASSERT(mailbox);
  84     TreeItemMsgList *msgList = dynamic_cast<TreeItemMsgList *>(mailbox->m_children[0]);
  85     Q_ASSERT(msgList);
  86
  87     msgList->m_fetchStatus = TreeItem::LOADING;
  88
  89     QMap<Parser *,ParserState>::iterator it = model->m_parsers.find(parser);
  90     Q_ASSERT(it != model->m_parsers.end());
  91
  92     QString qresyncArrived(QLatin1String("X-DRAFT-I01-QRESYNC-ARRIVED"));
  93
  94     oldSyncState = model->cache()->mailboxSyncState(mailbox->mailbox());
  95     if ((model->accessParser(parser).capabilities.contains(QLatin1String("QRESYNC")) ||
  96          model->accessParser(parser).capabilities.contains(qresyncArrived)
  97          ) && oldSyncState.isUsableForCondstore()) {
  98         Parser::SelectQresyncMode qresyncMode = model->accessParser(parser).capabilities.contains(qresyncArrived) ?
  99                     Parser::QRESYNC_ARRIVED :
 100                     Parser::QRESYNC_RFC5162;
 101         QList<uint> oldUidMap = model->cache()->uidMapping(mailbox->mailbox());
 102         if (oldUidMap.isEmpty()) {
 103             selectCmd = parser->selectQresync(qresyncMode, mailbox->mailbox(), oldSyncState.uidValidity(),
 104                                               oldSyncState.highestModSeq());
 105         } else {
 106             Sequence knownSeq, knownUid;
 107             int i = oldUidMap.size() / 2;
 108             while (i < oldUidMap.size()) {
 109                 // Message sequence number is one-based, our indexes are zero-based
 110                 knownSeq.add(i + 1);
 111                 knownUid.add(oldUidMap[i]);
 112                 i += (oldUidMap.size() - i) / 2 + 1;
 113             }
 114             m_usingQresync = true;
 115             // We absolutely want to maintain a complete UID->seq mapping at all times, which is why the known-uids shall remain
 116             // empty to indicate "anything".
 117             selectCmd = parser->selectQresync(qresyncMode, mailbox->mailbox(), oldSyncState.uidValidity(),
 118                                               oldSyncState.highestModSeq(), Sequence(), knownSeq, knownUid);
 119         }
 120     } else if (model->accessParser(parser).capabilities.contains(QLatin1String("CONDSTORE"))) {
 121         selectCmd = parser->select(mailbox->mailbox(), QList<QByteArray>() << "CONDSTORE");
 122     } else {
 123         selectCmd = parser->select(mailbox->mailbox());
 124     }
 125     mailbox->syncState = SyncState();
 126     status = STATE_SELECTING;
 127     log("Synchronizing mailbox", Common::LOG_MAILBOX_SYNC);
 128     emit model->mailboxSyncingProgress(mailboxIndex, status);
 129 }
 130
 131 bool ObtainSynchronizedMailboxTask::handleStateHelper(const Imap::Responses::State *const resp)
 132 {
 133     if (dieIfInvalidMailbox())
 134         return true;
 135
 136     if (handleResponseCodeInsideState(resp))
 137         return true;
 138
 139     if (resp->tag.isEmpty())
 140         return false;
 141
 142     if (_dead) {
 143         _failed("Asked to die");
 144         return true;
 145     }
 146     // We absolutely have to ignore the abort() request
 147
 148     if (resp->tag == selectCmd) {
 149
 150         if (resp->kind == Responses::OK) {
 151             //qDebug() << "received OK for selectCmd";
 152             Q_ASSERT(status == STATE_SELECTING);
 153             finalizeSelect();
 154         } else {
 155             _failed("SELECT failed");
 156             // FIXME: error handling
 157             model->changeConnectionState(parser, CONN_STATE_AUTHENTICATED);
 158         }
 159         return true;
 160     } else if (resp->tag == uidSyncingCmd) {
 161
 162         if (resp->kind == Responses::OK) {
 163             log("UIDs synchronized", Common::LOG_MAILBOX_SYNC);
 164             Q_ASSERT(status == STATE_SYNCING_FLAGS);
 165             Q_ASSERT(mailboxIndex.isValid());   // FIXME
 166             TreeItemMailbox *mailbox = dynamic_cast<TreeItemMailbox *>(static_cast<TreeItem *>(mailboxIndex.internalPointer()));
 167             Q_ASSERT(mailbox);
 168             syncFlags(mailbox);
 169         } else {
 170             _failed("UID syncing failed");
 171             // FIXME: error handling
 172         }
 173         return true;
 174     } else if (resp->tag == flagsCmd) {
 175
 176         if (resp->kind == Responses::OK) {
 177             //qDebug() << "received OK for flagsCmd";
 178             Q_ASSERT(status == STATE_SYNCING_FLAGS);
 179             Q_ASSERT(mailboxIndex.isValid());   // FIXME
 180             TreeItemMailbox *mailbox = dynamic_cast<TreeItemMailbox *>(static_cast<TreeItem *>(mailboxIndex.internalPointer()));
 181             Q_ASSERT(mailbox);
 182             status = STATE_DONE;
 183             log("Flags synchronized", Common::LOG_MAILBOX_SYNC);
 184             notifyInterestingMessages(mailbox);
 185             flagsCmd.clear();
 186
 187             if (newArrivalsFetch.isEmpty()) {
 188                 saveSyncState(mailbox);
 189                 _completed();
 190             } else {
 191                 log("Pending new arrival fetching, not terminating yet", Common::LOG_MAILBOX_SYNC);
 192             }
 193         } else {
 194             status = STATE_DONE;
 195             _failed("Flags synchronization failed");
 196             // FIXME: error handling
 197         }
 198         emit model->mailboxSyncingProgress(mailboxIndex, status);
 199         return true;
 200     } else if (newArrivalsFetch.contains(resp->tag)) {
 201
 202         if (resp->kind == Responses::OK) {
 203             newArrivalsFetch.removeOne(resp->tag);
 204
 205             if (newArrivalsFetch.isEmpty() && status == STATE_DONE && flagsCmd.isEmpty()) {
 206                 Q_ASSERT(mailboxIndex.isValid());   // FIXME
 207                 TreeItemMailbox *mailbox = dynamic_cast<TreeItemMailbox *>(static_cast<TreeItem *>(mailboxIndex.internalPointer()));
 208                 Q_ASSERT(mailbox);
 209                 saveSyncState(mailbox);
 210                 _completed();
 211             }
 212         } else {
 213             _failed("UID discovery of new arrivals after initial UID sync has failed");
 214         }
 215         return true;
 216
 217     } else {
 218         return false;
 219     }
 220 }
 221
 222 void ObtainSynchronizedMailboxTask::finalizeSelect()
 223 {
 224     Q_ASSERT(mailboxIndex.isValid());
 225     TreeItemMailbox *mailbox = dynamic_cast<TreeItemMailbox *>(static_cast<TreeItem *>(mailboxIndex.internalPointer()));
 226     Q_ASSERT(mailbox);
 227     TreeItemMsgList *list = dynamic_cast<TreeItemMsgList *>(mailbox->m_children[ 0 ]);
 228     Q_ASSERT(list);
 229
 230     model->changeConnectionState(parser, CONN_STATE_SYNCING);
 231     const SyncState &syncState = mailbox->syncState;
 232     oldSyncState = model->cache()->mailboxSyncState(mailbox->mailbox());
 233     list->m_totalMessageCount = syncState.exists();
 234     // Note: syncState.unSeen() is the NUMBER of the first unseen message, not their count!
 235
 236     uidMap = model->cache()->uidMapping(mailbox->mailbox());
 237
 238     if (static_cast<uint>(uidMap.size()) != oldSyncState.exists()) {
 239
 240         QString buf;
 241         QDebug dbg(&buf);
 242         dbg << "Inconsistent cache data, falling back to full sync (" << uidMap.size() << "in UID map," << oldSyncState.exists() <<
 243             "EXIST before)";
 244         log(buf, Common::LOG_MAILBOX_SYNC);
 245         fullMboxSync(mailbox, list);
 246     } else {
 247         if (syncState.isUsableForSyncing() && oldSyncState.isUsableForSyncing() && syncState.uidValidity() == oldSyncState.uidValidity()) {
 248             // Perform a nice re-sync
 249
 250             // Check the QRESYNC support and availability
 251             if (m_usingQresync && oldSyncState.isUsableForCondstore() && syncState.isUsableForCondstore()) {
 252                 // Looks like we can use QRESYNC for fast syncing
 253                 if (oldSyncState.highestModSeq() > syncState.highestModSeq()) {
 254                     // Looks like a corrupted cache or a server's bug
 255                     log("Yuck, recycled HIGHESTMODSEQ when trying to use QRESYNC", Common::LOG_MAILBOX_SYNC);
 256                     mailbox->syncState.setHighestModSeq(0);
 257                     model->cache()->clearAllMessages(mailbox->mailbox());
 258                     m_usingQresync = false;
 259                     fullMboxSync(mailbox, list);
 260                 } else {
 261                     if (oldSyncState.highestModSeq() == syncState.highestModSeq()) {
 262                         if (oldSyncState.exists() != syncState.exists()) {
 263                             log("Sync error: QRESYNC says no changes but EXISTS has changed", Common::LOG_MAILBOX_SYNC);
 264                             mailbox->syncState.setHighestModSeq(0);
 265                             model->cache()->clearAllMessages(mailbox->mailbox());
 266                             m_usingQresync = false;
 267                             fullMboxSync(mailbox, list);
 268                         } else if (oldSyncState.uidNext() != syncState.uidNext()) {
 269                             log("Sync error: QRESYNC says no changes but UIDNEXT has changed", Common::LOG_MAILBOX_SYNC);
 270                             mailbox->syncState.setHighestModSeq(0);
 271                             model->cache()->clearAllMessages(mailbox->mailbox());
 272                             m_usingQresync = false;
 273                             fullMboxSync(mailbox, list);
 274                         } else if (syncState.exists() != static_cast<uint>(list->m_children.size())) {
 275                             log(QString::fromUtf8("Sync error: constant HIGHESTMODSEQ, EXISTS says %1 messages but in fact "
 276                                                    "there are %2 when finalizing SELECT")
 277                                 .arg(QString::number(mailbox->syncState.exists()), QString::number(list->m_children.size())),
 278                                 Common::LOG_MAILBOX_SYNC);
 279                             mailbox->syncState.setHighestModSeq(0);
 280                             model->cache()->clearAllMessages(mailbox->mailbox());
 281                             m_usingQresync = false;
 282                             fullMboxSync(mailbox, list);
 283                         } else {
 284                             // This should be enough
 285                             list->m_fetchStatus = TreeItem::DONE;
 286                             notifyInterestingMessages(mailbox);
 287                             saveSyncState(mailbox);
 288                             _completed();
 289                         }
 290                         return;
 291                     }
 292
 293                     if (static_cast<uint>(list->m_children.size()) != mailbox->syncState.exists()) {
 294                         log(QString::fromUtf8("Sync error: EXISTS says %1 messages, msgList has %2")
 295                             .arg(QString::number(mailbox->syncState.exists()), QString::number(list->m_children.size())));
 296                         mailbox->syncState.setHighestModSeq(0);
 297                         model->cache()->clearAllMessages(mailbox->mailbox());
 298                         m_usingQresync = false;
 299                         fullMboxSync(mailbox, list);
 300                         return;
 301                     }
 302
 303
 304                     if (oldSyncState.uidNext() < syncState.uidNext()) {
 305                         list->m_fetchStatus = TreeItem::DONE;
 306                         int seqWithLowestUnknownUid = -1;
 307                         for (int i = 0; i < list->m_children.size(); ++i) {
 308                             TreeItemMessage *msg = static_cast<TreeItemMessage*>(list->m_children[i]);
 309                             if (!msg->uid()) {
 310                                 seqWithLowestUnknownUid = i;
 311                                 break;
 312                             }
 313                         }
 314                         if (seqWithLowestUnknownUid >= 0) {
 315                             // We've got some new arrivals, but unfortunately QRESYNC won't report them just yet :(
 316                             CommandHandle fetchCmd = parser->uidFetch(Sequence::startingAt(qMax(oldSyncState.uidNext(), 1u)),
 317                                                                       QStringList() << QLatin1String("FLAGS"));
 318                             newArrivalsFetch.append(fetchCmd);
 319                             status = STATE_DONE;
 320                         } else {
 321                             // All UIDs are known at this point, including the new arrivals, yay
 322                             notifyInterestingMessages(mailbox);
 323                             saveSyncState(mailbox);
 324                             _completed();
 325                         }
 326                     } else {
 327                         // This should be enough, the server should've sent the data already
 328                         list->m_fetchStatus = TreeItem::DONE;
 329                         notifyInterestingMessages(mailbox);
 330                         saveSyncState(mailbox);
 331                         _completed();
 332                     }
 333                 }
 334                 return;
 335             }
 336
 337             if (syncState.uidNext() == oldSyncState.uidNext()) {
 338                 // No new messages
 339
 340                 if (syncState.exists() == oldSyncState.exists()) {
 341                     // No deletions, either, so we resync only flag changes
 342                     syncNoNewNoDeletions(mailbox, list);
 343                 } else {
 344                     // Some messages got deleted, but there have been no additions
 345                     syncGeneric(mailbox, list);
 346                 }
 347
 348             } else if (syncState.uidNext() > oldSyncState.uidNext()) {
 349                 // Some new messages were delivered since we checked the last time.
 350                 // There's no guarantee they are still present, though.
 351
 352                 if (syncState.uidNext() - oldSyncState.uidNext() == syncState.exists() - oldSyncState.exists()) {
 353                     // Only some new arrivals, no deletions
 354                     syncOnlyAdditions(mailbox, list);
 355                 } else {
 356                     // Generic case; we don't know anything about which messages were deleted and which added
 357                     syncGeneric(mailbox, list);
 358                 }
 359             } else {
 360                 // The UIDNEXT has decreased while UIDVALIDITY remains the same. This is forbidden,
 361                 // so either a server's bug, or a completely invalid cache.
 362                 Q_ASSERT(syncState.uidNext() < oldSyncState.uidNext());
 363                 Q_ASSERT(syncState.uidValidity() == oldSyncState.uidValidity());
 364                 log("Yuck, UIDVALIDITY remains same but UIDNEXT decreased", Common::LOG_MAILBOX_SYNC);
 365                 model->cache()->clearAllMessages(mailbox->mailbox());
 366                 fullMboxSync(mailbox, list);
 367             }
 368         } else {
 369             // Forget everything, do a dumb sync
 370             model->cache()->clearAllMessages(mailbox->mailbox());
 371             fullMboxSync(mailbox, list);
 372         }
 373     }
 374 }
 375
 376 void ObtainSynchronizedMailboxTask::fullMboxSync(TreeItemMailbox *mailbox, TreeItemMsgList *list)
 377 {
 378     log("Full synchronization", Common::LOG_MAILBOX_SYNC);
 379
 380     QModelIndex parent = list->toIndex(model);
 381     if (! list->m_children.isEmpty()) {
 382         model->beginRemoveRows(parent, 0, list->m_children.size() - 1);
 383         QList<TreeItem*> oldItems = list->m_children;
 384         list->m_children.clear();
 385         model->endRemoveRows();
 386         qDeleteAll(oldItems);
 387     }
 388     if (mailbox->syncState.exists()) {
 389         model->beginInsertRows(parent, 0, mailbox->syncState.exists() - 1);
 390         for (uint i = 0; i < mailbox->syncState.exists(); ++i) {
 391             TreeItemMessage *msg = new TreeItemMessage(list);
 392             msg->m_offset = i;
 393             list->m_children << msg;
 394         }
 395         model->endInsertRows();
 396
 397         syncUids(mailbox);
 398         list->m_numberFetchingStatus = TreeItem::LOADING;
 399         list->m_unreadMessageCount = 0;
 400     } else {
 401         // No messages, we're done here
 402         list->m_totalMessageCount = 0;
 403         list->m_unreadMessageCount = 0;
 404         list->m_numberFetchingStatus = TreeItem::DONE;
 405         list->m_fetchStatus = TreeItem::DONE;
 406
 407         // The remote mailbox is empty -> we're done now
 408         model->changeConnectionState(parser, CONN_STATE_SELECTED);
 409         status = STATE_DONE;
 410         emit model->mailboxSyncingProgress(mailboxIndex, status);
 411         notifyInterestingMessages(mailbox);
 412         saveSyncState(mailbox);
 413         // Take care here: this call could invalidate our index (see test coverage)
 414         _completed();
 415     }
 416     // Our mailbox might have actually been invalidated by various callbacks activated above
 417     if (mailboxIndex.isValid()) {
 418         Q_ASSERT(mailboxIndex.internalPointer() == mailbox);
 419         model->emitMessageCountChanged(mailbox);
 420     }
 421 }
 422
 423 void ObtainSynchronizedMailboxTask::syncNoNewNoDeletions(TreeItemMailbox *mailbox, TreeItemMsgList *list)
 424 {
 425     Q_ASSERT(mailbox->syncState.exists() == static_cast<uint>(uidMap.size()));
 426     log("No arrivals or deletions since the last time", Common::LOG_MAILBOX_SYNC);
 427     if (mailbox->syncState.exists()) {
 428         // Verify that we indeed have all UIDs and not need them anymore
 429         bool uidsOk = true;
 430         for (int i = 0; i < list->m_children.size(); ++i) {
 431             if (! static_cast<TreeItemMessage *>(list->m_children[i])->uid()) {
 432                 uidsOk = false;
 433                 break;
 434             }
 435         }
 436         // FIXME: This assert can fail if the mailbox contained messages with missing UIDs even before we opened it now.
 437         Q_ASSERT(uidsOk);
 438     } else {
 439         list->m_unreadMessageCount = 0;
 440         list->m_totalMessageCount = 0;
 441         list->m_numberFetchingStatus = TreeItem::DONE;
 442     }
 443
 444     if (list->m_children.isEmpty()) {
 445         QList<TreeItem *> messages;
 446         for (uint i = 0; i < mailbox->syncState.exists(); ++i) {
 447             TreeItemMessage *msg = new TreeItemMessage(list);
 448             msg->m_offset = i;
 449             msg->m_uid = uidMap[ i ];
 450             messages << msg;
 451         }
 452         list->setChildren(messages);
 453
 454     } else {
 455         if (mailbox->syncState.exists() != static_cast<uint>(list->m_children.size())) {
 456             throw CantHappen("TreeItemMsgList has wrong number of "
 457                              "children, even though no change of "
 458                              "message count occured");
 459         }
 460     }
 461
 462     list->m_fetchStatus = TreeItem::DONE;
 463
 464     if (mailbox->syncState.exists()) {
 465         syncFlags(mailbox);
 466     } else {
 467         status = STATE_DONE;
 468         emit model->mailboxSyncingProgress(mailboxIndex, status);
 469         notifyInterestingMessages(mailbox);
 470
 471         if (newArrivalsFetch.isEmpty()) {
 472             saveSyncState(mailbox);
 473             _completed();
 474         }
 475     }
 476 }
 477
 478 void ObtainSynchronizedMailboxTask::syncOnlyAdditions(TreeItemMailbox *mailbox, TreeItemMsgList *list)
 479 {
 480     log("Syncing new arrivals", Common::LOG_MAILBOX_SYNC);
 481
 482     // So, we know that messages only got added to the mailbox and that none were removed,
 483     // neither those that we already know or those that got added while we weren't around.
 484     // Therefore we ask only for UIDs of new messages
 485
 486     firstUnknownUidOffset = oldSyncState.exists();
 487     list->m_numberFetchingStatus = TreeItem::LOADING;
 488     uidSyncingMode = UID_SYNC_ONLY_NEW;
 489     syncUids(mailbox, oldSyncState.uidNext());
 490 }
 491
 492 void ObtainSynchronizedMailboxTask::syncGeneric(TreeItemMailbox *mailbox, TreeItemMsgList *list)
 493 {
 494     log("generic synchronization from previous state", Common::LOG_MAILBOX_SYNC);
 495
 496     list->m_numberFetchingStatus = TreeItem::LOADING;
 497     list->m_unreadMessageCount = 0;
 498     uidSyncingMode = UID_SYNC_ALL;
 499     syncUids(mailbox);
 500 }
 501
 502 void ObtainSynchronizedMailboxTask::syncUids(TreeItemMailbox *mailbox, const uint lowestUidToQuery)
 503 {
 504     status = STATE_SYNCING_UIDS;
 505     log("Syncing UIDs", Common::LOG_MAILBOX_SYNC);
 506     QByteArray uidSpecification;
 507     if (lowestUidToQuery == 0) {
 508         uidSpecification = "ALL";
 509     } else {
 510         uidSpecification = QString::fromUtf8("UID %1:*").arg(QString::number(lowestUidToQuery)).toUtf8();
 511     }
 512     if (model->accessParser(parser).capabilities.contains(QLatin1String("ESEARCH"))) {
 513         uidSyncingCmd = parser->uidESearchUid(uidSpecification);
 514     } else {
 515         uidSyncingCmd = parser->uidSearchUid(uidSpecification);
 516     }
 517     model->cache()->clearUidMapping(mailbox->mailbox());
 518     emit model->mailboxSyncingProgress(mailboxIndex, status);
 519 }
 520
 521 void ObtainSynchronizedMailboxTask::syncFlags(TreeItemMailbox *mailbox)
 522 {
 523     status = STATE_SYNCING_FLAGS;
 524     log("Syncing flags", Common::LOG_MAILBOX_SYNC);
 525     TreeItemMsgList *list = dynamic_cast<TreeItemMsgList *>(mailbox->m_children[ 0 ]);
 526     Q_ASSERT(list);
 527
 528     // 0 => don't use it; >0 => use that as the old value
 529     quint64 useModSeq = 0;
 530     if ((model->accessParser(parser).capabilities.contains(QLatin1String("CONDSTORE")) ||
 531          model->accessParser(parser).capabilities.contains(QLatin1String("QRESYNC")) ||
 532          model->accessParser(parser).capabilities.contains(QLatin1String("X-DRAFT-I01-QRESYNC-ARRIVED"))) &&
 533             oldSyncState.highestModSeq() > 0 && mailbox->syncState.isUsableForCondstore() &&
 534             oldSyncState.uidValidity() == mailbox->syncState.uidValidity()) {
 535         // The CONDSTORE is available, UIDVALIDITY has not changed and the HIGHESTMODSEQ suggests that
 536         // it will be useful
 537         if (oldSyncState.highestModSeq() == mailbox->syncState.highestModSeq()) {
 538             // Looks like there were no changes in flags -- that's cool, we're done here,
 539             // but only after some sanity checks
 540             if (oldSyncState.exists() > mailbox->syncState.exists()) {
 541                 log("Some messages have arrived to the mailbox, but HIGHESTMODSEQ hasn't changed. "
 542                     "That's a bug in the server implementation.", Common::LOG_MAILBOX_SYNC);
 543                 // will issue the ordinary FETCH command for FLAGS
 544             } else if (oldSyncState.uidNext() != mailbox->syncState.uidNext()) {
 545                 log("UIDNEXT has changed, yet HIGHESTMODSEQ remained constant; that's server's bug", Common::LOG_MAILBOX_SYNC);
 546                 // and again, don't trust that HIGHESTMODSEQ
 547             } else {
 548                 // According to HIGHESTMODSEQ, there hasn't been any change. UIDNEXT and EXISTS do not contradict
 549                 // this interpretation, so we can go and call stuff finished.
 550                 if (newArrivalsFetch.isEmpty()) {
 551                     // No pending activity -> let's call it a day
 552                     status = STATE_DONE;
 553                     saveSyncState(mailbox);
 554                     _completed();
 555                     return;
 556                 } else {
 557                     // ...but there's still some pending activity; let's wait for its termination
 558                     status = STATE_DONE;
 559                 }
 560             }
 561         } else if (oldSyncState.highestModSeq() > mailbox->syncState.highestModSeq()) {
 562             // Clearly a bug
 563             log("HIGHESTMODSEQ decreased, that's a bug in the IMAP server", Common::LOG_MAILBOX_SYNC);
 564             // won't use HIGHESTMODSEQ
 565         } else {
 566             // Will use FETCH CHANGEDSINCE
 567             useModSeq = oldSyncState.highestModSeq();
 568         }
 569     }
 570     if (useModSeq > 0) {
 571         QMap<QByteArray, quint64> fetchModifier;
 572         fetchModifier["CHANGEDSINCE"] = oldSyncState.highestModSeq();
 573         flagsCmd = parser->fetch(Sequence(1, mailbox->syncState.exists()), QStringList() << QLatin1String("FLAGS"), fetchModifier);
 574     } else {
 575         flagsCmd = parser->fetch(Sequence(1, mailbox->syncState.exists()), QStringList() << QLatin1String("FLAGS"));
 576     }
 577     list->m_numberFetchingStatus = TreeItem::LOADING;
 578     emit model->mailboxSyncingProgress(mailboxIndex, status);
 579 }
 580
 581 bool ObtainSynchronizedMailboxTask::handleResponseCodeInsideState(const Imap::Responses::State *const resp)
 582 {
 583     if (dieIfInvalidMailbox())
 584         return true;
 585
 586     TreeItemMailbox *mailbox = Model::mailboxForSomeItem(mailboxIndex);
 587     Q_ASSERT(mailbox);
 588     bool res = false;
 589     switch (resp->respCode) {
 590     case Responses::UNSEEN:
 591     {
 592         const Responses::RespData<uint> *const num = dynamic_cast<const Responses::RespData<uint>* const>(resp->respCodeData.data());
 593         if (num) {
 594             mailbox->syncState.setUnSeenOffset(num->data);
 595             res = true;
 596         } else {
 597             throw CantHappen("State response has invalid UNSEEN respCodeData", *resp);
 598         }
 599         break;
 600     }
 601     case Responses::PERMANENTFLAGS:
 602     {
 603         const Responses::RespData<QStringList> *const num = dynamic_cast<const Responses::RespData<QStringList>* const>(resp->respCodeData.data());
 604         if (num) {
 605             mailbox->syncState.setPermanentFlags(num->data);
 606             res = true;
 607         } else {
 608             throw CantHappen("State response has invalid PERMANENTFLAGS respCodeData", *resp);
 609         }
 610         break;
 611     }
 612     case Responses::UIDNEXT:
 613     {
 614         const Responses::RespData<uint> *const num = dynamic_cast<const Responses::RespData<uint>* const>(resp->respCodeData.data());
 615         if (num) {
 616             mailbox->syncState.setUidNext(num->data);
 617             res = true;
 618         } else {
 619             throw CantHappen("State response has invalid UIDNEXT respCodeData", *resp);
 620         }
 621         break;
 622     }
 623     case Responses::UIDVALIDITY:
 624     {
 625         const Responses::RespData<uint> *const num = dynamic_cast<const Responses::RespData<uint>* const>(resp->respCodeData.data());
 626         if (num) {
 627             mailbox->syncState.setUidValidity(num->data);
 628             res = true;
 629         } else {
 630             throw CantHappen("State response has invalid UIDVALIDITY respCodeData", *resp);
 631         }
 632         break;
 633     }
 634     case Responses::NOMODSEQ:
 635         // NOMODSEQ means that this mailbox doesn't support CONDSTORE or QRESYNC. We have to avoid sending any fancy commands like
 636         // the FETCH CHANGEDSINCE etc.
 637         mailbox->syncState.setHighestModSeq(0);
 638         m_usingQresync = false;
 639         res = true;
 640         break;
 641
 642     case Responses::HIGHESTMODSEQ:
 643     {
 644         const Responses::RespData<quint64> *const num = dynamic_cast<const Responses::RespData<quint64>* const>(resp->respCodeData.data());
 645         Q_ASSERT(num);
 646         mailbox->syncState.setHighestModSeq(num->data);
 647         res = true;
 648         break;
 649     }
 650     case Responses::CLOSED:
 651         // FIXME: handle when supporting the qresync
 652         res = true;
 653         break;
 654     default:
 655         break;
 656     }
 657     return res;
 658 }
 659
 660 void ObtainSynchronizedMailboxTask::updateHighestKnownUid(TreeItemMailbox *mailbox, const TreeItemMsgList *list) const
 661 {
 662     uint highestKnownUid = 0;
 663     for (int i = list->m_children.size() - 1; ! highestKnownUid && i >= 0; --i) {
 664         highestKnownUid = static_cast<const TreeItemMessage *>(list->m_children[i])->uid();
 665     }
 666     if (highestKnownUid) {
 667         // If the UID walk return a usable number, remember that and use it for updating our idea of the UIDNEXT
 668         mailbox->syncState.setUidNext(qMax(mailbox->syncState.uidNext(), highestKnownUid + 1));
 669     }
 670 }
 671
 672 bool ObtainSynchronizedMailboxTask::handleNumberResponse(const Imap::Responses::NumberResponse *const resp)
 673 {
 674     if (dieIfInvalidMailbox())
 675         return true;
 676
 677     TreeItemMailbox *mailbox = Model::mailboxForSomeItem(mailboxIndex);
 678     Q_ASSERT(mailbox);
 679     TreeItemMsgList *list = dynamic_cast<TreeItemMsgList *>(mailbox->m_children[0]);
 680     Q_ASSERT(list);
 681     switch (resp->kind) {
 682     case Imap::Responses::EXISTS:
 683         switch (status) {
 684         case STATE_WAIT_FOR_CONN:
 685             Q_ASSERT(false);
 686             return false;
 687
 688         case STATE_SELECTING:
 689             if (m_usingQresync) {
 690                 // Because QRESYNC won't tell us anything about the new UIDs, we have to resort to this kludgy way of working.
 691                 // I really, really wonder why there's no such thing likt * ARRIVED to accompany * VANISHED. Oh well.
 692                 mailbox->syncState.setExists(resp->number);
 693                 int newArrivals = resp->number - list->m_children.size();
 694                 if (newArrivals > 0) {
 695                     // We have to add empty messages here
 696                     QModelIndex parent = list->toIndex(model);
 697                     int offset = list->m_children.size();
 698                     model->beginInsertRows(parent, offset, resp->number - 1);
 699                     for (int i = 0; i < newArrivals; ++i) {
 700                         TreeItemMessage *msg = new TreeItemMessage(list);
 701                         msg->m_offset = i + offset;
 702                         list->m_children << msg;
 703                         // yes, we really have to add this message with UID 0 :(
 704                     }
 705                     model->endInsertRows();
 706                     list->m_totalMessageCount = resp->number;
 707                 }
 708             } else {
 709                 // It's perfectly acceptable for the server to start its responses with EXISTS instead of UIDVALIDITY & UIDNEXT, so
 710                 // we really cannot do anything besides remembering this value for later.
 711                 mailbox->syncState.setExists(resp->number);
 712             }
 713             return true;
 714
 715         case STATE_SYNCING_UIDS:
 716             mailbox->handleExists(model, *resp);
 717             updateHighestKnownUid(mailbox, list);
 718             return true;
 719
 720         case STATE_SYNCING_FLAGS:
 721         case STATE_DONE:
 722             if (resp->number == static_cast<uint>(list->m_children.size())) {
 723                 // no changes
 724                 return true;
 725             }
 726             mailbox->handleExists(model, *resp);
 727             model->cache()->clearUidMapping(mailbox->mailbox());
 728             Q_ASSERT(list->m_children.size());
 729             updateHighestKnownUid(mailbox, list);
 730             CommandHandle fetchCmd = parser->uidFetch(Sequence::startingAt(
 731                                                     // prevent a possible invalid 0:*
 732                                                     qMax(mailbox->syncState.uidNext(), 1u)
 733                                                 ), QStringList() << QLatin1String("FLAGS"));
 734             newArrivalsFetch.append(fetchCmd);
 735             return true;
 736         }
 737         Q_ASSERT(false);
 738         return false;
 739
 740     case Imap::Responses::EXPUNGE:
 741
 742         if (mailbox->syncState.exists() > 0) {
 743             // Always update the number of expected messages
 744             mailbox->syncState.setExists(mailbox->syncState.exists() - 1);
 745         }
 746
 747         switch (status) {
 748         case STATE_SYNCING_FLAGS:
 749             // The UID mapping has been already established, but we don't have enough information for
 750             // an atomic state transition yet
 751             mailbox->handleExpunge(model, *resp);
 752             // The SyncState and the UID map will be saved later, along with the flags, when this task finishes
 753             return true;
 754
 755         case STATE_DONE:
 756             // The UID mapping has been already established, so we just want to handle the EXPUNGE as usual
 757             mailbox->handleExpunge(model, *resp);
 758             model->cache()->setMailboxSyncState(mailbox->mailbox(), mailbox->syncState);
 759             model->saveUidMap(list);
 760             return true;
 761
 762         default:
 763             // This is handled by the code below
 764             break;
 765         }
 766
 767         // We shall track updates to the place where the unknown UIDs resign
 768         if (resp->number < firstUnknownUidOffset + 1) {
 769             // The message which we're deleting has UID which is already known, ie. it isn't among those whose UIDs got requested
 770             // by an incremental UID SEARCH
 771             Q_ASSERT(firstUnknownUidOffset > 0);
 772             --firstUnknownUidOffset;
 773
 774             // The deleted message has previously been present; that means that we shall immediately signal about its expunge
 775             mailbox->handleExpunge(model, *resp);
 776         }
 777
 778         switch(status) {
 779         case STATE_WAIT_FOR_CONN:
 780             Q_ASSERT(false);
 781             return false;
 782
 783         case STATE_SELECTING:
 784             // The actual change will be handled by the UID syncing code
 785             return true;
 786
 787         case STATE_SYNCING_UIDS:
 788             // We shouldn't delete stuff at this point, it will be handled by the UID syncing.
 789             // The response shall be consumed, though.
 790             return true;
 791
 792         case STATE_SYNCING_FLAGS:
 793         case STATE_DONE:
 794             // Already handled above
 795             Q_ASSERT(false);
 796             return false;
 797
 798         }
 799
 800         break;
 801     case Imap::Responses::RECENT:
 802         mailbox->syncState.setRecent(resp->number);
 803         list->m_recentMessageCount = resp->number;
 804         return true;
 805         break;
 806     default:
 807         throw CantHappen("Got a NumberResponse of invalid kind. This is supposed to be handled in its constructor!", *resp);
 808     }
 809     return false;
 810 }
 811
 812 bool ObtainSynchronizedMailboxTask::handleVanished(const Imap::Responses::Vanished *const resp)
 813 {
 814     if (dieIfInvalidMailbox())
 815         return true;
 816
 817     TreeItemMailbox *mailbox = Model::mailboxForSomeItem(mailboxIndex);
 818     Q_ASSERT(mailbox);
 819
 820     switch (status) {
 821     case STATE_WAIT_FOR_CONN:
 822         Q_ASSERT(false);
 823         return false;
 824
 825     case STATE_SELECTING:
 826     case STATE_SYNCING_UIDS:
 827     case STATE_SYNCING_FLAGS:
 828     case STATE_DONE:
 829         mailbox->handleVanished(model, *resp);
 830         return true;
 831     }
 832
 833     Q_ASSERT(false);
 834     return false;
 835 }
 836
 837 bool ObtainSynchronizedMailboxTask::handleFlags(const Imap::Responses::Flags *const resp)
 838 {
 839     if (dieIfInvalidMailbox())
 840         return true;
 841
 842     TreeItemMailbox *mailbox = Model::mailboxForSomeItem(mailboxIndex);
 843     Q_ASSERT(mailbox);
 844     mailbox->syncState.setFlags(resp->flags);
 845     return true;
 846 }
 847
 848 bool ObtainSynchronizedMailboxTask::handleSearch(const Imap::Responses::Search *const resp)
 849 {
 850     if (dieIfInvalidMailbox())
 851         return true;
 852
 853     TreeItemMailbox *mailbox = Model::mailboxForSomeItem(mailboxIndex);
 854     Q_ASSERT(mailbox);
 855     TreeItemMsgList *list = dynamic_cast<TreeItemMsgList*>(mailbox->m_children[0]);
 856     Q_ASSERT(list);
 857
 858     if (!mailbox->syncState.isUsableForSyncing()) {
 859         throw MailboxException("Received SEARCH response but the syncState is not usable which means that we haven't asked for one", *resp);
 860     }
 861
 862     switch (uidSyncingMode) {
 863     case UID_SYNC_ALL:
 864         if (static_cast<uint>(resp->items.size()) != mailbox->syncState.exists()) {
 865             // The (possibly updated) EXISTS does not match what we received for UID SEARCH ALL. Please note that
 866             // it's the server's responsibility to feed us with valid data; scenarios like sending out-of-order responses
 867             // would clearly break this contract.
 868             std::ostringstream ss;
 869             ss << "Error when synchronizing all messages: server said that there are " << mailbox->syncState.exists() <<
 870                   " messages, but UID SEARCH ALL response contains " << resp->items.size() << " entries" << std::endl;
 871             ss.flush();
 872             throw MailboxException(ss.str().c_str(), *resp);
 873         }
 874         break;
 875     case UID_SYNC_ONLY_NEW:
 876     {
 877         // Be sure there really are some new messages
 878         const int newArrivals = mailbox->syncState.exists() - firstUnknownUidOffset;
 879         Q_ASSERT(newArrivals >= 0);
 880
 881         if (newArrivals != resp->items.size()) {
 882             std::ostringstream ss;
 883             ss << "Error when synchronizing new messages: server said that there are " << mailbox->syncState.exists() <<
 884                   " messages in total (" << newArrivals << " new), but UID SEARCH response contains " << resp->items.size() <<
 885                   " entries" << std::endl;
 886             ss.flush();
 887             throw MailboxException(ss.str().c_str(), *resp);
 888         }
 889         break;
 890     }
 891     }
 892     uidMap = resp->items;
 893     qSort(uidMap);
 894     if (!uidMap.isEmpty() && uidMap.front() == 0) {
 895         throw MailboxException("UID SEARCH response contains invalid UID zero", *resp);
 896     }
 897     applyUids(mailbox);
 898     uidMap.clear();
 899     updateHighestKnownUid(mailbox, list);
 900     status = STATE_SYNCING_FLAGS;
 901     return true;
 902 }
 903
 904 bool ObtainSynchronizedMailboxTask::handleESearch(const Imap::Responses::ESearch *const resp)
 905 {
 906     if (dieIfInvalidMailbox())
 907         return true;
 908
 909     TreeItemMailbox *mailbox = Model::mailboxForSomeItem(mailboxIndex);
 910     Q_ASSERT(mailbox);
 911     TreeItemMsgList *list = dynamic_cast<TreeItemMsgList*>(mailbox->m_children[0]);
 912     Q_ASSERT(list);
 913
 914
 915     if (resp->tag != uidSyncingCmd)
 916         return false;
 917
 918     if (resp->seqOrUids != Imap::Responses::ESearch::UIDS)
 919         throw UnexpectedResponseReceived("ESEARCH response with matching tag uses sequence numbers instead of UIDs", *resp);
 920
 921     // Yes, I just love templates.
 922     Responses::ESearch::CompareListDataIdentifier<Responses::ESearch::ListData_t> allComparator("ALL");
 923     Responses::ESearch::ListData_t::const_iterator listIterator =
 924             std::find_if(resp->listData.constBegin(), resp->listData.constEnd(), allComparator);
 925
 926     if (listIterator != resp->listData.constEnd()) {
 927         uidMap = listIterator->second;
 928         ++listIterator;
 929         if (std::find_if(listIterator, resp->listData.constEnd(), allComparator) != resp->listData.constEnd())
 930             throw UnexpectedResponseReceived("ESEARCH contains the ALL key too many times", *resp);
 931     } else {
 932         // If the ALL key is not present, the server is telling us that there are no messages matching the query
 933         uidMap.clear();
 934     }
 935
 936     switch (uidSyncingMode) {
 937     case UID_SYNC_ALL:
 938         if (static_cast<uint>(uidMap.size()) != mailbox->syncState.exists()) {
 939             // The (possibly updated) EXISTS does not match what we received for UID SEARCH ALL. Please note that
 940             // it's the server's responsibility to feed us with valid data; scenarios like sending out-of-order responses
 941             // would clearly break this contract.
 942             std::ostringstream ss;
 943             ss << "Error when synchronizing all messages: server said that there are " << mailbox->syncState.exists() <<
 944                   " messages, but UID ESEARCH ALL response contains " << uidMap.size() << " entries" << std::endl;
 945             ss.flush();
 946             throw MailboxException(ss.str().c_str(), *resp);
 947         }
 948         Q_ASSERT(mailbox->syncState.isUsableForSyncing());
 949         break;
 950     case UID_SYNC_ONLY_NEW:
 951     {
 952         // Be sure there really are some new messages
 953         const int newArrivals = mailbox->syncState.exists() - firstUnknownUidOffset;
 954         Q_ASSERT(newArrivals >= 0);
 955
 956         if (newArrivals != uidMap.size()) {
 957             std::ostringstream ss;
 958             ss << "Error when synchronizing new messages: server said that there are " << mailbox->syncState.exists() <<
 959                   " messages in total (" << newArrivals << " new), but UID ESEARCH response contains " << uidMap.size() <<
 960                   " entries" << std::endl;
 961             ss.flush();
 962             throw MailboxException(ss.str().c_str(), *resp);
 963         }
 964         break;
 965     }
 966     }
 967
 968     qSort(uidMap);
 969     if (!uidMap.isEmpty() && uidMap.front() == 0) {
 970         throw MailboxException("UID ESEARCH response contains invalid UID zero", *resp);
 971     }
 972     applyUids(mailbox);
 973     uidMap.clear();
 974     updateHighestKnownUid(mailbox, list);
 975     status = STATE_SYNCING_FLAGS;
 976     return true;
 977 }
 978
 979 bool ObtainSynchronizedMailboxTask::handleFetch(const Imap::Responses::Fetch *const resp)
 980 {
 981     if (dieIfInvalidMailbox())
 982         return true;
 983
 984     TreeItemMailbox *mailbox = Model::mailboxForSomeItem(mailboxIndex);
 985     Q_ASSERT(mailbox);
 986     QList<TreeItemPart *> changedParts;
 987     TreeItemMessage *changedMessage = 0;
 988     mailbox->handleFetchResponse(model, *resp, changedParts, changedMessage, false, m_usingQresync);
 989     if (changedMessage) {
 990         QModelIndex index = changedMessage->toIndex(model);
 991         emit model->dataChanged(index, index);
 992         if (mailbox->syncState.uidNext() <= changedMessage->uid()) {
 993             mailbox->syncState.setUidNext(changedMessage->uid() + 1);
 994         }
 995         // On the other hand, this one will be emitted at the very end
 996         // model->emitMessageCountChanged(mailbox);
 997     }
 998     if (!changedParts.isEmpty() && !m_usingQresync) {
 999         // On the other hand, with QRESYNC our code is ready to receive extra data that changes body parts...
1000         qDebug() << "Weird, FETCH when syncing has changed some body parts. We aren't ready for that.";
1001         log(QLatin1String("This response has changed some message parts. That should not have happened, as we're still syncing."));
1002     }
1003     return true;
1004 }
1005
1006 /** @short Apply the received UID map to the messages in mailbox
1007
1008 The @arg firstUnknownUidOffset corresponds to the offset of a message whose UID is specified by the first item in the UID map.
1009 */
1010 void ObtainSynchronizedMailboxTask::applyUids(TreeItemMailbox *mailbox)
1011 {
1012     TreeItemMsgList *list = dynamic_cast<TreeItemMsgList *>(mailbox->m_children[0]);
1013     Q_ASSERT(list);
1014     QModelIndex parent = list->toIndex(model);
1015
1016     int i = firstUnknownUidOffset;
1017     while (i < uidMap.size() + static_cast<int>(firstUnknownUidOffset)) {
1018         // Index inside the uidMap in which the UID of a message at offset i in the list->m_children can be found
1019         int uidOffset = i - firstUnknownUidOffset;
1020         Q_ASSERT(uidOffset >= 0);
1021         Q_ASSERT(uidOffset < uidMap.size());
1022
1023         // For each UID which is really supposed to be there...
1024
1025         Q_ASSERT(i <= list->m_children.size());
1026         if (i == list->m_children.size()) {
1027             // now we're just adding new messages to the end of the list
1028             const int futureTotalMessages = mailbox->syncState.exists();
1029             model->beginInsertRows(parent, i, futureTotalMessages - 1);
1030             for (/*nothing*/; i < futureTotalMessages; ++i) {
1031                 // Add all messages in one go
1032                 TreeItemMessage *msg = new TreeItemMessage(list);
1033                 msg->m_offset = i;
1034                 // We're iterating with i, so we got to update the uidOffset
1035                 uidOffset = i - firstUnknownUidOffset;
1036                 Q_ASSERT(uidOffset >= 0);
1037                 Q_ASSERT(uidOffset < uidMap.size());
1038                 msg->m_uid = uidMap[uidOffset];
1039                 list->m_children << msg;
1040             }
1041             model->endInsertRows();
1042             Q_ASSERT(i == list->m_children.size());
1043             Q_ASSERT(i == futureTotalMessages);
1044         } else if (dynamic_cast<TreeItemMessage *>(list->m_children[i])->m_uid == uidMap[uidOffset]) {
1045             // If the UID of the "current message" matches, we're okay
1046             dynamic_cast<TreeItemMessage *>(list->m_children[i])->m_offset = i;
1047             ++i;
1048         } else if (dynamic_cast<TreeItemMessage *>(list->m_children[i])->m_uid == 0) {
1049             // If the UID of the "current message" is zero, replace that with this message
1050             TreeItemMessage *msg = static_cast<TreeItemMessage*>(list->m_children[i]);
1051             msg->m_uid = uidMap[uidOffset];
1052             msg->m_offset = i;
1053             QModelIndex idx = model->createIndex(i, 0, msg);
1054             emit model->dataChanged(idx, idx);
1055             if (msg->m_fetchStatus == TreeItem::LOADING) {
1056                 // We've got to ask for the message metadata once again; the first attempt happened when the UID was still zero,
1057                 // so this is our chance
1058                 model->askForMsgMetadata(msg, Model::PRELOAD_PER_POLICY);
1059             }
1060             ++i;
1061         } else {
1062             // We've got an UID mismatch
1063             int pos = i;
1064             while (pos < list->m_children.size()) {
1065                 // Remove any messages which have non-zero UID which is at the same time different than the UID we want to add
1066                 // The key idea here is that IMAP guarantees that each and every new message will have greater UID than any
1067                 // other message already in the mailbox. Just for the sake of completeness, should an evil server send us a
1068                 // malformed response, we wouldn't care (or notice at this point), we'd just "needlessly" delete many "innocent"
1069                 // messages due to that one out-of-place arrival -- but we'd still remain correct and not crash.
1070                 TreeItemMessage *otherMessage = dynamic_cast<TreeItemMessage*>(list->m_children[pos]);
1071                 Q_ASSERT(otherMessage);
1072                 if (otherMessage->m_uid != 0 && otherMessage->m_uid != uidMap[uidOffset]) {
1073                     model->cache()->clearMessage(mailbox->mailbox(), otherMessage->uid());
1074                     ++pos;
1075                 } else {
1076                     break;
1077                 }
1078             }
1079             Q_ASSERT(pos > i);
1080             model->beginRemoveRows(parent, i, pos - 1);
1081             QList<TreeItem*> removedItems;
1082             for (int j = i; j < pos; ++j)
1083                 removedItems << list->m_children.takeAt(i);
1084             model->endRemoveRows();
1085             // the m_offset of all subsequent messages will be updated later, at the time *they* are processed
1086             qDeleteAll(removedItems);
1087             if (i == list->m_children.size()) {
1088                 // We're asked to add messages to the end of the list. That's something that's already implemented above,
1089                 // so let's reuse that code. That's why we do *not* want to increment the counter here.
1090             } else {
1091                 Q_ASSERT(i < list->m_children.size());
1092                 // But this case is also already implemented above, so we won't touch the counter from here, either,
1093                 // and let the existing code do its job
1094             }
1095         }
1096     }
1097
1098     if (i != list->m_children.size()) {
1099         // remove items at the end
1100         model->beginRemoveRows(parent, i, list->m_children.size() - 1);
1101         QList<TreeItem*> oldItems;
1102         for (/* nothing */; i < list->m_children.size(); ++i) {
1103             TreeItemMessage *message = static_cast<TreeItemMessage *>(list->m_children.takeAt(i));
1104             model->cache()->clearMessage(mailbox->mailbox(), message->uid());
1105             oldItems << message;
1106         }
1107         model->endRemoveRows();
1108         qDeleteAll(oldItems);
1109     }
1110
1111     uidMap.clear();
1112
1113     list->m_totalMessageCount = list->m_children.size();
1114     list->m_fetchStatus = TreeItem::DONE;
1115
1116     model->emitMessageCountChanged(mailbox);
1117     model->changeConnectionState(parser, CONN_STATE_SELECTED);
1118 }
1119
1120 QString ObtainSynchronizedMailboxTask::debugIdentification() const
1121 {
1122     if (! mailboxIndex.isValid())
1123         return QLatin1String("[invalid mailboxIndex]");
1124
1125     TreeItemMailbox *mailbox = dynamic_cast<TreeItemMailbox *>(static_cast<TreeItem *>(mailboxIndex.internalPointer()));
1126     Q_ASSERT(mailbox);
1127
1128     QString statusStr;
1129     switch (status) {
1130     case STATE_WAIT_FOR_CONN:
1131         statusStr = "STATE_WAIT_FOR_CONN";
1132         break;
1133     case STATE_SELECTING:
1134         statusStr = "STATE_SELECTING";
1135         break;
1136     case STATE_SYNCING_UIDS:
1137         statusStr = "STATE_SYNCING_UIDS";
1138         break;
1139     case STATE_SYNCING_FLAGS:
1140         statusStr = "STATE_SYNCING_FLAGS";
1141         break;
1142     case STATE_DONE:
1143         statusStr = "STATE_DONE";
1144         break;
1145     }
1146     return QString::fromUtf8("%1 %2").arg(statusStr, mailbox->mailbox());
1147 }
1148
1149 void ObtainSynchronizedMailboxTask::notifyInterestingMessages(TreeItemMailbox *mailbox)
1150 {
1151     Q_ASSERT(mailbox);
1152     TreeItemMsgList *list = dynamic_cast<Imap::Mailbox::TreeItemMsgList *>(mailbox->m_children[0]);
1153     Q_ASSERT(list);
1154     list->recalcVariousMessageCounts(model);
1155     QModelIndex listIndex = list->toIndex(model);
1156     Q_ASSERT(listIndex.isValid());
1157     QModelIndex firstInterestingMessage = model->index(
1158                 // remember, the offset has one-based indexing
1159                 mailbox->syncState.unSeenOffset() ? mailbox->syncState.unSeenOffset() - 1 : 0, 0, listIndex);
1160     if (!firstInterestingMessage.data(RoleMessageIsMarkedRecent).toBool() &&
1161             firstInterestingMessage.data(RoleMessageIsMarkedRead).toBool()) {
1162         // Clearly the reported value is utter nonsense. Let's just scroll to the end instead
1163         int offset = model->rowCount(listIndex) - 1;
1164         log(QString::fromUtf8("\"First interesting message\" doesn't look terribly interesting (%1), scrolling to the end at %2 instead")
1165             .arg(firstInterestingMessage.data(RoleMessageFlags).toStringList().join(QLatin1String(", ")),
1166                  QString::number(offset)), Common::LOG_MAILBOX_SYNC);
1167         firstInterestingMessage = model->index(offset, 0, listIndex);
1168     } else {
1169         log(QString::fromUtf8("First interesting message at %1 (%2)")
1170             .arg(QString::number(mailbox->syncState.unSeenOffset()),
1171                  firstInterestingMessage.data(RoleMessageFlags).toStringList().join(QLatin1String(", "))
1172                  ), Common::LOG_MAILBOX_SYNC);
1173     }
1174     emit model->mailboxFirstUnseenMessage(mailbox->toIndex(model), firstInterestingMessage);
1175 }
1176
1177 bool ObtainSynchronizedMailboxTask::dieIfInvalidMailbox()
1178 {
1179     if (mailboxIndex.isValid())
1180         return false;
1181
1182     // OK, so we are in trouble -- our mailbox has disappeared, but the IMAP server will likely keep us busy with its
1183     // status updates. This is bad, so we have to get out as fast as possible. All hands, evasive maneuvers!
1184
1185     log("Mailbox disappeared", Common::LOG_MAILBOX_SYNC);
1186
1187     if (!unSelectTask) {
1188         unSelectTask = model->m_taskFactory->createUnSelectTask(model, this);
1189         connect(unSelectTask, SIGNAL(completed(ImapTask *)), this, SLOT(slotUnSelectCompleted()));
1190         unSelectTask->perform();
1191     }
1192
1193     return true;
1194 }
1195
1196 void ObtainSynchronizedMailboxTask::slotUnSelectCompleted()
1197 {
1198     // Now, just finish and signal a failure
1199     _failed("Escaped from mailbox");
1200 }
1201
1202 QVariant ObtainSynchronizedMailboxTask::taskData(const int role) const
1203 {
1204     return role == RoleTaskCompactName ? QVariant(tr("Synchronizing mailbox")) : QVariant();
1205 }
1206
1207 void ObtainSynchronizedMailboxTask::saveSyncState(TreeItemMailbox *mailbox)
1208 {
1209     model->cache()->setMailboxSyncState(mailbox->mailbox(), mailbox->syncState);
1210     model->saveUidMap(dynamic_cast<TreeItemMsgList*>(mailbox->m_children[0]));
1211     // FIXME: Redmine#457, we should commit the data at this point, along with the flags.
1212     // The FETCH handler for flags shall be changes so that the persistent cache is updated only
1213     // when all data are here.
1214 }
1215
1216 }
1217 }