platform/vcs-impl/src/com/intellij/openapi/vcs/changes/committed/CommittedChangesCache.java

   1 /*
   2  * Copyright 2000-2009 JetBrains s.r.o.
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  */
  16 package com.intellij.openapi.vcs.changes.committed;
  17
  18 import com.intellij.concurrency.JobScheduler;
  19 import com.intellij.openapi.Disposable;
  20 import com.intellij.openapi.application.ApplicationManager;
  21 import com.intellij.openapi.application.ModalityState;
  22 import com.intellij.openapi.components.PersistentStateComponent;
  23 import com.intellij.openapi.components.RoamingType;
  24 import com.intellij.openapi.components.State;
  25 import com.intellij.openapi.components.Storage;
  26 import com.intellij.openapi.diagnostic.Logger;
  27 import com.intellij.openapi.progress.ProcessCanceledException;
  28 import com.intellij.openapi.progress.ProgressManagerQueue;
  29 import com.intellij.openapi.project.Project;
  30 import com.intellij.openapi.util.Computable;
  31 import com.intellij.openapi.util.Disposer;
  32 import com.intellij.openapi.util.Pair;
  33 import com.intellij.openapi.util.Ref;
  34 import com.intellij.openapi.vcs.*;
  35 import com.intellij.openapi.vcs.changes.Change;
  36 import com.intellij.openapi.vcs.update.UpdatedFiles;
  37 import com.intellij.openapi.vcs.versionBrowser.ChangeBrowserSettings;
  38 import com.intellij.openapi.vcs.versionBrowser.CommittedChangeList;
  39 import com.intellij.openapi.vfs.VirtualFile;
  40 import com.intellij.util.Consumer;
  41 import com.intellij.util.NotNullFunction;
  42 import com.intellij.util.containers.ConcurrentHashMap;
  43 import com.intellij.util.containers.MultiMap;
  44 import com.intellij.util.messages.MessageBus;
  45 import com.intellij.util.messages.Topic;
  46 import org.jetbrains.annotations.NonNls;
  47 import org.jetbrains.annotations.NotNull;
  48 import org.jetbrains.annotations.Nullable;
  49 import org.jetbrains.annotations.TestOnly;
  50
  51 import java.io.File;
  52 import java.io.IOException;
  53 import java.util.*;
  54 import java.util.concurrent.ScheduledFuture;
  55 import java.util.concurrent.TimeUnit;
  56
  57 /**
  58  * @author yole
  59  */
  60 @State(
  61   name="CommittedChangesCache",
  62   roamingType = RoamingType.DISABLED,
  63   storages= {
  64     @Storage(
  65       id="other",
  66       file = "$WORKSPACE_FILE$"
  67     )}
  68 )
  69 public class CommittedChangesCache implements PersistentStateComponent<CommittedChangesCache.State> {
  70   private static final Logger LOG = Logger.getInstance("#com.intellij.openapi.vcs.changes.committed.CommittedChangesCache");
  71
  72   private final Project myProject;
  73   private final MessageBus myBus;
  74   private final ProgressManagerQueue myTaskQueue;
  75   private boolean myRefreshingIncomingChanges = false;
  76   private int myPendingUpdateCount = 0;
  77   private State myState = new State();
  78   private ScheduledFuture myFuture;
  79   private List<CommittedChangeList> myCachedIncomingChangeLists;
  80   private final Set<CommittedChangeList> myNewIncomingChanges = new LinkedHashSet<CommittedChangeList>();
  81   private final ProjectLevelVcsManager myVcsManager;
  82
  83   public static final Change[] ALL_CHANGES = new Change[0];
  84   private MyRefreshRunnable myRefresnRunnable;
  85
  86   private final Map<String, Pair<Long, List<CommittedChangeList>>> myExternallyLoadedChangeLists;
  87   private final CachesHolder myCachesHolder;
  88   private final RepositoryLocationCache myLocationCache;
  89
  90   public static class State {
  91     private int myInitialCount = 500;
  92     private int myInitialDays = 90;
  93     private int myRefreshInterval = 30;
  94     private boolean myRefreshEnabled = true;
  95
  96     public int getInitialCount() {
  97       return myInitialCount;
  98     }
  99
 100     public void setInitialCount(final int initialCount) {
 101       myInitialCount = initialCount;
 102     }
 103
 104     public int getInitialDays() {
 105       return myInitialDays;
 106     }
 107
 108     public void setInitialDays(final int initialDays) {
 109       myInitialDays = initialDays;
 110     }
 111
 112     public int getRefreshInterval() {
 113       return myRefreshInterval;
 114     }
 115
 116     public void setRefreshInterval(final int refreshInterval) {
 117       myRefreshInterval = refreshInterval;
 118     }
 119
 120     public boolean isRefreshEnabled() {
 121       return myRefreshEnabled;
 122     }
 123
 124     public void setRefreshEnabled(final boolean refreshEnabled) {
 125       myRefreshEnabled = refreshEnabled;
 126     }
 127   }
 128
 129   public static final Topic<CommittedChangesListener> COMMITTED_TOPIC = new Topic<CommittedChangesListener>("committed changes updates",
 130                                                                                                             CommittedChangesListener.class);
 131
 132   public static CommittedChangesCache getInstance(Project project) {
 133     return project.getComponent(CommittedChangesCache.class);
 134   }
 135
 136   public CommittedChangesCache(final Project project, final MessageBus bus, final ProjectLevelVcsManager vcsManager) {
 137     myProject = project;
 138     myBus = bus;
 139     myLocationCache = new RepositoryLocationCache(project);
 140     myCachesHolder = new CachesHolder(project, myLocationCache);
 141     myTaskQueue = new ProgressManagerQueue(project, VcsBundle.message("committed.changes.refresh.progress"));
 142     myVcsManager = vcsManager;
 143     Disposer.register(project, new Disposable() {
 144       public void dispose() {
 145         cancelRefreshTimer();
 146       }
 147     });
 148     myExternallyLoadedChangeLists = new ConcurrentHashMap<String, Pair<Long, List<CommittedChangeList>>>();
 149   }
 150
 151   public MessageBus getMessageBus() {
 152     return myBus;
 153   }
 154
 155   public State getState() {
 156     return myState;
 157   }
 158
 159   public void loadState(State state) {
 160     myState = state;
 161     updateRefreshTimer();
 162   }
 163
 164   @Nullable
 165   public CommittedChangesProvider getProviderForProject() {
 166     final AbstractVcs[] vcss = myVcsManager.getAllActiveVcss();
 167     List<AbstractVcs> vcsWithProviders = new ArrayList<AbstractVcs>();
 168     for(AbstractVcs vcs: vcss) {
 169       if (vcs.getCommittedChangesProvider() != null) {
 170         vcsWithProviders.add(vcs);
 171       }
 172     }
 173     if (vcsWithProviders.isEmpty()) {
 174       return null;
 175     }
 176     if (vcsWithProviders.size() == 1) {
 177       return vcsWithProviders.get(0).getCommittedChangesProvider();
 178     }
 179     return new CompositeCommittedChangesProvider(myProject, vcsWithProviders.toArray(new AbstractVcs[vcsWithProviders.size()]));
 180   }
 181
 182   public boolean isMaxCountSupportedForProject() {
 183     for(AbstractVcs vcs: myVcsManager.getAllActiveVcss()) {
 184       final CommittedChangesProvider provider = vcs.getCommittedChangesProvider();
 185       if (provider instanceof CachingCommittedChangesProvider) {
 186         final CachingCommittedChangesProvider cachingProvider = (CachingCommittedChangesProvider)provider;
 187         if (!cachingProvider.isMaxCountSupported()) {
 188           return false;
 189         }
 190       }
 191     }
 192     return true;
 193   }
 194
 195   private class MyProjectChangesLoader implements Runnable {
 196     private final ChangeBrowserSettings mySettings;
 197     private final int myMaxCount;
 198     private final boolean myCacheOnly;
 199     private final Consumer<List<CommittedChangeList>> myConsumer;
 200     private final Consumer<List<VcsException>> myErrorConsumer;
 201
 202     private final LinkedHashSet<CommittedChangeList> myResult = new LinkedHashSet<CommittedChangeList>();
 203     private final List<VcsException> myExceptions = new ArrayList<VcsException>();
 204     private boolean myDisposed = false;
 205
 206     private MyProjectChangesLoader(ChangeBrowserSettings settings, int maxCount, boolean cacheOnly,
 207                                    Consumer<List<CommittedChangeList>> consumer, Consumer<List<VcsException>> errorConsumer) {
 208       mySettings = settings;
 209       myMaxCount = maxCount;
 210       myCacheOnly = cacheOnly;
 211       myConsumer = consumer;
 212       myErrorConsumer = errorConsumer;
 213     }
 214
 215     public void run() {
 216       for(AbstractVcs vcs: myVcsManager.getAllActiveVcss()) {
 217         final CommittedChangesProvider provider = vcs.getCommittedChangesProvider();
 218         if (provider == null) continue;
 219
 220         final VcsCommittedListsZipper vcsZipper = provider.getZipper();
 221         CommittedListsSequencesZipper zipper = null;
 222         if (vcsZipper != null) {
 223           zipper = new CommittedListsSequencesZipper(vcsZipper);
 224         }
 225         boolean zipSupported = zipper != null;
 226
 227         final Map<VirtualFile, RepositoryLocation> map = myCachesHolder.getAllRootsUnderVcs(vcs);
 228
 229         for (VirtualFile root : map.keySet()) {
 230           if (myProject.isDisposed())  return;
 231
 232           final RepositoryLocation location = map.get(root);
 233
 234           try {
 235             final List<CommittedChangeList> lists = getChanges(mySettings, root, vcs, myMaxCount, myCacheOnly, provider, location);
 236             if (lists != null) {
 237               if (zipSupported) {
 238                 zipper.add(location, lists);
 239               } else {
 240                 myResult.addAll(lists);
 241               }
 242             }
 243           }
 244           catch (VcsException e) {
 245             myExceptions.add(e);
 246           }
 247           catch(ProcessCanceledException e) {
 248             myDisposed = true;
 249           }
 250         }
 251
 252         if (zipSupported) {
 253           myResult.addAll(zipper.execute());
 254         }
 255       }
 256
 257       ApplicationManager.getApplication().invokeLater(new Runnable() {
 258         public void run() {
 259           LOG.info("FINISHED CommittedChangesCache.getProjectChangesAsync - execution in queue");
 260           if (myProject.isDisposed()) {
 261             return;
 262           }
 263           if (myExceptions.size() > 0) {
 264             myErrorConsumer.consume(myExceptions);
 265           }
 266           else if (!myDisposed) {
 267             myConsumer.consume(new ArrayList<CommittedChangeList>(myResult));
 268           }
 269         }
 270       }, ModalityState.NON_MODAL);
 271     }
 272   }
 273
 274   public void getProjectChangesAsync(final ChangeBrowserSettings settings,
 275                                      final int maxCount,
 276                                      final boolean cacheOnly,
 277                                      final Consumer<List<CommittedChangeList>> consumer,
 278                                      final Consumer<List<VcsException>> errorConsumer) {
 279     final MyProjectChangesLoader loader = new MyProjectChangesLoader(settings, maxCount, cacheOnly, consumer, errorConsumer);
 280     myTaskQueue.run(loader);
 281   }
 282
 283   @Nullable
 284   public List<CommittedChangeList> getChanges(ChangeBrowserSettings settings, final VirtualFile file, @NotNull final AbstractVcs vcs,
 285                                               final int maxCount, final boolean cacheOnly, final CommittedChangesProvider provider,
 286                                               final RepositoryLocation location) throws VcsException {
 287     if (settings instanceof CompositeCommittedChangesProvider.CompositeChangeBrowserSettings) {
 288       settings = ((CompositeCommittedChangesProvider.CompositeChangeBrowserSettings) settings).get(vcs);
 289     }
 290     if (provider instanceof CachingCommittedChangesProvider) {
 291       try {
 292         if (cacheOnly) {
 293           ChangesCacheFile cacheFile = myCachesHolder.getCacheFile(vcs, file, location);
 294           if (!cacheFile.isEmpty()) {
 295             return cacheFile.readChanges(settings, maxCount);
 296           }
 297           return null;
 298         }
 299         else {
 300           if (canGetFromCache(vcs, settings, file, location, maxCount)) {
 301             return getChangesWithCaching(vcs, settings, file, location, maxCount);
 302           }
 303         }
 304       }
 305       catch (IOException e) {
 306         LOG.error(e);
 307       }
 308     }
 309     //noinspection unchecked
 310     return provider.getCommittedChanges(settings, location, maxCount);
 311   }
 312
 313   private boolean canGetFromCache(final AbstractVcs vcs, final ChangeBrowserSettings settings,
 314                                   final VirtualFile root, final RepositoryLocation location, final int maxCount) throws IOException {
 315     ChangesCacheFile cacheFile = myCachesHolder.getCacheFile(vcs, root, location);
 316     if (cacheFile.isEmpty()) {
 317       return true;   // we'll initialize the cache and check again after that
 318     }
 319     if (settings.USE_DATE_BEFORE_FILTER && !settings.USE_DATE_AFTER_FILTER) {
 320       return cacheFile.hasCompleteHistory();
 321     }
 322     if (settings.USE_CHANGE_BEFORE_FILTER && !settings.USE_CHANGE_AFTER_FILTER) {
 323       return cacheFile.hasCompleteHistory();
 324     }
 325
 326     boolean hasDateFilter = settings.USE_DATE_AFTER_FILTER || settings.USE_DATE_BEFORE_FILTER || settings.USE_CHANGE_AFTER_FILTER || settings.USE_CHANGE_BEFORE_FILTER;
 327     boolean hasNonDateFilter = settings.isNonDateFilterSpecified();
 328     if (!hasDateFilter && hasNonDateFilter) {
 329       return cacheFile.hasCompleteHistory();
 330     }
 331     if (settings.USE_DATE_AFTER_FILTER && settings.getDateAfter().getTime() < cacheFile.getFirstCachedDate().getTime()) {
 332       return cacheFile.hasCompleteHistory();
 333     }
 334     if (settings.USE_CHANGE_AFTER_FILTER && settings.getChangeAfterFilter().longValue() < cacheFile.getFirstCachedChangelist()) {
 335       return cacheFile.hasCompleteHistory();
 336     }
 337     return true;
 338   }
 339
 340   public boolean hasCachesForAnyRoot() {
 341     try {
 342       return hasCachesWithEmptiness(false);
 343     }
 344     catch (ProcessCanceledException e) {
 345       return true;
 346     }
 347   }
 348
 349   public boolean hasEmptyCaches() {
 350     try {
 351       return hasCachesWithEmptiness(true);
 352     }
 353     catch (ProcessCanceledException e) {
 354       return false;
 355     }
 356   }
 357
 358   private boolean hasCachesWithEmptiness(final boolean emptiness) {
 359     final Ref<Boolean> resultRef = new Ref<Boolean>(Boolean.FALSE);
 360     myCachesHolder.iterateAllCaches(new NotNullFunction<ChangesCacheFile, Boolean>() {
 361       @NotNull
 362       public Boolean fun(final ChangesCacheFile changesCacheFile) {
 363         try {
 364           if (changesCacheFile.isEmpty() == emptiness) {
 365             resultRef.set(true);
 366             return true;
 367           }
 368         }
 369         catch (IOException e) {
 370           LOG.info(e);
 371         }
 372         return false;
 373       }
 374     });
 375     return resultRef.get();
 376   }
 377
 378   @Nullable
 379   public Iterator<ChangesBunch> getBackBunchedIterator(final AbstractVcs vcs, final VirtualFile root, final RepositoryLocation location, final int bunchSize) {
 380     final ChangesCacheFile cacheFile = myCachesHolder.getCacheFile(vcs, root, location);
 381     try {
 382       if (! cacheFile.isEmpty()) {
 383         return cacheFile.getBackBunchedIterator(bunchSize);
 384       }
 385     }
 386     catch (IOException e) {
 387       LOG.error(e);
 388     }
 389     return null;
 390   }
 391
 392   private List<CommittedChangeList> getChangesWithCaching(final AbstractVcs vcs,
 393                                                           final ChangeBrowserSettings settings,
 394                                                           final VirtualFile root,
 395                                                           final RepositoryLocation location,
 396                                                           final int maxCount) throws VcsException, IOException {
 397     ChangesCacheFile cacheFile = myCachesHolder.getCacheFile(vcs, root, location);
 398     if (cacheFile.isEmpty()) {
 399       List<CommittedChangeList> changes = initCache(cacheFile);
 400       if (canGetFromCache(vcs, settings, root, location, maxCount)) {
 401         settings.filterChanges(changes);
 402         return trimToSize(changes, maxCount);
 403       }
 404       //noinspection unchecked
 405       return cacheFile.getProvider().getCommittedChanges(settings, location, maxCount);
 406     }
 407     else {
 408       List<CommittedChangeList> changes = cacheFile.readChanges(settings, maxCount);
 409       List<CommittedChangeList> newChanges = refreshCache(cacheFile);
 410       settings.filterChanges(newChanges);
 411       changes.addAll(newChanges);
 412       return trimToSize(changes, maxCount);
 413     }
 414   }
 415
 416   @TestOnly
 417   public void refreshAllCaches() throws IOException, VcsException {
 418     final Collection<ChangesCacheFile> files = myCachesHolder.getAllCaches();
 419     for(ChangesCacheFile file: files) {
 420       if (file.isEmpty()) {
 421         initCache(file);
 422       }
 423       else {
 424         refreshCache(file);
 425       }
 426     }
 427   }
 428
 429   private List<CommittedChangeList> initCache(final ChangesCacheFile cacheFile) throws VcsException, IOException {
 430     debug("Initializing cache for " + cacheFile.getLocation());
 431     final CachingCommittedChangesProvider provider = cacheFile.getProvider();
 432     final RepositoryLocation location = cacheFile.getLocation();
 433     final ChangeBrowserSettings settings = provider.createDefaultSettings();
 434     int maxCount = 0;
 435     if (isMaxCountSupportedForProject()) {
 436       maxCount = myState.getInitialCount();
 437     }
 438     else {
 439       settings.USE_DATE_AFTER_FILTER = true;
 440       Calendar calendar = Calendar.getInstance();
 441       calendar.add(Calendar.DAY_OF_YEAR, -myState.getInitialDays());
 442       settings.setDateAfter(calendar.getTime());
 443     }
 444     //noinspection unchecked
 445     List<CommittedChangeList> changes = provider.getCommittedChanges(settings, location, maxCount);
 446     // when initially initializing cache, assume all changelists are locally available
 447     writeChangesInReadAction(cacheFile, changes); // this sorts changes in chronological order
 448     if (maxCount > 0 && changes.size() < myState.getInitialCount()) {
 449       cacheFile.setHaveCompleteHistory(true);
 450     }
 451     if (changes.size() > 0) {
 452       myBus.syncPublisher(COMMITTED_TOPIC).changesLoaded(location, changes);
 453     }
 454     return changes;
 455   }
 456
 457   // todo: fix - would externally loaded nesseccerily for file? i.e. just not efficient now
 458   private List<CommittedChangeList> refreshCache(final ChangesCacheFile cacheFile) throws VcsException, IOException {
 459     final List<CommittedChangeList> newLists = new ArrayList<CommittedChangeList>();
 460
 461     final CachingCommittedChangesProvider provider = cacheFile.getProvider();
 462     final RepositoryLocation location = cacheFile.getLocation();
 463
 464     final Pair<Long, List<CommittedChangeList>> externalLists = myExternallyLoadedChangeLists.get(location.getKey());
 465     final long latestChangeList = getLatestListForFile(cacheFile);
 466     if ((externalLists != null) && (latestChangeList == externalLists.first.longValue())) {
 467       newLists.addAll(appendLoadedChanges(cacheFile, location, externalLists.second));
 468       myExternallyLoadedChangeLists.clear();
 469     }
 470
 471     final ChangeBrowserSettings defaultSettings = provider.createDefaultSettings();
 472     int maxCount = 0;
 473     if (provider.refreshCacheByNumber()) {
 474       final long number = cacheFile.getLastCachedChangelist();
 475       debug("Refreshing cache for " + location + " since #" + number);
 476       if (number >= 0) {
 477         defaultSettings.CHANGE_AFTER = Long.toString(number);
 478         defaultSettings.USE_CHANGE_AFTER_FILTER = true;
 479       }
 480       else {
 481         maxCount = myState.getInitialCount();
 482       }
 483     }
 484     else {
 485       final Date date = cacheFile.getLastCachedDate();
 486       debug("Refreshing cache for " + location + " since " + date);
 487       defaultSettings.setDateAfter(date);
 488       defaultSettings.USE_DATE_AFTER_FILTER = true;
 489     }
 490     final List<CommittedChangeList> newChanges = provider.getCommittedChanges(defaultSettings, location, maxCount);
 491     debug("Loaded " + newChanges.size() + " new changelists");
 492     newLists.addAll(appendLoadedChanges(cacheFile, location, newChanges));
 493
 494     return newLists;
 495   }
 496
 497   private static void debug(@NonNls String message) {
 498     LOG.debug(message);
 499   }
 500
 501   private List<CommittedChangeList> appendLoadedChanges(final ChangesCacheFile cacheFile, final RepositoryLocation location,
 502                                                         final List<CommittedChangeList> newChanges) throws IOException {
 503     final List<CommittedChangeList> savedChanges = writeChangesInReadAction(cacheFile, newChanges);
 504     if (savedChanges.size() > 0) {
 505       myBus.syncPublisher(COMMITTED_TOPIC).changesLoaded(location, savedChanges);
 506     }
 507     return savedChanges;
 508   }
 509
 510   private static List<CommittedChangeList> writeChangesInReadAction(final ChangesCacheFile cacheFile,
 511                                                                     final List<CommittedChangeList> newChanges) throws IOException {
 512     // ensure that changes are loaded before taking read action, to avoid stalling UI
 513     for(CommittedChangeList changeList: newChanges) {
 514       changeList.getChanges();
 515     }
 516     final Ref<IOException> ref = new Ref<IOException>();
 517     final List<CommittedChangeList> savedChanges = ApplicationManager.getApplication().runReadAction(new Computable<List<CommittedChangeList>>() {
 518       public List<CommittedChangeList> compute() {
 519         try {
 520           return cacheFile.writeChanges(newChanges);    // skip duplicates;
 521         }
 522         catch (IOException e) {
 523           ref.set(e);
 524           return null;
 525         }
 526       }
 527     });
 528     if (!ref.isNull()) {
 529       throw ref.get();
 530     }
 531     return savedChanges;
 532   }
 533
 534   private static List<CommittedChangeList> trimToSize(final List<CommittedChangeList> changes, final int maxCount) {
 535     if (maxCount > 0) {
 536       while(changes.size() > maxCount) {
 537         changes.remove(0);
 538       }
 539     }
 540     return changes;
 541   }
 542
 543   private List<CommittedChangeList> loadIncomingChanges() {
 544     final List<CommittedChangeList> result = new ArrayList<CommittedChangeList>();
 545     final Collection<ChangesCacheFile> caches = myCachesHolder.getAllCaches();
 546
 547     final MultiMap<AbstractVcs, Pair<RepositoryLocation, List<CommittedChangeList>>> byVcs =
 548       new MultiMap<AbstractVcs, Pair<RepositoryLocation, List<CommittedChangeList>>>();
 549
 550     for(ChangesCacheFile cache: caches) {
 551       try {
 552         if (!cache.isEmpty()) {
 553           debug("Loading incoming changes for " + cache.getLocation());
 554           final List<CommittedChangeList> incomingChanges = cache.loadIncomingChanges();
 555           byVcs.putValue(cache.getVcs(), new Pair<RepositoryLocation, List<CommittedChangeList>>(cache.getLocation(), incomingChanges));
 556         }
 557       }
 558       catch (IOException e) {
 559         LOG.error(e);
 560       }
 561     }
 562
 563     for (AbstractVcs vcs : byVcs.keySet()) {
 564       final CommittedChangesProvider committedChangesProvider = vcs.getCommittedChangesProvider();
 565       VcsCommittedListsZipper vcsZipper = committedChangesProvider.getZipper();
 566       if (vcsZipper != null) {
 567         final VcsCommittedListsZipper incomingZipper = new IncomingListsZipper(vcsZipper);
 568         final CommittedListsSequencesZipper zipper = new CommittedListsSequencesZipper(incomingZipper);
 569         for (Pair<RepositoryLocation, List<CommittedChangeList>> pair : byVcs.get(vcs)) {
 570           zipper.add(pair.getFirst(), pair.getSecond());
 571         }
 572         result.addAll(zipper.execute());
 573       } else {
 574         for (Pair<RepositoryLocation, List<CommittedChangeList>> pair : byVcs.get(vcs)) {
 575           result.addAll(pair.getSecond());
 576         }
 577       }
 578     }
 579
 580     myCachedIncomingChangeLists = result;
 581     debug("Incoming changes loaded");
 582     notifyIncomingChangesUpdated(null);
 583     return result;
 584   }
 585
 586   private class IncomingListsZipper extends VcsCommittedListsZipperAdapter {
 587     private final VcsCommittedListsZipper myVcsZipper;
 588
 589     private IncomingListsZipper(final VcsCommittedListsZipper vcsZipper) {
 590       super(null);
 591       myVcsZipper = vcsZipper;
 592     }
 593
 594     public Pair<List<RepositoryLocationGroup>, List<RepositoryLocation>> groupLocations(final List<RepositoryLocation> in) {
 595       return myVcsZipper.groupLocations(in);
 596     }
 597
 598     @Override
 599     public CommittedChangeList zip(final RepositoryLocationGroup group, final List<CommittedChangeList> lists) {
 600       if (lists.size() == 1) {
 601         return lists.get(0);
 602       }
 603       final CommittedChangeList victim = lists.get(0) instanceof ReceivedChangeList ? (((ReceivedChangeList) lists.get(0)).getBaseList()) :
 604                                          lists.get(0);
 605       final ReceivedChangeList result = new ReceivedChangeList(victim);
 606       result.setForcePartial(false);
 607       final Set<Change> baseChanges = new HashSet<Change>();
 608
 609       for (CommittedChangeList list : lists) {
 610         baseChanges.addAll(list instanceof ReceivedChangeList ? ((ReceivedChangeList) list).getBaseList().getChanges() : list.getChanges());
 611
 612         final Collection<Change> changes = list.getChanges();
 613         for (Change change : changes) {
 614           if (! result.getChanges().contains(change)) {
 615             result.addChange(change);
 616           }
 617         }
 618       }
 619       result.setForcePartial(baseChanges.size() != result.getChanges().size());
 620       return result;
 621     }
 622
 623     @Override
 624     public long getNumber(final CommittedChangeList list) {
 625       return myVcsZipper.getNumber(list);
 626     }
 627   }
 628
 629   public void loadIncomingChangesAsync(@Nullable final Consumer<List<CommittedChangeList>> consumer) {
 630     debug("Loading incoming changes");
 631     final Runnable task = new Runnable() {
 632       public void run() {
 633         final List<CommittedChangeList> list = loadIncomingChanges();
 634         if (consumer != null) {
 635           consumer.consume(new ArrayList<CommittedChangeList>(list));
 636         }
 637       }
 638     };
 639     myTaskQueue.run(task);
 640   }
 641
 642   @Nullable
 643   public List<CommittedChangeList> getCachedIncomingChanges() {
 644     return myCachedIncomingChangeLists;
 645   }
 646
 647   public void processUpdatedFiles(final UpdatedFiles updatedFiles) {
 648     final Runnable task = new Runnable() {
 649       public void run() {
 650         debug("Processing updated files");
 651         final Collection<ChangesCacheFile> caches = myCachesHolder.getAllCaches();
 652         for(final ChangesCacheFile cache: caches) {
 653           myPendingUpdateCount++;
 654           try {
 655             if (cache.isEmpty()) {
 656               pendingUpdateProcessed();
 657               continue;
 658             }
 659             debug("Processing updated files in " + cache.getLocation());
 660             boolean needRefresh = cache.processUpdatedFiles(updatedFiles, myNewIncomingChanges);
 661             if (needRefresh) {
 662               debug("Found unaccounted files, requesting refresh");
 663               // todo do we need double-queueing here???
 664               processUpdatedFilesAfterRefresh(cache, updatedFiles);
 665             }
 666             else {
 667               debug("Clearing cached incoming changelists");
 668               myCachedIncomingChangeLists = null;
 669               pendingUpdateProcessed();
 670             }
 671           }
 672           catch (IOException e) {
 673             LOG.error(e);
 674           }
 675         }
 676       }
 677     };
 678     myTaskQueue.run(task);
 679   }
 680
 681   private void pendingUpdateProcessed() {
 682     myPendingUpdateCount--;
 683     if (myPendingUpdateCount == 0) {
 684       notifyIncomingChangesUpdated(myNewIncomingChanges);
 685       myNewIncomingChanges.clear();
 686     }
 687   }
 688
 689   private void processUpdatedFilesAfterRefresh(final ChangesCacheFile cache, final UpdatedFiles updatedFiles) {
 690     refreshCacheAsync(cache, false, new RefreshResultConsumer() {
 691       public void receivedChanges(final List<CommittedChangeList> committedChangeLists) {
 692         try {
 693           debug("Processing updated files after refresh in " + cache.getLocation());
 694           boolean result = true;
 695           if (committedChangeLists.size() > 0) {
 696             // received some new changelists, try to process updated files again
 697             result = cache.processUpdatedFiles(updatedFiles, myNewIncomingChanges);
 698           }
 699           debug(result ? "Still have unaccounted files" : "No more unaccounted files");
 700           // for svn, we won't get exact revision numbers in updatedFiles, so we have to double-check by
 701           // checking revisions we have locally
 702           if (result) {
 703             cache.refreshIncomingChanges();
 704             debug("Clearing cached incoming changelists");
 705             myCachedIncomingChangeLists = null;
 706           }
 707           pendingUpdateProcessed();
 708         }
 709         catch (IOException e) {
 710           LOG.error(e);
 711         }
 712         catch(VcsException e) {
 713           notifyRefreshError(e);
 714         }
 715       }
 716
 717       public void receivedError(VcsException ex) {
 718         notifyRefreshError(ex);
 719       }
 720     });
 721   }
 722
 723   private void notifyIncomingChangesUpdated(@Nullable final Collection<CommittedChangeList> receivedChanges) {
 724     final ArrayList<CommittedChangeList> listCopy = receivedChanges == null ? null : new ArrayList<CommittedChangeList>(receivedChanges);
 725     myBus.syncPublisher(COMMITTED_TOPIC).incomingChangesUpdated(listCopy);
 726   }
 727
 728   private void notifyRefreshError(final VcsException e) {
 729     myBus.syncPublisher(COMMITTED_TOPIC).refreshErrorStatusChanged(e);
 730   }
 731
 732   public boolean isRefreshingIncomingChanges() {
 733     return myRefreshingIncomingChanges;
 734   }
 735
 736   public boolean refreshIncomingChanges() {
 737     boolean hasChanges = false;
 738     final Collection<ChangesCacheFile> caches = myCachesHolder.getAllCaches();
 739     for(ChangesCacheFile file: caches) {
 740       try {
 741         if (file.isEmpty()) {
 742           continue;
 743         }
 744         debug("Refreshing incoming changes for " + file.getLocation());
 745         boolean changesForCache = file.refreshIncomingChanges();
 746         hasChanges |= changesForCache;
 747       }
 748       catch (IOException e) {
 749         LOG.error(e);
 750       }
 751       catch(VcsException e) {
 752         notifyRefreshError(e);
 753       }
 754     }
 755     return hasChanges;
 756   }
 757
 758   public void refreshIncomingChangesAsync() {
 759     debug("Refreshing incoming changes in background");
 760     myRefreshingIncomingChanges = true;
 761     final Runnable task = new Runnable() {
 762       public void run() {
 763         refreshIncomingChanges();
 764
 765         ApplicationManager.getApplication().invokeLater(new Runnable() {
 766           public void run() {
 767             myRefreshingIncomingChanges = false;
 768             debug("Incoming changes refresh complete, clearing cached incoming changes");
 769             notifyReloadIncomingChanges();
 770           }
 771         }, ModalityState.NON_MODAL);
 772       }
 773     };
 774     myTaskQueue.run(task);
 775   }
 776
 777   public void refreshAllCachesAsync(final boolean initIfEmpty) {
 778     final Runnable task = new Runnable() {
 779       public void run() {
 780         final List<ChangesCacheFile> files = myCachesHolder.getAllCaches();
 781         final RefreshResultConsumer notifyConsumer = new RefreshResultConsumer() {
 782           private VcsException myError = null;
 783           private int myCount = 0;
 784
 785           public void receivedChanges(List<CommittedChangeList> changes) {
 786             if (changes.size() > 0) {
 787               notifyReloadIncomingChanges();
 788             } else {
 789               myProject.getMessageBus().syncPublisher(CommittedChangesTreeBrowser.ITEMS_RELOADED).emptyRefresh();
 790             }
 791             checkDone();
 792           }
 793
 794           public void receivedError(VcsException ex) {
 795             myError = ex;
 796             checkDone();
 797           }
 798
 799           private void checkDone() {
 800             myCount++;
 801             if (myCount == files.size()) {
 802               notifyRefreshError(myError);
 803             }
 804           }
 805         };
 806         for(ChangesCacheFile file: files) {
 807           refreshCacheAsync(file, initIfEmpty, notifyConsumer, false);
 808         }
 809       }
 810     };
 811     myTaskQueue.run(task);
 812   }
 813
 814   private void notifyReloadIncomingChanges() {
 815     myCachedIncomingChangeLists = null;
 816     notifyIncomingChangesUpdated(null);
 817   }
 818
 819   private void refreshCacheAsync(final ChangesCacheFile cache, final boolean initIfEmpty,
 820                                  @Nullable final RefreshResultConsumer consumer) {
 821     refreshCacheAsync(cache, initIfEmpty, consumer, true);
 822   }
 823
 824   private void refreshCacheAsync(final ChangesCacheFile cache, final boolean initIfEmpty,
 825                                  @Nullable final RefreshResultConsumer consumer, final boolean asynch) {
 826     try {
 827       if (!initIfEmpty && cache.isEmpty()) {
 828         return;
 829       }
 830     }
 831     catch (IOException e) {
 832       LOG.error(e);
 833       return;
 834     }
 835     final Runnable task = new Runnable() {
 836       public void run() {
 837         try {
 838           final List<CommittedChangeList> list;
 839           if (initIfEmpty && cache.isEmpty()) {
 840             list = initCache(cache);
 841           }
 842           else {
 843             list = refreshCache(cache);
 844           }
 845           if (consumer != null) {
 846             consumer.receivedChanges(list);
 847           }
 848         }
 849         catch(ProcessCanceledException ex) {
 850           // ignore
 851         }
 852         catch (IOException e) {
 853           LOG.error(e);
 854         }
 855         catch (VcsException e) {
 856           if (consumer != null) {
 857             consumer.receivedError(e);
 858           }
 859         }
 860       }
 861     };
 862     if (asynch) {
 863       myTaskQueue.run(task);
 864     } else {
 865       task.run();
 866     }
 867   }
 868
 869   private void updateRefreshTimer() {
 870     cancelRefreshTimer();
 871     if (myState.isRefreshEnabled()) {
 872       myRefresnRunnable = new MyRefreshRunnable(this);
 873       // if "schedule with fixed rate" is used, then after waking up from stand-by mode, events are generated for inactive period
 874       // it does not make sense
 875       myFuture = JobScheduler.getScheduler().scheduleWithFixedDelay(myRefresnRunnable,
 876                                                                  myState.getRefreshInterval()*60, myState.getRefreshInterval()*60,
 877                                                                  TimeUnit.SECONDS);
 878     }
 879   }
 880
 881   private void cancelRefreshTimer() {
 882     if (myRefresnRunnable != null) {
 883       myRefresnRunnable.cancel();
 884       myRefresnRunnable = null;
 885     }
 886     if (myFuture != null) {
 887       myFuture.cancel(false);
 888       myFuture = null;
 889     }
 890   }
 891
 892   @Nullable
 893   public Pair<CommittedChangeList, Change> getIncomingChangeList(final VirtualFile file) {
 894     if (myCachedIncomingChangeLists != null) {
 895       File ioFile = new File(file.getPath());
 896       for(CommittedChangeList changeList: myCachedIncomingChangeLists) {
 897         for(Change change: changeList.getChanges()) {
 898           if (change.affectsFile(ioFile)) {
 899             return Pair.create(changeList, change);
 900           }
 901         }
 902       }
 903     }
 904     return null;
 905   }
 906
 907   private long getLatestListForFile(final ChangesCacheFile file) {
 908     try {
 909       if ((file == null) || (file.isEmpty())) {
 910         return -1;
 911       }
 912       return file.getLastCachedChangelist();
 913     }
 914     catch (IOException e) {
 915       return -1;
 916     }
 917   }
 918
 919   public CachesHolder getCachesHolder() {
 920     return myCachesHolder;
 921   }
 922
 923   public void submitExternallyLoaded(final RepositoryLocation location, final long myLastCl, final List<CommittedChangeList> lists) {
 924     myExternallyLoadedChangeLists.put(location.getKey(), new Pair<Long, List<CommittedChangeList>>(myLastCl, lists));
 925   }
 926
 927   private interface RefreshResultConsumer {
 928     void receivedChanges(List<CommittedChangeList> changes);
 929     void receivedError(VcsException ex);
 930   }
 931
 932   private static class MyRefreshRunnable implements Runnable {
 933     private CommittedChangesCache myCache;
 934
 935     private MyRefreshRunnable(final CommittedChangesCache cache) {
 936       myCache = cache;
 937     }
 938
 939     private void cancel() {
 940       myCache = null;
 941     }
 942
 943     public void run() {
 944       final CommittedChangesCache cache = myCache;
 945       if (cache == null) return;
 946       cache.refreshAllCachesAsync(false);
 947       final List<ChangesCacheFile> list = cache.getCachesHolder().getAllCaches();
 948       for(ChangesCacheFile file: list) {
 949         if (file.getProvider().refreshIncomingWithCommitted()) {
 950           cache.refreshIncomingChangesAsync();
 951           break;
 952         }
 953       }
 954     }
 955   }
 956
 957   public RepositoryLocationCache getLocationCache() {
 958     return myLocationCache;
 959   }
 960 }