?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Cleanup of collection syncing.
[acal.git] / src / com / morphoss / acal / service / SyncCollectionContents.java
blobbb6bbcb8cc7514ad188dd0cd33623b037196d47a
1 /*
2 * Copyright (C) 2011 Morphoss Ltd
3 *
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation, either version 3 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program. If not, see <http://www.gnu.org/licenses/>.
16 *
17 */
18
19 package com.morphoss.acal.service;
20
21 import java.io.BufferedReader;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.InputStreamReader;
25 import java.util.ArrayList;
26 import java.util.HashMap;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Set;
30 import java.util.regex.Matcher;
31
32 import javax.net.ssl.SSLException;
33
34 import org.apache.http.Header;
35
36 import android.content.ContentQueryMap;
37 import android.content.ContentResolver;
38 import android.content.ContentValues;
39 import android.content.Context;
40 import android.database.Cursor;
41 import android.net.ConnectivityManager;
42 import android.net.NetworkInfo;
43 import android.net.Uri;
44 import android.util.Log;
45
46 import com.morphoss.acal.AcalDebug;
47 import com.morphoss.acal.Constants;
48 import com.morphoss.acal.DatabaseChangedEvent;
49 import com.morphoss.acal.HashCodeUtil;
50 import com.morphoss.acal.ResourceModification;
51 import com.morphoss.acal.StaticHelpers;
52 import com.morphoss.acal.acaltime.AcalDateTime;
53 import com.morphoss.acal.providers.DavCollections;
54 import com.morphoss.acal.providers.DavResources;
55 import com.morphoss.acal.providers.Servers;
56 import com.morphoss.acal.service.SynchronisationJobs.WriteActions;
57 import com.morphoss.acal.service.connector.AcalRequestor;
58 import com.morphoss.acal.service.connector.ConnectionFailedException;
59 import com.morphoss.acal.service.connector.SendRequestFailedException;
60 import com.morphoss.acal.xml.DavNode;
61
62 public class SyncCollectionContents extends ServiceJob {
63
64 public static final String TAG = "aCal SyncCollectionContents";
65 private static final int nPerMultiget = 100;
66
67 private int collectionId = -5;
68 private int serverId = -5;
69 private String collectionPath = null;
70 private String syncToken = null;
71 private String oldSyncToken = null;
72 private boolean isAddressbook = false;
73
74 ContentValues collectionData;
75 private boolean collectionChanged = false;
76 ContentValues serverData;
77
78 private String dataType = "calendar";
79 private String multigetReportTag = "calendar-multiget";
80 private String nameSpace = Constants.NS_CALDAV;
81
82 // Note that this defines how often we wake up and see if we should be
83 // doing a sync. Not how often we actually wake up and hit the server
84 // with a request. Nevertheless we should not do this more than every
85 // minute or so in production.
86 private static final long minBetweenSyncs = (Constants.debugSyncCollectionContents || Constants.debugHeap ? 30000 : 300000); // milliseconds
87
88 private ContentResolver cr;
89 private aCalService context;
90 private boolean synchronisationForced = false;
91 private AcalRequestor requestor = null;
92 private boolean resourcesWereSynchronized;
93 private boolean syncWasCompleted;
94
95
96 /**
97 * <p>
98 * Constructor
99 * </p>
100 *
101 * @param collectionId2
102 * <p>
103 * The ID of the collection to be synced
104 * </p>
105 * @param context
106 * <p>
107 * The context to use for all those things contexts are used for.
108 * </p>
109 */
110 public SyncCollectionContents(int collectionId) {
111 this.collectionId = collectionId;
112 this.TIME_TO_EXECUTE = 0;
113 }
114
115
116 /**
117 * <p>
118 * Schedule a sync of the contents of a collection, potentially forcing it to happen now even
119 * if this would otherwise be considered too early according to the normal schedule.
120 * </p>
121 * @param collectionId
122 * @param forceSync
123 */
124 public SyncCollectionContents(int collectionId, boolean forceSync ) {
125 this.collectionId = collectionId;
126 this.synchronisationForced = forceSync;
127 this.TIME_TO_EXECUTE = 0;
128 }
129
130
131 @Override
132 public void run(aCalService context) {
133 if ( Constants.debugHeap ) AcalDebug.heapDebug(TAG, "SyncCollectionContents start");
134 this.context = context;
135 this.cr = context.getContentResolver();
136 if ( collectionId < 0 || !getCollectionInfo()) {
137 Log.w(TAG, "Could not read collection " + collectionId + " for server " + serverId
138 + " from collection table!");
139 return;
140 }
141
142 if (!(1 == serverData.getAsInteger(Servers.ACTIVE))) {
143 if (Constants.LOG_DEBUG) Log.println(Constants.LOGD,TAG, "Server is no longer active - sync cancelled: " + serverData.getAsInteger(Servers.ACTIVE)
144 + " " + serverData.getAsString(Servers.FRIENDLY_NAME));
145 return;
146 }
147
148 long start = System.currentTimeMillis();
149 try {
150 resourcesWereSynchronized = false;
151 syncWasCompleted = true;
152
153 // step 1 are there any 'needs_sync' in dav_resources?
154 Map<String, ContentValues> originalData = findSyncNeededResources();
155
156 if ( originalData.size() < 1 && ! timeToRun() ) {
157 scheduleNextInstance();
158 return;
159 }
160
161 if ( Constants.LOG_DEBUG )
162 Log.println(Constants.LOGD,TAG, "Starting sync on collection " + this.collectionPath + " (" + this.collectionId + ")");
163
164 aCalService.databaseDispatcher.dispatchEvent(new DatabaseChangedEvent(
165 DatabaseChangedEvent.DATABASE_BEGIN_RESOURCE_CHANGES, DavCollections.class, collectionData));
166
167 if (originalData.size() < 1) {
168 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
169 Log.println(Constants.LOGV,TAG, "No local resources marked as needing synchronisation.");
170 }
171 else
172 syncMarkedResources( originalData );
173
174 if ( (serverData.getAsInteger(Servers.HAS_SYNC) != null && (1 == serverData.getAsInteger(Servers.HAS_SYNC))
175 ? doRegularSyncReport()
176 : doRegularSyncPropfind() ) ) {
177 originalData = findSyncNeededResources();
178 syncMarkedResources(originalData);
179 }
180
181 String lastSynchronized = new AcalDateTime().setMillis(start).fmtIcal();
182 if ( syncWasCompleted ) {
183 // update last checked flag for collection
184 collectionData.put(DavCollections.LAST_SYNCHRONISED, lastSynchronized);
185 collectionData.put(DavCollections.NEEDS_SYNC, 0);
186 if ( syncToken != null ) {
187 collectionData.put(DavCollections.SYNC_TOKEN, syncToken);
188 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
189 Log.i(TAG,"Updated collection record with new sync token '"+syncToken+"' at "+lastSynchronized);
190 }
191 cr.update(DavCollections.CONTENT_URI, collectionData, DavCollections._ID + "=?",
192 new String[] { "" + collectionId });
193
194 }
195
196 if ( resourcesWereSynchronized ) {
197 aCalService.databaseDispatcher.dispatchEvent(
198 new DatabaseChangedEvent(DatabaseChangedEvent.DATABASE_RECORD_UPDATED,
199 DavCollections.class, collectionData)
200 );
201 }
202 }
203 catch (Exception e) {
204 Log.e(TAG, "Error syncing collection " + this.collectionId + ": " + e.getMessage());
205 Log.e(TAG, Log.getStackTraceString(e));
206 }
207 finally {
208 aCalService.databaseDispatcher.dispatchEvent(new DatabaseChangedEvent(
209 DatabaseChangedEvent.DATABASE_END_RESOURCE_CHANGES, DavCollections.class, collectionData));
210 }
211 long finish = System.currentTimeMillis();
212 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
213 Log.println(Constants.LOGV,TAG, "Collection sync finished in " + (finish - start) + "ms");
214
215 scheduleNextInstance();
216
217 this.collectionData = null;
218 this.serverData = null;
219 this.context = null;
220 this.requestor = null;
221 this.cr = null;
222 if ( Constants.debugHeap ) AcalDebug.heapDebug(TAG, "SyncCollectionContents end");
223 }
224
225 /**
226 * <p>
227 * Called from the constructor to initialise all of the collection-related data we need to be able to sync
228 * properly. This includes a bunch of server related data pulled into serverData
229 * </p>
230 *
231 * @return Whether it successfully read enough data to proceed
232 */
233 private boolean getCollectionInfo() {
234 long start = System.currentTimeMillis();
235 collectionData = DavCollections.getRow(collectionId, cr);
236 if ( collectionData == null ) {
237 Log.e(TAG, "Error getting collection data from DB for collection ID " + collectionId);
238 return false;
239 }
240
241 serverId = collectionData.getAsInteger(DavCollections.SERVER_ID);
242 collectionPath = collectionData.getAsString(DavCollections.COLLECTION_PATH);
243 oldSyncToken = collectionData.getAsString(DavCollections.SYNC_TOKEN);
244 isAddressbook = (1 == collectionData.getAsInteger(DavCollections.ACTIVE_ADDRESSBOOK));
245 dataType = "calendar";
246 multigetReportTag = "calendar-multiget";
247 nameSpace = Constants.NS_CALDAV;
248 if (isAddressbook) {
249 dataType = "address";
250 multigetReportTag = dataType + "book-multiget";
251 nameSpace = Constants.NS_CARDDAV;
252 }
253
254 try {
255 // get serverData
256 serverData = Servers.getRow(serverId, cr);
257 if (serverData == null) throw new Exception("No record for ID " + serverId);
258 requestor = AcalRequestor.fromServerValues(serverData);
259 requestor.setPath(collectionPath);
260 }
261 catch (Exception e) {
262 // Error getting data
263 Log.e(TAG, "Error getting server data: " + e.getMessage());
264 Log.e(TAG, "Deleting invalid collection Record.");
265 cr.delete(Uri.withAppendedPath(DavCollections.CONTENT_URI,Long.toString(collectionId)), null, null);
266 return false;
267 }
268
269 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
270 Log.println(Constants.LOGV,TAG,
271 "getCollectionInfo() completed in " + (System.currentTimeMillis() - start) + "ms");
272 return true;
273 }
274
275
276 /**
277 * <p>
278 * Do a sync run using a sync-collection REPORT against the collection, hopefully retrieving the -data
279 * pseudo-properties at the same time, but in any case getting a list of changed resources to process.
280 * Quick and light on the bandwidth, we hope.
281 * </p>
282 *
283 * @return true if we still need to syncMarkedResources() afterwards.
284 */
285 private boolean doRegularSyncReport() {
286 DavNode root = doCalendarRequest("REPORT", 1,
287 "<?xml version=\"1.0\" encoding=\"utf-8\" ?>"
288 + "<sync-collection xmlns=\"DAV:\">"
289 + "<prop>"
290 + "<getetag/>"
291 // + "<getlastmodified/>"
292 // + "<" + dataType + "-data xmlns=\"" + nameSpace + "\"/>"
293 + "</prop>"
294 + "<sync-token>" + oldSyncToken + "</sync-token>"
295 + "</sync-collection>"
296 );
297
298 boolean needSyncAfterwards = false;
299
300 if (root == null) {
301 Log.i(TAG, "Unable to sync collection " + this.collectionPath + " (ID:" + this.collectionId
302 + " - no data from server.");
303 syncWasCompleted = false;
304 return false;
305 }
306 /**
307 * SOGO's sync-response looks like this (as of draft-1):
308 *
309 <?xml version="1.0" encoding="utf-8"?>
310 <D:multistatus xmlns:D="DAV:">
311 <D:sync-response>
312 <D:href>/SOGo/dav/sogo2/Calendar/personal/351dc1af-2aa3-4d14-9704-eadbcfecaf7e.ics</D:href>
313 <D:status>HTTP/1.1 200 OK</D:status>
314 <D:propstat>
315 <D:prop>
316 <D:getetag>&quot;gcs00000001&quot;</D:getetag></D:prop>
317 <D:status>HTTP/1.1 200 OK</D:status>
318 </D:propstat>
319 </D:sync-response>
320 <D:sync-token>1322100412</D:sync-token>
321 </D:multistatus>
322 *
323 */
324 /**
325 * Correct sync-response looks like this (as of draft-2 and later):
326 *
327 <?xml version="1.0" encoding="utf-8" ?>
328 <multistatus xmlns="DAV:">
329 <response>
330 <href>/caldav.php/user1/home/DAYPARTY-77C6-4FB7-BDD3-6882E2F1BE74.ics</href>
331 <propstat>
332 <prop>
333 <getetag>"165746adbab8bc0c8336a63cc5332ff2"</getetag>
334 <getlastmodified>Dow, 01 Jan 2000 00:00:00 GMT</getlastmodified>
335 </prop>
336 <status>HTTP/1.1 200 OK</status>
337 </propstat>
338 </response>
339 </multistatus>
340 *
341 */
342
343 ArrayList<ResourceModification> changeList = new ArrayList<ResourceModification>();
344
345 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
346 Log.println(Constants.LOGV,TAG, "Start processing response");
347 List<DavNode> responses = root.getNodesFromPath("multistatus/response");
348 if ( responses.isEmpty() ) {
349 responses = root.getNodesFromPath("multistatus/sync-response");
350 if ( ! responses.isEmpty() ) {
351 Log.e("aCal","CalDAV Server at "+requestor.getHostName()+" uses obsolete draft sync-response syntax. Falling back to inefficient PROPFIND. Please upgrade your server.");
352 /*
353 * We won't write it back to the server, because at least we can use this much as an indication that
354 * something has changed, so we'll just fall through and do a PROPFIND sync.
355 *
356 serverData.put(Servers.HAS_SYNC,0);
357 Uri provider = ContentUris.withAppendedId(Servers.CONTENT_URI, serverData.getAsInteger(Servers._ID));
358 cr.update(provider, Servers.cloneValidColumns(serverData), null, null);
359 */
360 return doRegularSyncPropfind();
361 }
362 responses = root.getNodesFromPath("multistatus/sync-token");
363 if ( responses.isEmpty() ) {
364 Log.i("aCal","No sync-token in sync-report response. Falling back to PROPFIND.");
365 return doRegularSyncPropfind();
366 }
367
368 }
369 else {
370
371 for (DavNode response : responses) {
372 String responseHref = response.segmentFromFirstHref("href");
373 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
374 Log.println(Constants.LOGV,TAG, "Processing response for "+responseHref);
375 WriteActions action = WriteActions.UPDATE;
376
377 ContentValues cv = DavResources.getResourceInCollection( collectionId, responseHref, cr);
378 if ( cv == null ) {
379 cv = new ContentValues();
380 cv.put(DavResources.COLLECTION_ID, collectionId);
381 cv.put(DavResources.RESOURCE_NAME, responseHref);
382 cv.put(DavResources.NEEDS_SYNC, 1 );
383 action = WriteActions.INSERT;
384 }
385
386 List<DavNode> aNode = response.getNodesFromPath("status");
387 if ( aNode.isEmpty()
388 || aNode.get(0).getText().equalsIgnoreCase("HTTP/1.1 201 Created")
389 || aNode.get(0).getText().equalsIgnoreCase("HTTP/1.1 200 OK") ) {
390
391 if ( Constants.LOG_DEBUG )
392 Log.println(Constants.LOGD,TAG,"Updating node "+responseHref+" with "+action.toString() );
393 // We are dealing with an update or insert
394 if ( !parseResponseNode(response, cv) ) continue;
395 if ( cv.getAsInteger(DavResources.NEEDS_SYNC) == 1 ) needSyncAfterwards = true;
396
397 }
398 else if ( action == WriteActions.INSERT ) {
399 // It looked like an INSERT because it's not in our DB, but in fact
400 // the status message was not 200/201 so it's a DELETE that we're
401 // seeing reflected back at us.
402 Log.i(TAG,"Ignoring delete sync on node '"+responseHref+"' which is already deleted from our DB." );
403 }
404 else {
405 // This really *is* a DELETE, since the status could only
406 // have said so. Or we're getting invalid status messages
407 // and their events all deserve to die anyway!
408 if ( Constants.LOG_DEBUG )
409 Log.println(Constants.LOGD,TAG,"Deleting node '"+responseHref+"'with status: "+aNode.get(0).getText() );
410 action = WriteActions.DELETE;
411 }
412 root.removeSubTree(response);
413
414 changeList.add( new ResourceModification(action, cv, null) );
415
416 }
417 }
418
419 // Pull the syncToken we will update with.
420 syncToken = root.getFirstNodeText("multistatus/sync-token");
421 if ( Constants.LOG_DEBUG )
422 Log.println(Constants.LOGD,TAG,"Found sync token of '"+syncToken+"' in sync-report response." );
423
424 ResourceModification.commitChangeList(context, changeList);
425
426 return needSyncAfterwards;
427 }
428
429
430 /**
431 * <p>
432 * Do a sync run using a PROPFIND against the collection and a pass through the DB comparing all resources
433 * currently on file with the ones we got from the PROPFIND. Laborious and potentially bandwidth hogging.
434 * </p>
435 *
436 * @return true if we still need to syncMarkedResources() afterwards.
437 */
438 private boolean doRegularSyncPropfind() {
439 boolean needSyncAfterwards = false;
440 if ( !collectionTagChanged() ) return false;
441
442 DavNode root = doCalendarRequest("PROPFIND", 1,
443 "<?xml version=\"1.0\" encoding=\"utf-8\" ?>"
444 + "<propfind xmlns=\"DAV:\" xmlns:CS=\"http://calendarserver.org/ns/\">"
445 + "<prop>"
446 + "<getetag/>"
447 + "<CS:getctag/>"
448 + "</prop>"
449 + "</propfind>"
450 );
451
452 if (root == null ) {
453 Log.i(TAG, "Unable to PROPFIND collection " + this.collectionPath + " (ID:" + this.collectionId
454 + " - no data from server.");
455 syncWasCompleted = false;
456 return false;
457 }
458
459 Map<String, ContentValues> ourResourceMap = getCurrentResourceMap();
460 ArrayList<ResourceModification> changeList = new ArrayList<ResourceModification>();
461
462 try {
463 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
464 Log.println(Constants.LOGV,TAG, "Start processing PROPFIND response");
465 long start2 = System.currentTimeMillis();
466 List<DavNode> responses = root.getNodesFromPath("multistatus/response");
467
468 for (DavNode response : responses) {
469 String responseHref = response.segmentFromFirstHref("href");
470 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
471 Log.println(Constants.LOGV,TAG, "Processing response for "+responseHref);
472
473 ContentValues cv = null;
474 WriteActions action = WriteActions.UPDATE;
475 if ( ourResourceMap != null && ourResourceMap.containsKey(responseHref)) {
476 cv = ourResourceMap.get(responseHref);
477 ourResourceMap.remove(responseHref);
478 }
479 else {
480 cv = new ContentValues();
481 cv.put(DavResources.COLLECTION_ID, collectionId);
482 cv.put(DavResources.RESOURCE_NAME, responseHref);
483 cv.put(DavResources.NEEDS_SYNC, 1);
484 action = WriteActions.INSERT;
485 }
486
487 if ( !parseResponseNode(response, cv) ) continue;
488 if ( cv.getAsInteger(DavResources.NEEDS_SYNC) == 1 ) needSyncAfterwards = true;
489
490 needSyncAfterwards = true;
491
492 changeList.add( new ResourceModification(action, cv, null) );
493 }
494
495 if ( ourResourceMap != null ) {
496 // Delete any records still in ourResourceMap (hence not on server any longer)
497 Set<String> names = ourResourceMap.keySet();
498 for( String name : names ) {
499 ContentValues cv = ourResourceMap.get(name);
500 changeList.add( new ResourceModification(WriteActions.DELETE, cv, null) );
501 }
502 }
503
504 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
505 Log.println(Constants.LOGV,TAG, "Completed processing of PROPFIND sync in " + (System.currentTimeMillis() - start2) + "ms");
506 }
507 catch (Exception e) {
508 Log.e(TAG, "Exception processing PROPFIND response: " + e.getMessage());
509 Log.e(TAG, Log.getStackTraceString(e));
510 }
511
512 ResourceModification.commitChangeList(context, changeList);
513
514 return needSyncAfterwards;
515 }
516
517
518 /**
519 * Returns a Map of href to database record for the current database state.
520 * @return
521 */
522 private Map<String,ContentValues> getCurrentResourceMap() {
523 Cursor resourceCursor = cr.query(Uri.parse(DavResources.CONTENT_URI.toString() + "/collection/" + this.collectionId),
524 new String[] { DavResources._ID, DavResources.RESOURCE_NAME, DavResources.ETAG }, null, null, null);
525 if ( !resourceCursor.moveToFirst()) {
526 resourceCursor.close();
527 return new HashMap<String,ContentValues>();
528 }
529 ContentQueryMap cqm = new ContentQueryMap(resourceCursor, DavResources.RESOURCE_NAME, false, null);
530 cqm.requery();
531 Map<String, ContentValues> databaseList = cqm.getRows();
532 cqm.close();
533 resourceCursor.close();
534 return databaseList;
535 }
536
537
538 /**
539 * <p>
540 * Checks for an old CalendarServer-style ctag for this collection
541 * </p>
542 *
543 * @return <p>
544 * Returns true if the CTag is different from the previous one, or if either is null.
545 * </p>
546 */
547 private boolean collectionTagChanged() {
548 if (Constants.LOG_DEBUG) Log.println(Constants.LOGD,TAG, "Requesting CTag on collection.");
549 DavNode root = doCalendarRequest("PROPFIND", 0,
550 "<?xml version=\"1.0\" encoding=\"utf-8\" ?>"
551 + "<propfind xmlns=\"DAV:\" xmlns:CS=\"http://calendarserver.org/ns/\">"
552 + "<prop>"
553 + "<CS:getctag/>"
554 + "</prop>"
555 + "</propfind>"
556 );
557
558 if ( root == null ) {
559 Log.i(TAG,"No response from server - deferring sync.");
560 return false;
561 }
562
563 List<DavNode> responses = root.getNodesFromPath("multistatus/response");
564
565 for (DavNode response : responses) {
566
567 List<DavNode> propstats = response.getNodesFromPath("propstat");
568
569 for (DavNode propstat : propstats) {
570 if ( !propstat.getFirstNodeText("status").equalsIgnoreCase("HTTP/1.1 200 OK") )
571 continue;
572
573 DavNode prop = propstat.getNodesFromPath("prop").get(0);
574 String ctag = prop.getFirstNodeText("getctag");
575 String collectionTag = collectionData.getAsString(DavCollections.COLLECTION_TAG);
576
577 if ( ctag == null || collectionTag == null ) return true;
578 return ! ctag.equals(collectionTag);
579 }
580 }
581 return true;
582 }
583
584
585
586 /**
587 * <p>
588 * Does a request against the collection path
589 * </p>
590 *
591 * @return <p>
592 * A DavNode which is the root of the multistatus response.
593 * </p>
594 */
595 private DavNode doCalendarRequest( String method, int depth, String xml) {
596 DavNode root = requestor.doXmlRequest(method, collectionPath,
597 SynchronisationJobs.getReportHeaders(depth), xml);
598 if ( requestor.getStatusCode() == 404 ) {
599 Log.w(TAG,"Sync PROPFIND got 404 on "+collectionPath+" so a HomeSetsUpdate is being scheduled.");
600 ServiceJob sj = new HomeSetsUpdate(serverId);
601 context.addWorkerJob(sj);
602 return null;
603 }
604 return root;
605 }
606
607
608 /**
609 * Checks the resources we have in the DB currently flagged as needing synchronisation, and synchronises
610 * them if they are using an addressbook-multiget or calendar-multiget request, depending on the
611 * collection type.
612 */
613 private void syncMarkedResources( Map<String, ContentValues> originalData ) {
614
615 if (Constants.LOG_DEBUG)
616 Log.println(Constants.LOGD,TAG, "Found " + originalData.size() + " resources marked as needing synchronisation.");
617
618 Set<String> hrefSet = originalData.keySet();
619 Object[] hrefs = hrefSet.toArray();
620
621 if (serverData.getAsInteger(Servers.HAS_MULTIGET) != null && 1 == serverData.getAsInteger(Servers.HAS_MULTIGET)) {
622 syncWithMultiget(originalData, hrefs);
623 }
624 else {
625 syncWithGet(originalData, hrefs);
626 }
627 }
628
629 /**
630 * <p>
631 * Performs a sync using a series of multiget REPORT requests to retrieve the resources needing sync.
632 * </p>
633 *
634 * @param originalData
635 * <p>
636 * The href => Contentvalues map.
637 * </p>
638 * @param hrefs
639 * <p>
640 * The array of hrefs
641 * </p>
642 */
643 private void syncWithMultiget(Map<String, ContentValues> originalData, Object[] hrefs) {
644
645 String baseXml = "<?xml version=\"1.0\" encoding=\"utf-8\" ?>"
646 + "<" + multigetReportTag + " xmlns=\"" + nameSpace + "\" xmlns:D=\"DAV:\">\n"
647 + "<D:prop>\n"
648 + "<D:getetag/>\n"
649 + "<D:getcontenttype/>\n"
650 + "<D:getlastmodified/>\n"
651 + "<" + dataType + "-data/>\n"
652 + "</D:prop>\n"
653 + "%s"
654 + "</" + multigetReportTag + ">";
655
656 ArrayList<String> toBeRemoved = new ArrayList<String>(hrefs.length);
657 for( Object o : hrefs ) {
658 Matcher m = Constants.matchSegmentName.matcher(o.toString());
659 if ( m.find() ) toBeRemoved.add(m.group(1));
660 }
661
662 int limit;
663 StringBuilder hrefList;
664 for (int hrefIndex = 0; hrefIndex < hrefs.length; hrefIndex += nPerMultiget) {
665 limit = nPerMultiget + hrefIndex;
666 if ( limit > hrefs.length ) limit = hrefs.length;
667
668 hrefList = new StringBuilder();
669 for (int i = hrefIndex; i < limit; i++) {
670 hrefList.append(String.format("<D:href>%s</D:href>\n", collectionPath + hrefs[i].toString()));
671 }
672
673 if (Constants.LOG_DEBUG)
674 Log.println(Constants.LOGD,TAG, "Requesting " + multigetReportTag + " for " + nPerMultiget + " resources out of "+hrefs.length+"." );
675
676 DavNode root = doCalendarRequest( "REPORT", 1, String.format(baseXml,hrefList.toString()) );
677
678 if (root == null) {
679 Log.w(TAG, "Unable to sync collection " + this.collectionPath + " (ID:" + this.collectionId
680 + " - no data from server).");
681 return;
682 }
683
684 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
685 Log.println(Constants.LOGV,TAG, "Start processing response");
686 List<DavNode> responses = root.getNodesFromPath("multistatus/response");
687 List<ResourceModification> changeList = new ArrayList<ResourceModification>(hrefList.length());
688
689 for (DavNode response : responses) {
690 try { Thread.sleep(2); } catch ( InterruptedException e ) { } // Give the UI thread more of a chance to do stuff.
691 String name = response.segmentFromFirstHref("href");
692 if ( toBeRemoved.contains(name) ) {
693 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
694 Log.println(Constants.LOGV,TAG,"Found href in our list.");
695 toBeRemoved.remove(name);
696 }
697
698 ContentValues cv = originalData.get(name);
699 WriteActions action = WriteActions.UPDATE;
700 if ( cv == null ) {
701 cv = new ContentValues();
702 cv.put(DavResources.COLLECTION_ID, collectionId);
703 cv.put(DavResources.RESOURCE_NAME, name);
704 action = WriteActions.INSERT;
705 }
706 if ( !parseResponseNode(response, cv) ) continue;
707 if ( cv.getAsString("COLLECTION") != null ) continue;
708
709 if (Constants.LOG_DEBUG)
710 Log.println(Constants.LOGD,TAG, "Multiget response needs sync="+cv.getAsString(DavResources.NEEDS_SYNC)+" for "+name );
711
712 changeList.add( new ResourceModification(action, cv, null) );
713 }
714
715 ResourceModification.commitChangeList(context, changeList);
716
717 }
718
719 for( String href : toBeRemoved ) {
720 Log.println(Constants.LOGV,TAG,"Did not find +"+href+"+ in the list.");
721 }
722 if ( toBeRemoved.size() > 0 ) {
723 doRegularSyncPropfind();
724 return;
725 }
726
727 return;
728 }
729
730
731 /**
732 * <p>
733 * Finds the resources which have been marked as needing synchronisation in our local database.
734 * </p>
735 *
736 * @return A map of String/Data which are the hrefs we need to sync
737 */
738 private Map<String, ContentValues> findSyncNeededResources() {
739 long start = System.currentTimeMillis();
740 Map<String, ContentValues> originalData = null;
741
742 // step 1a get list of resources from db
743 start = System.currentTimeMillis();
744
745 Cursor mCursor = this.cr.query(
746 Uri.parse(DavResources.CONTENT_URI.toString() + "/collection/" + this.collectionId),
747 null,
748 DavResources.NEEDS_SYNC + " = 1 OR "+DavResources.RESOURCE_DATA+" IS NULL",
749 null, null);
750 ContentQueryMap cqm = new ContentQueryMap(mCursor, DavResources.RESOURCE_NAME, false, null);
751 cqm.requery();
752 originalData = cqm.getRows();
753 mCursor.close();
754 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
755 Log.println(Constants.LOGV,TAG, "DavCollections ContentQueryMap retrieved in " + (System.currentTimeMillis() - start) + "ms");
756 return originalData;
757 }
758
759 /**
760 * <p>
761 * Parse a single &lt;response&gt; node within a &lt;multistatus&gt;
762 * </p>
763 * @return true If we need to write to the database, false otherwise.
764 */
765 private boolean parseResponseNode(DavNode responseNode, ContentValues cv) {
766
767 DavNode prop = null;
768 for ( DavNode testPs : responseNode.getNodesFromPath("propstat")) {
769 String statusText = testPs.getFirstNodeText("status");
770 if ( statusText.equalsIgnoreCase("HTTP/1.1 200 OK") || statusText.equalsIgnoreCase("HTTP/1.1 201 Created")) {
771 prop = testPs.getNodesFromPath("prop").get(0);
772 break;
773 }
774 }
775
776 if ( prop == null ) {
777 return false;
778 }
779 else {
780 String s = prop.getFirstNodeText("getctag");
781 if ( s != null ) {
782 collectionChanged = (collectionData.getAsString(DavCollections.COLLECTION_TAG) == null
783 || s.equals(collectionData.getAsString(DavCollections.COLLECTION_TAG)));
784 if ( collectionChanged ) collectionData.put(DavCollections.COLLECTION_TAG, s);
785 cv.put("COLLECTION", true);
786 return false;
787 }
788 else {
789 String etag = prop.getFirstNodeText("getetag");
790
791 if ( etag != null ) {
792 String oldEtag = cv.getAsString(DavResources.ETAG);
793
794 if ( etag.equals(oldEtag) ) {
795 cv.put(DavResources.NEEDS_SYNC, 0);
796 if ( Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents ) Log.println(Constants.LOGD,TAG,
797 "Found etag "+etag+" in response. Old etag was "+oldEtag+". No sync needed.");
798 return true;
799 }
800 else {
801 if ( Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents ) Log.println(Constants.LOGD,TAG,
802 "Found etag "+etag+" in response. Old etags was "+oldEtag+". Sync will be needed.");
803 cv.put(DavResources.NEEDS_SYNC, 1);
804 }
805 }
806
807 String data = prop.getFirstNodeText(dataType + "-data");
808 if ( data != null ) {
809 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents ) {
810 Log.println(Constants.LOGV,TAG,"Found data in response:");
811 Log.println(Constants.LOGV,TAG,data);
812 Log.println(Constants.LOGV,TAG,StaticHelpers.toHexString(data.substring(0,40).getBytes()));
813 }
814 cv.put(DavResources.RESOURCE_DATA, data);
815 cv.put(DavResources.ETAG, etag);
816 cv.put(DavResources.NEEDS_SYNC, 0);
817 if ( Constants.LOG_DEBUG && Constants.debugSyncCollectionContents ) Log.println(Constants.LOGD,TAG,
818 "Found data for etag '"+etag+"' in response. Sync not needed.");
819 }
820 else if ( Constants.LOG_DEBUG && Constants.debugSyncCollectionContents ) Log.println(Constants.LOGD,TAG,
821 "Found no data for etag '"+etag+"' in response. Sync is needed.");
822
823
824 s = prop.getFirstNodeText("getlastmodified");
825 if ( s != null ) cv.put(DavResources.LAST_MODIFIED, s);
826
827 s = prop.getFirstNodeText("getcontenttype");
828 if ( s != null ) cv.put(DavResources.CONTENT_TYPE, s);
829 }
830 }
831
832 return true;
833 }
834
835 /**
836 * <p>
837 * Performs a sync using a series of GET requests to retrieve each resource needing sync. This is a
838 * fallback strategy and we really expect multiget to work in almost all circumstances.
839 * </p>
840 *
841 * @param originalData
842 * <p>
843 * The href => Contentvalues map.
844 * </p>
845 * @param hrefs
846 * <p>
847 * The array of hrefs
848 * </p>
849 */
850 private void syncWithGet(Map<String, ContentValues> originalData, Object[] hrefs) {
851 long fullMethod = System.currentTimeMillis();
852
853 Header[] headers = new Header[] {};
854 List<ResourceModification> changeList = new ArrayList<ResourceModification>(hrefs.length);
855 String path;
856 InputStream in;
857 int status;
858
859 for (int hrefIndex = 0; hrefIndex < hrefs.length; hrefIndex++) {
860 path = collectionPath + hrefs[hrefIndex];
861
862 try {
863 in = requestor.doRequest("GET", path, headers, "");
864 }
865 catch (ConnectionFailedException e) {
866 Log.i(TAG,"ConnectionFailedException ("+e.getMessage()+") on GET from "+path);
867 continue;
868 }
869 catch (SendRequestFailedException e) {
870 Log.i(TAG,"SendRequestFailedException ("+e.getMessage()+") on GET from "+path);
871 continue;
872 }
873 catch (SSLException e) {
874 Log.i(TAG,"SSLException on GET from "+path);
875 continue;
876 }
877 if (in == null) {
878 if (Constants.LOG_DEBUG) Log.println(Constants.LOGD,TAG, "Error - Unable to get data stream from server.");
879 continue;
880 }
881 else {
882 status = requestor.getStatusCode();
883 switch (status) {
884 case 200: // Status O.K.
885 StringBuilder resourceData = new StringBuilder();
886 BufferedReader r = new BufferedReader(new InputStreamReader(in),4096);
887 String line;
888 try {
889 while ((line = r.readLine() ) != null) {
890 resourceData.append(line);
891 resourceData.append("\n");
892 }
893 }
894 catch (IOException e) {
895 Log.i(TAG,Log.getStackTraceString(e));
896 }
897 ContentValues cv = originalData.get(hrefs[hrefIndex]);
898 cv.put(DavResources.RESOURCE_DATA, resourceData.toString() );
899 for (Header hdr : requestor.getResponseHeaders()) {
900 if (hdr.getName().equalsIgnoreCase("ETag")) {
901 cv.put(DavResources.ETAG, hdr.getValue());
902 break;
903 }
904 }
905 cv.put(DavResources.NEEDS_SYNC, 0);
906 changeList.add( new ResourceModification(WriteActions.UPDATE, cv, null) );
907 if (Constants.LOG_DEBUG)
908 Log.println(Constants.LOGD,TAG, "Get response for "+hrefs[hrefIndex] );
909 break;
910
911 default: // Unknown code
912 Log.w(TAG, "Status " + status + " on GET request for " + path);
913 }
914 }
915 }
916
917 ResourceModification.commitChangeList(context, changeList);
918
919 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
920 Log.println(Constants.LOGV,TAG, "syncWithGet() for " + hrefs.length + " resources took "
921 + (System.currentTimeMillis() - fullMethod) + "ms");
922 return;
923 }
924
925
926
927 private boolean timeToRun() {
928 if ( synchronisationForced ) {
929 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
930 Log.println(Constants.LOGV,TAG, "Synchronising now, since a sync has been forced.");
931 return true;
932 }
933 String needsSyncNow = collectionData.getAsString(DavCollections.NEEDS_SYNC);
934 if ( needsSyncNow == null || needsSyncNow.equals("1") ) {
935 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
936 Log.println(Constants.LOGV,TAG, "Synchronising now, since needs_sync is true.");
937 return true;
938 }
939 String lastSyncString = collectionData.getAsString(DavCollections.LAST_SYNCHRONISED);
940 if ( lastSyncString == null ) {
941 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
942 Log.println(Constants.LOGV,TAG, "Synchronising now, since last_sync is null.");
943 return true;
944 }
945 AcalDateTime lastRunTime = null;
946
947 lastRunTime = AcalDateTime.fromString(lastSyncString);
948 if ( lastRunTime == null ) return true;
949
950 lastRunTime.applyLocalTimeZone();
951
952 ConnectivityManager conMan = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE);
953 NetworkInfo netInfo = conMan.getActiveNetworkInfo();
954 long maxAgeMs = minBetweenSyncs;
955 if ( netInfo != null && netInfo.getType() == ConnectivityManager.TYPE_MOBILE )
956 maxAgeMs = collectionData.getAsLong(DavCollections.MAX_SYNC_AGE_3G);
957 else
958 maxAgeMs = collectionData.getAsLong(DavCollections.MAX_SYNC_AGE_WIFI);
959
960 if ( maxAgeMs < minBetweenSyncs ) maxAgeMs = minBetweenSyncs;
961
962 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
963 Log.println(Constants.LOGV,TAG, "Considering whether we are " + maxAgeMs / 1000 + "s past "
964 + lastRunTime.fmtIcal() + "("+lastRunTime.getMillis()+") yet? "
965 + "Now: " + new AcalDateTime().fmtIcal() + "("+System.currentTimeMillis()+")... So: "
966 + ((maxAgeMs + lastRunTime.getMillis() < System.currentTimeMillis()) ? "yes" : "no"));
967
968 return (maxAgeMs + lastRunTime.getMillis() < System.currentTimeMillis());
969 }
970
971
972 private void scheduleNextInstance() {
973 String lastSync = collectionData.getAsString(DavCollections.LAST_SYNCHRONISED);
974 long timeToWait = minBetweenSyncs;
975 if ( lastSync != null ) {
976 long maxAge3g = collectionData.getAsLong(DavCollections.MAX_SYNC_AGE_3G);
977 long maxAgeWifi = collectionData.getAsLong(DavCollections.MAX_SYNC_AGE_WIFI);
978 timeToWait = (maxAge3g > maxAgeWifi ? maxAgeWifi : maxAge3g);
979
980 AcalDateTime lastRunTime = null;
981 lastRunTime = AcalDateTime.fromString(lastSync);
982 lastRunTime.applyLocalTimeZone();
983 timeToWait += (lastRunTime.getMillis() - System.currentTimeMillis());
984
985 if ( minBetweenSyncs > timeToWait ) timeToWait = minBetweenSyncs;
986 }
987
988 if (Constants.LOG_VERBOSE && Constants.debugSyncCollectionContents )
989 Log.println(Constants.LOGV,TAG, "Scheduling next sync status check for "+collectionId+" - '"
990 + collectionData.getAsString(DavCollections.DISPLAYNAME)
991 +"' in " + Long.toString(timeToWait / 1000) + " seconds.");
992
993 this.TIME_TO_EXECUTE = timeToWait;
994 context.addWorkerJob(this);
995 }
996
997 public boolean equals(Object that) {
998 if (this == that) return true;
999 if (!(that instanceof SyncCollectionContents)) return false;
1000 SyncCollectionContents thatCis = (SyncCollectionContents) that;
1001 return this.collectionId == thatCis.collectionId;
1002 }
1003
1004 public int hashCode() {
1005 int result = HashCodeUtil.SEED;
1006 result = HashCodeUtil.hash(result, this.collectionId);
1007 return result;
1008 }
1009
1010 @Override
1011 public String getDescription() {
1012 return "Syncing collection contents of collection " + collectionId;
1013 }
1014
1015 }
aCal Android CalDAV Client