?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Push down extern handle variable to files that need it
[pacman-ng.git] / lib / libalpm / be_sync.c
blob2e472b3dddf4097a88c75eb8c601fdc6dc429650
1 /*
2 * be_sync.c
3 *
4 * Copyright (c) 2006-2011 Pacman Development Team <pacman-dev@archlinux.org>
5 * Copyright (c) 2002-2006 by Judd Vinet <jvinet@zeroflux.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "config.h"
22
23 #include <sys/stat.h>
24
25 /* libarchive */
26 #include <archive.h>
27 #include <archive_entry.h>
28
29 /* libalpm */
30 #include "util.h"
31 #include "log.h"
32 #include "alpm.h"
33 #include "alpm_list.h"
34 #include "package.h"
35 #include "handle.h"
36 #include "delta.h"
37 #include "deps.h"
38 #include "dload.h"
39
40 /* global handle variable */
41 extern pmhandle_t *handle;
42
43 /** Update a package database
44 *
45 * An update of the package database \a db will be attempted. Unless
46 * \a force is true, the update will only be performed if the remote
47 * database was modified since the last update.
48 *
49 * A transaction is necessary for this operation, in order to obtain a
50 * database lock. During this transaction the front-end will be informed
51 * of the download progress of the database via the download callback.
52 *
53 * Example:
54 * @code
55 * alpm_list_t *syncs = alpm_option_get_syncdbs();
56 * if(alpm_trans_init(0, NULL, NULL, NULL) == 0) {
57 * for(i = syncs; i; i = alpm_list_next(i)) {
58 * pmdb_t *db = alpm_list_getdata(i);
59 * result = alpm_db_update(0, db);
60 * alpm_trans_release();
61 *
62 * if(result < 0) {
63 * printf("Unable to update database: %s\n", alpm_strerrorlast());
64 * } else if(result == 1) {
65 * printf("Database already up to date\n");
66 * } else {
67 * printf("Database updated\n");
68 * }
69 * }
70 * }
71 * @endcode
72 *
73 * @ingroup alpm_databases
74 * @note After a successful update, the \link alpm_db_get_pkgcache()
75 * package cache \endlink will be invalidated
76 * @param force if true, then forces the update, otherwise update only in case
77 * the database isn't up to date
78 * @param db pointer to the package database to update
79 * @return 0 on success, -1 on error (pm_errno is set accordingly), 1 if up to
80 * to date
81 */
82 int SYMEXPORT alpm_db_update(int force, pmdb_t *db)
83 {
84 char *syncpath;
85 const char *dbpath;
86 alpm_list_t *i;
87 struct stat buf;
88 size_t len;
89 int ret = -1;
90 mode_t oldmask;
91 pgp_verify_t check_sig;
92
93 /* Sanity checks */
94 ASSERT(handle != NULL, RET_ERR(PM_ERR_HANDLE_NULL, -1));
95 ASSERT(db != NULL && db != handle->db_local, RET_ERR(PM_ERR_WRONG_ARGS, -1));
96 ASSERT(db->servers != NULL, RET_ERR(PM_ERR_SERVER_NONE, -1));
97
98 dbpath = alpm_option_get_dbpath();
99 len = strlen(dbpath) + 6;
100 MALLOC(syncpath, len, RET_ERR(PM_ERR_MEMORY, -1));
101 sprintf(syncpath, "%s%s", dbpath, "sync/");
102
103 /* make sure we have a sane umask */
104 oldmask = umask(0022);
105
106 if(stat(syncpath, &buf) != 0) {
107 _alpm_log(PM_LOG_DEBUG, "database dir '%s' does not exist, creating it\n",
108 syncpath);
109 if(_alpm_makepath(syncpath) != 0) {
110 free(syncpath);
111 RET_ERR(PM_ERR_SYSTEM, -1);
112 }
113 } else if(!S_ISDIR(buf.st_mode)) {
114 _alpm_log(PM_LOG_WARNING, _("removing invalid file: %s\n"), syncpath);
115 if(unlink(syncpath) != 0 || _alpm_makepath(syncpath) != 0) {
116 free(syncpath);
117 RET_ERR(PM_ERR_SYSTEM, -1);
118 }
119 }
120
121 check_sig = _alpm_db_get_sigverify_level(db);
122
123 for(i = db->servers; i; i = i->next) {
124 const char *server = i->data;
125 char *fileurl;
126 size_t len;
127 int sig_ret = 0;
128
129 /* print server + filename into a buffer (leave space for .sig) */
130 len = strlen(server) + strlen(db->treename) + 9;
131 CALLOC(fileurl, len, sizeof(char), RET_ERR(PM_ERR_MEMORY, -1));
132 snprintf(fileurl, len, "%s/%s.db", server, db->treename);
133
134 ret = _alpm_download(fileurl, syncpath, force, 0, 0);
135
136 if(ret == 0 && (check_sig == PM_PGP_VERIFY_ALWAYS ||
137 check_sig == PM_PGP_VERIFY_OPTIONAL)) {
138 int errors_ok = (check_sig == PM_PGP_VERIFY_OPTIONAL);
139 /* if we downloaded a DB, we want the .sig from the same server */
140 snprintf(fileurl, len, "%s/%s.db.sig", server, db->treename);
141
142 sig_ret = _alpm_download(fileurl, syncpath, 1, 0, errors_ok);
143 /* errors_ok suppresses error messages, but not the return code */
144 sig_ret = errors_ok ? 0 : sig_ret;
145 }
146
147 FREE(fileurl);
148 if(ret != -1 && sig_ret != -1) {
149 break;
150 }
151 }
152
153 if(ret == 1) {
154 /* files match, do nothing */
155 pm_errno = 0;
156 goto cleanup;
157 } else if(ret == -1) {
158 /* pm_errno was set by the download code */
159 _alpm_log(PM_LOG_DEBUG, "failed to sync db: %s\n", alpm_strerrorlast());
160 goto cleanup;
161 }
162
163 /* Cache needs to be rebuilt */
164 _alpm_db_free_pkgcache(db);
165
166 cleanup:
167
168 free(syncpath);
169 umask(oldmask);
170 return ret;
171 }
172
173 /* Forward decl so I don't reorganize the whole file right now */
174 static int sync_db_read(pmdb_t *db, struct archive *archive,
175 struct archive_entry *entry, pmpkg_t *likely_pkg);
176
177 /*
178 * This is the data table used to generate the estimating function below.
179 * "Weighted Avg" means averaging the bottom table values; thus each repo, big
180 * or small, will have equal influence. "Unweighted Avg" means averaging the
181 * sums of the top table columns, thus each package has equal influence. The
182 * final values are calculated by (surprise) averaging the averages, because
183 * why the hell not.
184 *
185 * Database Pkgs tar bz2 gz xz
186 * community 2096 5294080 256391 421227 301296
187 * core 180 460800 25257 36850 29356
188 * extra 2606 6635520 294647 470818 339392
189 * multilib 126 327680 16120 23261 18732
190 * testing 76 204800 10902 14348 12100
191 *
192 * Bytes Per Package
193 * community 2096 2525.80 122.32 200.97 143.75
194 * core 180 2560.00 140.32 204.72 163.09
195 * extra 2606 2546.25 113.06 180.67 130.23
196 * multilib 126 2600.63 127.94 184.61 148.67
197 * testing 76 2694.74 143.45 188.79 159.21
198
199 * Weighted Avg 2585.48 129.42 191.95 148.99
200 * Unweighted Avg 2543.39 118.74 190.16 137.93
201 * Average of Avgs 2564.44 124.08 191.06 143.46
202 */
203 static size_t estimate_package_count(struct stat *st, struct archive *archive)
204 {
205 unsigned int per_package;
206
207 switch(archive_compression(archive)) {
208 case ARCHIVE_COMPRESSION_NONE:
209 per_package = 2564;
210 break;
211 case ARCHIVE_COMPRESSION_GZIP:
212 per_package = 191;
213 break;
214 case ARCHIVE_COMPRESSION_BZIP2:
215 per_package = 124;
216 break;
217 case ARCHIVE_COMPRESSION_COMPRESS:
218 per_package = 193;
219 break;
220 case ARCHIVE_COMPRESSION_LZMA:
221 case ARCHIVE_COMPRESSION_XZ:
222 per_package = 143;
223 break;
224 #ifdef ARCHIVE_COMPRESSION_UU
225 case ARCHIVE_COMPRESSION_UU:
226 per_package = 3543;
227 break;
228 #endif
229 default:
230 /* assume it is at least somewhat compressed */
231 per_package = 200;
232 }
233 return (size_t)((st->st_size / per_package) + 1);
234 }
235
236 static int sync_db_populate(pmdb_t *db)
237 {
238 const char *dbpath;
239 size_t est_count;
240 int count = 0;
241 struct stat buf;
242 struct archive *archive;
243 struct archive_entry *entry;
244 pmpkg_t *pkg = NULL;
245
246 ASSERT(db != NULL, RET_ERR(PM_ERR_DB_NULL, -1));
247
248 if((archive = archive_read_new()) == NULL) {
249 RET_ERR(PM_ERR_LIBARCHIVE, -1);
250 }
251
252 archive_read_support_compression_all(archive);
253 archive_read_support_format_all(archive);
254
255 dbpath = _alpm_db_path(db);
256 if(!dbpath) {
257 /* pm_errno set in _alpm_db_path() */
258 return -1;
259 }
260
261 _alpm_log(PM_LOG_DEBUG, "opening database archive %s\n", dbpath);
262
263 if(archive_read_open_filename(archive, dbpath,
264 ARCHIVE_DEFAULT_BYTES_PER_BLOCK) != ARCHIVE_OK) {
265 _alpm_log(PM_LOG_ERROR, _("could not open file %s: %s\n"), dbpath,
266 archive_error_string(archive));
267 archive_read_finish(archive);
268 RET_ERR(PM_ERR_DB_OPEN, -1);
269 }
270 if(stat(dbpath, &buf) != 0) {
271 RET_ERR(PM_ERR_DB_OPEN, -1);
272 }
273 est_count = estimate_package_count(&buf, archive);
274
275 /* initialize hash at 66% full */
276 db->pkgcache = _alpm_pkghash_create(est_count * 3 / 2);
277 if(db->pkgcache == NULL) {
278 RET_ERR(PM_ERR_MEMORY, -1);
279 }
280
281 while(archive_read_next_header(archive, &entry) == ARCHIVE_OK) {
282 const struct stat *st;
283
284 st = archive_entry_stat(entry);
285
286 if(S_ISDIR(st->st_mode)) {
287 const char *name;
288
289 pkg = _alpm_pkg_new();
290 if(pkg == NULL) {
291 archive_read_finish(archive);
292 RET_ERR(PM_ERR_MEMORY, -1);
293 }
294
295 name = archive_entry_pathname(entry);
296
297 if(_alpm_splitname(name, pkg) != 0) {
298 _alpm_log(PM_LOG_ERROR, _("invalid name for database entry '%s'\n"),
299 name);
300 _alpm_pkg_free(pkg);
301 continue;
302 }
303
304 /* duplicated database entries are not allowed */
305 if(_alpm_pkghash_find(db->pkgcache, pkg->name)) {
306 _alpm_log(PM_LOG_ERROR, _("duplicated database entry '%s'\n"), pkg->name);
307 _alpm_pkg_free(pkg);
308 continue;
309 }
310
311 pkg->origin = PKG_FROM_SYNCDB;
312 pkg->origin_data.db = db;
313 pkg->ops = &default_pkg_ops;
314 pkg->handle = handle;
315
316 /* add to the collection */
317 _alpm_log(PM_LOG_FUNCTION, "adding '%s' to package cache for db '%s'\n",
318 pkg->name, db->treename);
319 db->pkgcache = _alpm_pkghash_add(db->pkgcache, pkg);
320 count++;
321 } else {
322 /* we have desc, depends or deltas - parse it */
323 sync_db_read(db, archive, entry, pkg);
324 }
325 }
326
327 if(count > 0) {
328 db->pkgcache->list = alpm_list_msort(db->pkgcache->list, (size_t)count, _alpm_pkg_cmp);
329 }
330 archive_read_finish(archive);
331 _alpm_log(PM_LOG_DEBUG, "added %d packages to package cache for db '%s'\n",
332 count, db->treename);
333
334 return count;
335 }
336
337 #define READ_NEXT(s) do { \
338 if(_alpm_archive_fgets(archive, &buf) != ARCHIVE_OK) goto error; \
339 s = _alpm_strtrim(buf.line); \
340 } while(0)
341
342 #define READ_AND_STORE(f) do { \
343 READ_NEXT(line); \
344 STRDUP(f, line, goto error); \
345 } while(0)
346
347 #define READ_AND_STORE_ALL(f) do { \
348 char *linedup; \
349 READ_NEXT(line); \
350 if(strlen(line) == 0) break; \
351 STRDUP(linedup, line, goto error); \
352 f = alpm_list_add(f, linedup); \
353 } while(1) /* note the while(1) and not (0) */
354
355 static int sync_db_read(pmdb_t *db, struct archive *archive,
356 struct archive_entry *entry, pmpkg_t *likely_pkg)
357 {
358 const char *entryname = NULL, *filename;
359 char *pkgname, *p, *q;
360 pmpkg_t *pkg;
361 struct archive_read_buffer buf;
362
363 if(db == NULL) {
364 RET_ERR(PM_ERR_DB_NULL, -1);
365 }
366
367 if(entry != NULL) {
368 entryname = archive_entry_pathname(entry);
369 }
370 if(entryname == NULL) {
371 _alpm_log(PM_LOG_DEBUG, "invalid archive entry provided to _alpm_sync_db_read, skipping\n");
372 return -1;
373 }
374
375 _alpm_log(PM_LOG_FUNCTION, "loading package data from archive entry %s\n",
376 entryname);
377
378 memset(&buf, 0, sizeof(buf));
379 /* 512K for a line length seems reasonable */
380 buf.max_line_size = 512 * 1024;
381
382 /* get package and db file names */
383 STRDUP(pkgname, entryname, RET_ERR(PM_ERR_MEMORY, -1));
384 p = pkgname + strlen(pkgname);
385 for(q = --p; *q && *q != '/'; q--);
386 filename = q + 1;
387 for(p = --q; *p && *p != '-'; p--);
388 for(q = --p; *q && *q != '-'; q--);
389 *q = '\0';
390
391 /* package is already in db due to parsing of directory name */
392 if(likely_pkg && strcmp(likely_pkg->name, pkgname) == 0) {
393 pkg = likely_pkg;
394 } else {
395 if(db->pkgcache == NULL) {
396 RET_ERR(PM_ERR_MEMORY, -1);
397 }
398 pkg = _alpm_pkghash_find(db->pkgcache, pkgname);
399 }
400 if(pkg == NULL) {
401 _alpm_log(PM_LOG_DEBUG, "package %s not found in %s sync database",
402 pkgname, db->treename);
403 return -1;
404 }
405
406 if(strcmp(filename, "desc") == 0 || strcmp(filename, "depends") == 0
407 || strcmp(filename, "deltas") == 0) {
408 while(_alpm_archive_fgets(archive, &buf) == ARCHIVE_OK) {
409 char *line = _alpm_strtrim(buf.line);
410
411 if(strcmp(line, "%NAME%") == 0) {
412 READ_NEXT(line);
413 if(strcmp(line, pkg->name) != 0) {
414 _alpm_log(PM_LOG_ERROR, _("%s database is inconsistent: name "
415 "mismatch on package %s\n"), db->treename, pkg->name);
416 }
417 } else if(strcmp(line, "%VERSION%") == 0) {
418 READ_NEXT(line);
419 if(strcmp(line, pkg->version) != 0) {
420 _alpm_log(PM_LOG_ERROR, _("%s database is inconsistent: version "
421 "mismatch on package %s\n"), db->treename, pkg->name);
422 }
423 } else if(strcmp(line, "%FILENAME%") == 0) {
424 READ_AND_STORE(pkg->filename);
425 } else if(strcmp(line, "%DESC%") == 0) {
426 READ_AND_STORE(pkg->desc);
427 } else if(strcmp(line, "%GROUPS%") == 0) {
428 READ_AND_STORE_ALL(pkg->groups);
429 } else if(strcmp(line, "%URL%") == 0) {
430 READ_AND_STORE(pkg->url);
431 } else if(strcmp(line, "%LICENSE%") == 0) {
432 READ_AND_STORE_ALL(pkg->licenses);
433 } else if(strcmp(line, "%ARCH%") == 0) {
434 READ_AND_STORE(pkg->arch);
435 } else if(strcmp(line, "%BUILDDATE%") == 0) {
436 READ_NEXT(line);
437 pkg->builddate = _alpm_parsedate(line);
438 } else if(strcmp(line, "%PACKAGER%") == 0) {
439 READ_AND_STORE(pkg->packager);
440 } else if(strcmp(line, "%CSIZE%") == 0) {
441 /* Note: the CSIZE and SIZE fields both share the "size" field in the
442 * pkginfo_t struct. This can be done b/c CSIZE is currently only used
443 * in sync databases, and SIZE is only used in local databases.
444 */
445 READ_NEXT(line);
446 pkg->size = atol(line);
447 /* also store this value to isize if isize is unset */
448 if(pkg->isize == 0) {
449 pkg->isize = pkg->size;
450 }
451 } else if(strcmp(line, "%ISIZE%") == 0) {
452 READ_NEXT(line);
453 pkg->isize = atol(line);
454 } else if(strcmp(line, "%MD5SUM%") == 0) {
455 READ_AND_STORE(pkg->md5sum);
456 } else if(strcmp(line, "%SHA256SUM%") == 0) {
457 /* we don't do anything with this value right now */
458 READ_NEXT(line);
459 } else if(strcmp(line, "%PGPSIG%") == 0) {
460 READ_AND_STORE(pkg->base64_sig);
461 } else if(strcmp(line, "%REPLACES%") == 0) {
462 READ_AND_STORE_ALL(pkg->replaces);
463 } else if(strcmp(line, "%DEPENDS%") == 0) {
464 /* Different than the rest because of the _alpm_splitdep call. */
465 while(1) {
466 READ_NEXT(line);
467 if(strlen(line) == 0) break;
468 pkg->depends = alpm_list_add(pkg->depends, _alpm_splitdep(line));
469 }
470 } else if(strcmp(line, "%OPTDEPENDS%") == 0) {
471 READ_AND_STORE_ALL(pkg->optdepends);
472 } else if(strcmp(line, "%CONFLICTS%") == 0) {
473 READ_AND_STORE_ALL(pkg->conflicts);
474 } else if(strcmp(line, "%PROVIDES%") == 0) {
475 READ_AND_STORE_ALL(pkg->provides);
476 } else if(strcmp(line, "%DELTAS%") == 0) {
477 /* Different than the rest because of the _alpm_delta_parse call. */
478 while(1) {
479 READ_NEXT(line);
480 if(strlen(line) == 0) break;
481 pkg->deltas = alpm_list_add(pkg->deltas, _alpm_delta_parse(line));
482 }
483 }
484 }
485 } else if(strcmp(filename, "files") == 0) {
486 /* currently do nothing with this file */
487 } else {
488 /* unknown database file */
489 _alpm_log(PM_LOG_DEBUG, "unknown database file: %s\n", filename);
490 }
491
492 error:
493 FREE(pkgname);
494 /* TODO: return 0 always? */
495 return 0;
496 }
497
498 static int sync_db_version(pmdb_t UNUSED *db)
499 {
500 return 2;
501 }
502
503 struct db_operations sync_db_ops = {
504 .populate = sync_db_populate,
505 .unregister = _alpm_db_unregister,
506 .version = sync_db_version,
507 };
508
509 pmdb_t *_alpm_db_register_sync(const char *treename)
510 {
511 pmdb_t *db;
512
513 _alpm_log(PM_LOG_DEBUG, "registering sync database '%s'\n", treename);
514
515 db = _alpm_db_new(treename, 0);
516 if(db == NULL) {
517 RET_ERR(PM_ERR_DB_CREATE, NULL);
518 }
519 db->ops = &sync_db_ops;
520 db->handle = handle;
521
522 handle->dbs_sync = alpm_list_add(handle->dbs_sync, db);
523 return db;
524 }
525
526
527 /* vim: set ts=2 sw=2 noet: */
ngaba's pacman repo