2 * be_sync.c : backend for sync databases
4 * Copyright (c) 2006-2012 Pacman Development Team <pacman-dev@archlinux.org>
5 * Copyright (c) 2002-2006 by Judd Vinet <jvinet@zeroflux.org>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
22 #include <sys/types.h>
29 #include <archive_entry.h>
35 #include "alpm_list.h"
42 static char *get_sync_dir(alpm_handle_t
*handle
)
44 size_t len
= strlen(handle
->dbpath
) + 6;
48 MALLOC(syncpath
, len
, RET_ERR(handle
, ALPM_ERR_MEMORY
, NULL
));
49 sprintf(syncpath
, "%s%s", handle
->dbpath
, "sync/");
51 if(stat(syncpath
, &buf
) != 0) {
52 _alpm_log(handle
, ALPM_LOG_DEBUG
, "database dir '%s' does not exist, creating it\n",
54 if(_alpm_makepath(syncpath
) != 0) {
56 RET_ERR(handle
, ALPM_ERR_SYSTEM
, NULL
);
58 } else if(!S_ISDIR(buf
.st_mode
)) {
59 _alpm_log(handle
, ALPM_LOG_WARNING
, _("removing invalid file: %s\n"), syncpath
);
60 if(unlink(syncpath
) != 0 || _alpm_makepath(syncpath
) != 0) {
62 RET_ERR(handle
, ALPM_ERR_SYSTEM
, NULL
);
69 static int sync_db_validate(alpm_db_t
*db
)
71 alpm_siglevel_t level
;
74 if(db
->status
& DB_STATUS_VALID
|| db
->status
& DB_STATUS_MISSING
) {
77 if(db
->status
& DB_STATUS_INVALID
) {
78 db
->handle
->pm_errno
= ALPM_ERR_DB_INVALID_SIG
;
82 dbpath
= _alpm_db_path(db
);
84 /* pm_errno set in _alpm_db_path() */
88 /* we can skip any validation if the database doesn't exist */
89 if(_alpm_access(db
->handle
, NULL
, dbpath
, R_OK
) != 0 && errno
== ENOENT
) {
90 db
->status
&= ~DB_STATUS_EXISTS
;
91 db
->status
|= DB_STATUS_MISSING
;
92 _alpm_log(db
->handle
, ALPM_LOG_WARNING
,
93 "database file for '%s' does not exist\n", db
->treename
);
96 db
->status
|= DB_STATUS_EXISTS
;
97 db
->status
&= ~DB_STATUS_MISSING
;
99 /* this takes into account the default verification level if UNKNOWN
100 * was assigned to this db */
101 level
= alpm_db_get_siglevel(db
);
103 if(level
& ALPM_SIG_DATABASE
) {
107 alpm_siglist_t
*siglist
;
108 ret
= _alpm_check_pgp_helper(db
->handle
, dbpath
, NULL
,
109 level
& ALPM_SIG_DATABASE_OPTIONAL
, level
& ALPM_SIG_DATABASE_MARGINAL_OK
,
110 level
& ALPM_SIG_DATABASE_UNKNOWN_OK
, &siglist
);
112 retry
= _alpm_process_siglist(db
->handle
, db
->treename
, siglist
,
113 level
& ALPM_SIG_DATABASE_OPTIONAL
, level
& ALPM_SIG_DATABASE_MARGINAL_OK
,
114 level
& ALPM_SIG_DATABASE_UNKNOWN_OK
);
116 alpm_siglist_cleanup(siglist
);
121 db
->status
&= ~DB_STATUS_VALID
;
122 db
->status
|= DB_STATUS_INVALID
;
123 db
->handle
->pm_errno
= ALPM_ERR_DB_INVALID_SIG
;
129 db
->status
|= DB_STATUS_VALID
;
130 db
->status
&= ~DB_STATUS_INVALID
;
134 /** Update a package database
136 * An update of the package database \a db will be attempted. Unless
137 * \a force is true, the update will only be performed if the remote
138 * database was modified since the last update.
140 * This operation requires a database lock, and will return an applicable error
141 * if the lock could not be obtained.
145 * alpm_list_t *syncs = alpm_option_get_syncdbs();
146 * for(i = syncs; i; i = alpm_list_next(i)) {
147 * alpm_db_t *db = alpm_list_getdata(i);
148 * result = alpm_db_update(0, db);
151 * printf("Unable to update database: %s\n", alpm_strerrorlast());
152 * } else if(result == 1) {
153 * printf("Database already up to date\n");
155 * printf("Database updated\n");
160 * @ingroup alpm_databases
161 * @note After a successful update, the \link alpm_db_get_pkgcache()
162 * package cache \endlink will be invalidated
163 * @param force if true, then forces the update, otherwise update only in case
164 * the database isn't up to date
165 * @param db pointer to the package database to update
166 * @return 0 on success, -1 on error (pm_errno is set accordingly), 1 if up to
169 int SYMEXPORT
alpm_db_update(int force
, alpm_db_t
*db
)
175 alpm_handle_t
*handle
;
176 alpm_siglevel_t level
;
179 ASSERT(db
!= NULL
, return -1);
181 handle
->pm_errno
= 0;
182 ASSERT(db
!= handle
->db_local
, RET_ERR(handle
, ALPM_ERR_WRONG_ARGS
, -1));
183 ASSERT(db
->servers
!= NULL
, RET_ERR(handle
, ALPM_ERR_SERVER_NONE
, -1));
185 syncpath
= get_sync_dir(handle
);
190 /* make sure we have a sane umask */
191 oldmask
= umask(0022);
193 level
= alpm_db_get_siglevel(db
);
195 /* attempt to grab a lock */
196 if(_alpm_handle_lock(handle
)) {
199 RET_ERR(handle
, ALPM_ERR_HANDLE_LOCK
, -1);
202 for(i
= db
->servers
; i
; i
= i
->next
) {
203 const char *server
= i
->data
;
204 struct dload_payload payload
;
208 memset(&payload
, 0, sizeof(struct dload_payload
));
210 /* set hard upper limit of 25MiB */
211 payload
.max_size
= 25 * 1024 * 1024;
213 /* print server + filename into a buffer */
214 len
= strlen(server
) + strlen(db
->treename
) + 5;
215 /* TODO fix leak syncpath and umask unset */
216 MALLOC(payload
.fileurl
, len
, RET_ERR(handle
, ALPM_ERR_MEMORY
, -1));
217 snprintf(payload
.fileurl
, len
, "%s/%s.db", server
, db
->treename
);
218 payload
.handle
= handle
;
219 payload
.force
= force
;
220 payload
.unlink_on_fail
= 1;
222 ret
= _alpm_download(&payload
, syncpath
, NULL
);
223 _alpm_dload_payload_reset(&payload
);
225 if(ret
== 0 && (level
& ALPM_SIG_DATABASE
)) {
226 /* an existing sig file is no good at this point */
227 char *sigpath
= _alpm_sigpath(handle
, _alpm_db_path(db
));
235 /* if we downloaded a DB, we want the .sig from the same server */
236 /* print server + filename into a buffer (leave space for .sig) */
237 len
= strlen(server
) + strlen(db
->treename
) + 9;
238 /* TODO fix leak syncpath and umask unset */
239 MALLOC(payload
.fileurl
, len
, RET_ERR(handle
, ALPM_ERR_MEMORY
, -1));
240 snprintf(payload
.fileurl
, len
, "%s/%s.db.sig", server
, db
->treename
);
241 payload
.handle
= handle
;
243 payload
.errors_ok
= (level
& ALPM_SIG_DATABASE_OPTIONAL
);
245 /* set hard upper limit of 16KiB */
246 payload
.max_size
= 16 * 1024;
248 sig_ret
= _alpm_download(&payload
, syncpath
, NULL
);
249 /* errors_ok suppresses error messages, but not the return code */
250 sig_ret
= payload
.errors_ok
? 0 : sig_ret
;
251 _alpm_dload_payload_reset(&payload
);
254 if(ret
!= -1 && sig_ret
!= -1) {
260 /* files match, do nothing */
261 handle
->pm_errno
= 0;
263 } else if(ret
== -1) {
264 /* pm_errno was set by the download code */
265 _alpm_log(handle
, ALPM_LOG_DEBUG
, "failed to sync db: %s\n",
266 alpm_strerror(handle
->pm_errno
));
270 /* Cache needs to be rebuilt */
271 _alpm_db_free_pkgcache(db
);
273 /* clear all status flags regarding validity/existence */
274 db
->status
&= ~DB_STATUS_VALID
;
275 db
->status
&= ~DB_STATUS_INVALID
;
276 db
->status
&= ~DB_STATUS_EXISTS
;
277 db
->status
&= ~DB_STATUS_MISSING
;
279 if(sync_db_validate(db
)) {
280 /* pm_errno should be set */
286 if(_alpm_handle_unlock(handle
)) {
287 _alpm_log(handle
, ALPM_LOG_WARNING
, _("could not remove lock file %s\n"),
295 /* Forward decl so I don't reorganize the whole file right now */
296 static int sync_db_read(alpm_db_t
*db
, struct archive
*archive
,
297 struct archive_entry
*entry
, alpm_pkg_t
**likely_pkg
);
299 static alpm_pkg_t
*load_pkg_for_entry(alpm_db_t
*db
, const char *entryname
,
300 const char **entry_filename
, alpm_pkg_t
*likely_pkg
)
302 char *pkgname
= NULL
, *pkgver
= NULL
;
303 unsigned long pkgname_hash
;
306 /* get package and db file names */
308 char *fname
= strrchr(entryname
, '/');
310 *entry_filename
= fname
+ 1;
312 *entry_filename
= NULL
;
315 if(_alpm_splitname(entryname
, &pkgname
, &pkgver
, &pkgname_hash
) != 0) {
316 _alpm_log(db
->handle
, ALPM_LOG_ERROR
,
317 _("invalid name for database entry '%s'\n"), entryname
);
321 if(likely_pkg
&& pkgname_hash
== likely_pkg
->name_hash
322 && strcmp(likely_pkg
->name
, pkgname
) == 0) {
325 pkg
= _alpm_pkghash_find(db
->pkgcache
, pkgname
);
328 pkg
= _alpm_pkg_new();
330 RET_ERR(db
->handle
, ALPM_ERR_MEMORY
, NULL
);
334 pkg
->version
= pkgver
;
335 pkg
->name_hash
= pkgname_hash
;
337 pkg
->origin
= PKG_FROM_SYNCDB
;
338 pkg
->origin_data
.db
= db
;
339 pkg
->ops
= &default_pkg_ops
;
340 pkg
->handle
= db
->handle
;
342 /* add to the collection */
343 _alpm_log(db
->handle
, ALPM_LOG_FUNCTION
, "adding '%s' to package cache for db '%s'\n",
344 pkg
->name
, db
->treename
);
345 db
->pkgcache
= _alpm_pkghash_add(db
->pkgcache
, pkg
);
354 /* This function doesn't work as well as one might think, as size of database
355 * entries varies considerably. Adding signatures nearly doubles the size of a
356 * single entry; deltas also can make for large variations in size. These
357 * current values are heavily influenced by Arch Linux; databases with no
358 * deltas and a single signature per package. */
359 static size_t estimate_package_count(struct stat
*st
, struct archive
*archive
)
363 switch(archive_compression(archive
)) {
364 case ARCHIVE_COMPRESSION_NONE
:
367 case ARCHIVE_COMPRESSION_GZIP
:
368 case ARCHIVE_COMPRESSION_COMPRESS
:
371 case ARCHIVE_COMPRESSION_BZIP2
:
374 case ARCHIVE_COMPRESSION_LZMA
:
375 case ARCHIVE_COMPRESSION_XZ
:
378 #ifdef ARCHIVE_COMPRESSION_UU
379 case ARCHIVE_COMPRESSION_UU
:
380 per_package
= 3015 * 4 / 3;
384 /* assume it is at least somewhat compressed */
387 return (size_t)((st
->st_size
/ per_package
) + 1);
390 static int sync_db_populate(alpm_db_t
*db
)
396 struct archive
*archive
;
397 struct archive_entry
*entry
;
398 alpm_pkg_t
*pkg
= NULL
;
400 if(db
->status
& DB_STATUS_INVALID
) {
401 RET_ERR(db
->handle
, ALPM_ERR_DB_INVALID
, -1);
403 if(db
->status
& DB_STATUS_MISSING
) {
404 RET_ERR(db
->handle
, ALPM_ERR_DB_NOT_FOUND
, -1);
406 dbpath
= _alpm_db_path(db
);
408 /* pm_errno set in _alpm_db_path() */
412 fd
= _alpm_open_archive(db
->handle
, dbpath
, &buf
,
413 &archive
, ALPM_ERR_DB_OPEN
);
417 est_count
= estimate_package_count(&buf
, archive
);
419 db
->pkgcache
= _alpm_pkghash_create(est_count
);
420 if(db
->pkgcache
== NULL
) {
421 db
->handle
->pm_errno
= ALPM_ERR_MEMORY
;
426 while(archive_read_next_header(archive
, &entry
) == ARCHIVE_OK
) {
427 mode_t mode
= archive_entry_mode(entry
);
431 /* we have desc, depends or deltas - parse it */
432 if(sync_db_read(db
, archive
, entry
, &pkg
) != 0) {
433 _alpm_log(db
->handle
, ALPM_LOG_ERROR
,
434 _("could not parse package description file '%s' from db '%s'\n"),
435 archive_entry_pathname(entry
), db
->treename
);
441 count
= alpm_list_count(db
->pkgcache
->list
);
443 db
->pkgcache
->list
= alpm_list_msort(db
->pkgcache
->list
,
444 (size_t)count
, _alpm_pkg_cmp
);
446 _alpm_log(db
->handle
, ALPM_LOG_DEBUG
,
447 "added %d packages to package cache for db '%s'\n",
448 count
, db
->treename
);
451 archive_read_finish(archive
);
458 #define READ_NEXT() do { \
459 if(_alpm_archive_fgets(archive, &buf) != ARCHIVE_OK) goto error; \
461 _alpm_strip_newline(line); \
464 #define READ_AND_STORE(f) do { \
466 STRDUP(f, line, goto error); \
469 #define READ_AND_STORE_ALL(f) do { \
471 if(_alpm_archive_fgets(archive, &buf) != ARCHIVE_OK) goto error; \
472 if(_alpm_strip_newline(buf.line) == 0) break; \
473 STRDUP(linedup, buf.line, goto error); \
474 f = alpm_list_add(f, linedup); \
475 } while(1) /* note the while(1) and not (0) */
477 #define READ_AND_SPLITDEP(f) do { \
478 if(_alpm_archive_fgets(archive, &buf) != ARCHIVE_OK) goto error; \
479 if(_alpm_strip_newline(buf.line) == 0) break; \
480 f = alpm_list_add(f, _alpm_splitdep(line)); \
481 } while(1) /* note the while(1) and not (0) */
483 static int sync_db_read(alpm_db_t
*db
, struct archive
*archive
,
484 struct archive_entry
*entry
, alpm_pkg_t
**likely_pkg
)
486 const char *entryname
, *filename
;
488 struct archive_read_buffer buf
;
490 entryname
= archive_entry_pathname(entry
);
491 if(entryname
== NULL
) {
492 _alpm_log(db
->handle
, ALPM_LOG_DEBUG
,
493 "invalid archive entry provided to _alpm_sync_db_read, skipping\n");
497 _alpm_log(db
->handle
, ALPM_LOG_FUNCTION
, "loading package data from archive entry %s\n",
500 memset(&buf
, 0, sizeof(buf
));
501 /* 512K for a line length seems reasonable */
502 buf
.max_line_size
= 512 * 1024;
504 pkg
= load_pkg_for_entry(db
, entryname
, &filename
, *likely_pkg
);
507 _alpm_log(db
->handle
, ALPM_LOG_DEBUG
,
508 "entry %s could not be loaded into %s sync database",
509 entryname
, db
->treename
);
513 if(strcmp(filename
, "desc") == 0 || strcmp(filename
, "depends") == 0
514 || strcmp(filename
, "deltas") == 0) {
516 while((ret
= _alpm_archive_fgets(archive
, &buf
)) == ARCHIVE_OK
) {
517 char *line
= buf
.line
;
518 if(_alpm_strip_newline(line
) == 0) {
519 /* length of stripped line was zero */
523 if(strcmp(line
, "%NAME%") == 0) {
525 if(strcmp(line
, pkg
->name
) != 0) {
526 _alpm_log(db
->handle
, ALPM_LOG_ERROR
, _("%s database is inconsistent: name "
527 "mismatch on package %s\n"), db
->treename
, pkg
->name
);
529 } else if(strcmp(line
, "%VERSION%") == 0) {
531 if(strcmp(line
, pkg
->version
) != 0) {
532 _alpm_log(db
->handle
, ALPM_LOG_ERROR
, _("%s database is inconsistent: version "
533 "mismatch on package %s\n"), db
->treename
, pkg
->name
);
535 } else if(strcmp(line
, "%FILENAME%") == 0) {
536 READ_AND_STORE(pkg
->filename
);
537 } else if(strcmp(line
, "%DESC%") == 0) {
538 READ_AND_STORE(pkg
->desc
);
539 } else if(strcmp(line
, "%GROUPS%") == 0) {
540 READ_AND_STORE_ALL(pkg
->groups
);
541 } else if(strcmp(line
, "%URL%") == 0) {
542 READ_AND_STORE(pkg
->url
);
543 } else if(strcmp(line
, "%LICENSE%") == 0) {
544 READ_AND_STORE_ALL(pkg
->licenses
);
545 } else if(strcmp(line
, "%ARCH%") == 0) {
546 READ_AND_STORE(pkg
->arch
);
547 } else if(strcmp(line
, "%BUILDDATE%") == 0) {
549 pkg
->builddate
= _alpm_parsedate(line
);
550 } else if(strcmp(line
, "%PACKAGER%") == 0) {
551 READ_AND_STORE(pkg
->packager
);
552 } else if(strcmp(line
, "%CSIZE%") == 0) {
554 pkg
->size
= _alpm_strtoofft(line
);
555 } else if(strcmp(line
, "%ISIZE%") == 0) {
557 pkg
->isize
= _alpm_strtoofft(line
);
558 } else if(strcmp(line
, "%MD5SUM%") == 0) {
559 READ_AND_STORE(pkg
->md5sum
);
560 } else if(strcmp(line
, "%SHA256SUM%") == 0) {
561 READ_AND_STORE(pkg
->sha256sum
);
562 } else if(strcmp(line
, "%PGPSIG%") == 0) {
563 READ_AND_STORE(pkg
->base64_sig
);
564 } else if(strcmp(line
, "%REPLACES%") == 0) {
565 READ_AND_SPLITDEP(pkg
->replaces
);
566 } else if(strcmp(line
, "%DEPENDS%") == 0) {
567 READ_AND_SPLITDEP(pkg
->depends
);
568 } else if(strcmp(line
, "%OPTDEPENDS%") == 0) {
569 READ_AND_SPLITDEP(pkg
->optdepends
);
570 } else if(strcmp(line
, "%CONFLICTS%") == 0) {
571 READ_AND_SPLITDEP(pkg
->conflicts
);
572 } else if(strcmp(line
, "%PROVIDES%") == 0) {
573 READ_AND_SPLITDEP(pkg
->provides
);
574 } else if(strcmp(line
, "%DELTAS%") == 0) {
575 /* Different than the rest because of the _alpm_delta_parse call. */
578 if(strlen(line
) == 0) break;
579 pkg
->deltas
= alpm_list_add(pkg
->deltas
,
580 _alpm_delta_parse(db
->handle
, line
));
584 if(ret
!= ARCHIVE_EOF
) {
588 } else if(strcmp(filename
, "files") == 0) {
589 /* currently do nothing with this file */
591 /* unknown database file */
592 _alpm_log(db
->handle
, ALPM_LOG_DEBUG
, "unknown database file: %s\n", filename
);
598 _alpm_log(db
->handle
, ALPM_LOG_DEBUG
, "error parsing database file: %s\n", filename
);
602 struct db_operations sync_db_ops
= {
603 .validate
= sync_db_validate
,
604 .populate
= sync_db_populate
,
605 .unregister
= _alpm_db_unregister
,
608 alpm_db_t
*_alpm_db_register_sync(alpm_handle_t
*handle
, const char *treename
,
609 alpm_siglevel_t level
)
613 _alpm_log(handle
, ALPM_LOG_DEBUG
, "registering sync database '%s'\n", treename
);
615 #ifndef HAVE_LIBGPGME
616 if(level
!= 0 && level
!= ALPM_SIG_USE_DEFAULT
) {
617 RET_ERR(handle
, ALPM_ERR_WRONG_ARGS
, NULL
);
621 db
= _alpm_db_new(treename
, 0);
623 RET_ERR(handle
, ALPM_ERR_DB_CREATE
, NULL
);
625 db
->ops
= &sync_db_ops
;
627 db
->siglevel
= level
;
629 sync_db_validate(db
);
631 handle
->dbs_sync
= alpm_list_add(handle
->dbs_sync
, db
);
635 /* vim: set ts=2 sw=2 noet: */