?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Correctly parse %DELTAS% entries in sync DB
[pacman-ng.git] / lib / libalpm / be_sync.c
blobca76105fe7036e306065c15fe097f48e392a2e03
1 /*
2 * be_sync.c
3 *
4 * Copyright (c) 2006-2011 Pacman Development Team <pacman-dev@archlinux.org>
5 * Copyright (c) 2002-2006 by Judd Vinet <jvinet@zeroflux.org>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21 #include "config.h"
22
23 #include <errno.h>
24 #include <limits.h>
25
26 /* libarchive */
27 #include <archive.h>
28 #include <archive_entry.h>
29
30 /* libalpm */
31 #include "util.h"
32 #include "log.h"
33 #include "alpm.h"
34 #include "alpm_list.h"
35 #include "package.h"
36 #include "handle.h"
37 #include "delta.h"
38 #include "deps.h"
39 #include "dload.h"
40
41 /** Update a package database
42 *
43 * An update of the package database \a db will be attempted. Unless
44 * \a force is true, the update will only be performed if the remote
45 * database was modified since the last update.
46 *
47 * A transaction is necessary for this operation, in order to obtain a
48 * database lock. During this transaction the front-end will be informed
49 * of the download progress of the database via the download callback.
50 *
51 * Example:
52 * @code
53 * alpm_list_t *syncs = alpm_option_get_syncdbs();
54 * if(alpm_trans_init(0, NULL, NULL, NULL) == 0) {
55 * for(i = syncs; i; i = alpm_list_next(i)) {
56 * pmdb_t *db = alpm_list_getdata(i);
57 * result = alpm_db_update(0, db);
58 * alpm_trans_release();
59 *
60 * if(result < 0) {
61 * printf("Unable to update database: %s\n", alpm_strerrorlast());
62 * } else if(result == 1) {
63 * printf("Database already up to date\n");
64 * } else {
65 * printf("Database updated\n");
66 * }
67 * }
68 * }
69 * @endcode
70 *
71 * @ingroup alpm_databases
72 * @note After a successful update, the \link alpm_db_get_pkgcache()
73 * package cache \endlink will be invalidated
74 * @param force if true, then forces the update, otherwise update only in case
75 * the database isn't up to date
76 * @param db pointer to the package database to update
77 * @return 0 on success, -1 on error (pm_errno is set accordingly), 1 if up to
78 * to date
79 */
80 int SYMEXPORT alpm_db_update(int force, pmdb_t *db)
81 {
82 char *dbfile, *syncpath;
83 const char *dbpath;
84 struct stat buf;
85 size_t len;
86 int ret;
87
88 ALPM_LOG_FUNC;
89
90 /* Sanity checks */
91 ASSERT(handle != NULL, RET_ERR(PM_ERR_HANDLE_NULL, -1));
92 ASSERT(db != NULL && db != handle->db_local, RET_ERR(PM_ERR_WRONG_ARGS, -1));
93
94 if(!alpm_list_find_ptr(handle->dbs_sync, db)) {
95 RET_ERR(PM_ERR_DB_NOT_FOUND, -1);
96 }
97
98 len = strlen(db->treename) + 4;
99 MALLOC(dbfile, len, RET_ERR(PM_ERR_MEMORY, -1));
100 sprintf(dbfile, "%s.db", db->treename);
101
102 dbpath = alpm_option_get_dbpath();
103 len = strlen(dbpath) + 6;
104 MALLOC(syncpath, len, RET_ERR(PM_ERR_MEMORY, -1));
105 sprintf(syncpath, "%s%s", dbpath, "sync/");
106
107 if(stat(syncpath, &buf) != 0) {
108 _alpm_log(PM_LOG_DEBUG, "database dir '%s' does not exist, creating it\n",
109 syncpath);
110 if(_alpm_makepath(syncpath) != 0) {
111 free(dbfile);
112 free(syncpath);
113 RET_ERR(PM_ERR_SYSTEM, -1);
114 }
115 } else if(!S_ISDIR(buf.st_mode)) {
116 _alpm_log(PM_LOG_WARNING, _("removing invalid file: %s\n"), syncpath);
117 if(unlink(syncpath) != 0 || _alpm_makepath(syncpath) != 0) {
118 free(dbfile);
119 free(syncpath);
120 RET_ERR(PM_ERR_SYSTEM, -1);
121 }
122 }
123
124 ret = _alpm_download_single_file(dbfile, db->servers, syncpath, force);
125 free(dbfile);
126 free(syncpath);
127
128 if(ret == 1) {
129 /* files match, do nothing */
130 pm_errno = 0;
131 return(1);
132 } else if(ret == -1) {
133 /* pm_errno was set by the download code */
134 _alpm_log(PM_LOG_DEBUG, "failed to sync db: %s\n", alpm_strerrorlast());
135 return(-1);
136 }
137
138 /* Cache needs to be rebuilt */
139 _alpm_db_free_pkgcache(db);
140
141 return(0);
142 }
143
144 /* Forward decl so I don't reorganize the whole file right now */
145 static int sync_db_read(pmdb_t *db, struct archive *archive,
146 struct archive_entry *entry, pmpkg_t *likely_pkg);
147
148 /*
149 * This is the data table used to generate the estimating function below.
150 * "Weighted Avg" means averaging the bottom table values; thus each repo, big
151 * or small, will have equal influence. "Unweighted Avg" means averaging the
152 * sums of the top table columns, thus each package has equal influence. The
153 * final values are calculated by (surprise) averaging the averages, because
154 * why the hell not.
155 *
156 * Database Pkgs tar bz2 gz xz
157 * community 2096 5294080 256391 421227 301296
158 * core 180 460800 25257 36850 29356
159 * extra 2606 6635520 294647 470818 339392
160 * multilib 126 327680 16120 23261 18732
161 * testing 76 204800 10902 14348 12100
162 *
163 * Bytes Per Package
164 * community 2096 2525.80 122.32 200.97 143.75
165 * core 180 2560.00 140.32 204.72 163.09
166 * extra 2606 2546.25 113.06 180.67 130.23
167 * multilib 126 2600.63 127.94 184.61 148.67
168 * testing 76 2694.74 143.45 188.79 159.21
169
170 * Weighted Avg 2585.48 129.42 191.95 148.99
171 * Unweighted Avg 2543.39 118.74 190.16 137.93
172 * Average of Avgs 2564.44 124.08 191.06 143.46
173 */
174 static size_t estimate_package_count(struct stat *st, struct archive *archive)
175 {
176 unsigned int per_package;
177
178 switch(archive_compression(archive)) {
179 case ARCHIVE_COMPRESSION_NONE:
180 per_package = 2564;
181 break;
182 case ARCHIVE_COMPRESSION_GZIP:
183 per_package = 191;
184 break;
185 case ARCHIVE_COMPRESSION_BZIP2:
186 per_package = 124;
187 break;
188 case ARCHIVE_COMPRESSION_COMPRESS:
189 per_package = 193;
190 break;
191 case ARCHIVE_COMPRESSION_LZMA:
192 case ARCHIVE_COMPRESSION_XZ:
193 per_package = 143;
194 break;
195 case ARCHIVE_COMPRESSION_UU:
196 per_package = 3543;
197 break;
198 default:
199 /* assume it is at least somewhat compressed */
200 per_package = 200;
201 }
202 return((size_t)(st->st_size / per_package) + 1);
203 }
204
205 static int sync_db_populate(pmdb_t *db)
206 {
207 size_t est_count;
208 int count = 0;
209 struct stat buf;
210 struct archive *archive;
211 struct archive_entry *entry;
212 pmpkg_t *pkg = NULL;
213
214 ALPM_LOG_FUNC;
215
216 ASSERT(db != NULL, RET_ERR(PM_ERR_DB_NULL, -1));
217
218 if((archive = archive_read_new()) == NULL)
219 RET_ERR(PM_ERR_LIBARCHIVE, 1);
220
221 archive_read_support_compression_all(archive);
222 archive_read_support_format_all(archive);
223
224 if(archive_read_open_filename(archive, _alpm_db_path(db),
225 ARCHIVE_DEFAULT_BYTES_PER_BLOCK) != ARCHIVE_OK) {
226 _alpm_log(PM_LOG_ERROR, _("could not open file %s: %s\n"), _alpm_db_path(db),
227 archive_error_string(archive));
228 archive_read_finish(archive);
229 RET_ERR(PM_ERR_DB_OPEN, 1);
230 }
231 if(stat(_alpm_db_path(db), &buf) != 0) {
232 RET_ERR(PM_ERR_DB_OPEN, 1);
233 }
234 est_count = estimate_package_count(&buf, archive);
235
236 /* initialize hash at 66% full */
237 db->pkgcache = _alpm_pkghash_create(est_count * 3 / 2);
238 if(db->pkgcache == NULL) {
239 RET_ERR(PM_ERR_MEMORY, -1);
240 }
241
242 while(archive_read_next_header(archive, &entry) == ARCHIVE_OK) {
243 const struct stat *st;
244
245 st = archive_entry_stat(entry);
246
247 if(S_ISDIR(st->st_mode)) {
248 const char *name;
249
250 pkg = _alpm_pkg_new();
251 if(pkg == NULL) {
252 archive_read_finish(archive);
253 RET_ERR(PM_ERR_MEMORY, -1);
254 }
255
256 name = archive_entry_pathname(entry);
257
258 if(_alpm_splitname(name, pkg) != 0) {
259 _alpm_log(PM_LOG_ERROR, _("invalid name for database entry '%s'\n"),
260 name);
261 _alpm_pkg_free(pkg);
262 continue;
263 }
264
265 /* duplicated database entries are not allowed */
266 if(_alpm_pkghash_find(db->pkgcache, pkg->name)) {
267 _alpm_log(PM_LOG_ERROR, _("duplicated database entry '%s'\n"), pkg->name);
268 _alpm_pkg_free(pkg);
269 continue;
270 }
271
272 pkg->origin = PKG_FROM_SYNCDB;
273 pkg->ops = &default_pkg_ops;
274 pkg->origin_data.db = db;
275
276 /* add to the collection */
277 _alpm_log(PM_LOG_FUNCTION, "adding '%s' to package cache for db '%s'\n",
278 pkg->name, db->treename);
279 db->pkgcache = _alpm_pkghash_add(db->pkgcache, pkg);
280 count++;
281 } else {
282 /* we have desc, depends or deltas - parse it */
283 sync_db_read(db, archive, entry, pkg);
284 }
285 }
286
287 if(count > 0) {
288 db->pkgcache->list = alpm_list_msort(db->pkgcache->list, (size_t)count, _alpm_pkg_cmp);
289 }
290 archive_read_finish(archive);
291
292 return(count);
293 }
294
295 #define READ_NEXT(s) do { \
296 if(_alpm_archive_fgets(archive, &buf) != ARCHIVE_OK) goto error; \
297 s = _alpm_strtrim(buf.line); \
298 } while(0)
299
300 #define READ_AND_STORE(f) do { \
301 READ_NEXT(line); \
302 STRDUP(f, line, goto error); \
303 } while(0)
304
305 #define READ_AND_STORE_ALL(f) do { \
306 char *linedup; \
307 READ_NEXT(line); \
308 if(strlen(line) == 0) break; \
309 STRDUP(linedup, line, goto error); \
310 f = alpm_list_add(f, linedup); \
311 } while(1) /* note the while(1) and not (0) */
312
313 static int sync_db_read(pmdb_t *db, struct archive *archive,
314 struct archive_entry *entry, pmpkg_t *likely_pkg)
315 {
316 const char *entryname = NULL, *filename;
317 char *pkgname, *p, *q;
318 pmpkg_t *pkg;
319 struct archive_read_buffer buf;
320
321 ALPM_LOG_FUNC;
322
323 if(db == NULL) {
324 RET_ERR(PM_ERR_DB_NULL, -1);
325 }
326
327 if(entry != NULL) {
328 entryname = archive_entry_pathname(entry);
329 }
330 if(entryname == NULL) {
331 _alpm_log(PM_LOG_DEBUG, "invalid archive entry provided to _alpm_sync_db_read, skipping\n");
332 return(-1);
333 }
334
335 _alpm_log(PM_LOG_FUNCTION, "loading package data from archive entry %s\n",
336 entryname);
337
338 memset(&buf, 0, sizeof(buf));
339 /* 512K for a line length seems reasonable */
340 buf.max_line_size = 512 * 1024;
341
342 /* get package and db file names */
343 STRDUP(pkgname, entryname, RET_ERR(PM_ERR_MEMORY, -1));
344 p = pkgname + strlen(pkgname);
345 for(q = --p; *q && *q != '/'; q--);
346 filename = q + 1;
347 for(p = --q; *p && *p != '-'; p--);
348 for(q = --p; *q && *q != '-'; q--);
349 *q = '\0';
350
351 /* package is already in db due to parsing of directory name */
352 if(likely_pkg && strcmp(likely_pkg->name, pkgname) == 0) {
353 pkg = likely_pkg;
354 } else {
355 if(db->pkgcache == NULL) {
356 RET_ERR(PM_ERR_MEMORY, -1);
357 }
358 pkg = _alpm_pkghash_find(db->pkgcache, pkgname);
359 }
360 if(pkg == NULL) {
361 _alpm_log(PM_LOG_DEBUG, "package %s not found in %s sync database",
362 pkgname, db->treename);
363 return(-1);
364 }
365
366 if(strcmp(filename, "desc") == 0 || strcmp(filename, "depends") == 0
367 || strcmp(filename, "deltas") == 0) {
368 while(_alpm_archive_fgets(archive, &buf) == ARCHIVE_OK) {
369 char *line = _alpm_strtrim(buf.line);
370
371 if(strcmp(line, "%NAME%") == 0) {
372 READ_NEXT(line);
373 if(strcmp(line, pkg->name) != 0) {
374 _alpm_log(PM_LOG_ERROR, _("%s database is inconsistent: name "
375 "mismatch on package %s\n"), db->treename, pkg->name);
376 }
377 } else if(strcmp(line, "%VERSION%") == 0) {
378 READ_NEXT(line);
379 if(strcmp(line, pkg->version) != 0) {
380 _alpm_log(PM_LOG_ERROR, _("%s database is inconsistent: version "
381 "mismatch on package %s\n"), db->treename, pkg->name);
382 }
383 } else if(strcmp(line, "%FILENAME%") == 0) {
384 READ_AND_STORE(pkg->filename);
385 } else if(strcmp(line, "%DESC%") == 0) {
386 READ_AND_STORE(pkg->desc);
387 } else if(strcmp(line, "%GROUPS%") == 0) {
388 READ_AND_STORE_ALL(pkg->groups);
389 } else if(strcmp(line, "%URL%") == 0) {
390 READ_AND_STORE(pkg->url);
391 } else if(strcmp(line, "%LICENSE%") == 0) {
392 READ_AND_STORE_ALL(pkg->licenses);
393 } else if(strcmp(line, "%ARCH%") == 0) {
394 READ_AND_STORE(pkg->arch);
395 } else if(strcmp(line, "%BUILDDATE%") == 0) {
396 READ_NEXT(line);
397 pkg->builddate = _alpm_parsedate(line);
398 } else if(strcmp(line, "%PACKAGER%") == 0) {
399 READ_AND_STORE(pkg->packager);
400 } else if(strcmp(line, "%CSIZE%") == 0) {
401 /* Note: the CSIZE and SIZE fields both share the "size" field in the
402 * pkginfo_t struct. This can be done b/c CSIZE is currently only used
403 * in sync databases, and SIZE is only used in local databases.
404 */
405 READ_NEXT(line);
406 pkg->size = atol(line);
407 /* also store this value to isize if isize is unset */
408 if(pkg->isize == 0) {
409 pkg->isize = pkg->size;
410 }
411 } else if(strcmp(line, "%ISIZE%") == 0) {
412 READ_NEXT(line);
413 pkg->isize = atol(line);
414 } else if(strcmp(line, "%MD5SUM%") == 0) {
415 READ_AND_STORE(pkg->md5sum);
416 } else if(strcmp(line, "%REPLACES%") == 0) {
417 READ_AND_STORE_ALL(pkg->replaces);
418 } else if(strcmp(line, "%DEPENDS%") == 0) {
419 /* Different than the rest because of the _alpm_splitdep call. */
420 while(1) {
421 READ_NEXT(line);
422 if(strlen(line) == 0) break;
423 pkg->depends = alpm_list_add(pkg->depends, _alpm_splitdep(line));
424 }
425 } else if(strcmp(line, "%OPTDEPENDS%") == 0) {
426 READ_AND_STORE_ALL(pkg->optdepends);
427 } else if(strcmp(line, "%CONFLICTS%") == 0) {
428 READ_AND_STORE_ALL(pkg->conflicts);
429 } else if(strcmp(line, "%PROVIDES%") == 0) {
430 READ_AND_STORE_ALL(pkg->provides);
431 } else if(strcmp(line, "%DELTAS%") == 0) {
432 /* Different than the rest because of the _alpm_delta_parse call. */
433 while(1) {
434 READ_NEXT(line);
435 if(strlen(line) == 0) break;
436 pkg->deltas = alpm_list_add(pkg->deltas, _alpm_delta_parse(line));
437 }
438 }
439 }
440 } else if(strcmp(filename, "files") == 0) {
441 /* currently do nothing with this file */
442 } else {
443 /* unknown database file */
444 _alpm_log(PM_LOG_DEBUG, "unknown database file: %s\n", filename);
445 }
446
447 error:
448 FREE(pkgname);
449 /* TODO: return 0 always? */
450 return(0);
451 }
452
453 static int sync_db_version(pmdb_t *db)
454 {
455 return(2);
456 }
457
458 struct db_operations sync_db_ops = {
459 .populate = sync_db_populate,
460 .unregister = _alpm_db_unregister,
461 .version = sync_db_version,
462 };
463
464 pmdb_t *_alpm_db_register_sync(const char *treename)
465 {
466 pmdb_t *db;
467 alpm_list_t *i;
468
469 ALPM_LOG_FUNC;
470
471 for(i = handle->dbs_sync; i; i = i->next) {
472 pmdb_t *sdb = i->data;
473 if(strcmp(treename, sdb->treename) == 0) {
474 _alpm_log(PM_LOG_DEBUG, "attempt to re-register the '%s' database, using existing\n", sdb->treename);
475 return sdb;
476 }
477 }
478
479 _alpm_log(PM_LOG_DEBUG, "registering sync database '%s'\n", treename);
480
481 db = _alpm_db_new(treename, 0);
482 if(db == NULL) {
483 RET_ERR(PM_ERR_DB_CREATE, NULL);
484 }
485 db->ops = &sync_db_ops;
486
487 handle->dbs_sync = alpm_list_add(handle->dbs_sync, db);
488 return(db);
489 }
490
491
492 /* vim: set ts=2 sw=2 noet: */
ngaba's pacman repo