2 * Copyright (c) 2019-2020 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * This code uses concepts and configuration based on 'synth', by
8 * John R. Marino <draco@marino.st>, which was written in ada.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in
18 * the documentation and/or other materials provided with the
20 * 3. Neither the name of The DragonFly Project nor the names of its
21 * contributors may be used to endorse or promote products derived
22 * from this software without specific, prior written permission.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
38 #include <openssl/md5.h>
40 typedef struct pinfo
{
47 static void removePackagesMetaRecurse(pkg_t
*pkg
);
48 static int pinfocmp(const void *s1
, const void *s2
);
49 static void scanit(const char *path
, const char *subpath
,
50 int *countp
, pinfo_t
***list_tailp
,
52 pinfo_t
*pinfofind(pinfo_t
**ary
, int count
, char *spath
);
53 static void childRebuildRepo(bulk_t
*bulk
);
54 static void scandeletenew(const char *path
);
56 static void rebuildTerminateSignal(int signo
);
57 static char *md5lkfile(char *rpath
, int which
);
58 static int lkdircount(char *buf
);
60 static char *RebuildRemovePath
;
63 DoRebuildRepo(int ask
)
72 if (askyn("Rebuild the repository? ") == 0)
77 * Scan the repository for temporary .new files and delete them.
79 scandeletenew(RepositoryPath
);
82 * Generate temporary file
84 snprintf(tpath
, sizeof(tpath
), "/tmp/meta.XXXXXXXX.conf");
86 signal(SIGTERM
, rebuildTerminateSignal
);
87 signal(SIGINT
, rebuildTerminateSignal
);
88 signal(SIGHUP
, rebuildTerminateSignal
);
90 RebuildRemovePath
= tpath
;
93 fd
= mkostemps(tpath
, 5, 0);
95 dfatal_errno("Cannot create %s", tpath
);
97 fprintf(fp
, "version = %d;\n", MetaVersion
);
98 fprintf(fp
, "packing_format = \"%s\";\n", sufx
+ 1);
102 * Run the operation under our bulk infrastructure to
103 * get the correct environment.
105 initbulk(childRebuildRepo
, 1);
106 queuebulk(tpath
, NULL
, NULL
, NULL
);
110 printf("Rebuild succeeded\n");
112 printf("Rebuild failed\n");
119 repackage(const char *basepath
, const char *basefile
,
120 const char *decomp_suffix
, const char *comp_suffix
,
121 const char *decomp
, const char *comp
);
124 childRebuildRepo(bulk_t
*bulk
)
130 const char *cav
[MAXCAC
];
133 int repackage_mode
= 0;
136 * We have to use the pkg-static that we built as part of the
137 * build process to rebuild the repo because the system pkg might
138 * not be compatible with the repo format changes made in 1.17.
140 asprintf(&pkg_path
, "%s/Template/usr/local/sbin/pkg-static", BuildBase
);
143 cav
[cac
++] = pkg_path
;
146 cav
[cac
++] = bulk
->s1
;
148 cav
[cac
++] = PackagesPath
;
151 * The yaml needs to generate paths relative to PackagePath
153 if (strncmp(PackagesPath
, RepositoryPath
, strlen(PackagesPath
)) == 0)
154 cav
[cac
++] = PackagesPath
;
156 cav
[cac
++] = RepositoryPath
;
158 printf("pkg repo -m %s -o %s %s\n", bulk
->s1
, cav
[cac
-2], cav
[cac
-1]);
160 fp
= dexec_open(NULL
, cav
, cac
, &pid
, NULL
, 1, 0);
161 while ((ptr
= fgetln(fp
, &len
)) != NULL
)
162 fwrite(ptr
, 1, len
, stdout
);
163 if (dexec_close(fp
, pid
) == 0)
164 bulk
->r1
= strdup("");
167 * Check package version. Pkg version 1.12 and later generates
168 * the proper repo compression format. Prior to that version
169 * the repo directive always generated .txz files.
172 cav
[cac
++] = pkg_path
;
174 fp
= dexec_open(NULL
, cav
, cac
, &pid
, NULL
, 1, 0);
175 if ((ptr
= fgetln(fp
, &len
)) != NULL
&& len
> 0) {
180 if (sscanf(ptr
, "%d.%d", &v1
, &v2
) == 2) {
181 printf("pkg repo - pkg version: %d.%d\n", v1
, v2
);
182 if (v1
> 1 || (v1
== 1 && v2
>= 12))
186 dexec_close(fp
, pid
);
189 * Repackage the .txz files created by pkg repo if necessary
191 if (repackage_mode
== 0 && strcmp(UsePkgSufx
, ".txz") != 0) {
195 printf("pkg repo - recompressing digests and packagesite\n");
197 if (strcmp(UsePkgSufx
, ".tar") == 0) {
200 } else if (strcmp(UsePkgSufx
, ".tgz") == 0) {
203 } else if (strcmp(UsePkgSufx
, ".tbz") == 0) {
206 } else if (strcmp(UsePkgSufx
, ".tzst") == 0) {
210 dfatal("recompressing as %s not supported",
215 repackage(PackagesPath
, "digests",
218 repackage(PackagesPath
, "packagesite",
221 } else if (repackage_mode
== 1 && strcmp(UsePkgSufx
, ".txz") != 0) {
225 printf("pkg repo - recompressing meta\n");
227 if (strcmp(UsePkgSufx
, ".tar") == 0) {
230 } else if (strcmp(UsePkgSufx
, ".tgz") == 0) {
233 } else if (strcmp(UsePkgSufx
, ".tbz") == 0) {
236 } else if (strcmp(UsePkgSufx
, ".tzst") == 0) {
240 dfatal("recompressing from %s not supported",
245 repackage(PackagesPath
, "meta",
254 repackage(const char *basepath
, const char *basefile
,
255 const char *decomp_suffix
, const char *comp_suffix
,
256 const char *decomp
, const char *comp
)
260 asprintf(&buf
, "%s < %s/%s%s | %s > %s/%s%s",
261 decomp
, basepath
, basefile
, decomp_suffix
,
262 comp
, basepath
, basefile
, comp_suffix
);
263 if (system(buf
) != 0) {
264 dfatal("command failed: %s", buf
);
270 DoUpgradePkgs(pkg_t
*pkgs __unused
, int ask __unused
)
275 PurgeDistfiles(pkg_t
*pkgs
)
287 printf("Scanning distfiles... ");
292 scanit(DistFilesPath
, NULL
, &count
, &list_tail
, 0);
293 printf("Checking %d distfiles\n", count
);
296 ary
= calloc(count
, sizeof(pinfo_t
*));
297 for (i
= 0; i
< count
; ++i
) {
301 ddassert(list
== NULL
);
302 qsort(ary
, count
, sizeof(pinfo_t
*), pinfocmp
);
304 for (; pkgs
; pkgs
= pkgs
->bnext
) {
305 if (pkgs
->distfiles
== NULL
|| pkgs
->distfiles
[0] == 0)
307 ddprintf(0, "distfiles %s\n", pkgs
->distfiles
);
308 dstr
= strtok(pkgs
->distfiles
, " \t");
314 if (pkgs
->distsubdir
) {
315 asprintf(&buf
, "%s/%s",
316 pkgs
->distsubdir
, dstr
);
320 item
= pinfofind(ary
, count
, buf
);
323 if (item
&& item
->inlocks
== 0) {
325 * Look for the lock file
329 scount
= lkdircount(buf
);
331 for (i
= 0; i
<= scount
; ++i
) {
332 item
= pinfofind(ary
, count
,
340 * Cleanup and iterate
346 if (strrchr(dstr
, ':') == NULL
)
348 *strrchr(dstr
, ':') = 0;
350 dstr
= strtok(NULL
, " \t");
355 for (i
= 0; i
< count
; ++i
) {
357 if (item
->foundit
== 0) {
362 printf("No obsolete source files out of %d found\n", count
);
363 } else if (askyn("Delete %d of %d items? ", delcount
, count
)) {
364 printf("Deleting %d/%d obsolete source distfiles\n",
366 for (i
= 0; i
< count
; ++i
) {
368 if (item
->foundit
== 0) {
369 asprintf(&buf
, "%s/%s",
370 DistFilesPath
, item
->spath
);
372 printf("Cannot delete %s\n", buf
);
374 printf("Deleted %s\n", item
->spath
);
385 RemovePackages(pkg_t
*list
)
390 for (scan
= list
; scan
; scan
= scan
->bnext
) {
391 if ((scan
->flags
& PKGF_MANUALSEL
) == 0)
394 scan
->flags
&= ~PKGF_PACKAGED
;
395 scan
->pkgfile_size
= 0;
396 asprintf(&path
, "%s/%s", RepositoryPath
, scan
->pkgfile
);
397 if (remove(path
) == 0)
398 printf("Removed: %s\n", path
);
401 if (scan
->pkgfile
== NULL
||
402 (scan
->flags
& (PKGF_DUMMY
| PKGF_META
))) {
403 removePackagesMetaRecurse(scan
);
409 removePackagesMetaRecurse(pkg_t
*pkg
)
415 PKGLIST_FOREACH(link
, &pkg
->idepon_list
) {
419 if (scan
->pkgfile
== NULL
||
420 (scan
->flags
& (PKGF_DUMMY
| PKGF_META
))) {
421 removePackagesMetaRecurse(scan
);
424 scan
->flags
&= ~PKGF_PACKAGED
;
425 scan
->pkgfile_size
= 0;
427 asprintf(&path
, "%s/%s", RepositoryPath
, scan
->pkgfile
);
428 if (remove(path
) == 0)
429 printf("Removed: %s\n", path
);
435 pinfocmp(const void *s1
, const void *s2
)
437 const pinfo_t
*item1
= *(const pinfo_t
*const*)s1
;
438 const pinfo_t
*item2
= *(const pinfo_t
*const*)s2
;
440 return (strcmp(item1
->spath
, item2
->spath
));
444 pinfofind(pinfo_t
**ary
, int count
, char *spath
)
457 res
= strcmp(spath
, item
->spath
);
470 scanit(const char *path
, const char *subpath
,
471 int *countp
, pinfo_t
***list_tailp
,
481 if ((dir
= opendir(path
)) != NULL
) {
482 while ((den
= readdir(dir
)) != NULL
) {
483 if (den
->d_namlen
== 1 && den
->d_name
[0] == '.')
485 if (den
->d_namlen
== 2 && den
->d_name
[0] == '.' &&
486 den
->d_name
[1] == '.')
488 asprintf(&npath
, "%s/%s", path
, den
->d_name
);
489 if (lstat(npath
, &st
) < 0) {
493 if (S_ISDIR(st
.st_mode
)) {
497 (strcmp(den
->d_name
, ".locks") == 0);
500 asprintf(&spath
, "%s/%s",
501 subpath
, den
->d_name
);
502 scanit(npath
, spath
, countp
,
503 list_tailp
, sublocks
);
506 scanit(npath
, den
->d_name
, countp
,
507 list_tailp
, sublocks
);
509 } else if (S_ISREG(st
.st_mode
)) {
510 item
= calloc(1, sizeof(*item
));
512 asprintf(&item
->spath
, "%s/%s",
513 subpath
, den
->d_name
);
515 item
->spath
= strdup(den
->d_name
);
517 item
->inlocks
= inlocks
;
520 *list_tailp
= &item
->next
;
522 ddprintf(0, "scan %s\n", item
->spath
);
531 * This removes any .new files left over in the repo. These can wind
532 * being left around when dsynth is killed.
535 scandeletenew(const char *path
)
542 if ((dir
= opendir(path
)) == NULL
)
543 dfatal_errno("Cannot scan directory %s", path
);
544 while ((den
= readdir(dir
)) != NULL
) {
545 if ((ptr
= strrchr(den
->d_name
, '.')) != NULL
&&
546 strcmp(ptr
, ".new") == 0) {
547 asprintf(&buf
, "%s/%s", path
, den
->d_name
);
549 dfatal_errno("remove: Garbage %s\n", buf
);
550 printf("Deleted Garbage %s\n", buf
);
558 rebuildTerminateSignal(int signo __unused
)
560 if (RebuildRemovePath
)
561 remove(RebuildRemovePath
);
567 * There will be a .locks sub-directory in /usr/distfiles and also
568 * in each sub-directory underneath it containing the MD5 sums for
569 * the files in that subdirectory.
571 * This is a bit of a mess. Sometimes the .locks/ for a subdirectory
572 * are in parentdir/.locks and not parentdir/subdir/.locks. The invocation
573 * of do-fetch can be a bit messy so we look for a .locks subdir everywhere.
575 * The /usr/dports/Mk/Scripts/do-fetch.sh script uses 'echo blah | md5',
576 * so we have to add a newline to the buffer being md5'd.
578 * The pass-in rpath is relative to the distfiles base.
581 md5lkfile(char *rpath
, int which_slash
)
583 static char mstr
[128];
584 static char lkfile
[128];
585 uint8_t digest
[MD5_DIGEST_LENGTH
];
590 for (i
= 0; i
< which_slash
; ++i
) {
591 while (rpath
[bplen
] && rpath
[bplen
] != '/')
596 snprintf(mstr
, sizeof(mstr
), "%s\n", rpath
+ bplen
);
597 MD5(mstr
, strlen(mstr
), digest
);
599 snprintf(lkfile
, sizeof(lkfile
),
601 "%02x%02x%02x%02x%02x%02x%02x%02x"
602 "%02x%02x%02x%02x%02x%02x%02x%02x"
605 digest
[0], digest
[1], digest
[2], digest
[3],
606 digest
[4], digest
[5], digest
[6], digest
[7],
607 digest
[8], digest
[9], digest
[10], digest
[11],
608 digest
[12], digest
[13], digest
[14], digest
[15]);
614 lkdircount(char *buf
)
620 for (i
= 0; buf
[i
]; ++i
) {