2 * Copyright (c) 2019 The DragonFly Project. All rights reserved.
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
7 * This code uses concepts and configuration based on 'synth', by
8 * John R. Marino <draco@marino.st>, which was written in ada.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in
18 * the documentation and/or other materials provided with the
20 * 3. Neither the name of The DragonFly Project nor the names of its
21 * contributors may be used to endorse or promote products derived
22 * from this software without specific, prior written permission.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
40 static int CheckAddReExec(int lkfd
);
41 static void DoAddReExec(int lkfd
, int ac
, char **oldav
);
42 static void DoInit(void);
43 static void usage(int ecode
) __dead2
;
46 int OverridePkgDeleteOpt
;
53 int SlowStartOpt
= -1;
54 int CapabilityRestrictions
;
55 long PkgDepMemoryTarget
;
56 long PkgDepScaleTarget
= 100; /* 1.00 */
58 char *ProfileOverrideOpt
;
62 main(int ac
, char **av
)
72 #if defined(__DragonFly__)
74 * The system is expected to have capabilities
77 size_t len
= sizeof(CapabilityRestrictions
);
78 sysctlbyname("kern.caps_available",
79 &CapabilityRestrictions
, &len
, NULL
, 0);
80 if (CapabilityRestrictions
== 0)
81 fprintf(stderr
, "caps restrictions unavailable\n");
86 * Get our exec path so we can self-exec clean WORKER
92 CTL_KERN
, KERN_PROC
, KERN_PROC_PATHNAME
, -1,
94 if (sysctl(name
, 4, NULL
, &len
, NULL
, 0) < 0)
95 dfatal_errno("Cannot get binary path");
96 DSynthExecPath
= malloc(len
+ 1);
97 if (sysctl(name
, 4, DSynthExecPath
, &len
, NULL
, 0) < 0)
98 dfatal_errno("Cannot get binary path");
99 DSynthExecPath
[len
] = 0;
103 * Override profile in dsynth.ini (can be further overridden
104 * with the -p profile option).
106 ProfileOverrideOpt
= getenv("DSYNTH_PROFILE");
109 * Process options and make sure the directive is present
112 while ((c
= getopt(ac
, av
, "dfhm:p:vxys:DPM:NS")) != -1) {
118 ++OverridePkgDeleteOpt
;
124 WorkerProcFlags
|= WORKER_PROC_DEVELOPER
;
127 WorkerProcFlags
|= WORKER_PROC_CHECK_PLIST
;
147 printf("dsynth %s\n", DSYNTH_VERSION
);
151 * Start with N jobs, increasing to the configured
152 * maximum slowly. 0 to disable (starts with the
155 SlowStartOpt
= strtol(optarg
, NULL
, 0);
158 PkgDepMemoryTarget
= strtoul(optarg
, NULL
, 0);
159 PkgDepMemoryTarget
*= ONEGB
;
162 PkgDepScaleTarget
= strtod(optarg
, NULL
) * 100;
163 if (PkgDepScaleTarget
< 1)
164 PkgDepScaleTarget
= 1;
165 if (PkgDepScaleTarget
> 9900)
166 PkgDepScaleTarget
= 9900;
169 ProfileOverrideOpt
= optarg
;
172 fprintf(stderr
, "Unknown option: %c\n", c
);
182 fprintf(stderr
, "Missing directive\n");
188 * Directives which do not require a working configuration
190 if (strcmp(av
[0], "init") == 0) {
195 if (strcmp(av
[0], "help") == 0) {
200 if (strcmp(av
[0], "version") == 0) {
201 printf("dsynth %s\n", DSYNTH_VERSION
);
209 if (strcmp(av
[0], "WORKER") == 0) {
215 signal(SIGPIPE
, SIG_IGN
);
216 ParseConfiguration(isworker
);
219 * Lock file path (also contains any 'add' directives thrown in
222 asprintf(&lkpath
, "%s/.lock", BuildBase
);
225 * Setup some environment for bulk operations (pkglist scan).
226 * These are not used by the builder (the builder will replicate
229 * NOTE: PKG_SUFX - pkg versions older than 1.17
230 * PKG_COMPRESSION_FORMAT - pkg versions >= 1.17
232 addbuildenv("PORTSDIR", DPortsPath
,
233 BENV_ENVIRONMENT
| BENV_PKGLIST
);
234 addbuildenv("BATCH", "yes",
235 BENV_ENVIRONMENT
| BENV_PKGLIST
);
236 addbuildenv("PKG_COMPRESSION_FORMAT", UsePkgSufx
,
237 BENV_ENVIRONMENT
| BENV_PKGLIST
);
238 addbuildenv("PKG_SUFX", UsePkgSufx
,
239 BENV_ENVIRONMENT
| BENV_PKGLIST
);
240 addbuildenv("PACKAGE_BUILDING", "yes",
241 BENV_ENVIRONMENT
| BENV_PKGLIST
);
242 addbuildenv("ARCH", ArchitectureName
,
243 BENV_ENVIRONMENT
| BENV_PKGLIST
);
249 addbuildenv("OSTYPE", OperatingSystemName
,
250 BENV_ENVIRONMENT
| BENV_PKGLIST
);
251 addbuildenv("MACHTYPE", MachineName
,
252 BENV_ENVIRONMENT
| BENV_PKGLIST
);
255 * SlowStart auto adjust. We nominally start with 1 job and increase
256 * it to the maximum every 5 seconds to give various dynamic management
257 * parameters time to stabilize.
259 * This can take a while on a many-core box with a high jobs setting,
260 * so increase the initial jobs in such cases.
262 if (SlowStartOpt
> MaxWorkers
)
263 SlowStartOpt
= MaxWorkers
;
264 if (SlowStartOpt
< 0) {
268 SlowStartOpt
= MaxWorkers
/ 4;
272 * Special directive for when dsynth execs itself to manage
276 WorkerProcess(ac
, av
);
281 * Build initialization and directive handling
286 * Directives that use the configuration but are not interlocked
287 * against a running dsynth.
289 if (strcmp(av
[0], "monitor") == 0) {
294 asprintf(&spath
, "%s/%s", StatsBase
, STATS_FILE
);
295 asprintf(&lpath
, "%s/%s", StatsBase
, STATS_LOCKFILE
);
296 MonitorDirective(spath
, lpath
);
300 MonitorDirective(av
[1], NULL
);
304 } else if (strcmp(av
[0], "add") == 0) {
310 * The lock check is a bit racey XXX
312 fd
= open(lkpath
, O_RDWR
| O_CREAT
| O_APPEND
, 0644);
313 if (flock(fd
, LOCK_EX
| LOCK_NB
) == 0) {
314 dfatal("No dsynth running to add ports to");
317 for (i
= 1; i
< ac
; ++i
) {
318 asprintf(&buf
, "%s\n", av
[i
]);
319 write(fd
, buf
, strlen(buf
));
320 printf("added to run: %s\n", av
[i
]);
327 * Front-end exec (not a WORKER exec), normal startup. We have
328 * the configuration so the first thing we need to do is check
331 lkfd
= open(lkpath
, O_RDWR
| O_CREAT
| O_CLOEXEC
, 0644);
333 dfatal_errno("Unable to create %s", lkpath
);
334 if (flock(lkfd
, LOCK_EX
| LOCK_NB
) < 0) {
335 dfatal("Another dsynth is using %s, exiting",
340 * Starting a new run cleans out any prior add directives
341 * that may have been pending.
344 /* leave descriptor open */
348 if (strcmp(av
[0], "debug") == 0) {
351 pkgs
= ParsePackageList(ac
- 1, av
+ 1, 1);
352 RemovePackages(pkgs
);
355 } else if (strcmp(av
[0], "status") == 0) {
358 pkgs
= ParsePackageList(ac
- 1, av
+ 1, 0);
360 pkgs
= GetLocalPackageList();
362 } else if (strcmp(av
[0], "cleanup") == 0) {
364 } else if (strcmp(av
[0], "configure") == 0) {
367 } else if (strcmp(av
[0], "fetch-only") == 0) {
368 if (SlowStartOpt
== -1)
370 if (PkgDepScaleTarget
== 100)
371 PkgDepScaleTarget
= 999;
374 WorkerProcFlags
|= WORKER_PROC_FETCHONLY
;
377 if (ac
== 2 && strcmp(av
[1], "everything") == 0) {
379 pkgs
= GetFullPackageList();
381 pkgs
= ParsePackageList(ac
- 1, av
+ 1, 0);
385 } else if (strcmp(av
[0], "list-system") == 0) {
390 pkgs
= GetLocalPackageList();
391 if ((fp
= fopen("build.txt", "w")) != NULL
) {
393 fprintf(fp
, "%s\n", pkgs
->portdir
);
397 printf("list written to build.txt\n");
399 fprintf(stderr
, "Cannot create 'build.txt'\n");
402 } else if (strcmp(av
[0], "upgrade-system") == 0) {
405 pkgs
= GetLocalPackageList();
408 DoUpgradePkgs(pkgs
, 0);
409 dfatal("NOTE: you have to pkg upgrade manually");
410 } else if (strcmp(av
[0], "prepare-system") == 0) {
411 DeleteObsoletePkgs
= 1;
414 pkgs
= GetLocalPackageList();
417 } else if (strcmp(av
[0], "rebuild-repository") == 0) {
420 } else if (strcmp(av
[0], "purge-distfiles") == 0) {
422 pkgs
= GetFullPackageList();
423 PurgeDistfiles(pkgs
);
424 } else if (strcmp(av
[0], "reset-db") == 0) {
427 asprintf(&dbmpath
, "%s/ports_crc.db", BuildBase
);
429 printf("%s reset, will be regenerated on next build\n",
432 } else if (strcmp(av
[0], "status-everything") == 0) {
434 pkgs
= GetFullPackageList();
436 } else if (strcmp(av
[0], "everything") == 0) {
437 if (WorkerProcFlags
& WORKER_PROC_DEVELOPER
)
438 WorkerProcFlags
|= WORKER_PROC_CHECK_PLIST
;
440 DeleteObsoletePkgs
= 1;
443 pkgs
= GetFullPackageList();
445 DoRebuildRepo(!CheckAddReExec(lkfd
));
446 } else if (strcmp(av
[0], "build") == 0) {
449 pkgs
= ParsePackageList(ac
- 1, av
+ 1, 0);
451 DoRebuildRepo(!CheckAddReExec(lkfd
));
452 DoUpgradePkgs(pkgs
, 1);
454 } else if (strcmp(av
[0], "just-build") == 0) {
457 pkgs
= ParsePackageList(ac
- 1, av
+ 1, 0);
460 } else if (strcmp(av
[0], "install") == 0) {
463 pkgs
= ParsePackageList(ac
- 1, av
+ 1, 0);
466 DoUpgradePkgs(pkgs
, 0);
468 } else if (strcmp(av
[0], "force") == 0) {
471 pkgs
= ParsePackageList(ac
- 1, av
+ 1, 0);
472 RemovePackages(pkgs
);
474 DoRebuildRepo(!CheckAddReExec(lkfd
));
475 DoUpgradePkgs(pkgs
, 1);
477 } else if (strcmp(av
[0], "test") == 0) {
478 WorkerProcFlags
|= WORKER_PROC_CHECK_PLIST
|
479 WORKER_PROC_INSTALL
|
480 WORKER_PROC_DEINSTALL
;
483 pkgs
= ParsePackageList(ac
- 1, av
+ 1, 0);
484 RemovePackages(pkgs
);
485 WorkerProcFlags
|= WORKER_PROC_DEVELOPER
;
489 fprintf(stderr
, "Unknown directive '%s'\n", av
[0]);
494 * For directives that support the 'add' directive, check for
495 * additions and re-exec.
497 * Note that the lockfile is O_CLOEXEC and will be remade on exec.
499 * XXX a bit racey vs adds done just as we are finishing
501 if (doadds
&& CheckAddReExec(lkfd
))
502 DoAddReExec(lkfd
, optind
+ 1, av
- optind
);
508 * If the 'add' directive was issued while a dsynth build was in
509 * progress, we re-exec dsynth with its original options and
510 * directive along with the added ports.
513 CheckAddReExec(int lkfd
)
517 if (fstat(lkfd
, &st
) < 0 || st
.st_size
== 0)
523 DoAddReExec(int lkfd
, int ac
, char **oldav
)
533 if (fstat(lkfd
, &st
) < 0 || st
.st_size
== 0)
535 buf
= malloc(st
.st_size
+ 1);
536 if (read(lkfd
, buf
, st
.st_size
) != st
.st_size
) {
543 for (i
= 0; i
< (size_t)st
.st_size
; ++i
) {
544 if (buf
[i
] == '\n' || buf
[i
] == 0) {
550 av
= calloc(ac
+ nadd
+ 1, sizeof(char *));
552 for (n
= 0; n
< ac
; ++n
)
557 for (i
= 0; i
< (size_t)st
.st_size
; ++i
) {
559 av
[ac
+ nadd
] = buf
+ bi
;
565 printf("dsynth re-exec'ing additionally added packages\n");
566 for (n
= 0; n
< ac
+ nadd
; ++n
)
567 printf(" %s", av
[n
]);
571 execv(DSynthExecPath
, av
);
581 if (stat(ConfigBase1
, &st
) == 0) {
582 dfatal("init will not overwrite %s", ConfigBase1
);
584 if (stat(ConfigBase2
, &st
) == 0) {
585 dfatal("init will not create %s if %s exists",
586 ConfigBase2
, ConfigBase1
);
588 if (mkdir(ConfigBase1
, 0755) < 0)
589 dfatal_errno("Unable to mkdir %s", ConfigBase1
);
591 asprintf(&path
, "%s/dsynth.ini", ConfigBase1
);
592 fp
= fopen(path
, "w");
593 dassert_errno(fp
, "Unable to create %s", path
);
595 "; This Synth configuration file is automatically generated\n"
596 "; Take care when hand editing!\n"
598 "[Global Configuration]\n"
599 "profile_selected= LiveSystem\n"
602 "Operating_system= DragonFly\n"
603 "Directory_packages= /build/synth/live_packages\n"
604 "Directory_repository= /build/synth/live_packages/All\n"
605 "Directory_portsdir= /build/synth/dports\n"
606 "Directory_options= /build/synth/options\n"
607 "Directory_distfiles= /build/synth/distfiles\n"
608 "Directory_buildbase= /build/synth/build\n"
609 "Directory_logs= /build/synth/logs\n"
610 "Directory_ccache= disabled\n"
611 "Directory_system= /\n"
612 "Package_suffix= .txz\n"
613 "Number_of_builders= 0\n"
614 "Max_jobs_per_builder= 0\n"
615 "Tmpfs_workdir= true\n"
616 "Tmpfs_localbase= true\n"
617 "Display_with_ncurses= true\n"
618 "leverage_prebuilt= false\n"
619 "; Meta_version= 2\n"
620 "; Check_plist= false\n"
621 "; Numa_setsize= 2\n"
624 dfatal_errno("Unable to write to %s\n", ConfigBase1
);
627 asprintf(&path
, "%s/LiveSystem-make.conf", ConfigBase1
);
628 fp
= fopen(path
, "w");
629 dassert_errno(fp
, "Unable to create %s", path
);
632 "# Various dports options that might be of interest\n"
634 "#LICENSES_ACCEPTED= NONE\n"
635 "#DISABLE_LICENSES= yes\n"
636 "#DEFAULT_VERSIONS= ssl=openssl\n"
637 "#FORCE_PACKAGE= yes\n"
638 "#DPORTS_BUILDER= yes\n"
640 "# Turn these on to generate debug binaries. However, these\n"
641 "# options will seriously bloat memory use and storage use,\n"
642 "# do not use lightly\n"
648 dfatal_errno("Unable to write to %s\n", ConfigBase1
);
656 fprintf(stderr
, "Run 'dsynth help' for usage\n");
661 "dsynth [options] directive\n"
662 " -d - Debug verbosity (-dd disables ncurses)\n"
663 " -f - Force (for purge-distfiles)\n"
664 " -h - Display this screen and exit\n"
665 " -m gb - Load management based on pkgdep memory\n"
666 " -p profile - Override profile selected in dsynth.ini\n"
667 " -s n - Set initial DynamicMaxWorkers\n"
668 " -v - Print version info and exit\n"
669 " -x - Do not rebuild packages with dependencies\n"
670 " which require rebuilding\n"
671 " -xx - Do not rebuild packages whos dports trees\n"
673 " -y - Automatically answer yes to dsynth questions\n"
674 " -D - Enable DEVELOPER mode\n"
675 " -P - Include the check-plist stage\n"
676 " -S - Disable ncurses\n"
677 " -N - Do not nice-up sub-processes (else nice +10)\n"
679 " init - Initialize /etc/dsynth\n"
680 " status - Dry-run of 'upgrade-system'\n"
681 " cleanup - Clean-up mounts\n"
682 " configure - Bring up configuration menu\n"
683 " list-system - Just generate the build list to build.txt\n"
684 " upgrade-system - Incremental build and upgrade using pkg list\n"
685 " from local system, then upgrade the local\n"
687 " prepare-system - 'upgrade-system' but stops after building\n"
688 " rebuild-repository - Rebuild database files for current repository\n"
689 " purge-distfiles - Delete obsolete source distribution files\n"
690 " reset-db - Delete ports_crc.db, regenerate next build\n"
691 " status-everything - Dry-run of 'everything'\n"
692 " everything - Build entire dports tree and repo database\n"
693 " (-D everything infers -P)\n"
694 " version - Print version info and exit\n"
695 " help - Display this screen and exit\n"
696 " status [ports] - Dry-run of 'build' with given list\n"
697 " build [ports] - Incrementally build dports based on the given\n"
698 " list, but asks before updating the repo\n"
699 " database and system\n"
700 " just-build [ports] - 'build' but skips post-build steps\n"
701 " install [ports] - 'build' but upgrades system without asking\n"
702 " force [ports] - 'build' but deletes existing packages first\n"
703 " test [ports] - 'build' w/DEVELOPER=yes and pre-deletes pkgs\n"
704 " (also infers -P)\n"
705 " debug [ports] - like 'test' but leaves mounts intact\n"
706 " fetch-only [ports] - Fetch src dists only ('everything' ok)\n"
707 " monitor [datfile] - Monitor a running dsynth\n"
709 " [ports] is a space-delimited list of origins, e.g. editors/joe. It\n"
710 " may also be a path to a file containing one origin per line.\n"