| blob | 042dfab67f0b0b0159e3ab0070d57c2e86cdfbec |
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
3 ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org>
4 ;;;
5 ;;; This file is part of GNU Guix.
6 ;;;
7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
8 ;;; under the terms of the GNU General Public License as published by
9 ;;; the Free Software Foundation; either version 3 of the License, or (at
10 ;;; your option) any later version.
11 ;;;
12 ;;; GNU Guix is distributed in the hope that it will be useful, but
13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ;;; GNU General Public License for more details.
16 ;;;
17 ;;; You should have received a copy of the GNU General Public License
18 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
20 (define-module (guix store)
21 #:use-module (guix utils)
22 #:use-module (guix config)
23 #:use-module (guix memoization)
24 #:use-module (guix serialization)
25 #:use-module (guix monads)
26 #:use-module (guix records)
27 #:use-module (guix base16)
28 #:use-module (guix base32)
29 #:use-module (gcrypt hash)
30 #:use-module (guix profiling)
31 #:autoload (guix build syscalls) (terminal-columns)
32 #:use-module (rnrs bytevectors)
33 #:use-module (ice-9 binary-ports)
34 #:use-module ((ice-9 control) #:select (let/ec))
35 #:use-module (srfi srfi-1)
36 #:use-module (srfi srfi-9)
37 #:use-module (srfi srfi-9 gnu)
38 #:use-module (srfi srfi-11)
39 #:use-module (srfi srfi-26)
40 #:use-module (srfi srfi-34)
41 #:use-module (srfi srfi-35)
42 #:use-module (srfi srfi-39)
43 #:use-module (ice-9 match)
44 #:use-module (ice-9 regex)
45 #:use-module (ice-9 vlist)
46 #:use-module (ice-9 popen)
47 #:use-module (ice-9 threads)
48 #:use-module (ice-9 format)
49 #:use-module (web uri)
50 #:export (%daemon-socket-uri
51 %gc-roots-directory
52 %default-substitute-urls
54 nix-server?
55 nix-server-version
56 nix-server-major-version
57 nix-server-minor-version
58 nix-server-socket
59 current-store-protocol-version ;for internal use
60 mcached
62 &nix-error nix-error?
63 &nix-connection-error nix-connection-error?
64 nix-connection-error-file
65 nix-connection-error-code
66 &nix-protocol-error nix-protocol-error?
67 nix-protocol-error-message
68 nix-protocol-error-status
70 hash-algo
71 build-mode
73 open-connection
74 port->connection
75 close-connection
76 with-store
77 set-build-options
78 set-build-options*
79 valid-path?
80 query-path-hash
81 hash-part->path
82 query-path-info
83 add-data-to-store
84 add-text-to-store
85 add-to-store
86 add-file-tree-to-store
87 binary-file
88 build-things
89 build
90 query-failed-paths
91 clear-failed-paths
92 add-temp-root
93 add-indirect-root
94 add-permanent-root
95 remove-permanent-root
97 substitutable?
98 substitutable-path
99 substitutable-deriver
100 substitutable-references
101 substitutable-download-size
102 substitutable-nar-size
103 has-substitutes?
104 substitutable-paths
105 substitutable-path-info
107 path-info?
108 path-info-deriver
109 path-info-hash
110 path-info-references
111 path-info-registration-time
112 path-info-nar-size
114 built-in-builders
115 references
116 references/substitutes
117 references*
118 query-path-info*
119 requisites
120 referrers
121 optimize-store
122 verify-store
123 topologically-sorted
124 valid-derivers
125 query-derivation-outputs
126 live-paths
127 dead-paths
128 collect-garbage
129 delete-paths
130 import-paths
131 export-paths
133 current-build-output-port
135 %store-monad
136 store-bind
137 store-return
138 store-lift
139 store-lower
140 run-with-store
141 %guile-for-build
142 current-system
143 set-current-system
144 text-file
145 interned-file
146 interned-file-tree
148 %store-prefix
149 store-path
150 output-path
151 fixed-output-path
152 store-path?
153 direct-store-path?
154 derivation-path?
155 store-path-package-name
156 store-path-hash-part
157 direct-store-path
158 derivation-log-file
159 log-file))
161 (define %protocol-version #x163)
163 (define %worker-magic-1 #x6e697863) ; "nixc"
164 (define %worker-magic-2 #x6478696f) ; "dxio"
166 (define (protocol-major magic)
167 (logand magic #xff00))
168 (define (protocol-minor magic)
169 (logand magic #x00ff))
170 (define (protocol-version major minor)
171 (logior major minor))
173 (define-syntax define-enumerate-type
174 (syntax-rules ()
175 ((_ name->int (name id) ...)
176 (define-syntax name->int
177 (syntax-rules (name ...)
178 ((_ name) id) ...)))))
180 (define-enumerate-type operation-id
181 ;; operation numbers from worker-protocol.hh
182 (quit 0)
183 (valid-path? 1)
184 (has-substitutes? 3)
185 (query-path-hash 4)
186 (query-references 5)
187 (query-referrers 6)
188 (add-to-store 7)
189 (add-text-to-store 8)
190 (build-things 9)
191 (ensure-path 10)
192 (add-temp-root 11)
193 (add-indirect-root 12)
194 (sync-with-gc 13)
195 (find-roots 14)
196 (export-path 16)
197 (query-deriver 18)
198 (set-options 19)
199 (collect-garbage 20)
200 ;;(query-substitutable-path-info 21) ; obsolete as of #x10c
201 (query-derivation-outputs 22)
202 (query-all-valid-paths 23)
203 (query-failed-paths 24)
204 (clear-failed-paths 25)
205 (query-path-info 26)
206 (import-paths 27)
207 (query-derivation-output-names 28)
208 (query-path-from-hash-part 29)
209 (query-substitutable-path-infos 30)
210 (query-valid-paths 31)
211 (query-substitutable-paths 32)
212 (query-valid-derivers 33)
213 (optimize-store 34)
214 (verify-store 35)
215 (built-in-builders 80))
217 (define-enumerate-type hash-algo
218 ;; hash.hh
219 (md5 1)
220 (sha1 2)
221 (sha256 3))
223 (define-enumerate-type build-mode
224 ;; store-api.hh
225 (normal 0)
226 (repair 1)
227 (check 2))
229 (define-enumerate-type gc-action
230 ;; store-api.hh
231 (return-live 0)
232 (return-dead 1)
233 (delete-dead 2)
234 (delete-specific 3))
236 (define %default-socket-path
237 (string-append %state-directory "/daemon-socket/socket"))
239 (define %daemon-socket-uri
240 ;; URI or file name of the socket the daemon listens too.
241 (make-parameter (or (getenv "GUIX_DAEMON_SOCKET")
242 %default-socket-path)))
245 \f
246 ;; Information about a substitutable store path.
247 (define-record-type <substitutable>
248 (substitutable path deriver refs dl-size nar-size)
249 substitutable?
250 (path substitutable-path)
251 (deriver substitutable-deriver)
252 (refs substitutable-references)
253 (dl-size substitutable-download-size)
254 (nar-size substitutable-nar-size))
256 (define (read-substitutable-path-list p)
257 (let loop ((len (read-int p))
258 (result '()))
259 (if (zero? len)
260 (reverse result)
261 (let ((path (read-store-path p))
262 (deriver (read-store-path p))
263 (refs (read-store-path-list p))
264 (dl-size (read-long-long p))
265 (nar-size (read-long-long p)))
266 (loop (- len 1)
267 (cons (substitutable path deriver refs dl-size nar-size)
268 result))))))
270 ;; Information about a store path.
271 (define-record-type <path-info>
272 (path-info deriver hash references registration-time nar-size)
273 path-info?
274 (deriver path-info-deriver) ;string | #f
275 (hash path-info-hash)
276 (references path-info-references)
277 (registration-time path-info-registration-time)
278 (nar-size path-info-nar-size))
280 (define (read-path-info p)
281 (let ((deriver (match (read-store-path p)
282 ("" #f)
283 (x x)))
284 (hash (base16-string->bytevector (read-string p)))
285 (refs (read-store-path-list p))
286 (registration-time (read-int p))
287 (nar-size (read-long-long p)))
288 (path-info deriver hash refs registration-time nar-size)))
290 (define-syntax write-arg
291 (syntax-rules (integer boolean bytevector
292 string string-list string-pairs
293 store-path store-path-list base16)
294 ((_ integer arg p)
295 (write-int arg p))
296 ((_ boolean arg p)
297 (write-int (if arg 1 0) p))
298 ((_ bytevector arg p)
299 (write-bytevector arg p))
300 ((_ string arg p)
301 (write-string arg p))
302 ((_ string-list arg p)
303 (write-string-list arg p))
304 ((_ string-pairs arg p)
305 (write-string-pairs arg p))
306 ((_ store-path arg p)
307 (write-store-path arg p))
308 ((_ store-path-list arg p)
309 (write-store-path-list arg p))
310 ((_ base16 arg p)
311 (write-string (bytevector->base16-string arg) p))))
313 (define-syntax read-arg
314 (syntax-rules (integer boolean string store-path store-path-list string-list
315 substitutable-path-list path-info base16)
316 ((_ integer p)
317 (read-int p))
318 ((_ boolean p)
319 (not (zero? (read-int p))))
320 ((_ string p)
321 (read-string p))
322 ((_ store-path p)
323 (read-store-path p))
324 ((_ store-path-list p)
325 (read-store-path-list p))
326 ((_ string-list p)
327 (read-string-list p))
328 ((_ substitutable-path-list p)
329 (read-substitutable-path-list p))
330 ((_ path-info p)
331 (read-path-info p))
332 ((_ base16 p)
333 (base16-string->bytevector (read-string p)))))
335 \f
336 ;; remote-store.cc
338 (define-record-type* <nix-server> nix-server %make-nix-server
339 nix-server?
340 (socket nix-server-socket)
341 (major nix-server-major-version)
342 (minor nix-server-minor-version)
344 (buffer nix-server-output-port) ;output port
345 (flush nix-server-flush-output) ;thunk
347 ;; Caches. We keep them per-connection, because store paths build
348 ;; during the session are temporary GC roots kept for the duration of
349 ;; the session.
350 (ats-cache nix-server-add-to-store-cache)
351 (atts-cache nix-server-add-text-to-store-cache)
352 (object-cache nix-server-object-cache
353 (default vlist-null))) ;vhash
355 (set-record-type-printer! <nix-server>
356 (lambda (obj port)
357 (format port "#<build-daemon ~a.~a ~a>"
358 (nix-server-major-version obj)
359 (nix-server-minor-version obj)
360 (number->string (object-address obj)
361 16))))
363 (define-condition-type &nix-error &error
364 nix-error?)
366 (define-condition-type &nix-connection-error &nix-error
367 nix-connection-error?
368 (file nix-connection-error-file)
369 (errno nix-connection-error-code))
371 (define-condition-type &nix-protocol-error &nix-error
372 nix-protocol-error?
373 (message nix-protocol-error-message)
374 (status nix-protocol-error-status))
376 (define-syntax-rule (system-error-to-connection-error file exp ...)
377 "Catch 'system-error' exceptions and translate them to
378 '&nix-connection-error'."
379 (catch 'system-error
380 (lambda ()
381 exp ...)
382 (lambda args
383 (let ((errno (system-error-errno args)))
384 (raise (condition (&nix-connection-error
385 (file file)
386 (errno errno))))))))
388 (define (open-unix-domain-socket file)
389 "Connect to the Unix-domain socket at FILE and return it. Raise a
390 '&nix-connection-error' upon error."
391 (let ((s (with-fluids ((%default-port-encoding #f))
392 ;; This trick allows use of the `scm_c_read' optimization.
393 (socket PF_UNIX SOCK_STREAM 0)))
394 (a (make-socket-address PF_UNIX file)))
396 (system-error-to-connection-error file
397 (connect s a)
398 s)))
400 (define %default-guix-port
401 ;; Default port when connecting to a daemon over TCP/IP.
402 44146)
404 (define (open-inet-socket host port)
405 "Connect to the Unix-domain socket at HOST:PORT and return it. Raise a
406 '&nix-connection-error' upon error."
407 ;; Define 'TCP_NODELAY' on Guile 2.0. The value is the same on all GNU
408 ;; systems.
409 (cond-expand (guile-2.2 #t)
410 (else (define TCP_NODELAY 1)))
412 (let ((sock (with-fluids ((%default-port-encoding #f))
413 ;; This trick allows use of the `scm_c_read' optimization.
414 (socket PF_UNIX SOCK_STREAM 0))))
415 (define addresses
416 (getaddrinfo host
417 (if (number? port) (number->string port) port)
418 (if (number? port)
419 (logior AI_ADDRCONFIG AI_NUMERICSERV)
420 AI_ADDRCONFIG)
421 0 ;any address family
422 SOCK_STREAM)) ;TCP only
424 (let loop ((addresses addresses))
425 (match addresses
426 ((ai rest ...)
427 (let ((s (socket (addrinfo:fam ai)
428 ;; TCP/IP only
429 SOCK_STREAM IPPROTO_IP)))
431 (catch 'system-error
432 (lambda ()
433 (connect s (addrinfo:addr ai))
435 ;; Setting this option makes a dramatic difference because it
436 ;; avoids the "ACK delay" on our RPC messages.
437 (setsockopt s IPPROTO_TCP TCP_NODELAY 1)
438 s)
439 (lambda args
440 ;; Connection failed, so try one of the other addresses.
441 (close s)
442 (if (null? rest)
443 (raise (condition (&nix-connection-error
444 (file host)
445 (errno (system-error-errno args)))))
446 (loop rest))))))))))
448 (define (connect-to-daemon uri)
449 "Connect to the daemon at URI, a string that may be an actual URI or a file
450 name."
451 (define (not-supported)
452 (raise (condition (&nix-connection-error
453 (file uri)
454 (errno ENOTSUP)))))
456 (define connect
457 (match (string->uri uri)
458 (#f ;URI is a file name
459 open-unix-domain-socket)
460 ((? uri? uri)
461 (match (uri-scheme uri)
462 ((or #f 'file 'unix)
463 (lambda (_)
464 (open-unix-domain-socket (uri-path uri))))
465 ('guix
466 (lambda (_)
467 (open-inet-socket (uri-host uri)
468 (or (uri-port uri) %default-guix-port))))
469 ((? symbol? scheme)
470 ;; Try to dynamically load a module for SCHEME.
471 ;; XXX: Errors are swallowed.
472 (match (false-if-exception
473 (resolve-interface `(guix store ,scheme)))
474 ((? module? module)
475 (match (false-if-exception
476 (module-ref module 'connect-to-daemon))
477 ((? procedure? connect)
478 (lambda (_)
479 (connect uri)))
480 (x (not-supported))))
481 (#f (not-supported))))
482 (x
483 (not-supported))))))
485 (connect uri))
487 (define* (open-connection #:optional (uri (%daemon-socket-uri))
488 #:key port (reserve-space? #t) cpu-affinity)
489 "Connect to the daemon at URI (a string), or, if PORT is not #f, use it as
490 the I/O port over which to communicate to a build daemon.
492 When RESERVE-SPACE? is true, instruct it to reserve a little bit of extra
493 space on the file system so that the garbage collector can still operate,
494 should the disk become full. When CPU-AFFINITY is true, it must be an integer
495 corresponding to an OS-level CPU number to which the daemon's worker process
496 for this connection will be pinned. Return a server object."
497 (guard (c ((nar-error? c)
498 ;; One of the 'write-' or 'read-' calls below failed, but this is
499 ;; really a connection error.
500 (raise (condition
501 (&nix-connection-error (file (or port uri))
502 (errno EPROTO))
503 (&message (message "build daemon handshake failed"))))))
504 (let*-values (((port)
505 (or port (connect-to-daemon uri)))
506 ((output flush)
507 (buffering-output-port port
508 (make-bytevector 8192))))
509 (write-int %worker-magic-1 port)
510 (let ((r (read-int port)))
511 (and (eqv? r %worker-magic-2)
512 (let ((v (read-int port)))
513 (and (eqv? (protocol-major %protocol-version)
514 (protocol-major v))
515 (begin
516 (write-int %protocol-version port)
517 (when (>= (protocol-minor v) 14)
518 (write-int (if cpu-affinity 1 0) port)
519 (when cpu-affinity
520 (write-int cpu-affinity port)))
521 (when (>= (protocol-minor v) 11)
522 (write-int (if reserve-space? 1 0) port))
523 (let ((conn (%make-nix-server port
524 (protocol-major v)
525 (protocol-minor v)
526 output flush
527 (make-hash-table 100)
528 (make-hash-table 100)
529 vlist-null)))
530 (let loop ((done? (process-stderr conn)))
531 (or done? (process-stderr conn)))
532 conn)))))))))
534 (define* (port->connection port
535 #:key (version %protocol-version))
536 "Assimilate PORT, an input/output port, and return a connection to the
537 daemon, assuming the given protocol VERSION.
539 Warning: this procedure assumes that the initial handshake with the daemon has
540 already taken place on PORT and that we're just continuing on this established
541 connection. Use with care."
542 (let-values (((output flush)
543 (buffering-output-port port (make-bytevector 8192))))
544 (%make-nix-server port
545 (protocol-major version)
546 (protocol-minor version)
547 output flush
548 (make-hash-table 100)
549 (make-hash-table 100)
550 vlist-null)))
552 (define (nix-server-version store)
553 "Return the protocol version of STORE as an integer."
554 (protocol-version (nix-server-major-version store)
555 (nix-server-minor-version store)))
557 (define (write-buffered-output server)
558 "Flush SERVER's output port."
559 (force-output (nix-server-output-port server))
560 ((nix-server-flush-output server)))
562 (define (close-connection server)
563 "Close the connection to SERVER."
564 (close (nix-server-socket server)))
566 (define-syntax-rule (with-store store exp ...)
567 "Bind STORE to an open connection to the store and evaluate EXPs;
568 automatically close the store when the dynamic extent of EXP is left."
569 (let ((store (open-connection)))
570 (dynamic-wind
571 (const #f)
572 (lambda ()
573 (parameterize ((current-store-protocol-version
574 (nix-server-version store)))
575 exp) ...)
576 (lambda ()
577 (false-if-exception (close-connection store))))))
579 (define current-store-protocol-version
580 ;; Protocol version of the store currently used. XXX: This is a hack to
581 ;; communicate the protocol version to the build output port. It's a hack
582 ;; because it could be inaccurrate, for instance if there's code that
583 ;; manipulates several store connections at once; it works well for the
584 ;; purposes of (guix status) though.
585 (make-parameter #f))
587 (define current-build-output-port
588 ;; The port where build output is sent.
589 (make-parameter (current-error-port)))
591 (define* (dump-port in out
592 #:optional len
593 #:key (buffer-size 16384))
594 "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it
595 to OUT, using chunks of BUFFER-SIZE bytes."
596 (define buffer
597 (make-bytevector buffer-size))
599 (let loop ((total 0)
600 (bytes (get-bytevector-n! in buffer 0
601 (if len
602 (min len buffer-size)
603 buffer-size))))
604 (or (eof-object? bytes)
605 (and len (= total len))
606 (let ((total (+ total bytes)))
607 (put-bytevector out buffer 0 bytes)
608 (loop total
609 (get-bytevector-n! in buffer 0
610 (if len
611 (min (- len total) buffer-size)
612 buffer-size)))))))
614 (define %newlines
615 ;; Newline characters triggering a flush of 'current-build-output-port'.
616 ;; Unlike Guile's _IOLBF, we flush upon #\return so that progress reports
617 ;; that use that trick are correctly displayed.
618 (char-set #\newline #\return))
620 (define* (process-stderr server #:optional user-port)
621 "Read standard output and standard error from SERVER, writing it to
622 CURRENT-BUILD-OUTPUT-PORT. Return #t when SERVER is done sending data, and
623 #f otherwise; in the latter case, the caller should call `process-stderr'
624 again until #t is returned or an error is raised.
626 Since the build process's output cannot be assumed to be UTF-8, we
627 conservatively consider it to be Latin-1, thereby avoiding possible
628 encoding conversion errors."
629 (define p
630 (nix-server-socket server))
632 ;; magic cookies from worker-protocol.hh
633 (define %stderr-next #x6f6c6d67) ; "olmg", build log
634 (define %stderr-read #x64617461) ; "data", data needed from source
635 (define %stderr-write #x64617416) ; "dat\x16", data for sink
636 (define %stderr-last #x616c7473) ; "alts", we're done
637 (define %stderr-error #x63787470) ; "cxtp", error reporting
639 (let ((k (read-int p)))
640 (cond ((= k %stderr-write)
641 ;; Write a byte stream to USER-PORT.
642 (let* ((len (read-int p))
643 (m (modulo len 8)))
644 (dump-port p user-port len
645 #:buffer-size (if (<= len 16384) 16384 65536))
646 (unless (zero? m)
647 ;; Consume padding, as for strings.
648 (get-bytevector-n p (- 8 m))))
649 #f)
650 ((= k %stderr-read)
651 ;; Read a byte stream from USER-PORT.
652 ;; Note: Avoid 'get-bytevector-n' to work around
653 ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11.
654 (let* ((max-len (read-int p))
655 (data (make-bytevector max-len))
656 (len (get-bytevector-n! user-port data 0 max-len)))
657 (write-bytevector data p len)
658 #f))
659 ((= k %stderr-next)
660 ;; Log a string. Build logs are usually UTF-8-encoded, but they
661 ;; may also contain arbitrary byte sequences that should not cause
662 ;; this to fail. Thus, use the permissive
663 ;; 'read-maybe-utf8-string'.
664 (let ((s (read-maybe-utf8-string p)))
665 (display s (current-build-output-port))
666 (when (string-any %newlines s)
667 (force-output (current-build-output-port)))
668 #f))
669 ((= k %stderr-error)
670 ;; Report an error.
671 (let ((error (read-maybe-utf8-string p))
672 ;; Currently the daemon fails to send a status code for early
673 ;; errors like DB schema version mismatches, so check for EOF.
674 (status (if (and (>= (nix-server-minor-version server) 8)
675 (not (eof-object? (lookahead-u8 p))))
676 (read-int p)
677 1)))
678 (raise (condition (&nix-protocol-error
679 (message error)
680 (status status))))))
681 ((= k %stderr-last)
682 ;; The daemon is done (see `stopWork' in `nix-worker.cc'.)
683 #t)
684 (else
685 (raise (condition (&nix-protocol-error
686 (message "invalid error code")
687 (status k))))))))
689 (define %default-substitute-urls
690 ;; Default list of substituters. This is *not* the list baked in
691 ;; 'guix-daemon', but it is used by 'guix-service-type' and and a couple of
692 ;; clients ('guix build --log-file' uses it.)
693 (map (if (false-if-exception (resolve-interface '(gnutls)))
694 (cut string-append "https://" <>)
695 (cut string-append "http://" <>))
696 '("ci.guix.info")))
698 (define* (set-build-options server
699 #:key keep-failed? keep-going? fallback?
700 (verbosity 0)
701 rounds ;number of build rounds
702 max-build-jobs
703 timeout
704 max-silent-time
705 (use-build-hook? #t)
706 (build-verbosity 0)
707 (log-type 0)
708 (print-build-trace #t)
710 ;; When true, provide machine-readable "build
711 ;; traces" for use by (guix status). Old clients
712 ;; are unable to make sense, which is why it's
713 ;; disabled by default.
714 print-extended-build-trace?
716 ;; When true, the daemon prefixes builder output
717 ;; with "@ build-log" traces so we can
718 ;; distinguish it from daemon output, and we can
719 ;; distinguish each builder's output
720 ;; (PRINT-BUILD-TRACE must be true as well.) The
721 ;; latter is particularly useful when
722 ;; MAX-BUILD-JOBS > 1.
723 multiplexed-build-output?
725 build-cores
726 (use-substitutes? #t)
728 ;; Client-provided substitute URLs. If it is #f,
729 ;; the daemon's settings are used. Otherwise, it
730 ;; overrides the daemons settings; see 'guix
731 ;; substitute'.
732 (substitute-urls #f)
734 ;; Number of columns in the client's terminal.
735 (terminal-columns (terminal-columns))
737 ;; Locale of the client.
738 (locale (false-if-exception (setlocale LC_ALL))))
739 ;; Must be called after `open-connection'.
741 (define socket
742 (nix-server-socket server))
744 (let-syntax ((send (syntax-rules ()
745 ((_ (type option) ...)
746 (begin
747 (write-arg type option socket)
748 ...)))))
749 (write-int (operation-id set-options) socket)
750 (send (boolean keep-failed?) (boolean keep-going?)
751 (boolean fallback?) (integer verbosity))
752 (when (< (nix-server-minor-version server) #x61)
753 (let ((max-build-jobs (or max-build-jobs 1))
754 (max-silent-time (or max-silent-time 3600)))
755 (send (integer max-build-jobs) (integer max-silent-time))))
756 (when (>= (nix-server-minor-version server) 2)
757 (send (boolean use-build-hook?)))
758 (when (>= (nix-server-minor-version server) 4)
759 (send (integer build-verbosity) (integer log-type)
760 (boolean print-build-trace)))
761 (when (and (>= (nix-server-minor-version server) 6)
762 (< (nix-server-minor-version server) #x61))
763 (let ((build-cores (or build-cores (current-processor-count))))
764 (send (integer build-cores))))
765 (when (>= (nix-server-minor-version server) 10)
766 (send (boolean use-substitutes?)))
767 (when (>= (nix-server-minor-version server) 12)
768 (let ((pairs `(;; This option is honored by 'guix substitute' et al.
769 ,@(if print-build-trace
770 `(("print-extended-build-trace"
771 . ,(if print-extended-build-trace? "1" "0")))
772 '())
773 ,@(if multiplexed-build-output?
774 `(("multiplexed-build-output"
775 . ,(if multiplexed-build-output? "true" "false")))
776 '())
777 ,@(if timeout
778 `(("build-timeout" . ,(number->string timeout)))
779 '())
780 ,@(if max-silent-time
781 `(("build-max-silent-time"
782 . ,(number->string max-silent-time)))
783 '())
784 ,@(if max-build-jobs
785 `(("build-max-jobs"
786 . ,(number->string max-build-jobs)))
787 '())
788 ,@(if build-cores
789 `(("build-cores" . ,(number->string build-cores)))
790 '())
791 ,@(if substitute-urls
792 `(("substitute-urls"
793 . ,(string-join substitute-urls)))
794 '())
795 ,@(if rounds
796 `(("build-repeat"
797 . ,(number->string (max 0 (1- rounds)))))
798 '())
799 ,@(if terminal-columns
800 `(("terminal-columns"
801 . ,(number->string terminal-columns)))
802 '())
803 ,@(if locale
804 `(("locale" . ,locale))
805 '()))))
806 (send (string-pairs pairs))))
807 (let loop ((done? (process-stderr server)))
808 (or done? (process-stderr server)))))
810 (define (buffering-output-port port buffer)
811 "Return two value: an output port wrapped around PORT that uses BUFFER (a
812 bytevector) as its internal buffer, and a thunk to flush this output port."
813 ;; Note: In Guile 2.2.2, custom binary output ports already have their own
814 ;; 4K internal buffer.
815 (define size
816 (bytevector-length buffer))
818 (define total 0)
820 (define (flush)
821 (put-bytevector port buffer 0 total)
822 (force-output port)
823 (set! total 0))
825 (define (write bv offset count)
826 (if (zero? count) ;end of file
827 (flush)
828 (let loop ((offset offset)
829 (count count)
830 (written 0))
831 (cond ((= total size)
832 (flush)
833 (loop offset count written))
834 ((zero? count)
835 written)
836 (else
837 (let ((to-copy (min count (- size total))))
838 (bytevector-copy! bv offset buffer total to-copy)
839 (set! total (+ total to-copy))
840 (loop (+ offset to-copy) (- count to-copy)
841 (+ written to-copy))))))))
843 ;; Note: We need to return FLUSH because the custom binary port has no way
844 ;; to be notified of a 'force-output' call on itself.
845 (values (make-custom-binary-output-port "buffering-output-port"
846 write #f #f flush)
847 flush))
849 (define profiled?
850 (let ((profiled
851 (or (and=> (getenv "GUIX_PROFILING") string-tokenize)
852 '())))
853 (lambda (component)
854 "Return true if COMPONENT profiling is active."
855 (member component profiled))))
857 (define %rpc-calls
858 ;; Mapping from RPC names (symbols) to invocation counts.
859 (make-hash-table))
861 (define* (show-rpc-profile #:optional (port (current-error-port)))
862 "Write to PORT a summary of the RPCs that have been made."
863 (let ((profile (sort (hash-fold alist-cons '() %rpc-calls)
864 (lambda (rpc1 rpc2)
865 (< (cdr rpc1) (cdr rpc2))))))
866 (format port "Remote procedure call summary: ~a RPCs~%"
867 (match profile
868 (((names . counts) ...)
869 (reduce + 0 counts))))
870 (for-each (match-lambda
871 ((rpc . count)
872 (format port " ~30a ... ~5@a~%" rpc count)))
873 profile)))
875 (define record-operation
876 ;; Optionally, increment the number of calls of the given RPC.
877 (if (profiled? "rpc")
878 (begin
879 (register-profiling-hook! "rpc" show-rpc-profile)
880 (lambda (name)
881 (let ((count (or (hashq-ref %rpc-calls name) 0)))
882 (hashq-set! %rpc-calls name (+ count 1)))))
883 (lambda (_)
884 #t)))
886 (define-syntax operation
887 (syntax-rules ()
888 "Define a client-side RPC stub for the given operation."
889 ((_ (name (type arg) ...) docstring return ...)
890 (lambda (server arg ...)
891 docstring
892 (let* ((s (nix-server-socket server))
893 (buffered (nix-server-output-port server)))
894 (record-operation 'name)
895 (write-int (operation-id name) buffered)
896 (write-arg type arg buffered)
897 ...
898 (write-buffered-output server)
900 ;; Loop until the server is done sending error output.
901 (let loop ((done? (process-stderr server)))
902 (or done? (loop (process-stderr server))))
903 (values (read-arg return s) ...))))))
905 (define-syntax-rule (define-operation (name args ...)
906 docstring return ...)
907 (define name
908 (operation (name args ...) docstring return ...)))
910 (define-operation (valid-path? (string path))
911 "Return #t when PATH designates a valid store item and #f otherwise (an
912 invalid item may exist on disk but still be invalid, for instance because it
913 is the result of an aborted or failed build.)
915 A '&nix-protocol-error' condition is raised if PATH is not prefixed by the
916 store directory (/gnu/store)."
917 boolean)
919 (define-operation (query-path-hash (store-path path))
920 "Return the SHA256 hash of the nar serialization of PATH as a bytevector."
921 base16)
923 (define hash-part->path
924 (let ((query-path-from-hash-part
925 (operation (query-path-from-hash-part (string hash))
926 #f
927 store-path)))
928 (lambda (server hash-part)
929 "Return the store path whose hash part is HASH-PART (a nix-base32
930 string). Raise an error if no such path exists."
931 ;; This RPC is primarily used by Hydra to reply to HTTP GETs of
932 ;; /HASH.narinfo.
933 (query-path-from-hash-part server hash-part))))
935 (define-operation (query-path-info (store-path path))
936 "Return the info (hash, references, etc.) for PATH."
937 path-info)
939 (define add-data-to-store
940 ;; A memoizing version of `add-to-store', to avoid repeated RPCs with
941 ;; the very same arguments during a given session.
942 (let ((add-text-to-store
943 (operation (add-text-to-store (string name) (bytevector text)
944 (string-list references))
945 #f
946 store-path)))
947 (lambda* (server name bytes #:optional (references '()))
948 "Add BYTES under file NAME in the store, and return its store path.
949 REFERENCES is the list of store paths referred to by the resulting store
950 path."
951 (let* ((args `(,bytes ,name ,references))
952 (cache (nix-server-add-text-to-store-cache server)))
953 (or (hash-ref cache args)
954 (let ((path (add-text-to-store server name bytes references)))
955 (hash-set! cache args path)
956 path))))))
958 (define* (add-text-to-store store name text #:optional (references '()))
959 "Add TEXT under file NAME in the store, and return its store path.
960 REFERENCES is the list of store paths referred to by the resulting store
961 path."
962 (add-data-to-store store name (string->utf8 text) references))
964 (define true
965 ;; Define it once and for all since we use it as a default value for
966 ;; 'add-to-store' and want to make sure two default values are 'eq?' for the
967 ;; purposes or memoization.
968 (lambda (file stat)
969 #t))
971 (define add-to-store
972 ;; A memoizing version of `add-to-store'. This is important because
973 ;; `add-to-store' leads to huge data transfers to the server, and
974 ;; because it's often called many times with the very same argument.
975 (let ((add-to-store
976 (lambda* (server basename recursive? hash-algo file-name
977 #:key (select? true))
978 ;; We don't use the 'operation' macro so we can pass SELECT? to
979 ;; 'write-file'.
980 (record-operation 'add-to-store)
981 (let ((port (nix-server-socket server)))
982 (write-int (operation-id add-to-store) port)
983 (write-string basename port)
984 (write-int 1 port) ;obsolete, must be #t
985 (write-int (if recursive? 1 0) port)
986 (write-string hash-algo port)
987 (write-file file-name port #:select? select?)
988 (write-buffered-output server)
989 (let loop ((done? (process-stderr server)))
990 (or done? (loop (process-stderr server))))
991 (read-store-path port)))))
992 (lambda* (server basename recursive? hash-algo file-name
993 #:key (select? true))
994 "Add the contents of FILE-NAME under BASENAME to the store. When
995 RECURSIVE? is false, FILE-NAME must designate a regular file--not a directory
996 nor a symlink. When RECURSIVE? is true and FILE-NAME designates a directory,
997 the contents of FILE-NAME are added recursively; if FILE-NAME designates a
998 flat file and RECURSIVE? is true, its contents are added, and its permission
999 bits are kept. HASH-ALGO must be a string such as \"sha256\".
1001 When RECURSIVE? is true, call (SELECT? FILE STAT) for each directory entry,
1002 where FILE is the entry's absolute file name and STAT is the result of
1003 'lstat'; exclude entries for which SELECT? does not return true."
1004 ;; Note: We don't stat FILE-NAME at each call, and thus we assume that
1005 ;; the file remains unchanged for the lifetime of SERVER.
1006 (let* ((args `(,file-name ,basename ,recursive? ,hash-algo ,select?))
1007 (cache (nix-server-add-to-store-cache server)))
1008 (or (hash-ref cache args)
1009 (let ((path (add-to-store server basename recursive?
1010 hash-algo file-name
1011 #:select? select?)))
1012 (hash-set! cache args path)
1013 path))))))
1015 (define %not-slash
1016 (char-set-complement (char-set #\/)))
1018 (define* (add-file-tree-to-store server tree
1019 #:key
1020 (hash-algo "sha256")
1021 (recursive? #t))
1022 "Add the given TREE to the store on SERVER. TREE must be an entry such as:
1024 (\"my-tree\" directory
1025 (\"a\" regular (data \"hello\"))
1026 (\"b\" symlink \"a\")
1027 (\"c\" directory
1028 (\"d\" executable (file \"/bin/sh\"))))
1030 This is a generalized version of 'add-to-store'. It allows you to reproduce
1031 an arbitrary directory layout in the store without creating a derivation."
1033 ;; Note: The format of TREE was chosen to allow trees to be compared with
1034 ;; 'equal?', which in turn allows us to memoize things.
1036 (define root
1037 ;; TREE is a single entry.
1038 (list tree))
1040 (define basename
1041 (match tree
1042 ((name . _) name)))
1044 (define (lookup file)
1045 (let loop ((components (string-tokenize file %not-slash))
1046 (tree root))
1047 (match components
1048 ((basename)
1049 (assoc basename tree))
1050 ((head . rest)
1051 (loop rest
1052 (match (assoc-ref tree head)
1053 (('directory . entries) entries)))))))
1055 (define (file-type+size file)
1056 (match (lookup file)
1057 ((_ (and type (or 'directory 'symlink)) . _)
1058 (values type 0))
1059 ((_ type ('file file))
1060 (values type (stat:size (stat file))))
1061 ((_ type ('data (? string? data)))
1062 (values type (string-length data)))
1063 ((_ type ('data (? bytevector? data)))
1064 (values type (bytevector-length data)))))
1066 (define (file-port file)
1067 (match (lookup file)
1068 ((_ (or 'regular 'executable) content)
1069 (match content
1070 (('file (? string? file))
1071 (open-file file "r0b"))
1072 (('data (? string? str))
1073 (open-input-string str))
1074 (('data (? bytevector? bv))
1075 (open-bytevector-input-port bv))))))
1077 (define (symlink-target file)
1078 (match (lookup file)
1079 ((_ 'symlink target) target)))
1081 (define (directory-entries directory)
1082 (match (lookup directory)
1083 ((_ 'directory (names . _) ...) names)))
1085 (define cache
1086 (nix-server-add-to-store-cache server))
1088 (or (hash-ref cache tree)
1089 (begin
1090 ;; We don't use the 'operation' macro so we can use 'write-file-tree'
1091 ;; instead of 'write-file'.
1092 (record-operation 'add-to-store/tree)
1093 (let ((port (nix-server-socket server)))
1094 (write-int (operation-id add-to-store) port)
1095 (write-string basename port)
1096 (write-int 1 port) ;obsolete, must be #t
1097 (write-int (if recursive? 1 0) port)
1098 (write-string hash-algo port)
1099 (write-file-tree basename port
1100 #:file-type+size file-type+size
1101 #:file-port file-port
1102 #:symlink-target symlink-target
1103 #:directory-entries directory-entries)
1104 (write-buffered-output server)
1105 (let loop ((done? (process-stderr server)))
1106 (or done? (loop (process-stderr server))))
1107 (let ((result (read-store-path port)))
1108 (hash-set! cache tree result)
1109 result)))))
1111 (define build-things
1112 (let ((build (operation (build-things (string-list things)
1113 (integer mode))
1114 "Do it!"
1115 boolean))
1116 (build/old (operation (build-things (string-list things))
1117 "Do it!"
1118 boolean)))
1119 (lambda* (store things #:optional (mode (build-mode normal)))
1120 "Build THINGS, a list of store items which may be either '.drv' files or
1121 outputs, and return when the worker is done building them. Elements of THINGS
1122 that are not derivations can only be substituted and not built locally.
1123 Return #t on success."
1124 (parameterize ((current-store-protocol-version
1125 (nix-server-version store)))
1126 (if (>= (nix-server-minor-version store) 15)
1127 (build store things mode)
1128 (if (= mode (build-mode normal))
1129 (build/old store things)
1130 (raise (condition (&nix-protocol-error
1131 (message "unsupported build mode")
1132 (status 1))))))))))
1134 (define-operation (add-temp-root (store-path path))
1135 "Make PATH a temporary root for the duration of the current session.
1136 Return #t."
1137 boolean)
1139 (define-operation (add-indirect-root (string file-name))
1140 "Make the symlink FILE-NAME an indirect root for the garbage collector:
1141 whatever store item FILE-NAME points to will not be collected. Return #t on
1142 success.
1144 FILE-NAME can be anywhere on the file system, but it must be an absolute file
1145 name--it is the caller's responsibility to ensure that it is an absolute file
1146 name."
1147 boolean)
1149 (define %gc-roots-directory
1150 ;; The place where garbage collector roots (symlinks) are kept.
1151 (string-append %state-directory "/gcroots"))
1153 (define (add-permanent-root target)
1154 "Add a garbage collector root pointing to TARGET, an element of the store,
1155 preventing TARGET from even being collected. This can also be used if TARGET
1156 does not exist yet.
1158 Raise an error if the caller does not have write access to the GC root
1159 directory."
1160 (let* ((root (string-append %gc-roots-directory "/" (basename target))))
1161 (catch 'system-error
1162 (lambda ()
1163 (symlink target root))
1164 (lambda args
1165 ;; If ROOT already exists, this is fine; otherwise, re-throw.
1166 (unless (= EEXIST (system-error-errno args))
1167 (apply throw args))))))
1169 (define (remove-permanent-root target)
1170 "Remove the permanent garbage collector root pointing to TARGET. Raise an
1171 error if there is no such root."
1172 (delete-file (string-append %gc-roots-directory "/" (basename target))))
1174 (define references
1175 (operation (query-references (store-path path))
1176 "Return the list of references of PATH."
1177 store-path-list))
1179 (define %reference-cache
1180 ;; Brute-force cache mapping store items to their list of references.
1181 ;; Caching matters because when building a profile in the presence of
1182 ;; grafts, we keep calling 'graft-derivation', which in turn calls
1183 ;; 'references/substitutes' many times with the same arguments. Ideally we
1184 ;; would use a cache associated with the daemon connection instead (XXX).
1185 (make-hash-table 100))
1187 (define (references/substitutes store items)
1188 "Return the list of list of references of ITEMS; the result has the same
1189 length as ITEMS. Query substitute information for any item missing from the
1190 store at once. Raise a '&nix-protocol-error' exception if reference
1191 information for one of ITEMS is missing."
1192 (let* ((requested items)
1193 (local-refs (map (lambda (item)
1194 (or (hash-ref %reference-cache item)
1195 (guard (c ((nix-protocol-error? c) #f))
1196 (references store item))))
1197 items))
1198 (missing (fold-right (lambda (item local-ref result)
1199 (if local-ref
1200 result
1201 (cons item result)))
1202 '()
1203 items local-refs))
1205 ;; Query all the substitutes at once to minimize the cost of
1206 ;; launching 'guix substitute' and making HTTP requests.
1207 (substs (if (null? missing)
1208 '()
1209 (substitutable-path-info store missing))))
1210 (when (< (length substs) (length missing))
1211 (raise (condition (&nix-protocol-error
1212 (message "cannot determine \
1213 the list of references")
1214 (status 1)))))
1216 ;; Intersperse SUBSTS and LOCAL-REFS.
1217 (let loop ((items items)
1218 (local-refs local-refs)
1219 (result '()))
1220 (match items
1221 (()
1222 (let ((result (reverse result)))
1223 (for-each (cut hash-set! %reference-cache <> <>)
1224 requested result)
1225 result))
1226 ((item items ...)
1227 (match local-refs
1228 ((#f tail ...)
1229 (loop items tail
1230 (cons (any (lambda (subst)
1231 (and (string=? (substitutable-path subst) item)
1232 (substitutable-references subst)))
1233 substs)
1234 result)))
1235 ((head tail ...)
1236 (loop items tail
1237 (cons head result)))))))))
1239 (define* (fold-path store proc seed paths
1240 #:optional (relatives (cut references store <>)))
1241 "Call PROC for each of the RELATIVES of PATHS, exactly once, and return the
1242 result formed from the successive calls to PROC, the first of which is passed
1243 SEED."
1244 (let loop ((paths paths)
1245 (result seed)
1246 (seen vlist-null))
1247 (match paths
1248 ((path rest ...)
1249 (if (vhash-assoc path seen)
1250 (loop rest result seen)
1251 (let ((seen (vhash-cons path #t seen))
1252 (rest (append rest (relatives path)))
1253 (result (proc path result)))
1254 (loop rest result seen))))
1255 (()
1256 result))))
1258 (define (requisites store paths)
1259 "Return the requisites of PATHS, including PATHS---i.e., their closures (all
1260 its references, recursively)."
1261 (fold-path store cons '() paths))
1263 (define (topologically-sorted store paths)
1264 "Return a list containing PATHS and all their references sorted in
1265 topological order."
1266 (define (traverse)
1267 ;; Do a simple depth-first traversal of all of PATHS.
1268 (let loop ((paths paths)
1269 (visited vlist-null)
1270 (result '()))
1271 (define (visit n)
1272 (vhash-cons n #t visited))
1274 (define (visited? n)
1275 (vhash-assoc n visited))
1277 (match paths
1278 ((head tail ...)
1279 (if (visited? head)
1280 (loop tail visited result)
1281 (call-with-values
1282 (lambda ()
1283 (loop (references store head)
1284 (visit head)
1285 result))
1286 (lambda (visited result)
1287 (loop tail
1288 visited
1289 (cons head result))))))
1290 (()
1291 (values visited result)))))
1293 (call-with-values traverse
1294 (lambda (_ result)
1295 (reverse result))))
1297 (define referrers
1298 (operation (query-referrers (store-path path))
1299 "Return the list of path that refer to PATH."
1300 store-path-list))
1302 (define valid-derivers
1303 (operation (query-valid-derivers (store-path path))
1304 "Return the list of valid \"derivers\" of PATH---i.e., all the
1305 .drv present in the store that have PATH among their outputs."
1306 store-path-list))
1308 (define query-derivation-outputs ; avoid name clash with `derivation-outputs'
1309 (operation (query-derivation-outputs (store-path path))
1310 "Return the list of outputs of PATH, a .drv file."
1311 store-path-list))
1313 (define-operation (has-substitutes? (store-path path))
1314 "Return #t if binary substitutes are available for PATH, and #f otherwise."
1315 boolean)
1317 (define substitutable-paths
1318 (operation (query-substitutable-paths (store-path-list paths))
1319 "Return the subset of PATHS that is substitutable."
1320 store-path-list))
1322 (define substitutable-path-info
1323 (operation (query-substitutable-path-infos (store-path-list paths))
1324 "Return information about the subset of PATHS that is
1325 substitutable. For each substitutable path, a `substitutable?' object is
1326 returned; thus, the resulting list can be shorter than PATHS. Furthermore,
1327 that there is no guarantee that the order of the resulting list matches the
1328 order of PATHS."
1329 substitutable-path-list))
1331 (define built-in-builders
1332 (let ((builders (operation (built-in-builders)
1333 "Return the built-in builders."
1334 string-list)))
1335 (lambda (store)
1336 "Return the names of the supported built-in derivation builders
1337 supported by STORE."
1338 ;; Check whether STORE's version supports this RPC and built-in
1339 ;; derivation builders in general, which appeared in Guix > 0.11.0.
1340 ;; Return the empty list if it doesn't. Note that this RPC does not
1341 ;; exist in 'nix-daemon'.
1342 (if (or (> (nix-server-major-version store) #x100)
1343 (and (= (nix-server-major-version store) #x100)
1344 (>= (nix-server-minor-version store) #x60)))
1345 (builders store)
1346 '()))))
1348 (define-operation (optimize-store)
1349 "Optimize the store by hard-linking identical files (\"deduplication\".)
1350 Return #t on success."
1351 ;; Note: the daemon in Guix <= 0.8.2 does not implement this RPC.
1352 boolean)
1354 (define verify-store
1355 (let ((verify (operation (verify-store (boolean check-contents?)
1356 (boolean repair?))
1357 "Verify the store."
1358 boolean)))
1359 (lambda* (store #:key check-contents? repair?)
1360 "Verify the integrity of the store and return false if errors remain,
1361 and true otherwise. When REPAIR? is true, repair any missing or altered store
1362 items by substituting them (this typically requires root privileges because it
1363 is not an atomic operation.) When CHECK-CONTENTS? is true, check the contents
1364 of store items; this can take a lot of time."
1365 (not (verify store check-contents? repair?)))))
1367 (define (run-gc server action to-delete min-freed)
1368 "Perform the garbage-collector operation ACTION, one of the
1369 `gc-action' values. When ACTION is `delete-specific', the TO-DELETE is
1370 the list of store paths to delete. IGNORE-LIVENESS? should always be
1371 #f. MIN-FREED is the minimum amount of disk space to be freed, in
1372 bytes, before the GC can stop. Return the list of store paths delete,
1373 and the number of bytes freed."
1374 (let ((s (nix-server-socket server)))
1375 (write-int (operation-id collect-garbage) s)
1376 (write-int action s)
1377 (write-store-path-list to-delete s)
1378 (write-arg boolean #f s) ; ignore-liveness?
1379 (write-long-long min-freed s)
1380 (write-int 0 s) ; obsolete
1381 (when (>= (nix-server-minor-version server) 5)
1382 ;; Obsolete `use-atime' and `max-atime' parameters.
1383 (write-int 0 s)
1384 (write-int 0 s))
1386 ;; Loop until the server is done sending error output.
1387 (let loop ((done? (process-stderr server)))
1388 (or done? (loop (process-stderr server))))
1390 (let ((paths (read-store-path-list s))
1391 (freed (read-long-long s))
1392 (obsolete (read-long-long s)))
1393 (unless (null? paths)
1394 ;; To be on the safe side, completely invalidate both caches.
1395 ;; Otherwise we could end up returning store paths that are no longer
1396 ;; valid.
1397 (hash-clear! (nix-server-add-to-store-cache server))
1398 (hash-clear! (nix-server-add-text-to-store-cache server)))
1400 (values paths freed))))
1402 (define-syntax-rule (%long-long-max)
1403 ;; Maximum unsigned 64-bit integer.
1404 (- (expt 2 64) 1))
1406 (define (live-paths server)
1407 "Return the list of live store paths---i.e., store paths still
1408 referenced, and thus not subject to being garbage-collected."
1409 (run-gc server (gc-action return-live) '() (%long-long-max)))
1411 (define (dead-paths server)
1412 "Return the list of dead store paths---i.e., store paths no longer
1413 referenced, and thus subject to being garbage-collected."
1414 (run-gc server (gc-action return-dead) '() (%long-long-max)))
1416 (define* (collect-garbage server #:optional (min-freed (%long-long-max)))
1417 "Collect garbage from the store at SERVER. If MIN-FREED is non-zero,
1418 then collect at least MIN-FREED bytes. Return the paths that were
1419 collected, and the number of bytes freed."
1420 (run-gc server (gc-action delete-dead) '() min-freed))
1422 (define* (delete-paths server paths #:optional (min-freed (%long-long-max)))
1423 "Delete PATHS from the store at SERVER, if they are no longer
1424 referenced. If MIN-FREED is non-zero, then stop after at least
1425 MIN-FREED bytes have been collected. Return the paths that were
1426 collected, and the number of bytes freed."
1427 (run-gc server (gc-action delete-specific) paths min-freed))
1429 (define (import-paths server port)
1430 "Import the set of store paths read from PORT into SERVER's store. An error
1431 is raised if the set of paths read from PORT is not signed (as per
1432 'export-path #:sign? #t'.) Return the list of store paths imported."
1433 (let ((s (nix-server-socket server)))
1434 (write-int (operation-id import-paths) s)
1435 (let loop ((done? (process-stderr server port)))
1436 (or done? (loop (process-stderr server port))))
1437 (read-store-path-list s)))
1439 (define* (export-path server path port #:key (sign? #t))
1440 "Export PATH to PORT. When SIGN? is true, sign it."
1441 (let ((s (nix-server-socket server)))
1442 (write-int (operation-id export-path) s)
1443 (write-store-path path s)
1444 (write-arg boolean sign? s)
1445 (let loop ((done? (process-stderr server port)))
1446 (or done? (loop (process-stderr server port))))
1447 (= 1 (read-int s))))
1449 (define* (export-paths server paths port #:key (sign? #t) recursive?)
1450 "Export the store paths listed in PATHS to PORT, in topological order,
1451 signing them if SIGN? is true. When RECURSIVE? is true, export the closure of
1452 PATHS---i.e., PATHS and all their dependencies."
1453 (define ordered
1454 (let ((sorted (topologically-sorted server paths)))
1455 ;; When RECURSIVE? is #f, filter out the references of PATHS.
1456 (if recursive?
1457 sorted
1458 (filter (cut member <> paths) sorted))))
1460 (let loop ((paths ordered))
1461 (match paths
1462 (()
1463 (write-int 0 port))
1464 ((head tail ...)
1465 (write-int 1 port)
1466 (and (export-path server head port #:sign? sign?)
1467 (loop tail))))))
1469 (define-operation (query-failed-paths)
1470 "Return the list of store items for which a build failure is cached.
1472 The result is always the empty list unless the daemon was started with
1473 '--cache-failures'."
1474 store-path-list)
1476 (define-operation (clear-failed-paths (store-path-list items))
1477 "Remove ITEMS from the list of cached build failures.
1479 This makes sense only when the daemon was started with '--cache-failures'."
1480 boolean)
1482 \f
1483 ;;;
1484 ;;; Store monad.
1485 ;;;
1487 (define-syntax-rule (define-alias new old)
1488 (define-syntax new (identifier-syntax old)))
1490 ;; The store monad allows us to (1) build sequences of operations in the
1491 ;; store, and (2) make the store an implicit part of the execution context,
1492 ;; rather than a parameter of every single function.
1493 (define-alias %store-monad %state-monad)
1494 (define-alias store-return state-return)
1495 (define-alias store-bind state-bind)
1497 ;; Instantiate templates for %STORE-MONAD since it's syntactically different
1498 ;; from %STATE-MONAD.
1499 (template-directory instantiations %store-monad)
1501 (define* (cache-object-mapping object keys result)
1502 "Augment the store's object cache with a mapping from OBJECT/KEYS to RESULT.
1503 KEYS is a list of additional keys to match against, for instance a (SYSTEM
1504 TARGET) tuple.
1506 OBJECT is typically a high-level object such as a <package> or an <origin>,
1507 and RESULT is typically its derivation."
1508 (lambda (store)
1509 (values result
1510 (nix-server
1511 (inherit store)
1512 (object-cache (vhash-consq object (cons result keys)
1513 (nix-server-object-cache store)))))))
1515 (define record-cache-lookup!
1516 (if (profiled? "object-cache")
1517 (let ((fresh 0)
1518 (lookups 0)
1519 (hits 0))
1520 (register-profiling-hook!
1521 "object-cache"
1522 (lambda ()
1523 (format (current-error-port) "Store object cache:
1524 fresh caches: ~5@a
1525 lookups: ~5@a
1526 hits: ~5@a (~,1f%)~%"
1527 fresh lookups hits
1528 (if (zero? lookups)
1529 100.
1530 (* 100. (/ hits lookups))))))
1532 (lambda (hit? cache)
1533 (set! fresh
1534 (if (eq? cache vlist-null)
1535 (+ 1 fresh)
1536 fresh))
1537 (set! lookups (+ 1 lookups))
1538 (set! hits (if hit? (+ hits 1) hits))))
1539 (lambda (x y)
1540 #t)))
1542 (define* (lookup-cached-object object #:optional (keys '()))
1543 "Return the cached object in the store connection corresponding to OBJECT
1544 and KEYS. KEYS is a list of additional keys to match against, and which are
1545 compared with 'equal?'. Return #f on failure and the cached result
1546 otherwise."
1547 (lambda (store)
1548 (let* ((cache (nix-server-object-cache store))
1550 ;; Escape as soon as we find the result. This avoids traversing
1551 ;; the whole vlist chain and significantly reduces the number of
1552 ;; 'hashq' calls.
1553 (value (let/ec return
1554 (vhash-foldq* (lambda (item result)
1555 (match item
1556 ((value . keys*)
1557 (if (equal? keys keys*)
1558 (return value)
1559 result))))
1560 #f object
1561 cache))))
1562 (record-cache-lookup! value cache)
1563 (values value store))))
1565 (define* (%mcached mthunk object #:optional (keys '()))
1566 "Bind the monadic value returned by MTHUNK, which supposedly corresponds to
1567 OBJECT/KEYS, or return its cached value."
1568 (mlet %store-monad ((cached (lookup-cached-object object keys)))
1569 (if cached
1570 (return cached)
1571 (>>= (mthunk)
1572 (lambda (result)
1573 (cache-object-mapping object keys result))))))
1575 (define-syntax-rule (mcached mvalue object keys ...)
1576 "Run MVALUE, which corresponds to OBJECT/KEYS, and cache it; or return the
1577 value associated with OBJECT/KEYS in the store's object cache if there is
1578 one."
1579 (%mcached (lambda () mvalue)
1580 object (list keys ...)))
1582 (define (preserve-documentation original proc)
1583 "Return PROC with documentation taken from ORIGINAL."
1584 (set-object-property! proc 'documentation
1585 (procedure-property original 'documentation))
1586 proc)
1588 (define (store-lift proc)
1589 "Lift PROC, a procedure whose first argument is a connection to the store,
1590 in the store monad."
1591 (preserve-documentation proc
1592 (lambda args
1593 (lambda (store)
1594 (values (apply proc store args) store)))))
1596 (define (store-lower proc)
1597 "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure
1598 taking the store as its first argument."
1599 (preserve-documentation proc
1600 (lambda (store . args)
1601 (run-with-store store (apply proc args)))))
1603 ;;
1604 ;; Store monad operators.
1605 ;;
1607 (define* (binary-file name
1608 data ;bytevector
1609 #:optional (references '()))
1610 "Return as a monadic value the absolute file name in the store of the file
1611 containing DATA, a bytevector. REFERENCES is a list of store items that the
1612 resulting text file refers to; it defaults to the empty list."
1613 (lambda (store)
1614 (values (add-data-to-store store name data references)
1615 store)))
1617 (define* (text-file name
1618 text ;string
1619 #:optional (references '()))
1620 "Return as a monadic value the absolute file name in the store of the file
1621 containing TEXT, a string. REFERENCES is a list of store items that the
1622 resulting text file refers to; it defaults to the empty list."
1623 (lambda (store)
1624 (values (add-text-to-store store name text references)
1625 store)))
1627 (define* (interned-file file #:optional name
1628 #:key (recursive? #t) (select? true))
1629 "Return the name of FILE once interned in the store. Use NAME as its store
1630 name, or the basename of FILE if NAME is omitted.
1632 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
1633 designates a flat file and RECURSIVE? is true, its contents are added, and its
1634 permission bits are kept.
1636 When RECURSIVE? is true, call (SELECT? FILE STAT) for each directory entry,
1637 where FILE is the entry's absolute file name and STAT is the result of
1638 'lstat'; exclude entries for which SELECT? does not return true."
1639 (lambda (store)
1640 (values (add-to-store store (or name (basename file))
1641 recursive? "sha256" file
1642 #:select? select?)
1643 store)))
1645 (define interned-file-tree
1646 (store-lift add-file-tree-to-store))
1648 (define build
1649 ;; Monadic variant of 'build-things'.
1650 (store-lift build-things))
1652 (define set-build-options*
1653 (store-lift set-build-options))
1655 (define references*
1656 (store-lift references))
1658 (define (query-path-info* item)
1659 "Monadic version of 'query-path-info' that returns #f when ITEM is not in
1660 the store."
1661 (lambda (store)
1662 (guard (c ((nix-protocol-error? c)
1663 ;; ITEM is not in the store; return #f.
1664 (values #f store)))
1665 (values (query-path-info store item) store))))
1667 (define-inlinable (current-system)
1668 ;; Consult the %CURRENT-SYSTEM fluid at bind time. This is equivalent to
1669 ;; (lift0 %current-system %store-monad), but inlinable, thus avoiding
1670 ;; closure allocation in some cases.
1671 (lambda (state)
1672 (values (%current-system) state)))
1674 (define-inlinable (set-current-system system)
1675 ;; Set the %CURRENT-SYSTEM fluid at bind time.
1676 (lambda (state)
1677 (values (%current-system system) state)))
1679 (define %guile-for-build
1680 ;; The derivation of the Guile to be used within the build environment,
1681 ;; when using 'gexp->derivation' and co.
1682 (make-parameter #f))
1684 (define* (run-with-store store mval
1685 #:key
1686 (guile-for-build (%guile-for-build))
1687 (system (%current-system))
1688 (target #f))
1689 "Run MVAL, a monadic value in the store monad, in STORE, an open store
1690 connection, and return the result."
1691 ;; Initialize the dynamic bindings here to avoid bad surprises. The
1692 ;; difficulty lies in the fact that dynamic bindings are resolved at
1693 ;; bind-time and not at call time, which can be disconcerting.
1694 (parameterize ((%guile-for-build guile-for-build)
1695 (%current-system system)
1696 (%current-target-system target))
1697 (call-with-values (lambda ()
1698 (run-with-state mval store))
1699 (lambda (result store)
1700 ;; Discard the state.
1701 result))))
1703 \f
1704 ;;;
1705 ;;; Store paths.
1706 ;;;
1708 (define %store-prefix
1709 ;; Absolute path to the Nix store.
1710 (make-parameter %store-directory))
1712 (define (compressed-hash bv size) ; `compressHash'
1713 "Given the hash stored in BV, return a compressed version thereof that fits
1714 in SIZE bytes."
1715 (define new (make-bytevector size 0))
1716 (define old-size (bytevector-length bv))
1717 (let loop ((i 0))
1718 (if (= i old-size)
1719 new
1720 (let* ((j (modulo i size))
1721 (o (bytevector-u8-ref new j)))
1722 (bytevector-u8-set! new j
1723 (logxor o (bytevector-u8-ref bv i)))
1724 (loop (+ 1 i))))))
1726 (define (store-path type hash name) ; makeStorePath
1727 "Return the store path for NAME/HASH/TYPE."
1728 (let* ((s (string-append type ":sha256:"
1729 (bytevector->base16-string hash) ":"
1730 (%store-prefix) ":" name))
1731 (h (sha256 (string->utf8 s)))
1732 (c (compressed-hash h 20)))
1733 (string-append (%store-prefix) "/"
1734 (bytevector->nix-base32-string c) "-"
1735 name)))
1737 (define (output-path output hash name) ; makeOutputPath
1738 "Return an output path for OUTPUT (the name of the output as a string) of
1739 the derivation called NAME with hash HASH."
1740 (store-path (string-append "output:" output) hash
1741 (if (string=? output "out")
1742 name
1743 (string-append name "-" output))))
1745 (define* (fixed-output-path name hash
1746 #:key
1747 (output "out")
1748 (hash-algo 'sha256)
1749 (recursive? #t))
1750 "Return an output path for the fixed output OUTPUT defined by HASH of type
1751 HASH-ALGO, of the derivation NAME. RECURSIVE? has the same meaning as for
1752 'add-to-store'."
1753 (if (and recursive? (eq? hash-algo 'sha256))
1754 (store-path "source" hash name)
1755 (let ((tag (string-append "fixed:" output ":"
1756 (if recursive? "r:" "")
1757 (symbol->string hash-algo) ":"
1758 (bytevector->base16-string hash) ":")))
1759 (store-path (string-append "output:" output)
1760 (sha256 (string->utf8 tag))
1761 name))))
1763 (define (store-path? path)
1764 "Return #t if PATH is a store path."
1765 ;; This is a lightweight check, compared to using a regexp, but this has to
1766 ;; be fast as it's called often in `derivation', for instance.
1767 ;; `isStorePath' in Nix does something similar.
1768 (string-prefix? (%store-prefix) path))
1770 (define (direct-store-path? path)
1771 "Return #t if PATH is a store path, and not a sub-directory of a store path.
1772 This predicate is sometimes needed because files *under* a store path are not
1773 valid inputs."
1774 (and (store-path? path)
1775 (not (string=? path (%store-prefix)))
1776 (let ((len (+ 1 (string-length (%store-prefix)))))
1777 (not (string-index (substring path len) #\/)))))
1779 (define (direct-store-path path)
1780 "Return the direct store path part of PATH, stripping components after
1781 '/gnu/store/xxxx-foo'."
1782 (let ((prefix-length (+ (string-length (%store-prefix)) 35)))
1783 (if (> (string-length path) prefix-length)
1784 (let ((slash (string-index path #\/ prefix-length)))
1785 (if slash (string-take path slash) path))
1786 path)))
1788 (define (derivation-path? path)
1789 "Return #t if PATH is a derivation path."
1790 (and (store-path? path) (string-suffix? ".drv" path)))
1792 (define store-regexp*
1793 ;; The substituter makes repeated calls to 'store-path-hash-part', hence
1794 ;; this optimization.
1795 (mlambda (store)
1796 "Return a regexp matching a file in STORE."
1797 (make-regexp (string-append "^" (regexp-quote store)
1798 "/([0-9a-df-np-sv-z]{32})-([^/]+)$"))))
1800 (define (store-path-package-name path)
1801 "Return the package name part of PATH, a file name in the store."
1802 (let ((path-rx (store-regexp* (%store-prefix))))
1803 (and=> (regexp-exec path-rx path)
1804 (cut match:substring <> 2))))
1806 (define (store-path-hash-part path)
1807 "Return the hash part of PATH as a base32 string, or #f if PATH is not a
1808 syntactically valid store path."
1809 (and (string-prefix? (%store-prefix) path)
1810 (let ((base (string-drop path (+ 1 (string-length (%store-prefix))))))
1811 (and (> (string-length base) 33)
1812 (let ((hash (string-take base 32)))
1813 (and (string-every %nix-base32-charset hash)
1814 hash))))))
1816 (define (derivation-log-file drv)
1817 "Return the build log file for DRV, a derivation file name, or #f if it
1818 could not be found."
1819 (let* ((base (basename drv))
1820 (log (string-append (dirname %state-directory) ; XXX
1821 "/log/guix/drvs/"
1822 (string-take base 2) "/"
1823 (string-drop base 2)))
1824 (log.gz (string-append log ".gz"))
1825 (log.bz2 (string-append log ".bz2")))
1826 (cond ((file-exists? log.gz) log.gz)
1827 ((file-exists? log.bz2) log.bz2)
1828 ((file-exists? log) log)
1829 (else #f))))
1831 (define (log-file store file)
1832 "Return the build log file for FILE, or #f if none could be found. FILE
1833 must be an absolute store file name, or a derivation file name."
1834 (cond ((derivation-path? file)
1835 (derivation-log-file file))
1836 (else
1837 (match (valid-derivers store file)
1838 ((derivers ...)
1839 ;; Return the first that works.
1840 (any (cut log-file store <>) derivers))
1841 (_ #f)))))
1843 ;;; Local Variables:
1844 ;;; eval: (put 'system-error-to-connection-error 'scheme-indent-function 1)
1845 ;;; End: