| blob | d7c603898ca0bb04c224df7b3f90426f121c57a2 |
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
3 ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org>
4 ;;;
5 ;;; This file is part of GNU Guix.
6 ;;;
7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
8 ;;; under the terms of the GNU General Public License as published by
9 ;;; the Free Software Foundation; either version 3 of the License, or (at
10 ;;; your option) any later version.
11 ;;;
12 ;;; GNU Guix is distributed in the hope that it will be useful, but
13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ;;; GNU General Public License for more details.
16 ;;;
17 ;;; You should have received a copy of the GNU General Public License
18 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
20 (define-module (guix store)
21 #:use-module (guix utils)
22 #:use-module (guix config)
23 #:use-module (guix deprecation)
24 #:use-module (guix memoization)
25 #:use-module (guix serialization)
26 #:use-module (guix monads)
27 #:use-module (guix records)
28 #:use-module (guix base16)
29 #:use-module (guix base32)
30 #:use-module (gcrypt hash)
31 #:use-module (guix profiling)
32 #:autoload (guix build syscalls) (terminal-columns)
33 #:use-module (rnrs bytevectors)
34 #:use-module (ice-9 binary-ports)
35 #:use-module ((ice-9 control) #:select (let/ec))
36 #:use-module (srfi srfi-1)
37 #:use-module (srfi srfi-9)
38 #:use-module (srfi srfi-9 gnu)
39 #:use-module (srfi srfi-11)
40 #:use-module (srfi srfi-26)
41 #:use-module (srfi srfi-34)
42 #:use-module (srfi srfi-35)
43 #:use-module (srfi srfi-39)
44 #:use-module (ice-9 match)
45 #:use-module (ice-9 regex)
46 #:use-module (ice-9 vlist)
47 #:use-module (ice-9 popen)
48 #:use-module (ice-9 threads)
49 #:use-module (ice-9 format)
50 #:use-module (web uri)
51 #:export (%daemon-socket-uri
52 %gc-roots-directory
53 %default-substitute-urls
55 store-connection?
56 store-connection-version
57 store-connection-major-version
58 store-connection-minor-version
59 store-connection-socket
61 ;; Deprecated forms for 'store-connection'.
62 nix-server?
63 nix-server-version
64 nix-server-major-version
65 nix-server-minor-version
66 nix-server-socket
68 current-store-protocol-version ;for internal use
69 mcached
71 &store-error store-error?
72 &store-connection-error store-connection-error?
73 store-connection-error-file
74 store-connection-error-code
75 &store-protocol-error store-protocol-error?
76 store-protocol-error-message
77 store-protocol-error-status
79 ;; Deprecated forms for '&store-error' et al.
80 &nix-error nix-error?
81 &nix-connection-error nix-connection-error?
82 nix-connection-error-file
83 nix-connection-error-code
84 &nix-protocol-error nix-protocol-error?
85 nix-protocol-error-message
86 nix-protocol-error-status
88 hash-algo
89 build-mode
91 open-connection
92 port->connection
93 close-connection
94 with-store
95 set-build-options
96 set-build-options*
97 valid-path?
98 query-path-hash
99 hash-part->path
100 query-path-info
101 add-data-to-store
102 add-text-to-store
103 add-to-store
104 add-file-tree-to-store
105 binary-file
106 build-things
107 build
108 query-failed-paths
109 clear-failed-paths
110 add-temp-root
111 add-indirect-root
112 add-permanent-root
113 remove-permanent-root
115 substitutable?
116 substitutable-path
117 substitutable-deriver
118 substitutable-references
119 substitutable-download-size
120 substitutable-nar-size
121 has-substitutes?
122 substitutable-paths
123 substitutable-path-info
125 path-info?
126 path-info-deriver
127 path-info-hash
128 path-info-references
129 path-info-registration-time
130 path-info-nar-size
132 built-in-builders
133 references
134 references/substitutes
135 references*
136 query-path-info*
137 requisites
138 referrers
139 optimize-store
140 verify-store
141 topologically-sorted
142 valid-derivers
143 query-derivation-outputs
144 live-paths
145 dead-paths
146 collect-garbage
147 delete-paths
148 import-paths
149 export-paths
151 current-build-output-port
153 %store-monad
154 store-bind
155 store-return
156 store-lift
157 store-lower
158 run-with-store
159 %guile-for-build
160 current-system
161 set-current-system
162 text-file
163 interned-file
164 interned-file-tree
166 %store-prefix
167 store-path
168 output-path
169 fixed-output-path
170 store-path?
171 direct-store-path?
172 derivation-path?
173 store-path-package-name
174 store-path-hash-part
175 direct-store-path
176 derivation-log-file
177 log-file))
179 (define %protocol-version #x163)
181 (define %worker-magic-1 #x6e697863) ; "nixc"
182 (define %worker-magic-2 #x6478696f) ; "dxio"
184 (define (protocol-major magic)
185 (logand magic #xff00))
186 (define (protocol-minor magic)
187 (logand magic #x00ff))
188 (define (protocol-version major minor)
189 (logior major minor))
191 (define-syntax define-enumerate-type
192 (syntax-rules ()
193 ((_ name->int (name id) ...)
194 (define-syntax name->int
195 (syntax-rules (name ...)
196 ((_ name) id) ...)))))
198 (define-enumerate-type operation-id
199 ;; operation numbers from worker-protocol.hh
200 (quit 0)
201 (valid-path? 1)
202 (has-substitutes? 3)
203 (query-path-hash 4)
204 (query-references 5)
205 (query-referrers 6)
206 (add-to-store 7)
207 (add-text-to-store 8)
208 (build-things 9)
209 (ensure-path 10)
210 (add-temp-root 11)
211 (add-indirect-root 12)
212 (sync-with-gc 13)
213 (find-roots 14)
214 (export-path 16)
215 (query-deriver 18)
216 (set-options 19)
217 (collect-garbage 20)
218 ;;(query-substitutable-path-info 21) ; obsolete as of #x10c
219 (query-derivation-outputs 22)
220 (query-all-valid-paths 23)
221 (query-failed-paths 24)
222 (clear-failed-paths 25)
223 (query-path-info 26)
224 (import-paths 27)
225 (query-derivation-output-names 28)
226 (query-path-from-hash-part 29)
227 (query-substitutable-path-infos 30)
228 (query-valid-paths 31)
229 (query-substitutable-paths 32)
230 (query-valid-derivers 33)
231 (optimize-store 34)
232 (verify-store 35)
233 (built-in-builders 80))
235 (define-enumerate-type hash-algo
236 ;; hash.hh
237 (md5 1)
238 (sha1 2)
239 (sha256 3))
241 (define-enumerate-type build-mode
242 ;; store-api.hh
243 (normal 0)
244 (repair 1)
245 (check 2))
247 (define-enumerate-type gc-action
248 ;; store-api.hh
249 (return-live 0)
250 (return-dead 1)
251 (delete-dead 2)
252 (delete-specific 3))
254 (define %default-socket-path
255 (string-append %state-directory "/daemon-socket/socket"))
257 (define %daemon-socket-uri
258 ;; URI or file name of the socket the daemon listens too.
259 (make-parameter (or (getenv "GUIX_DAEMON_SOCKET")
260 %default-socket-path)))
263 \f
264 ;; Information about a substitutable store path.
265 (define-record-type <substitutable>
266 (substitutable path deriver refs dl-size nar-size)
267 substitutable?
268 (path substitutable-path)
269 (deriver substitutable-deriver)
270 (refs substitutable-references)
271 (dl-size substitutable-download-size)
272 (nar-size substitutable-nar-size))
274 (define (read-substitutable-path-list p)
275 (let loop ((len (read-int p))
276 (result '()))
277 (if (zero? len)
278 (reverse result)
279 (let ((path (read-store-path p))
280 (deriver (read-store-path p))
281 (refs (read-store-path-list p))
282 (dl-size (read-long-long p))
283 (nar-size (read-long-long p)))
284 (loop (- len 1)
285 (cons (substitutable path deriver refs dl-size nar-size)
286 result))))))
288 ;; Information about a store path.
289 (define-record-type <path-info>
290 (path-info deriver hash references registration-time nar-size)
291 path-info?
292 (deriver path-info-deriver) ;string | #f
293 (hash path-info-hash)
294 (references path-info-references)
295 (registration-time path-info-registration-time)
296 (nar-size path-info-nar-size))
298 (define (read-path-info p)
299 (let ((deriver (match (read-store-path p)
300 ("" #f)
301 (x x)))
302 (hash (base16-string->bytevector (read-string p)))
303 (refs (read-store-path-list p))
304 (registration-time (read-int p))
305 (nar-size (read-long-long p)))
306 (path-info deriver hash refs registration-time nar-size)))
308 (define-syntax write-arg
309 (syntax-rules (integer boolean bytevector
310 string string-list string-pairs
311 store-path store-path-list base16)
312 ((_ integer arg p)
313 (write-int arg p))
314 ((_ boolean arg p)
315 (write-int (if arg 1 0) p))
316 ((_ bytevector arg p)
317 (write-bytevector arg p))
318 ((_ string arg p)
319 (write-string arg p))
320 ((_ string-list arg p)
321 (write-string-list arg p))
322 ((_ string-pairs arg p)
323 (write-string-pairs arg p))
324 ((_ store-path arg p)
325 (write-store-path arg p))
326 ((_ store-path-list arg p)
327 (write-store-path-list arg p))
328 ((_ base16 arg p)
329 (write-string (bytevector->base16-string arg) p))))
331 (define-syntax read-arg
332 (syntax-rules (integer boolean string store-path store-path-list string-list
333 substitutable-path-list path-info base16)
334 ((_ integer p)
335 (read-int p))
336 ((_ boolean p)
337 (not (zero? (read-int p))))
338 ((_ string p)
339 (read-string p))
340 ((_ store-path p)
341 (read-store-path p))
342 ((_ store-path-list p)
343 (read-store-path-list p))
344 ((_ string-list p)
345 (read-string-list p))
346 ((_ substitutable-path-list p)
347 (read-substitutable-path-list p))
348 ((_ path-info p)
349 (read-path-info p))
350 ((_ base16 p)
351 (base16-string->bytevector (read-string p)))))
353 \f
354 ;; remote-store.cc
356 (define-record-type* <store-connection> store-connection %make-store-connection
357 store-connection?
358 (socket store-connection-socket)
359 (major store-connection-major-version)
360 (minor store-connection-minor-version)
362 (buffer store-connection-output-port) ;output port
363 (flush store-connection-flush-output) ;thunk
365 ;; Caches. We keep them per-connection, because store paths build
366 ;; during the session are temporary GC roots kept for the duration of
367 ;; the session.
368 (ats-cache store-connection-add-to-store-cache)
369 (atts-cache store-connection-add-text-to-store-cache)
370 (object-cache store-connection-object-cache
371 (default vlist-null)) ;vhash
372 (built-in-builders store-connection-built-in-builders
373 (default (delay '())))) ;promise
375 (set-record-type-printer! <store-connection>
376 (lambda (obj port)
377 (format port "#<store-connection ~a.~a ~a>"
378 (store-connection-major-version obj)
379 (store-connection-minor-version obj)
380 (number->string (object-address obj)
381 16))))
383 (define-deprecated/alias nix-server? store-connection?)
384 (define-deprecated/alias nix-server-major-version
385 store-connection-major-version)
386 (define-deprecated/alias nix-server-minor-version
387 store-connection-minor-version)
388 (define-deprecated/alias nix-server-socket store-connection-socket)
391 (define-condition-type &store-error &error
392 store-error?)
394 (define-condition-type &store-connection-error &store-error
395 store-connection-error?
396 (file store-connection-error-file)
397 (errno store-connection-error-code))
399 (define-condition-type &store-protocol-error &store-error
400 store-protocol-error?
401 (message store-protocol-error-message)
402 (status store-protocol-error-status))
404 (define-deprecated/alias &nix-error &store-error)
405 (define-deprecated/alias nix-error? store-error?)
406 (define-deprecated/alias &nix-connection-error &store-connection-error)
407 (define-deprecated/alias nix-connection-error? store-connection-error?)
408 (define-deprecated/alias nix-connection-error-file
409 store-connection-error-file)
410 (define-deprecated/alias nix-connection-error-code
411 store-connection-error-code)
412 (define-deprecated/alias &nix-protocol-error &store-protocol-error)
413 (define-deprecated/alias nix-protocol-error? store-protocol-error?)
414 (define-deprecated/alias nix-protocol-error-message
415 store-protocol-error-message)
416 (define-deprecated/alias nix-protocol-error-status
417 store-protocol-error-status)
420 (define-syntax-rule (system-error-to-connection-error file exp ...)
421 "Catch 'system-error' exceptions and translate them to
422 '&store-connection-error'."
423 (catch 'system-error
424 (lambda ()
425 exp ...)
426 (lambda args
427 (let ((errno (system-error-errno args)))
428 (raise (condition (&store-connection-error
429 (file file)
430 (errno errno))))))))
432 (define (open-unix-domain-socket file)
433 "Connect to the Unix-domain socket at FILE and return it. Raise a
434 '&store-connection-error' upon error."
435 (let ((s (with-fluids ((%default-port-encoding #f))
436 ;; This trick allows use of the `scm_c_read' optimization.
437 (socket PF_UNIX SOCK_STREAM 0)))
438 (a (make-socket-address PF_UNIX file)))
440 (system-error-to-connection-error file
441 (connect s a)
442 s)))
444 (define %default-guix-port
445 ;; Default port when connecting to a daemon over TCP/IP.
446 44146)
448 (define (open-inet-socket host port)
449 "Connect to the Unix-domain socket at HOST:PORT and return it. Raise a
450 '&store-connection-error' upon error."
451 (let ((sock (with-fluids ((%default-port-encoding #f))
452 ;; This trick allows use of the `scm_c_read' optimization.
453 (socket PF_UNIX SOCK_STREAM 0))))
454 (define addresses
455 (getaddrinfo host
456 (if (number? port) (number->string port) port)
457 (if (number? port)
458 (logior AI_ADDRCONFIG AI_NUMERICSERV)
459 AI_ADDRCONFIG)
460 0 ;any address family
461 SOCK_STREAM)) ;TCP only
463 (let loop ((addresses addresses))
464 (match addresses
465 ((ai rest ...)
466 (let ((s (socket (addrinfo:fam ai)
467 ;; TCP/IP only
468 SOCK_STREAM IPPROTO_IP)))
470 (catch 'system-error
471 (lambda ()
472 (connect s (addrinfo:addr ai))
474 ;; Setting this option makes a dramatic difference because it
475 ;; avoids the "ACK delay" on our RPC messages.
476 (setsockopt s IPPROTO_TCP TCP_NODELAY 1)
477 s)
478 (lambda args
479 ;; Connection failed, so try one of the other addresses.
480 (close s)
481 (if (null? rest)
482 (raise (condition (&store-connection-error
483 (file host)
484 (errno (system-error-errno args)))))
485 (loop rest))))))))))
487 (define (connect-to-daemon uri)
488 "Connect to the daemon at URI, a string that may be an actual URI or a file
489 name."
490 (define (not-supported)
491 (raise (condition (&store-connection-error
492 (file uri)
493 (errno ENOTSUP)))))
495 (define connect
496 (match (string->uri uri)
497 (#f ;URI is a file name
498 open-unix-domain-socket)
499 ((? uri? uri)
500 (match (uri-scheme uri)
501 ((or #f 'file 'unix)
502 (lambda (_)
503 (open-unix-domain-socket (uri-path uri))))
504 ('guix
505 (lambda (_)
506 (open-inet-socket (uri-host uri)
507 (or (uri-port uri) %default-guix-port))))
508 ((? symbol? scheme)
509 ;; Try to dynamically load a module for SCHEME.
510 ;; XXX: Errors are swallowed.
511 (match (false-if-exception
512 (resolve-interface `(guix store ,scheme)))
513 ((? module? module)
514 (match (false-if-exception
515 (module-ref module 'connect-to-daemon))
516 ((? procedure? connect)
517 (lambda (_)
518 (connect uri)))
519 (x (not-supported))))
520 (#f (not-supported))))
521 (x
522 (not-supported))))))
524 (connect uri))
526 (define* (open-connection #:optional (uri (%daemon-socket-uri))
527 #:key port (reserve-space? #t) cpu-affinity)
528 "Connect to the daemon at URI (a string), or, if PORT is not #f, use it as
529 the I/O port over which to communicate to a build daemon.
531 When RESERVE-SPACE? is true, instruct it to reserve a little bit of extra
532 space on the file system so that the garbage collector can still operate,
533 should the disk become full. When CPU-AFFINITY is true, it must be an integer
534 corresponding to an OS-level CPU number to which the daemon's worker process
535 for this connection will be pinned. Return a server object."
536 (guard (c ((nar-error? c)
537 ;; One of the 'write-' or 'read-' calls below failed, but this is
538 ;; really a connection error.
539 (raise (condition
540 (&store-connection-error (file (or port uri))
541 (errno EPROTO))
542 (&message (message "build daemon handshake failed"))))))
543 (let*-values (((port)
544 (or port (connect-to-daemon uri)))
545 ((output flush)
546 (buffering-output-port port
547 (make-bytevector 8192))))
548 (write-int %worker-magic-1 port)
549 (let ((r (read-int port)))
550 (and (eqv? r %worker-magic-2)
551 (let ((v (read-int port)))
552 (and (eqv? (protocol-major %protocol-version)
553 (protocol-major v))
554 (begin
555 (write-int %protocol-version port)
556 (when (>= (protocol-minor v) 14)
557 (write-int (if cpu-affinity 1 0) port)
558 (when cpu-affinity
559 (write-int cpu-affinity port)))
560 (when (>= (protocol-minor v) 11)
561 (write-int (if reserve-space? 1 0) port))
562 (letrec* ((built-in-builders
563 (delay (%built-in-builders conn)))
564 (conn
565 (%make-store-connection port
566 (protocol-major v)
567 (protocol-minor v)
568 output flush
569 (make-hash-table 100)
570 (make-hash-table 100)
571 vlist-null
572 built-in-builders)))
573 (let loop ((done? (process-stderr conn)))
574 (or done? (process-stderr conn)))
575 conn)))))))))
577 (define* (port->connection port
578 #:key (version %protocol-version))
579 "Assimilate PORT, an input/output port, and return a connection to the
580 daemon, assuming the given protocol VERSION.
582 Warning: this procedure assumes that the initial handshake with the daemon has
583 already taken place on PORT and that we're just continuing on this established
584 connection. Use with care."
585 (let-values (((output flush)
586 (buffering-output-port port (make-bytevector 8192))))
587 (define connection
588 (%make-store-connection port
589 (protocol-major version)
590 (protocol-minor version)
591 output flush
592 (make-hash-table 100)
593 (make-hash-table 100)
594 vlist-null
595 (delay (%built-in-builders connection))))
597 connection))
599 (define (store-connection-version store)
600 "Return the protocol version of STORE as an integer."
601 (protocol-version (store-connection-major-version store)
602 (store-connection-minor-version store)))
604 (define-deprecated/alias nix-server-version store-connection-version)
606 (define (write-buffered-output server)
607 "Flush SERVER's output port."
608 (force-output (store-connection-output-port server))
609 ((store-connection-flush-output server)))
611 (define (close-connection server)
612 "Close the connection to SERVER."
613 (close (store-connection-socket server)))
615 (define (call-with-store proc)
616 "Call PROC with an open store connection."
617 (let ((store (open-connection)))
618 (dynamic-wind
619 (const #f)
620 (lambda ()
621 (parameterize ((current-store-protocol-version
622 (store-connection-version store)))
623 (proc store)))
624 (lambda ()
625 (false-if-exception (close-connection store))))))
627 (define-syntax-rule (with-store store exp ...)
628 "Bind STORE to an open connection to the store and evaluate EXPs;
629 automatically close the store when the dynamic extent of EXP is left."
630 (call-with-store (lambda (store) exp ...)))
632 (define current-store-protocol-version
633 ;; Protocol version of the store currently used. XXX: This is a hack to
634 ;; communicate the protocol version to the build output port. It's a hack
635 ;; because it could be inaccurrate, for instance if there's code that
636 ;; manipulates several store connections at once; it works well for the
637 ;; purposes of (guix status) though.
638 (make-parameter #f))
640 (define current-build-output-port
641 ;; The port where build output is sent.
642 (make-parameter (current-error-port)))
644 (define* (dump-port in out
645 #:optional len
646 #:key (buffer-size 16384))
647 "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it
648 to OUT, using chunks of BUFFER-SIZE bytes."
649 (define buffer
650 (make-bytevector buffer-size))
652 (let loop ((total 0)
653 (bytes (get-bytevector-n! in buffer 0
654 (if len
655 (min len buffer-size)
656 buffer-size))))
657 (or (eof-object? bytes)
658 (and len (= total len))
659 (let ((total (+ total bytes)))
660 (put-bytevector out buffer 0 bytes)
661 (loop total
662 (get-bytevector-n! in buffer 0
663 (if len
664 (min (- len total) buffer-size)
665 buffer-size)))))))
667 (define %newlines
668 ;; Newline characters triggering a flush of 'current-build-output-port'.
669 ;; Unlike Guile's 'line, we flush upon #\return so that progress reports
670 ;; that use that trick are correctly displayed.
671 (char-set #\newline #\return))
673 (define* (process-stderr server #:optional user-port)
674 "Read standard output and standard error from SERVER, writing it to
675 CURRENT-BUILD-OUTPUT-PORT. Return #t when SERVER is done sending data, and
676 #f otherwise; in the latter case, the caller should call `process-stderr'
677 again until #t is returned or an error is raised.
679 Since the build process's output cannot be assumed to be UTF-8, we
680 conservatively consider it to be Latin-1, thereby avoiding possible
681 encoding conversion errors."
682 (define p
683 (store-connection-socket server))
685 ;; magic cookies from worker-protocol.hh
686 (define %stderr-next #x6f6c6d67) ; "olmg", build log
687 (define %stderr-read #x64617461) ; "data", data needed from source
688 (define %stderr-write #x64617416) ; "dat\x16", data for sink
689 (define %stderr-last #x616c7473) ; "alts", we're done
690 (define %stderr-error #x63787470) ; "cxtp", error reporting
692 (let ((k (read-int p)))
693 (cond ((= k %stderr-write)
694 ;; Write a byte stream to USER-PORT.
695 (let* ((len (read-int p))
696 (m (modulo len 8)))
697 (dump-port p user-port len
698 #:buffer-size (if (<= len 16384) 16384 65536))
699 (unless (zero? m)
700 ;; Consume padding, as for strings.
701 (get-bytevector-n p (- 8 m))))
702 #f)
703 ((= k %stderr-read)
704 ;; Read a byte stream from USER-PORT.
705 ;; Note: Avoid 'get-bytevector-n' to work around
706 ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11.
707 (let* ((max-len (read-int p))
708 (data (make-bytevector max-len))
709 (len (get-bytevector-n! user-port data 0 max-len)))
710 (write-bytevector data p len)
711 #f))
712 ((= k %stderr-next)
713 ;; Log a string. Build logs are usually UTF-8-encoded, but they
714 ;; may also contain arbitrary byte sequences that should not cause
715 ;; this to fail. Thus, use the permissive
716 ;; 'read-maybe-utf8-string'.
717 (let ((s (read-maybe-utf8-string p)))
718 (display s (current-build-output-port))
719 (when (string-any %newlines s)
720 (force-output (current-build-output-port)))
721 #f))
722 ((= k %stderr-error)
723 ;; Report an error.
724 (let ((error (read-maybe-utf8-string p))
725 ;; Currently the daemon fails to send a status code for early
726 ;; errors like DB schema version mismatches, so check for EOF.
727 (status (if (and (>= (store-connection-minor-version server) 8)
728 (not (eof-object? (lookahead-u8 p))))
729 (read-int p)
730 1)))
731 (raise (condition (&store-protocol-error
732 (message error)
733 (status status))))))
734 ((= k %stderr-last)
735 ;; The daemon is done (see `stopWork' in `nix-worker.cc'.)
736 #t)
737 (else
738 (raise (condition (&store-protocol-error
739 (message "invalid error code")
740 (status k))))))))
742 (define %default-substitute-urls
743 ;; Default list of substituters. This is *not* the list baked in
744 ;; 'guix-daemon', but it is used by 'guix-service-type' and and a couple of
745 ;; clients ('guix build --log-file' uses it.)
746 (map (if (false-if-exception (resolve-interface '(gnutls)))
747 (cut string-append "https://" <>)
748 (cut string-append "http://" <>))
749 '("ci.guix.gnu.org")))
751 (define* (set-build-options server
752 #:key keep-failed? keep-going? fallback?
753 (verbosity 0)
754 rounds ;number of build rounds
755 max-build-jobs
756 timeout
757 max-silent-time
758 (use-build-hook? #t)
759 (build-verbosity 0)
760 (log-type 0)
761 (print-build-trace #t)
763 ;; When true, provide machine-readable "build
764 ;; traces" for use by (guix status). Old clients
765 ;; are unable to make sense, which is why it's
766 ;; disabled by default.
767 print-extended-build-trace?
769 ;; When true, the daemon prefixes builder output
770 ;; with "@ build-log" traces so we can
771 ;; distinguish it from daemon output, and we can
772 ;; distinguish each builder's output
773 ;; (PRINT-BUILD-TRACE must be true as well.) The
774 ;; latter is particularly useful when
775 ;; MAX-BUILD-JOBS > 1.
776 multiplexed-build-output?
778 build-cores
779 (use-substitutes? #t)
781 ;; Client-provided substitute URLs. If it is #f,
782 ;; the daemon's settings are used. Otherwise, it
783 ;; overrides the daemons settings; see 'guix
784 ;; substitute'.
785 (substitute-urls #f)
787 ;; Number of columns in the client's terminal.
788 (terminal-columns (terminal-columns))
790 ;; Locale of the client.
791 (locale (false-if-exception (setlocale LC_ALL))))
792 ;; Must be called after `open-connection'.
794 (define socket
795 (store-connection-socket server))
797 (let-syntax ((send (syntax-rules ()
798 ((_ (type option) ...)
799 (begin
800 (write-arg type option socket)
801 ...)))))
802 (write-int (operation-id set-options) socket)
803 (send (boolean keep-failed?) (boolean keep-going?)
804 (boolean fallback?) (integer verbosity))
805 (when (< (store-connection-minor-version server) #x61)
806 (let ((max-build-jobs (or max-build-jobs 1))
807 (max-silent-time (or max-silent-time 3600)))
808 (send (integer max-build-jobs) (integer max-silent-time))))
809 (when (>= (store-connection-minor-version server) 2)
810 (send (boolean use-build-hook?)))
811 (when (>= (store-connection-minor-version server) 4)
812 (send (integer build-verbosity) (integer log-type)
813 (boolean print-build-trace)))
814 (when (and (>= (store-connection-minor-version server) 6)
815 (< (store-connection-minor-version server) #x61))
816 (let ((build-cores (or build-cores (current-processor-count))))
817 (send (integer build-cores))))
818 (when (>= (store-connection-minor-version server) 10)
819 (send (boolean use-substitutes?)))
820 (when (>= (store-connection-minor-version server) 12)
821 (let ((pairs `(;; This option is honored by 'guix substitute' et al.
822 ,@(if print-build-trace
823 `(("print-extended-build-trace"
824 . ,(if print-extended-build-trace? "1" "0")))
825 '())
826 ,@(if multiplexed-build-output?
827 `(("multiplexed-build-output"
828 . ,(if multiplexed-build-output? "true" "false")))
829 '())
830 ,@(if timeout
831 `(("build-timeout" . ,(number->string timeout)))
832 '())
833 ,@(if max-silent-time
834 `(("build-max-silent-time"
835 . ,(number->string max-silent-time)))
836 '())
837 ,@(if max-build-jobs
838 `(("build-max-jobs"
839 . ,(number->string max-build-jobs)))
840 '())
841 ,@(if build-cores
842 `(("build-cores" . ,(number->string build-cores)))
843 '())
844 ,@(if substitute-urls
845 `(("substitute-urls"
846 . ,(string-join substitute-urls)))
847 '())
848 ,@(if rounds
849 `(("build-repeat"
850 . ,(number->string (max 0 (1- rounds)))))
851 '())
852 ,@(if terminal-columns
853 `(("terminal-columns"
854 . ,(number->string terminal-columns)))
855 '())
856 ,@(if locale
857 `(("locale" . ,locale))
858 '()))))
859 (send (string-pairs pairs))))
860 (let loop ((done? (process-stderr server)))
861 (or done? (process-stderr server)))))
863 (define (buffering-output-port port buffer)
864 "Return two value: an output port wrapped around PORT that uses BUFFER (a
865 bytevector) as its internal buffer, and a thunk to flush this output port."
866 ;; Note: In Guile 2.2.2, custom binary output ports already have their own
867 ;; 4K internal buffer.
868 (define size
869 (bytevector-length buffer))
871 (define total 0)
873 (define (flush)
874 (put-bytevector port buffer 0 total)
875 (force-output port)
876 (set! total 0))
878 (define (write bv offset count)
879 (if (zero? count) ;end of file
880 (flush)
881 (let loop ((offset offset)
882 (count count)
883 (written 0))
884 (cond ((= total size)
885 (flush)
886 (loop offset count written))
887 ((zero? count)
888 written)
889 (else
890 (let ((to-copy (min count (- size total))))
891 (bytevector-copy! bv offset buffer total to-copy)
892 (set! total (+ total to-copy))
893 (loop (+ offset to-copy) (- count to-copy)
894 (+ written to-copy))))))))
896 ;; Note: We need to return FLUSH because the custom binary port has no way
897 ;; to be notified of a 'force-output' call on itself.
898 (values (make-custom-binary-output-port "buffering-output-port"
899 write #f #f flush)
900 flush))
902 (define profiled?
903 (let ((profiled
904 (or (and=> (getenv "GUIX_PROFILING") string-tokenize)
905 '())))
906 (lambda (component)
907 "Return true if COMPONENT profiling is active."
908 (member component profiled))))
910 (define %rpc-calls
911 ;; Mapping from RPC names (symbols) to invocation counts.
912 (make-hash-table))
914 (define* (show-rpc-profile #:optional (port (current-error-port)))
915 "Write to PORT a summary of the RPCs that have been made."
916 (let ((profile (sort (hash-fold alist-cons '() %rpc-calls)
917 (lambda (rpc1 rpc2)
918 (< (cdr rpc1) (cdr rpc2))))))
919 (format port "Remote procedure call summary: ~a RPCs~%"
920 (match profile
921 (((names . counts) ...)
922 (reduce + 0 counts))))
923 (for-each (match-lambda
924 ((rpc . count)
925 (format port " ~30a ... ~5@a~%" rpc count)))
926 profile)))
928 (define record-operation
929 ;; Optionally, increment the number of calls of the given RPC.
930 (if (profiled? "rpc")
931 (begin
932 (register-profiling-hook! "rpc" show-rpc-profile)
933 (lambda (name)
934 (let ((count (or (hashq-ref %rpc-calls name) 0)))
935 (hashq-set! %rpc-calls name (+ count 1)))))
936 (lambda (_)
937 #t)))
939 (define-syntax operation
940 (syntax-rules ()
941 "Define a client-side RPC stub for the given operation."
942 ((_ (name (type arg) ...) docstring return ...)
943 (lambda (server arg ...)
944 docstring
945 (let* ((s (store-connection-socket server))
946 (buffered (store-connection-output-port server)))
947 (record-operation 'name)
948 (write-int (operation-id name) buffered)
949 (write-arg type arg buffered)
950 ...
951 (write-buffered-output server)
953 ;; Loop until the server is done sending error output.
954 (let loop ((done? (process-stderr server)))
955 (or done? (loop (process-stderr server))))
956 (values (read-arg return s) ...))))))
958 (define-syntax-rule (define-operation (name args ...)
959 docstring return ...)
960 (define name
961 (operation (name args ...) docstring return ...)))
963 (define-operation (valid-path? (string path))
964 "Return #t when PATH designates a valid store item and #f otherwise (an
965 invalid item may exist on disk but still be invalid, for instance because it
966 is the result of an aborted or failed build.)
968 A '&store-protocol-error' condition is raised if PATH is not prefixed by the
969 store directory (/gnu/store)."
970 boolean)
972 (define-operation (query-path-hash (store-path path))
973 "Return the SHA256 hash of the nar serialization of PATH as a bytevector."
974 base16)
976 (define hash-part->path
977 (let ((query-path-from-hash-part
978 (operation (query-path-from-hash-part (string hash))
979 #f
980 store-path)))
981 (lambda (server hash-part)
982 "Return the store path whose hash part is HASH-PART (a nix-base32
983 string). Return the empty string if no such path exists."
984 ;; This RPC is primarily used by Hydra to reply to HTTP GETs of
985 ;; /HASH.narinfo.
986 (query-path-from-hash-part server hash-part))))
988 (define-operation (query-path-info (store-path path))
989 "Return the info (hash, references, etc.) for PATH."
990 path-info)
992 (define add-data-to-store
993 ;; A memoizing version of `add-to-store', to avoid repeated RPCs with
994 ;; the very same arguments during a given session.
995 (let ((add-text-to-store
996 (operation (add-text-to-store (string name) (bytevector text)
997 (string-list references))
998 #f
999 store-path))
1000 (lookup (if (profiled? "add-data-to-store-cache")
1001 (let ((lookups 0)
1002 (hits 0)
1003 (drv 0)
1004 (scheme 0))
1005 (define (show-stats)
1006 (define (% n)
1007 (if (zero? lookups)
1008 100.
1009 (* 100. (/ n lookups))))
1011 (format (current-error-port) "
1012 'add-data-to-store' cache:
1013 lookups: ~5@a
1014 hits: ~5@a (~,1f%)
1015 .drv files: ~5@a (~,1f%)
1016 Scheme files: ~5@a (~,1f%)~%"
1017 lookups hits (% hits)
1018 drv (% drv)
1019 scheme (% scheme)))
1021 (register-profiling-hook! "add-data-to-store-cache"
1022 show-stats)
1023 (lambda (cache args)
1024 (let ((result (hash-ref cache args)))
1025 (set! lookups (+ 1 lookups))
1026 (when result
1027 (set! hits (+ 1 hits)))
1028 (match args
1029 ((_ name _)
1030 (cond ((string-suffix? ".drv" name)
1031 (set! drv (+ drv 1)))
1032 ((string-suffix? "-builder" name)
1033 (set! scheme (+ scheme 1)))
1034 ((string-suffix? ".scm" name)
1035 (set! scheme (+ scheme 1))))))
1036 result)))
1037 hash-ref)))
1038 (lambda* (server name bytes #:optional (references '()))
1039 "Add BYTES under file NAME in the store, and return its store path.
1040 REFERENCES is the list of store paths referred to by the resulting store
1041 path."
1042 (let* ((args `(,bytes ,name ,references))
1043 (cache (store-connection-add-text-to-store-cache server)))
1044 (or (lookup cache args)
1045 (let ((path (add-text-to-store server name bytes references)))
1046 (hash-set! cache args path)
1047 path))))))
1049 (define* (add-text-to-store store name text #:optional (references '()))
1050 "Add TEXT under file NAME in the store, and return its store path.
1051 REFERENCES is the list of store paths referred to by the resulting store
1052 path."
1053 (add-data-to-store store name (string->utf8 text) references))
1055 (define true
1056 ;; Define it once and for all since we use it as a default value for
1057 ;; 'add-to-store' and want to make sure two default values are 'eq?' for the
1058 ;; purposes or memoization.
1059 (lambda (file stat)
1060 #t))
1062 (define add-to-store
1063 ;; A memoizing version of `add-to-store'. This is important because
1064 ;; `add-to-store' leads to huge data transfers to the server, and
1065 ;; because it's often called many times with the very same argument.
1066 (let ((add-to-store
1067 (lambda* (server basename recursive? hash-algo file-name
1068 #:key (select? true))
1069 ;; We don't use the 'operation' macro so we can pass SELECT? to
1070 ;; 'write-file'.
1071 (record-operation 'add-to-store)
1072 (let ((port (store-connection-socket server)))
1073 (write-int (operation-id add-to-store) port)
1074 (write-string basename port)
1075 (write-int 1 port) ;obsolete, must be #t
1076 (write-int (if recursive? 1 0) port)
1077 (write-string hash-algo port)
1078 (write-file file-name port #:select? select?)
1079 (write-buffered-output server)
1080 (let loop ((done? (process-stderr server)))
1081 (or done? (loop (process-stderr server))))
1082 (read-store-path port)))))
1083 (lambda* (server basename recursive? hash-algo file-name
1084 #:key (select? true))
1085 "Add the contents of FILE-NAME under BASENAME to the store. When
1086 RECURSIVE? is false, FILE-NAME must designate a regular file--not a directory
1087 nor a symlink. When RECURSIVE? is true and FILE-NAME designates a directory,
1088 the contents of FILE-NAME are added recursively; if FILE-NAME designates a
1089 flat file and RECURSIVE? is true, its contents are added, and its permission
1090 bits are kept. HASH-ALGO must be a string such as \"sha256\".
1092 When RECURSIVE? is true, call (SELECT? FILE STAT) for each directory entry,
1093 where FILE is the entry's absolute file name and STAT is the result of
1094 'lstat'; exclude entries for which SELECT? does not return true."
1095 ;; Note: We don't stat FILE-NAME at each call, and thus we assume that
1096 ;; the file remains unchanged for the lifetime of SERVER.
1097 (let* ((args `(,file-name ,basename ,recursive? ,hash-algo ,select?))
1098 (cache (store-connection-add-to-store-cache server)))
1099 (or (hash-ref cache args)
1100 (let ((path (add-to-store server basename recursive?
1101 hash-algo file-name
1102 #:select? select?)))
1103 (hash-set! cache args path)
1104 path))))))
1106 (define %not-slash
1107 (char-set-complement (char-set #\/)))
1109 (define* (add-file-tree-to-store server tree
1110 #:key
1111 (hash-algo "sha256")
1112 (recursive? #t))
1113 "Add the given TREE to the store on SERVER. TREE must be an entry such as:
1115 (\"my-tree\" directory
1116 (\"a\" regular (data \"hello\"))
1117 (\"b\" symlink \"a\")
1118 (\"c\" directory
1119 (\"d\" executable (file \"/bin/sh\"))))
1121 This is a generalized version of 'add-to-store'. It allows you to reproduce
1122 an arbitrary directory layout in the store without creating a derivation."
1124 ;; Note: The format of TREE was chosen to allow trees to be compared with
1125 ;; 'equal?', which in turn allows us to memoize things.
1127 (define root
1128 ;; TREE is a single entry.
1129 (list tree))
1131 (define basename
1132 (match tree
1133 ((name . _) name)))
1135 (define (lookup file)
1136 (let loop ((components (string-tokenize file %not-slash))
1137 (tree root))
1138 (match components
1139 ((basename)
1140 (assoc basename tree))
1141 ((head . rest)
1142 (loop rest
1143 (match (assoc-ref tree head)
1144 (('directory . entries) entries)))))))
1146 (define (file-type+size file)
1147 (match (lookup file)
1148 ((_ (and type (or 'directory 'symlink)) . _)
1149 (values type 0))
1150 ((_ type ('file file))
1151 (values type (stat:size (stat file))))
1152 ((_ type ('data (? string? data)))
1153 (values type (string-length data)))
1154 ((_ type ('data (? bytevector? data)))
1155 (values type (bytevector-length data)))))
1157 (define (file-port file)
1158 (match (lookup file)
1159 ((_ (or 'regular 'executable) content)
1160 (match content
1161 (('file (? string? file))
1162 (open-file file "r0b"))
1163 (('data (? string? str))
1164 (open-input-string str))
1165 (('data (? bytevector? bv))
1166 (open-bytevector-input-port bv))))))
1168 (define (symlink-target file)
1169 (match (lookup file)
1170 ((_ 'symlink target) target)))
1172 (define (directory-entries directory)
1173 (match (lookup directory)
1174 ((_ 'directory (names . _) ...) names)))
1176 (define cache
1177 (store-connection-add-to-store-cache server))
1179 (or (hash-ref cache tree)
1180 (begin
1181 ;; We don't use the 'operation' macro so we can use 'write-file-tree'
1182 ;; instead of 'write-file'.
1183 (record-operation 'add-to-store/tree)
1184 (let ((port (store-connection-socket server)))
1185 (write-int (operation-id add-to-store) port)
1186 (write-string basename port)
1187 (write-int 1 port) ;obsolete, must be #t
1188 (write-int (if recursive? 1 0) port)
1189 (write-string hash-algo port)
1190 (write-file-tree basename port
1191 #:file-type+size file-type+size
1192 #:file-port file-port
1193 #:symlink-target symlink-target
1194 #:directory-entries directory-entries)
1195 (write-buffered-output server)
1196 (let loop ((done? (process-stderr server)))
1197 (or done? (loop (process-stderr server))))
1198 (let ((result (read-store-path port)))
1199 (hash-set! cache tree result)
1200 result)))))
1202 (define build-things
1203 (let ((build (operation (build-things (string-list things)
1204 (integer mode))
1205 "Do it!"
1206 boolean))
1207 (build/old (operation (build-things (string-list things))
1208 "Do it!"
1209 boolean)))
1210 (lambda* (store things #:optional (mode (build-mode normal)))
1211 "Build THINGS, a list of store items which may be either '.drv' files or
1212 outputs, and return when the worker is done building them. Elements of THINGS
1213 that are not derivations can only be substituted and not built locally.
1214 Alternately, an element of THING can be a derivation/output name pair, in
1215 which case the daemon will attempt to substitute just the requested output of
1216 the derivation. Return #t on success."
1217 (let ((things (map (match-lambda
1218 ((drv . output) (string-append drv "!" output))
1219 (thing thing))
1220 things)))
1221 (parameterize ((current-store-protocol-version
1222 (store-connection-version store)))
1223 (if (>= (store-connection-minor-version store) 15)
1224 (build store things mode)
1225 (if (= mode (build-mode normal))
1226 (build/old store things)
1227 (raise (condition (&store-protocol-error
1228 (message "unsupported build mode")
1229 (status 1)))))))))))
1231 (define-operation (add-temp-root (store-path path))
1232 "Make PATH a temporary root for the duration of the current session.
1233 Return #t."
1234 boolean)
1236 (define-operation (add-indirect-root (string file-name))
1237 "Make the symlink FILE-NAME an indirect root for the garbage collector:
1238 whatever store item FILE-NAME points to will not be collected. Return #t on
1239 success.
1241 FILE-NAME can be anywhere on the file system, but it must be an absolute file
1242 name--it is the caller's responsibility to ensure that it is an absolute file
1243 name."
1244 boolean)
1246 (define %gc-roots-directory
1247 ;; The place where garbage collector roots (symlinks) are kept.
1248 (string-append %state-directory "/gcroots"))
1250 (define (add-permanent-root target)
1251 "Add a garbage collector root pointing to TARGET, an element of the store,
1252 preventing TARGET from even being collected. This can also be used if TARGET
1253 does not exist yet.
1255 Raise an error if the caller does not have write access to the GC root
1256 directory."
1257 (let* ((root (string-append %gc-roots-directory "/" (basename target))))
1258 (catch 'system-error
1259 (lambda ()
1260 (symlink target root))
1261 (lambda args
1262 ;; If ROOT already exists, this is fine; otherwise, re-throw.
1263 (unless (= EEXIST (system-error-errno args))
1264 (apply throw args))))))
1266 (define (remove-permanent-root target)
1267 "Remove the permanent garbage collector root pointing to TARGET. Raise an
1268 error if there is no such root."
1269 (delete-file (string-append %gc-roots-directory "/" (basename target))))
1271 (define references
1272 (operation (query-references (store-path path))
1273 "Return the list of references of PATH."
1274 store-path-list))
1276 (define %reference-cache
1277 ;; Brute-force cache mapping store items to their list of references.
1278 ;; Caching matters because when building a profile in the presence of
1279 ;; grafts, we keep calling 'graft-derivation', which in turn calls
1280 ;; 'references/substitutes' many times with the same arguments. Ideally we
1281 ;; would use a cache associated with the daemon connection instead (XXX).
1282 (make-hash-table 100))
1284 (define (references/substitutes store items)
1285 "Return the list of list of references of ITEMS; the result has the same
1286 length as ITEMS. Query substitute information for any item missing from the
1287 store at once. Raise a '&store-protocol-error' exception if reference
1288 information for one of ITEMS is missing."
1289 (let* ((requested items)
1290 (local-refs (map (lambda (item)
1291 (or (hash-ref %reference-cache item)
1292 (guard (c ((store-protocol-error? c) #f))
1293 (references store item))))
1294 items))
1295 (missing (fold-right (lambda (item local-ref result)
1296 (if local-ref
1297 result
1298 (cons item result)))
1299 '()
1300 items local-refs))
1302 ;; Query all the substitutes at once to minimize the cost of
1303 ;; launching 'guix substitute' and making HTTP requests.
1304 (substs (if (null? missing)
1305 '()
1306 (substitutable-path-info store missing))))
1307 (when (< (length substs) (length missing))
1308 (raise (condition (&store-protocol-error
1309 (message "cannot determine \
1310 the list of references")
1311 (status 1)))))
1313 ;; Intersperse SUBSTS and LOCAL-REFS.
1314 (let loop ((items items)
1315 (local-refs local-refs)
1316 (result '()))
1317 (match items
1318 (()
1319 (let ((result (reverse result)))
1320 (for-each (cut hash-set! %reference-cache <> <>)
1321 requested result)
1322 result))
1323 ((item items ...)
1324 (match local-refs
1325 ((#f tail ...)
1326 (loop items tail
1327 (cons (any (lambda (subst)
1328 (and (string=? (substitutable-path subst) item)
1329 (substitutable-references subst)))
1330 substs)
1331 result)))
1332 ((head tail ...)
1333 (loop items tail
1334 (cons head result)))))))))
1336 (define* (fold-path store proc seed paths
1337 #:optional (relatives (cut references store <>)))
1338 "Call PROC for each of the RELATIVES of PATHS, exactly once, and return the
1339 result formed from the successive calls to PROC, the first of which is passed
1340 SEED."
1341 (let loop ((paths paths)
1342 (result seed)
1343 (seen vlist-null))
1344 (match paths
1345 ((path rest ...)
1346 (if (vhash-assoc path seen)
1347 (loop rest result seen)
1348 (let ((seen (vhash-cons path #t seen))
1349 (rest (append rest (relatives path)))
1350 (result (proc path result)))
1351 (loop rest result seen))))
1352 (()
1353 result))))
1355 (define (requisites store paths)
1356 "Return the requisites of PATHS, including PATHS---i.e., their closures (all
1357 its references, recursively)."
1358 (fold-path store cons '() paths))
1360 (define (topologically-sorted store paths)
1361 "Return a list containing PATHS and all their references sorted in
1362 topological order."
1363 (define (traverse)
1364 ;; Do a simple depth-first traversal of all of PATHS.
1365 (let loop ((paths paths)
1366 (visited vlist-null)
1367 (result '()))
1368 (define (visit n)
1369 (vhash-cons n #t visited))
1371 (define (visited? n)
1372 (vhash-assoc n visited))
1374 (match paths
1375 ((head tail ...)
1376 (if (visited? head)
1377 (loop tail visited result)
1378 (call-with-values
1379 (lambda ()
1380 (loop (references store head)
1381 (visit head)
1382 result))
1383 (lambda (visited result)
1384 (loop tail
1385 visited
1386 (cons head result))))))
1387 (()
1388 (values visited result)))))
1390 (call-with-values traverse
1391 (lambda (_ result)
1392 (reverse result))))
1394 (define referrers
1395 (operation (query-referrers (store-path path))
1396 "Return the list of path that refer to PATH."
1397 store-path-list))
1399 (define valid-derivers
1400 (operation (query-valid-derivers (store-path path))
1401 "Return the list of valid \"derivers\" of PATH---i.e., all the
1402 .drv present in the store that have PATH among their outputs."
1403 store-path-list))
1405 (define query-derivation-outputs ; avoid name clash with `derivation-outputs'
1406 (operation (query-derivation-outputs (store-path path))
1407 "Return the list of outputs of PATH, a .drv file."
1408 store-path-list))
1410 (define-operation (has-substitutes? (store-path path))
1411 "Return #t if binary substitutes are available for PATH, and #f otherwise."
1412 boolean)
1414 (define substitutable-paths
1415 (operation (query-substitutable-paths (store-path-list paths))
1416 "Return the subset of PATHS that is substitutable."
1417 store-path-list))
1419 (define substitutable-path-info
1420 (operation (query-substitutable-path-infos (store-path-list paths))
1421 "Return information about the subset of PATHS that is
1422 substitutable. For each substitutable path, a `substitutable?' object is
1423 returned; thus, the resulting list can be shorter than PATHS. Furthermore,
1424 that there is no guarantee that the order of the resulting list matches the
1425 order of PATHS."
1426 substitutable-path-list))
1428 (define %built-in-builders
1429 (let ((builders (operation (built-in-builders)
1430 "Return the built-in builders."
1431 string-list)))
1432 (lambda (store)
1433 "Return the names of the supported built-in derivation builders
1434 supported by STORE. The result is memoized for STORE."
1435 ;; Check whether STORE's version supports this RPC and built-in
1436 ;; derivation builders in general, which appeared in Guix > 0.11.0.
1437 ;; Return the empty list if it doesn't. Note that this RPC does not
1438 ;; exist in 'nix-daemon'.
1439 (if (or (> (store-connection-major-version store) #x100)
1440 (and (= (store-connection-major-version store) #x100)
1441 (>= (store-connection-minor-version store) #x60)))
1442 (builders store)
1443 '()))))
1445 (define (built-in-builders store)
1446 "Return the names of the supported built-in derivation builders
1447 supported by STORE."
1448 (force (store-connection-built-in-builders store)))
1450 (define-operation (optimize-store)
1451 "Optimize the store by hard-linking identical files (\"deduplication\".)
1452 Return #t on success."
1453 ;; Note: the daemon in Guix <= 0.8.2 does not implement this RPC.
1454 boolean)
1456 (define verify-store
1457 (let ((verify (operation (verify-store (boolean check-contents?)
1458 (boolean repair?))
1459 "Verify the store."
1460 boolean)))
1461 (lambda* (store #:key check-contents? repair?)
1462 "Verify the integrity of the store and return false if errors remain,
1463 and true otherwise. When REPAIR? is true, repair any missing or altered store
1464 items by substituting them (this typically requires root privileges because it
1465 is not an atomic operation.) When CHECK-CONTENTS? is true, check the contents
1466 of store items; this can take a lot of time."
1467 (not (verify store check-contents? repair?)))))
1469 (define (run-gc server action to-delete min-freed)
1470 "Perform the garbage-collector operation ACTION, one of the
1471 `gc-action' values. When ACTION is `delete-specific', the TO-DELETE is
1472 the list of store paths to delete. IGNORE-LIVENESS? should always be
1473 #f. MIN-FREED is the minimum amount of disk space to be freed, in
1474 bytes, before the GC can stop. Return the list of store paths delete,
1475 and the number of bytes freed."
1476 (let ((s (store-connection-socket server)))
1477 (write-int (operation-id collect-garbage) s)
1478 (write-int action s)
1479 (write-store-path-list to-delete s)
1480 (write-arg boolean #f s) ; ignore-liveness?
1481 (write-long-long min-freed s)
1482 (write-int 0 s) ; obsolete
1483 (when (>= (store-connection-minor-version server) 5)
1484 ;; Obsolete `use-atime' and `max-atime' parameters.
1485 (write-int 0 s)
1486 (write-int 0 s))
1488 ;; Loop until the server is done sending error output.
1489 (let loop ((done? (process-stderr server)))
1490 (or done? (loop (process-stderr server))))
1492 (let ((paths (read-store-path-list s))
1493 (freed (read-long-long s))
1494 (obsolete (read-long-long s)))
1495 (unless (null? paths)
1496 ;; To be on the safe side, completely invalidate both caches.
1497 ;; Otherwise we could end up returning store paths that are no longer
1498 ;; valid.
1499 (hash-clear! (store-connection-add-to-store-cache server))
1500 (hash-clear! (store-connection-add-text-to-store-cache server)))
1502 (values paths freed))))
1504 (define-syntax-rule (%long-long-max)
1505 ;; Maximum unsigned 64-bit integer.
1506 (- (expt 2 64) 1))
1508 (define (live-paths server)
1509 "Return the list of live store paths---i.e., store paths still
1510 referenced, and thus not subject to being garbage-collected."
1511 (run-gc server (gc-action return-live) '() (%long-long-max)))
1513 (define (dead-paths server)
1514 "Return the list of dead store paths---i.e., store paths no longer
1515 referenced, and thus subject to being garbage-collected."
1516 (run-gc server (gc-action return-dead) '() (%long-long-max)))
1518 (define* (collect-garbage server #:optional (min-freed (%long-long-max)))
1519 "Collect garbage from the store at SERVER. If MIN-FREED is non-zero,
1520 then collect at least MIN-FREED bytes. Return the paths that were
1521 collected, and the number of bytes freed."
1522 (run-gc server (gc-action delete-dead) '() min-freed))
1524 (define* (delete-paths server paths #:optional (min-freed (%long-long-max)))
1525 "Delete PATHS from the store at SERVER, if they are no longer
1526 referenced. If MIN-FREED is non-zero, then stop after at least
1527 MIN-FREED bytes have been collected. Return the paths that were
1528 collected, and the number of bytes freed."
1529 (run-gc server (gc-action delete-specific) paths min-freed))
1531 (define (import-paths server port)
1532 "Import the set of store paths read from PORT into SERVER's store. An error
1533 is raised if the set of paths read from PORT is not signed (as per
1534 'export-path #:sign? #t'.) Return the list of store paths imported."
1535 (let ((s (store-connection-socket server)))
1536 (write-int (operation-id import-paths) s)
1537 (let loop ((done? (process-stderr server port)))
1538 (or done? (loop (process-stderr server port))))
1539 (read-store-path-list s)))
1541 (define* (export-path server path port #:key (sign? #t))
1542 "Export PATH to PORT. When SIGN? is true, sign it."
1543 (let ((s (store-connection-socket server)))
1544 (write-int (operation-id export-path) s)
1545 (write-store-path path s)
1546 (write-arg boolean sign? s)
1547 (let loop ((done? (process-stderr server port)))
1548 (or done? (loop (process-stderr server port))))
1549 (= 1 (read-int s))))
1551 (define* (export-paths server paths port #:key (sign? #t) recursive?)
1552 "Export the store paths listed in PATHS to PORT, in topological order,
1553 signing them if SIGN? is true. When RECURSIVE? is true, export the closure of
1554 PATHS---i.e., PATHS and all their dependencies."
1555 (define ordered
1556 (let ((sorted (topologically-sorted server paths)))
1557 ;; When RECURSIVE? is #f, filter out the references of PATHS.
1558 (if recursive?
1559 sorted
1560 (filter (cut member <> paths) sorted))))
1562 (let loop ((paths ordered))
1563 (match paths
1564 (()
1565 (write-int 0 port))
1566 ((head tail ...)
1567 (write-int 1 port)
1568 (and (export-path server head port #:sign? sign?)
1569 (loop tail))))))
1571 (define-operation (query-failed-paths)
1572 "Return the list of store items for which a build failure is cached.
1574 The result is always the empty list unless the daemon was started with
1575 '--cache-failures'."
1576 store-path-list)
1578 (define-operation (clear-failed-paths (store-path-list items))
1579 "Remove ITEMS from the list of cached build failures.
1581 This makes sense only when the daemon was started with '--cache-failures'."
1582 boolean)
1584 \f
1585 ;;;
1586 ;;; Store monad.
1587 ;;;
1589 (define-syntax-rule (define-alias new old)
1590 (define-syntax new (identifier-syntax old)))
1592 ;; The store monad allows us to (1) build sequences of operations in the
1593 ;; store, and (2) make the store an implicit part of the execution context,
1594 ;; rather than a parameter of every single function.
1595 (define-alias %store-monad %state-monad)
1596 (define-alias store-return state-return)
1597 (define-alias store-bind state-bind)
1599 ;; Instantiate templates for %STORE-MONAD since it's syntactically different
1600 ;; from %STATE-MONAD.
1601 (template-directory instantiations %store-monad)
1603 (define* (cache-object-mapping object keys result)
1604 "Augment the store's object cache with a mapping from OBJECT/KEYS to RESULT.
1605 KEYS is a list of additional keys to match against, for instance a (SYSTEM
1606 TARGET) tuple.
1608 OBJECT is typically a high-level object such as a <package> or an <origin>,
1609 and RESULT is typically its derivation."
1610 (lambda (store)
1611 (values result
1612 (store-connection
1613 (inherit store)
1614 (object-cache (vhash-consq object (cons result keys)
1615 (store-connection-object-cache store)))))))
1617 (define record-cache-lookup!
1618 (if (profiled? "object-cache")
1619 (let ((fresh 0)
1620 (lookups 0)
1621 (hits 0))
1622 (register-profiling-hook!
1623 "object-cache"
1624 (lambda ()
1625 (format (current-error-port) "Store object cache:
1626 fresh caches: ~5@a
1627 lookups: ~5@a
1628 hits: ~5@a (~,1f%)~%"
1629 fresh lookups hits
1630 (if (zero? lookups)
1631 100.
1632 (* 100. (/ hits lookups))))))
1634 (lambda (hit? cache)
1635 (set! fresh
1636 (if (eq? cache vlist-null)
1637 (+ 1 fresh)
1638 fresh))
1639 (set! lookups (+ 1 lookups))
1640 (set! hits (if hit? (+ hits 1) hits))))
1641 (lambda (x y)
1642 #t)))
1644 (define* (lookup-cached-object object #:optional (keys '()))
1645 "Return the cached object in the store connection corresponding to OBJECT
1646 and KEYS. KEYS is a list of additional keys to match against, and which are
1647 compared with 'equal?'. Return #f on failure and the cached result
1648 otherwise."
1649 (lambda (store)
1650 (let* ((cache (store-connection-object-cache store))
1652 ;; Escape as soon as we find the result. This avoids traversing
1653 ;; the whole vlist chain and significantly reduces the number of
1654 ;; 'hashq' calls.
1655 (value (let/ec return
1656 (vhash-foldq* (lambda (item result)
1657 (match item
1658 ((value . keys*)
1659 (if (equal? keys keys*)
1660 (return value)
1661 result))))
1662 #f object
1663 cache))))
1664 (record-cache-lookup! value cache)
1665 (values value store))))
1667 (define* (%mcached mthunk object #:optional (keys '()))
1668 "Bind the monadic value returned by MTHUNK, which supposedly corresponds to
1669 OBJECT/KEYS, or return its cached value."
1670 (mlet %store-monad ((cached (lookup-cached-object object keys)))
1671 (if cached
1672 (return cached)
1673 (>>= (mthunk)
1674 (lambda (result)
1675 (cache-object-mapping object keys result))))))
1677 (define-syntax-rule (mcached mvalue object keys ...)
1678 "Run MVALUE, which corresponds to OBJECT/KEYS, and cache it; or return the
1679 value associated with OBJECT/KEYS in the store's object cache if there is
1680 one."
1681 (%mcached (lambda () mvalue)
1682 object (list keys ...)))
1684 (define (preserve-documentation original proc)
1685 "Return PROC with documentation taken from ORIGINAL."
1686 (set-object-property! proc 'documentation
1687 (procedure-property original 'documentation))
1688 proc)
1690 (define (store-lift proc)
1691 "Lift PROC, a procedure whose first argument is a connection to the store,
1692 in the store monad."
1693 (preserve-documentation proc
1694 (lambda args
1695 (lambda (store)
1696 (values (apply proc store args) store)))))
1698 (define (store-lower proc)
1699 "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure
1700 taking the store as its first argument."
1701 (preserve-documentation proc
1702 (lambda (store . args)
1703 (run-with-store store (apply proc args)))))
1705 ;;
1706 ;; Store monad operators.
1707 ;;
1709 (define* (binary-file name
1710 data ;bytevector
1711 #:optional (references '()))
1712 "Return as a monadic value the absolute file name in the store of the file
1713 containing DATA, a bytevector. REFERENCES is a list of store items that the
1714 resulting text file refers to; it defaults to the empty list."
1715 (lambda (store)
1716 (values (add-data-to-store store name data references)
1717 store)))
1719 (define* (text-file name
1720 text ;string
1721 #:optional (references '()))
1722 "Return as a monadic value the absolute file name in the store of the file
1723 containing TEXT, a string. REFERENCES is a list of store items that the
1724 resulting text file refers to; it defaults to the empty list."
1725 (lambda (store)
1726 (values (add-text-to-store store name text references)
1727 store)))
1729 (define* (interned-file file #:optional name
1730 #:key (recursive? #t) (select? true))
1731 "Return the name of FILE once interned in the store. Use NAME as its store
1732 name, or the basename of FILE if NAME is omitted.
1734 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
1735 designates a flat file and RECURSIVE? is true, its contents are added, and its
1736 permission bits are kept.
1738 When RECURSIVE? is true, call (SELECT? FILE STAT) for each directory entry,
1739 where FILE is the entry's absolute file name and STAT is the result of
1740 'lstat'; exclude entries for which SELECT? does not return true."
1741 (lambda (store)
1742 (values (add-to-store store (or name (basename file))
1743 recursive? "sha256" file
1744 #:select? select?)
1745 store)))
1747 (define interned-file-tree
1748 (store-lift add-file-tree-to-store))
1750 (define build
1751 ;; Monadic variant of 'build-things'.
1752 (store-lift build-things))
1754 (define set-build-options*
1755 (store-lift set-build-options))
1757 (define references*
1758 (store-lift references))
1760 (define (query-path-info* item)
1761 "Monadic version of 'query-path-info' that returns #f when ITEM is not in
1762 the store."
1763 (lambda (store)
1764 (guard (c ((store-protocol-error? c)
1765 ;; ITEM is not in the store; return #f.
1766 (values #f store)))
1767 (values (query-path-info store item) store))))
1769 (define-inlinable (current-system)
1770 ;; Consult the %CURRENT-SYSTEM fluid at bind time. This is equivalent to
1771 ;; (lift0 %current-system %store-monad), but inlinable, thus avoiding
1772 ;; closure allocation in some cases.
1773 (lambda (state)
1774 (values (%current-system) state)))
1776 (define-inlinable (set-current-system system)
1777 ;; Set the %CURRENT-SYSTEM fluid at bind time.
1778 (lambda (state)
1779 (values (%current-system system) state)))
1781 (define %guile-for-build
1782 ;; The derivation of the Guile to be used within the build environment,
1783 ;; when using 'gexp->derivation' and co.
1784 (make-parameter #f))
1786 (define set-store-connection-object-cache!
1787 (record-modifier <store-connection> 'object-cache))
1789 (define* (run-with-store store mval
1790 #:key
1791 (guile-for-build (%guile-for-build))
1792 (system (%current-system))
1793 (target #f))
1794 "Run MVAL, a monadic value in the store monad, in STORE, an open store
1795 connection, and return the result."
1796 ;; Initialize the dynamic bindings here to avoid bad surprises. The
1797 ;; difficulty lies in the fact that dynamic bindings are resolved at
1798 ;; bind-time and not at call time, which can be disconcerting.
1799 (parameterize ((%guile-for-build guile-for-build)
1800 (%current-system system)
1801 (%current-target-system target))
1802 (call-with-values (lambda ()
1803 (run-with-state mval store))
1804 (lambda (result new-store)
1805 (when (and store new-store)
1806 ;; Copy the object cache from NEW-STORE so we don't fully discard
1807 ;; the state.
1808 (let ((cache (store-connection-object-cache new-store)))
1809 (set-store-connection-object-cache! store cache)))
1810 result))))
1812 \f
1813 ;;;
1814 ;;; Store paths.
1815 ;;;
1817 (define %store-prefix
1818 ;; Absolute path to the Nix store.
1819 (make-parameter %store-directory))
1821 (define (compressed-hash bv size) ; `compressHash'
1822 "Given the hash stored in BV, return a compressed version thereof that fits
1823 in SIZE bytes."
1824 (define new (make-bytevector size 0))
1825 (define old-size (bytevector-length bv))
1826 (let loop ((i 0))
1827 (if (= i old-size)
1828 new
1829 (let* ((j (modulo i size))
1830 (o (bytevector-u8-ref new j)))
1831 (bytevector-u8-set! new j
1832 (logxor o (bytevector-u8-ref bv i)))
1833 (loop (+ 1 i))))))
1835 (define (store-path type hash name) ; makeStorePath
1836 "Return the store path for NAME/HASH/TYPE."
1837 (let* ((s (string-append type ":sha256:"
1838 (bytevector->base16-string hash) ":"
1839 (%store-prefix) ":" name))
1840 (h (sha256 (string->utf8 s)))
1841 (c (compressed-hash h 20)))
1842 (string-append (%store-prefix) "/"
1843 (bytevector->nix-base32-string c) "-"
1844 name)))
1846 (define (output-path output hash name) ; makeOutputPath
1847 "Return an output path for OUTPUT (the name of the output as a string) of
1848 the derivation called NAME with hash HASH."
1849 (store-path (string-append "output:" output) hash
1850 (if (string=? output "out")
1851 name
1852 (string-append name "-" output))))
1854 (define* (fixed-output-path name hash
1855 #:key
1856 (output "out")
1857 (hash-algo 'sha256)
1858 (recursive? #t))
1859 "Return an output path for the fixed output OUTPUT defined by HASH of type
1860 HASH-ALGO, of the derivation NAME. RECURSIVE? has the same meaning as for
1861 'add-to-store'."
1862 (if (and recursive? (eq? hash-algo 'sha256))
1863 (store-path "source" hash name)
1864 (let ((tag (string-append "fixed:" output ":"
1865 (if recursive? "r:" "")
1866 (symbol->string hash-algo) ":"
1867 (bytevector->base16-string hash) ":")))
1868 (store-path (string-append "output:" output)
1869 (sha256 (string->utf8 tag))
1870 name))))
1872 (define (store-path? path)
1873 "Return #t if PATH is a store path."
1874 ;; This is a lightweight check, compared to using a regexp, but this has to
1875 ;; be fast as it's called often in `derivation', for instance.
1876 ;; `isStorePath' in Nix does something similar.
1877 (string-prefix? (%store-prefix) path))
1879 (define (direct-store-path? path)
1880 "Return #t if PATH is a store path, and not a sub-directory of a store path.
1881 This predicate is sometimes needed because files *under* a store path are not
1882 valid inputs."
1883 (and (store-path? path)
1884 (not (string=? path (%store-prefix)))
1885 (let ((len (+ 1 (string-length (%store-prefix)))))
1886 (not (string-index (substring path len) #\/)))))
1888 (define (direct-store-path path)
1889 "Return the direct store path part of PATH, stripping components after
1890 '/gnu/store/xxxx-foo'."
1891 (let ((prefix-length (+ (string-length (%store-prefix)) 35)))
1892 (if (> (string-length path) prefix-length)
1893 (let ((slash (string-index path #\/ prefix-length)))
1894 (if slash (string-take path slash) path))
1895 path)))
1897 (define (derivation-path? path)
1898 "Return #t if PATH is a derivation path."
1899 (and (store-path? path) (string-suffix? ".drv" path)))
1901 (define store-regexp*
1902 ;; The substituter makes repeated calls to 'store-path-hash-part', hence
1903 ;; this optimization.
1904 (mlambda (store)
1905 "Return a regexp matching a file in STORE."
1906 (make-regexp (string-append "^" (regexp-quote store)
1907 "/([0-9a-df-np-sv-z]{32})-([^/]+)$"))))
1909 (define (store-path-package-name path)
1910 "Return the package name part of PATH, a file name in the store."
1911 (let ((path-rx (store-regexp* (%store-prefix))))
1912 (and=> (regexp-exec path-rx path)
1913 (cut match:substring <> 2))))
1915 (define (store-path-hash-part path)
1916 "Return the hash part of PATH as a base32 string, or #f if PATH is not a
1917 syntactically valid store path."
1918 (and (string-prefix? (%store-prefix) path)
1919 (let ((base (string-drop path (+ 1 (string-length (%store-prefix))))))
1920 (and (> (string-length base) 33)
1921 (let ((hash (string-take base 32)))
1922 (and (string-every %nix-base32-charset hash)
1923 hash))))))
1925 (define (derivation-log-file drv)
1926 "Return the build log file for DRV, a derivation file name, or #f if it
1927 could not be found."
1928 (let* ((base (basename drv))
1929 (log (string-append (or (getenv "GUIX_LOG_DIRECTORY")
1930 (string-append %localstatedir "/log/guix"))
1931 "/drvs/"
1932 (string-take base 2) "/"
1933 (string-drop base 2)))
1934 (log.gz (string-append log ".gz"))
1935 (log.bz2 (string-append log ".bz2")))
1936 (cond ((file-exists? log.gz) log.gz)
1937 ((file-exists? log.bz2) log.bz2)
1938 ((file-exists? log) log)
1939 (else #f))))
1941 (define (log-file store file)
1942 "Return the build log file for FILE, or #f if none could be found. FILE
1943 must be an absolute store file name, or a derivation file name."
1944 (cond ((derivation-path? file)
1945 (derivation-log-file file))
1946 (else
1947 (match (valid-derivers store file)
1948 ((derivers ...)
1949 ;; Return the first that works.
1950 (any (cut log-file store <>) derivers))
1951 (_ #f)))))
1953 ;;; Local Variables:
1954 ;;; eval: (put 'system-error-to-connection-error 'scheme-indent-function 1)
1955 ;;; End: