| blob | 132b8a3ac437af2aeee5621a6c9b73ac0bac5abb |
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2012, 2013, 2014, 2015 Ludovic Courtès <ludo@gnu.org>
3 ;;;
4 ;;; This file is part of GNU Guix.
5 ;;;
6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
7 ;;; under the terms of the GNU General Public License as published by
8 ;;; the Free Software Foundation; either version 3 of the License, or (at
9 ;;; your option) any later version.
10 ;;;
11 ;;; GNU Guix is distributed in the hope that it will be useful, but
12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 ;;; GNU General Public License for more details.
15 ;;;
16 ;;; You should have received a copy of the GNU General Public License
17 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
19 (define-module (guix store)
20 #:use-module (guix utils)
21 #:use-module (guix config)
22 #:use-module (guix serialization)
23 #:use-module (guix monads)
24 #:autoload (guix base32) (bytevector->base32-string)
25 #:use-module (rnrs bytevectors)
26 #:use-module (rnrs io ports)
27 #:use-module (srfi srfi-1)
28 #:use-module (srfi srfi-9)
29 #:use-module (srfi srfi-9 gnu)
30 #:use-module (srfi srfi-26)
31 #:use-module (srfi srfi-34)
32 #:use-module (srfi srfi-35)
33 #:use-module (srfi srfi-39)
34 #:use-module (ice-9 match)
35 #:use-module (ice-9 regex)
36 #:use-module (ice-9 vlist)
37 #:use-module (ice-9 popen)
38 #:export (%daemon-socket-file
39 %gc-roots-directory
40 %default-substitute-urls
42 nix-server?
43 nix-server-major-version
44 nix-server-minor-version
45 nix-server-socket
47 &nix-error nix-error?
48 &nix-connection-error nix-connection-error?
49 nix-connection-error-file
50 nix-connection-error-code
51 &nix-protocol-error nix-protocol-error?
52 nix-protocol-error-message
53 nix-protocol-error-status
55 hash-algo
57 open-connection
58 close-connection
59 with-store
60 set-build-options
61 valid-path?
62 query-path-hash
63 hash-part->path
64 query-path-info
65 add-text-to-store
66 add-to-store
67 build-things
68 build
69 add-temp-root
70 add-indirect-root
71 add-permanent-root
72 remove-permanent-root
74 substitutable?
75 substitutable-path
76 substitutable-deriver
77 substitutable-references
78 substitutable-download-size
79 substitutable-nar-size
80 has-substitutes?
81 substitutable-paths
82 substitutable-path-info
84 path-info?
85 path-info-deriver
86 path-info-hash
87 path-info-references
88 path-info-registration-time
89 path-info-nar-size
91 references
92 requisites
93 referrers
94 optimize-store
95 verify-store
96 topologically-sorted
97 valid-derivers
98 query-derivation-outputs
99 live-paths
100 dead-paths
101 collect-garbage
102 delete-paths
103 import-paths
104 export-paths
106 current-build-output-port
108 register-path
110 %store-monad
111 store-bind
112 store-return
113 store-lift
114 store-lower
115 run-with-store
116 %guile-for-build
117 text-file
118 interned-file
120 %store-prefix
121 store-path?
122 direct-store-path?
123 derivation-path?
124 store-path-package-name
125 store-path-hash-part
126 direct-store-path
127 log-file))
129 (define %protocol-version #x10c)
131 (define %worker-magic-1 #x6e697863) ; "nixc"
132 (define %worker-magic-2 #x6478696f) ; "dxio"
134 (define (protocol-major magic)
135 (logand magic #xff00))
136 (define (protocol-minor magic)
137 (logand magic #x00ff))
139 (define-syntax define-enumerate-type
140 (syntax-rules ()
141 ((_ name->int (name id) ...)
142 (define-syntax name->int
143 (syntax-rules (name ...)
144 ((_ name) id) ...)))))
146 (define-enumerate-type operation-id
147 ;; operation numbers from worker-protocol.hh
148 (quit 0)
149 (valid-path? 1)
150 (has-substitutes? 3)
151 (query-path-hash 4)
152 (query-references 5)
153 (query-referrers 6)
154 (add-to-store 7)
155 (add-text-to-store 8)
156 (build-things 9)
157 (ensure-path 10)
158 (add-temp-root 11)
159 (add-indirect-root 12)
160 (sync-with-gc 13)
161 (find-roots 14)
162 (export-path 16)
163 (query-deriver 18)
164 (set-options 19)
165 (collect-garbage 20)
166 ;;(query-substitutable-path-info 21) ; obsolete as of #x10c
167 (query-derivation-outputs 22)
168 (query-all-valid-paths 23)
169 (query-failed-paths 24)
170 (clear-failed-paths 25)
171 (query-path-info 26)
172 (import-paths 27)
173 (query-derivation-output-names 28)
174 (query-path-from-hash-part 29)
175 (query-substitutable-path-infos 30)
176 (query-valid-paths 31)
177 (query-substitutable-paths 32)
178 (query-valid-derivers 33)
179 (optimize-store 34)
180 (verify-store 35))
182 (define-enumerate-type hash-algo
183 ;; hash.hh
184 (md5 1)
185 (sha1 2)
186 (sha256 3))
188 (define-enumerate-type gc-action
189 ;; store-api.hh
190 (return-live 0)
191 (return-dead 1)
192 (delete-dead 2)
193 (delete-specific 3))
195 (define %default-socket-path
196 (string-append %state-directory "/daemon-socket/socket"))
198 (define %daemon-socket-file
199 ;; File name of the socket the daemon listens too.
200 (make-parameter (or (getenv "GUIX_DAEMON_SOCKET")
201 %default-socket-path)))
204 \f
205 ;; Information about a substitutable store path.
206 (define-record-type <substitutable>
207 (substitutable path deriver refs dl-size nar-size)
208 substitutable?
209 (path substitutable-path)
210 (deriver substitutable-deriver)
211 (refs substitutable-references)
212 (dl-size substitutable-download-size)
213 (nar-size substitutable-nar-size))
215 (define (read-substitutable-path-list p)
216 (let loop ((len (read-int p))
217 (result '()))
218 (if (zero? len)
219 (reverse result)
220 (let ((path (read-store-path p))
221 (deriver (read-store-path p))
222 (refs (read-store-path-list p))
223 (dl-size (read-long-long p))
224 (nar-size (read-long-long p)))
225 (loop (- len 1)
226 (cons (substitutable path deriver refs dl-size nar-size)
227 result))))))
229 ;; Information about a store path.
230 (define-record-type <path-info>
231 (path-info deriver hash references registration-time nar-size)
232 path-info?
233 (deriver path-info-deriver)
234 (hash path-info-hash)
235 (references path-info-references)
236 (registration-time path-info-registration-time)
237 (nar-size path-info-nar-size))
239 (define (read-path-info p)
240 (let ((deriver (read-store-path p))
241 (hash (base16-string->bytevector (read-string p)))
242 (refs (read-store-path-list p))
243 (registration-time (read-int p))
244 (nar-size (read-long-long p)))
245 (path-info deriver hash refs registration-time nar-size)))
247 (define-syntax write-arg
248 (syntax-rules (integer boolean file string string-list string-pairs
249 store-path store-path-list base16)
250 ((_ integer arg p)
251 (write-int arg p))
252 ((_ boolean arg p)
253 (write-int (if arg 1 0) p))
254 ((_ file arg p)
255 (write-file arg p))
256 ((_ string arg p)
257 (write-string arg p))
258 ((_ string-list arg p)
259 (write-string-list arg p))
260 ((_ string-pairs arg p)
261 (write-string-pairs arg p))
262 ((_ store-path arg p)
263 (write-store-path arg p))
264 ((_ store-path-list arg p)
265 (write-store-path-list arg p))
266 ((_ base16 arg p)
267 (write-string (bytevector->base16-string arg) p))))
269 (define-syntax read-arg
270 (syntax-rules (integer boolean string store-path store-path-list
271 substitutable-path-list path-info base16)
272 ((_ integer p)
273 (read-int p))
274 ((_ boolean p)
275 (not (zero? (read-int p))))
276 ((_ string p)
277 (read-string p))
278 ((_ store-path p)
279 (read-store-path p))
280 ((_ store-path-list p)
281 (read-store-path-list p))
282 ((_ substitutable-path-list p)
283 (read-substitutable-path-list p))
284 ((_ path-info p)
285 (read-path-info p))
286 ((_ base16 p)
287 (base16-string->bytevector (read-string p)))))
289 \f
290 ;; remote-store.cc
292 (define-record-type <nix-server>
293 (%make-nix-server socket major minor
294 ats-cache atts-cache)
295 nix-server?
296 (socket nix-server-socket)
297 (major nix-server-major-version)
298 (minor nix-server-minor-version)
300 ;; Caches. We keep them per-connection, because store paths build
301 ;; during the session are temporary GC roots kept for the duration of
302 ;; the session.
303 (ats-cache nix-server-add-to-store-cache)
304 (atts-cache nix-server-add-text-to-store-cache))
306 (set-record-type-printer! <nix-server>
307 (lambda (obj port)
308 (format port "#<build-daemon ~a.~a ~a>"
309 (nix-server-major-version obj)
310 (nix-server-minor-version obj)
311 (number->string (object-address obj)
312 16))))
314 (define-condition-type &nix-error &error
315 nix-error?)
317 (define-condition-type &nix-connection-error &nix-error
318 nix-connection-error?
319 (file nix-connection-error-file)
320 (errno nix-connection-error-code))
322 (define-condition-type &nix-protocol-error &nix-error
323 nix-protocol-error?
324 (message nix-protocol-error-message)
325 (status nix-protocol-error-status))
327 (define* (open-connection #:optional (file (%daemon-socket-file))
328 #:key (reserve-space? #t))
329 "Connect to the daemon over the Unix-domain socket at FILE. When
330 RESERVE-SPACE? is true, instruct it to reserve a little bit of extra
331 space on the file system so that the garbage collector can still
332 operate, should the disk become full. Return a server object."
333 (let ((s (with-fluids ((%default-port-encoding #f))
334 ;; This trick allows use of the `scm_c_read' optimization.
335 (socket PF_UNIX SOCK_STREAM 0)))
336 (a (make-socket-address PF_UNIX file)))
338 (catch 'system-error
339 (cut connect s a)
340 (lambda args
341 ;; Translate the error to something user-friendly.
342 (let ((errno (system-error-errno args)))
343 (raise (condition (&nix-connection-error
344 (file file)
345 (errno errno)))))))
347 (write-int %worker-magic-1 s)
348 (let ((r (read-int s)))
349 (and (eqv? r %worker-magic-2)
350 (let ((v (read-int s)))
351 (and (eqv? (protocol-major %protocol-version)
352 (protocol-major v))
353 (begin
354 (write-int %protocol-version s)
355 (if (>= (protocol-minor v) 11)
356 (write-int (if reserve-space? 1 0) s))
357 (let ((s (%make-nix-server s
358 (protocol-major v)
359 (protocol-minor v)
360 (make-hash-table 100)
361 (make-hash-table 100))))
362 (let loop ((done? (process-stderr s)))
363 (or done? (process-stderr s)))
364 s))))))))
366 (define (close-connection server)
367 "Close the connection to SERVER."
368 (close (nix-server-socket server)))
370 (define-syntax-rule (with-store store exp ...)
371 "Bind STORE to an open connection to the store and evaluate EXPs;
372 automatically close the store when the dynamic extent of EXP is left."
373 (let ((store (open-connection)))
374 (dynamic-wind
375 (const #f)
376 (lambda ()
377 exp ...)
378 (lambda ()
379 (false-if-exception (close-connection store))))))
381 (define current-build-output-port
382 ;; The port where build output is sent.
383 (make-parameter (current-error-port)))
385 (define* (dump-port in out
386 #:optional len
387 #:key (buffer-size 16384))
388 "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it
389 to OUT, using chunks of BUFFER-SIZE bytes."
390 (define buffer
391 (make-bytevector buffer-size))
393 (let loop ((total 0)
394 (bytes (get-bytevector-n! in buffer 0
395 (if len
396 (min len buffer-size)
397 buffer-size))))
398 (or (eof-object? bytes)
399 (and len (= total len))
400 (let ((total (+ total bytes)))
401 (put-bytevector out buffer 0 bytes)
402 (loop total
403 (get-bytevector-n! in buffer 0
404 (if len
405 (min (- len total) buffer-size)
406 buffer-size)))))))
408 (define %newlines
409 ;; Newline characters triggering a flush of 'current-build-output-port'.
410 ;; Unlike Guile's _IOLBF, we flush upon #\return so that progress reports
411 ;; that use that trick are correctly displayed.
412 (char-set #\newline #\return))
414 (define* (process-stderr server #:optional user-port)
415 "Read standard output and standard error from SERVER, writing it to
416 CURRENT-BUILD-OUTPUT-PORT. Return #t when SERVER is done sending data, and
417 #f otherwise; in the latter case, the caller should call `process-stderr'
418 again until #t is returned or an error is raised.
420 Since the build process's output cannot be assumed to be UTF-8, we
421 conservatively consider it to be Latin-1, thereby avoiding possible
422 encoding conversion errors."
423 (define p
424 (nix-server-socket server))
426 ;; magic cookies from worker-protocol.hh
427 (define %stderr-next #x6f6c6d67) ; "olmg", build log
428 (define %stderr-read #x64617461) ; "data", data needed from source
429 (define %stderr-write #x64617416) ; "dat\x16", data for sink
430 (define %stderr-last #x616c7473) ; "alts", we're done
431 (define %stderr-error #x63787470) ; "cxtp", error reporting
433 (let ((k (read-int p)))
434 (cond ((= k %stderr-write)
435 ;; Write a byte stream to USER-PORT.
436 (let* ((len (read-int p))
437 (m (modulo len 8)))
438 (dump-port p user-port len)
439 (unless (zero? m)
440 ;; Consume padding, as for strings.
441 (get-bytevector-n p (- 8 m))))
442 #f)
443 ((= k %stderr-read)
444 ;; Read a byte stream from USER-PORT.
445 ;; Note: Avoid 'get-bytevector-n' to work around
446 ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11.
447 (let* ((max-len (read-int p))
448 (data (make-bytevector max-len))
449 (len (get-bytevector-n! user-port data 0 max-len)))
450 (write-int len p)
451 (put-bytevector p data 0 len)
452 (write-padding len p)
453 #f))
454 ((= k %stderr-next)
455 ;; Log a string. Build logs are usually UTF-8-encoded, but they
456 ;; may also contain arbitrary byte sequences that should not cause
457 ;; this to fail. Thus, use the permissive
458 ;; 'read-maybe-utf8-string'.
459 (let ((s (read-maybe-utf8-string p)))
460 (display s (current-build-output-port))
461 (when (string-any %newlines s)
462 (flush-output-port (current-build-output-port)))
463 #f))
464 ((= k %stderr-error)
465 ;; Report an error.
466 (let ((error (read-maybe-utf8-string p))
467 ;; Currently the daemon fails to send a status code for early
468 ;; errors like DB schema version mismatches, so check for EOF.
469 (status (if (and (>= (nix-server-minor-version server) 8)
470 (not (eof-object? (lookahead-u8 p))))
471 (read-int p)
472 1)))
473 (raise (condition (&nix-protocol-error
474 (message error)
475 (status status))))))
476 ((= k %stderr-last)
477 ;; The daemon is done (see `stopWork' in `nix-worker.cc'.)
478 #t)
479 (else
480 (raise (condition (&nix-protocol-error
481 (message "invalid error code")
482 (status k))))))))
484 (define %default-substitute-urls
485 ;; Default list of substituters.
486 '("http://hydra.gnu.org"))
488 (define* (set-build-options server
489 #:key keep-failed? keep-going? fallback?
490 (verbosity 0)
491 (max-build-jobs 1)
492 timeout
493 (max-silent-time 3600)
494 (use-build-hook? #t)
495 (build-verbosity 0)
496 (log-type 0)
497 (print-build-trace #t)
498 (build-cores (current-processor-count))
499 (use-substitutes? #t)
501 ;; Client-provided substitute URLs. For
502 ;; unprivileged clients, these are considered
503 ;; "untrusted"; for "trusted" users, they override
504 ;; the daemon's settings.
505 (substitute-urls %default-substitute-urls))
506 ;; Must be called after `open-connection'.
508 (define socket
509 (nix-server-socket server))
511 (let-syntax ((send (syntax-rules ()
512 ((_ (type option) ...)
513 (begin
514 (write-arg type option socket)
515 ...)))))
516 (write-int (operation-id set-options) socket)
517 (send (boolean keep-failed?) (boolean keep-going?)
518 (boolean fallback?) (integer verbosity)
519 (integer max-build-jobs) (integer max-silent-time))
520 (when (>= (nix-server-minor-version server) 2)
521 (send (boolean use-build-hook?)))
522 (when (>= (nix-server-minor-version server) 4)
523 (send (integer build-verbosity) (integer log-type)
524 (boolean print-build-trace)))
525 (when (>= (nix-server-minor-version server) 6)
526 (send (integer build-cores)))
527 (when (>= (nix-server-minor-version server) 10)
528 (send (boolean use-substitutes?)))
529 (when (>= (nix-server-minor-version server) 12)
530 (let ((pairs `(,@(if timeout
531 `(("build-timeout" . ,(number->string timeout)))
532 '())
533 ("substitute-urls" . ,(string-join substitute-urls)))))
534 (send (string-pairs pairs))))
535 (let loop ((done? (process-stderr server)))
536 (or done? (process-stderr server)))))
538 (define-syntax operation
539 (syntax-rules ()
540 "Define a client-side RPC stub for the given operation."
541 ((_ (name (type arg) ...) docstring return ...)
542 (lambda (server arg ...)
543 docstring
544 (let ((s (nix-server-socket server)))
545 (write-int (operation-id name) s)
546 (write-arg type arg s)
547 ...
548 ;; Loop until the server is done sending error output.
549 (let loop ((done? (process-stderr server)))
550 (or done? (loop (process-stderr server))))
551 (values (read-arg return s) ...))))))
553 (define-syntax-rule (define-operation (name args ...)
554 docstring return ...)
555 (define name
556 (operation (name args ...) docstring return ...)))
558 (define-operation (valid-path? (string path))
559 "Return #t when PATH is a valid store path."
560 boolean)
562 (define-operation (query-path-hash (store-path path))
563 "Return the SHA256 hash of PATH as a bytevector."
564 base16)
566 (define hash-part->path
567 (let ((query-path-from-hash-part
568 (operation (query-path-from-hash-part (string hash))
569 #f
570 store-path)))
571 (lambda (server hash-part)
572 "Return the store path whose hash part is HASH-PART (a nix-base32
573 string). Raise an error if no such path exists."
574 ;; This RPC is primarily used by Hydra to reply to HTTP GETs of
575 ;; /HASH.narinfo.
576 (query-path-from-hash-part server hash-part))))
578 (define-operation (query-path-info (store-path path))
579 "Return the info (hash, references, etc.) for PATH."
580 path-info)
582 (define add-text-to-store
583 ;; A memoizing version of `add-to-store', to avoid repeated RPCs with
584 ;; the very same arguments during a given session.
585 (let ((add-text-to-store
586 (operation (add-text-to-store (string name) (string text)
587 (string-list references))
588 #f
589 store-path)))
590 (lambda* (server name text #:optional (references '()))
591 "Add TEXT under file NAME in the store, and return its store path.
592 REFERENCES is the list of store paths referred to by the resulting store
593 path."
594 (let ((args `(,text ,name ,references))
595 (cache (nix-server-add-text-to-store-cache server)))
596 (or (hash-ref cache args)
597 (let ((path (add-text-to-store server name text references)))
598 (hash-set! cache args path)
599 path))))))
601 (define add-to-store
602 ;; A memoizing version of `add-to-store'. This is important because
603 ;; `add-to-store' leads to huge data transfers to the server, and
604 ;; because it's often called many times with the very same argument.
605 (let ((add-to-store (operation (add-to-store (string basename)
606 (boolean fixed?) ; obsolete, must be #t
607 (boolean recursive?)
608 (string hash-algo)
609 (file file-name))
610 #f
611 store-path)))
612 (lambda (server basename recursive? hash-algo file-name)
613 "Add the contents of FILE-NAME under BASENAME to the store. When
614 RECURSIVE? is false, FILE-NAME must designate a regular file--not a directory
615 nor a symlink. When RECURSIVE? is true and FILE-NAME designates a directory,
616 the contents of FILE-NAME are added recursively; if FILE-NAME designates a
617 flat file and RECURSIVE? is true, its contents are added, and its permission
618 bits are kept. HASH-ALGO must be a string such as \"sha256\"."
619 (let* ((st (false-if-exception (lstat file-name)))
620 (args `(,st ,basename ,recursive? ,hash-algo))
621 (cache (nix-server-add-to-store-cache server)))
622 (or (and st (hash-ref cache args))
623 (let ((path (add-to-store server basename #t recursive?
624 hash-algo file-name)))
625 (hash-set! cache args path)
626 path))))))
628 (define-operation (build-things (string-list things))
629 "Build THINGS, a list of store items which may be either '.drv' files or
630 outputs, and return when the worker is done building them. Elements of THINGS
631 that are not derivations can only be substituted and not built locally.
632 Return #t on success."
633 boolean)
635 (define-operation (add-temp-root (store-path path))
636 "Make PATH a temporary root for the duration of the current session.
637 Return #t."
638 boolean)
640 (define-operation (add-indirect-root (string file-name))
641 "Make the symlink FILE-NAME an indirect root for the garbage collector:
642 whatever store item FILE-NAME points to will not be collected. Return #t on
643 success.
645 FILE-NAME can be anywhere on the file system, but it must be an absolute file
646 name--it is the caller's responsibility to ensure that it is an absolute file
647 name."
648 boolean)
650 (define %gc-roots-directory
651 ;; The place where garbage collector roots (symlinks) are kept.
652 (string-append %state-directory "/gcroots"))
654 (define (add-permanent-root target)
655 "Add a garbage collector root pointing to TARGET, an element of the store,
656 preventing TARGET from even being collected. This can also be used if TARGET
657 does not exist yet.
659 Raise an error if the caller does not have write access to the GC root
660 directory."
661 (let* ((root (string-append %gc-roots-directory "/" (basename target))))
662 (catch 'system-error
663 (lambda ()
664 (symlink target root))
665 (lambda args
666 ;; If ROOT already exists, this is fine; otherwise, re-throw.
667 (unless (= EEXIST (system-error-errno args))
668 (apply throw args))))))
670 (define (remove-permanent-root target)
671 "Remove the permanent garbage collector root pointing to TARGET. Raise an
672 error if there is no such root."
673 (delete-file (string-append %gc-roots-directory "/" (basename target))))
675 (define references
676 (operation (query-references (store-path path))
677 "Return the list of references of PATH."
678 store-path-list))
680 (define* (fold-path store proc seed path
681 #:optional (relatives (cut references store <>)))
682 "Call PROC for each of the RELATIVES of PATH, exactly once, and return the
683 result formed from the successive calls to PROC, the first of which is passed
684 SEED."
685 (let loop ((paths (list path))
686 (result seed)
687 (seen vlist-null))
688 (match paths
689 ((path rest ...)
690 (if (vhash-assoc path seen)
691 (loop rest result seen)
692 (let ((seen (vhash-cons path #t seen))
693 (rest (append rest (relatives path)))
694 (result (proc path result)))
695 (loop rest result seen))))
696 (()
697 result))))
699 (define (requisites store path)
700 "Return the requisites of PATH, including PATH---i.e., its closure (all its
701 references, recursively)."
702 (fold-path store cons '() path))
704 (define (topologically-sorted store paths)
705 "Return a list containing PATHS and all their references sorted in
706 topological order."
707 (define (traverse)
708 ;; Do a simple depth-first traversal of all of PATHS.
709 (let loop ((paths paths)
710 (visited vlist-null)
711 (result '()))
712 (define (visit n)
713 (vhash-cons n #t visited))
715 (define (visited? n)
716 (vhash-assoc n visited))
718 (match paths
719 ((head tail ...)
720 (if (visited? head)
721 (loop tail visited result)
722 (call-with-values
723 (lambda ()
724 (loop (references store head)
725 (visit head)
726 result))
727 (lambda (visited result)
728 (loop tail
729 visited
730 (cons head result))))))
731 (()
732 (values visited result)))))
734 (call-with-values traverse
735 (lambda (_ result)
736 (reverse result))))
738 (define referrers
739 (operation (query-referrers (store-path path))
740 "Return the list of path that refer to PATH."
741 store-path-list))
743 (define valid-derivers
744 (operation (query-valid-derivers (store-path path))
745 "Return the list of valid \"derivers\" of PATH---i.e., all the
746 .drv present in the store that have PATH among their outputs."
747 store-path-list))
749 (define query-derivation-outputs ; avoid name clash with `derivation-outputs'
750 (operation (query-derivation-outputs (store-path path))
751 "Return the list of outputs of PATH, a .drv file."
752 store-path-list))
754 (define-operation (has-substitutes? (store-path path))
755 "Return #t if binary substitutes are available for PATH, and #f otherwise."
756 boolean)
758 (define substitutable-paths
759 (operation (query-substitutable-paths (store-path-list paths))
760 "Return the subset of PATHS that is substitutable."
761 store-path-list))
763 (define substitutable-path-info
764 (operation (query-substitutable-path-infos (store-path-list paths))
765 "Return information about the subset of PATHS that is
766 substitutable. For each substitutable path, a `substitutable?' object is
767 returned."
768 substitutable-path-list))
770 (define-operation (optimize-store)
771 "Optimize the store by hard-linking identical files (\"deduplication\".)
772 Return #t on success."
773 ;; Note: the daemon in Guix <= 0.8.2 does not implement this RPC.
774 boolean)
776 (define verify-store
777 (let ((verify (operation (verify-store (boolean check-contents?)
778 (boolean repair?))
779 "Verify the store."
780 boolean)))
781 (lambda* (store #:key check-contents? repair?)
782 "Verify the integrity of the store and return false if errors remain,
783 and true otherwise. When REPAIR? is true, repair any missing or altered store
784 items by substituting them (this typically requires root privileges because it
785 is not an atomic operation.) When CHECK-CONTENTS? is true, check the contents
786 of store items; this can take a lot of time."
787 (not (verify store check-contents? repair?)))))
789 (define (run-gc server action to-delete min-freed)
790 "Perform the garbage-collector operation ACTION, one of the
791 `gc-action' values. When ACTION is `delete-specific', the TO-DELETE is
792 the list of store paths to delete. IGNORE-LIVENESS? should always be
793 #f. MIN-FREED is the minimum amount of disk space to be freed, in
794 bytes, before the GC can stop. Return the list of store paths delete,
795 and the number of bytes freed."
796 (let ((s (nix-server-socket server)))
797 (write-int (operation-id collect-garbage) s)
798 (write-int action s)
799 (write-store-path-list to-delete s)
800 (write-arg boolean #f s) ; ignore-liveness?
801 (write-long-long min-freed s)
802 (write-int 0 s) ; obsolete
803 (when (>= (nix-server-minor-version server) 5)
804 ;; Obsolete `use-atime' and `max-atime' parameters.
805 (write-int 0 s)
806 (write-int 0 s))
808 ;; Loop until the server is done sending error output.
809 (let loop ((done? (process-stderr server)))
810 (or done? (loop (process-stderr server))))
812 (let ((paths (read-store-path-list s))
813 (freed (read-long-long s))
814 (obsolete (read-long-long s)))
815 (unless (null? paths)
816 ;; To be on the safe side, completely invalidate both caches.
817 ;; Otherwise we could end up returning store paths that are no longer
818 ;; valid.
819 (hash-clear! (nix-server-add-to-store-cache server))
820 (hash-clear! (nix-server-add-text-to-store-cache server)))
822 (values paths freed))))
824 (define-syntax-rule (%long-long-max)
825 ;; Maximum unsigned 64-bit integer.
826 (- (expt 2 64) 1))
828 (define (live-paths server)
829 "Return the list of live store paths---i.e., store paths still
830 referenced, and thus not subject to being garbage-collected."
831 (run-gc server (gc-action return-live) '() (%long-long-max)))
833 (define (dead-paths server)
834 "Return the list of dead store paths---i.e., store paths no longer
835 referenced, and thus subject to being garbage-collected."
836 (run-gc server (gc-action return-dead) '() (%long-long-max)))
838 (define* (collect-garbage server #:optional (min-freed (%long-long-max)))
839 "Collect garbage from the store at SERVER. If MIN-FREED is non-zero,
840 then collect at least MIN-FREED bytes. Return the paths that were
841 collected, and the number of bytes freed."
842 (run-gc server (gc-action delete-dead) '() min-freed))
844 (define* (delete-paths server paths #:optional (min-freed (%long-long-max)))
845 "Delete PATHS from the store at SERVER, if they are no longer
846 referenced. If MIN-FREED is non-zero, then stop after at least
847 MIN-FREED bytes have been collected. Return the paths that were
848 collected, and the number of bytes freed."
849 (run-gc server (gc-action delete-specific) paths min-freed))
851 (define (import-paths server port)
852 "Import the set of store paths read from PORT into SERVER's store. An error
853 is raised if the set of paths read from PORT is not signed (as per
854 'export-path #:sign? #t'.) Return the list of store paths imported."
855 (let ((s (nix-server-socket server)))
856 (write-int (operation-id import-paths) s)
857 (let loop ((done? (process-stderr server port)))
858 (or done? (loop (process-stderr server port))))
859 (read-store-path-list s)))
861 (define* (export-path server path port #:key (sign? #t))
862 "Export PATH to PORT. When SIGN? is true, sign it."
863 (let ((s (nix-server-socket server)))
864 (write-int (operation-id export-path) s)
865 (write-store-path path s)
866 (write-arg boolean sign? s)
867 (let loop ((done? (process-stderr server port)))
868 (or done? (loop (process-stderr server port))))
869 (= 1 (read-int s))))
871 (define* (export-paths server paths port #:key (sign? #t) recursive?)
872 "Export the store paths listed in PATHS to PORT, in topological order,
873 signing them if SIGN? is true. When RECURSIVE? is true, export the closure of
874 PATHS---i.e., PATHS and all their dependencies."
875 (define ordered
876 (let ((sorted (topologically-sorted server paths)))
877 ;; When RECURSIVE? is #f, filter out the references of PATHS.
878 (if recursive?
879 sorted
880 (filter (cut member <> paths) sorted))))
882 (let loop ((paths ordered))
883 (match paths
884 (()
885 (write-int 0 port))
886 ((head tail ...)
887 (write-int 1 port)
888 (and (export-path server head port #:sign? sign?)
889 (loop tail))))))
891 (define* (register-path path
892 #:key (references '()) deriver prefix
893 state-directory)
894 "Register PATH as a valid store file, with REFERENCES as its list of
895 references, and DERIVER as its deriver (.drv that led to it.) If PREFIX is
896 not #f, it must be the name of the directory containing the new store to
897 initialize; if STATE-DIRECTORY is not #f, it must be a string containing the
898 absolute file name to the state directory of the store being initialized.
899 Return #t on success.
901 Use with care as it directly modifies the store! This is primarily meant to
902 be used internally by the daemon's build hook."
903 ;; Currently this is implemented by calling out to the fine C++ blob.
904 (catch 'system-error
905 (lambda ()
906 (let ((pipe (apply open-pipe* OPEN_WRITE %guix-register-program
907 `(,@(if prefix
908 `("--prefix" ,prefix)
909 '())
910 ,@(if state-directory
911 `("--state-directory" ,state-directory)
912 '())))))
913 (and pipe
914 (begin
915 (format pipe "~a~%~a~%~a~%"
916 path (or deriver "") (length references))
917 (for-each (cut format pipe "~a~%" <>) references)
918 (zero? (close-pipe pipe))))))
919 (lambda args
920 ;; Failed to run %GUIX-REGISTER-PROGRAM.
921 #f)))
923 \f
924 ;;;
925 ;;; Store monad.
926 ;;;
928 (define-syntax-rule (define-alias new old)
929 (define-syntax new (identifier-syntax old)))
931 ;; The store monad allows us to (1) build sequences of operations in the
932 ;; store, and (2) make the store an implicit part of the execution context,
933 ;; rather than a parameter of every single function.
934 (define-alias %store-monad %state-monad)
935 (define-alias store-return state-return)
936 (define-alias store-bind state-bind)
938 (define (preserve-documentation original proc)
939 "Return PROC with documentation taken from ORIGINAL."
940 (set-object-property! proc 'documentation
941 (procedure-property original 'documentation))
942 proc)
944 (define (store-lift proc)
945 "Lift PROC, a procedure whose first argument is a connection to the store,
946 in the store monad."
947 (preserve-documentation proc
948 (lambda args
949 (lambda (store)
950 (values (apply proc store args) store)))))
952 (define (store-lower proc)
953 "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure
954 taking the store as its first argument."
955 (preserve-documentation proc
956 (lambda (store . args)
957 (run-with-store store (apply proc args)))))
959 ;;
960 ;; Store monad operators.
961 ;;
963 (define* (text-file name text
964 #:optional (references '()))
965 "Return as a monadic value the absolute file name in the store of the file
966 containing TEXT, a string. REFERENCES is a list of store items that the
967 resulting text file refers to; it defaults to the empty list."
968 (lambda (store)
969 (values (add-text-to-store store name text references)
970 store)))
972 (define* (interned-file file #:optional name
973 #:key (recursive? #t))
974 "Return the name of FILE once interned in the store. Use NAME as its store
975 name, or the basename of FILE if NAME is omitted.
977 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
978 designates a flat file and RECURSIVE? is true, its contents are added, and its
979 permission bits are kept."
980 (lambda (store)
981 (values (add-to-store store (or name (basename file))
982 recursive? "sha256" file)
983 store)))
985 (define build
986 ;; Monadic variant of 'build-things'.
987 (store-lift build-things))
989 (define %guile-for-build
990 ;; The derivation of the Guile to be used within the build environment,
991 ;; when using 'gexp->derivation' and co.
992 (make-parameter #f))
994 (define* (run-with-store store mval
995 #:key
996 (guile-for-build (%guile-for-build))
997 (system (%current-system)))
998 "Run MVAL, a monadic value in the store monad, in STORE, an open store
999 connection, and return the result."
1000 ;; Initialize the dynamic bindings here to avoid bad surprises. The
1001 ;; difficulty lies in the fact that dynamic bindings are resolved at
1002 ;; bind-time and not at call time, which can be disconcerting.
1003 (parameterize ((%guile-for-build guile-for-build)
1004 (%current-system system)
1005 (%current-target-system #f))
1006 (call-with-values (lambda ()
1007 (run-with-state mval store))
1008 (lambda (result store)
1009 ;; Discard the state.
1010 result))))
1012 \f
1013 ;;;
1014 ;;; Store paths.
1015 ;;;
1017 (define %store-prefix
1018 ;; Absolute path to the Nix store.
1019 (make-parameter %store-directory))
1021 (define (store-path? path)
1022 "Return #t if PATH is a store path."
1023 ;; This is a lightweight check, compared to using a regexp, but this has to
1024 ;; be fast as it's called often in `derivation', for instance.
1025 ;; `isStorePath' in Nix does something similar.
1026 (string-prefix? (%store-prefix) path))
1028 (define (direct-store-path? path)
1029 "Return #t if PATH is a store path, and not a sub-directory of a store path.
1030 This predicate is sometimes needed because files *under* a store path are not
1031 valid inputs."
1032 (and (store-path? path)
1033 (not (string=? path (%store-prefix)))
1034 (let ((len (+ 1 (string-length (%store-prefix)))))
1035 (not (string-index (substring path len) #\/)))))
1037 (define (direct-store-path path)
1038 "Return the direct store path part of PATH, stripping components after
1039 '/gnu/store/xxxx-foo'."
1040 (let ((prefix-length (+ (string-length (%store-prefix)) 35)))
1041 (if (> (string-length path) prefix-length)
1042 (let ((slash (string-index path #\/ prefix-length)))
1043 (if slash (string-take path slash) path))
1044 path)))
1046 (define (derivation-path? path)
1047 "Return #t if PATH is a derivation path."
1048 (and (store-path? path) (string-suffix? ".drv" path)))
1050 (define store-regexp*
1051 ;; The substituter makes repeated calls to 'store-path-hash-part', hence
1052 ;; this optimization.
1053 (memoize
1054 (lambda (store)
1055 "Return a regexp matching a file in STORE."
1056 (make-regexp (string-append "^" (regexp-quote store)
1057 "/([0-9a-df-np-sv-z]{32})-([^/]+)$")))))
1059 (define (store-path-package-name path)
1060 "Return the package name part of PATH, a file name in the store."
1061 (let ((path-rx (store-regexp* (%store-prefix))))
1062 (and=> (regexp-exec path-rx path)
1063 (cut match:substring <> 2))))
1065 (define (store-path-hash-part path)
1066 "Return the hash part of PATH as a base32 string, or #f if PATH is not a
1067 syntactically valid store path."
1068 (let ((path-rx (store-regexp* (%store-prefix))))
1069 (and=> (regexp-exec path-rx path)
1070 (cut match:substring <> 1))))
1072 (define (log-file store file)
1073 "Return the build log file for FILE, or #f if none could be found. FILE
1074 must be an absolute store file name, or a derivation file name."
1075 (cond ((derivation-path? file)
1076 (let* ((base (basename file))
1077 (log (string-append (dirname %state-directory) ; XXX
1078 "/log/guix/drvs/"
1079 (string-take base 2) "/"
1080 (string-drop base 2)))
1081 (log.bz2 (string-append log ".bz2")))
1082 (cond ((file-exists? log.bz2) log.bz2)
1083 ((file-exists? log) log)
1084 (else #f))))
1085 (else
1086 (match (valid-derivers store file)
1087 ((derivers ...)
1088 ;; Return the first that works.
1089 (any (cut log-file store <>) derivers))
1090 (_ #f)))))