| blob | b5dfdab32fc3340f717b32c75019d21d57725e5c |
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2015 David Thompson <davet@gnu.org>
3 ;;; Copyright © 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
4 ;;;
5 ;;; This file is part of GNU Guix.
6 ;;;
7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
8 ;;; under the terms of the GNU General Public License as published by
9 ;;; the Free Software Foundation; either version 3 of the License, or (at
10 ;;; your option) any later version.
11 ;;;
12 ;;; GNU Guix is distributed in the hope that it will be useful, but
13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ;;; GNU General Public License for more details.
16 ;;;
17 ;;; You should have received a copy of the GNU General Public License
18 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
20 (define-module (guix scripts publish)
21 #:use-module ((system repl server) #:prefix repl:)
22 #:use-module (ice-9 binary-ports)
23 #:use-module (ice-9 format)
24 #:use-module (ice-9 match)
25 #:use-module (ice-9 regex)
26 #:use-module (ice-9 rdelim)
27 #:use-module (ice-9 threads)
28 #:use-module (rnrs bytevectors)
29 #:use-module (srfi srfi-1)
30 #:use-module (srfi srfi-2)
31 #:use-module (srfi srfi-9)
32 #:use-module (srfi srfi-9 gnu)
33 #:use-module (srfi srfi-19)
34 #:use-module (srfi srfi-26)
35 #:use-module (srfi srfi-34)
36 #:use-module (srfi srfi-37)
37 #:use-module (web http)
38 #:use-module (web request)
39 #:use-module (web response)
40 #:use-module (web server)
41 #:use-module (web uri)
42 #:autoload (sxml simple) (sxml->xml)
43 #:use-module (guix base32)
44 #:use-module (guix base64)
45 #:use-module (guix config)
46 #:use-module (guix derivations)
47 #:use-module (guix hash)
48 #:use-module (guix pki)
49 #:use-module (guix pk-crypto)
50 #:use-module (guix workers)
51 #:use-module (guix store)
52 #:use-module ((guix serialization) #:select (write-file))
53 #:use-module (guix zlib)
54 #:use-module (guix cache)
55 #:use-module (guix ui)
56 #:use-module (guix scripts)
57 #:use-module ((guix utils)
58 #:select (with-atomic-file-output compressed-file?))
59 #:use-module ((guix build utils)
60 #:select (dump-port mkdir-p find-files))
61 #:use-module ((guix build syscalls) #:select (set-thread-name))
62 #:export (%public-key
63 %private-key
65 guix-publish))
67 (define (show-help)
68 (format #t (G_ "Usage: guix publish [OPTION]...
69 Publish ~a over HTTP.\n") %store-directory)
70 (display (G_ "
71 -p, --port=PORT listen on PORT"))
72 (display (G_ "
73 --listen=HOST listen on the network interface for HOST"))
74 (display (G_ "
75 -u, --user=USER change privileges to USER as soon as possible"))
76 (display (G_ "
77 -C, --compression[=LEVEL]
78 compress archives at LEVEL"))
79 (display (G_ "
80 -c, --cache=DIRECTORY cache published items to DIRECTORY"))
81 (display (G_ "
82 --workers=N use N workers to bake items"))
83 (display (G_ "
84 --ttl=TTL announce narinfos can be cached for TTL seconds"))
85 (display (G_ "
86 --nar-path=PATH use PATH as the prefix for nar URLs"))
87 (display (G_ "
88 --public-key=FILE use FILE as the public key for signatures"))
89 (display (G_ "
90 --private-key=FILE use FILE as the private key for signatures"))
91 (display (G_ "
92 -r, --repl[=PORT] spawn REPL server on PORT"))
93 (newline)
94 (display (G_ "
95 -h, --help display this help and exit"))
96 (display (G_ "
97 -V, --version display version information and exit"))
98 (newline)
99 (show-bug-report-information))
101 (define (getaddrinfo* host)
102 "Like 'getaddrinfo', but properly report errors."
103 (catch 'getaddrinfo-error
104 (lambda ()
105 (getaddrinfo host))
106 (lambda (key error)
107 (leave (G_ "lookup of host '~a' failed: ~a~%")
108 host (gai-strerror error)))))
110 ;; Nar compression parameters.
111 (define-record-type <compression>
112 (compression type level)
113 compression?
114 (type compression-type)
115 (level compression-level))
117 (define %no-compression
118 (compression 'none 0))
120 (define %default-gzip-compression
121 ;; Since we compress on the fly, default to fast compression.
122 (compression 'gzip 3))
124 (define (actual-compression item requested)
125 "Return the actual compression used for ITEM, which may be %NO-COMPRESSION
126 if ITEM is already compressed."
127 (if (compressed-file? item)
128 %no-compression
129 requested))
131 (define %options
132 (list (option '(#\h "help") #f #f
133 (lambda _
134 (show-help)
135 (exit 0)))
136 (option '(#\V "version") #f #f
137 (lambda _
138 (show-version-and-exit "guix publish")))
139 (option '(#\u "user") #t #f
140 (lambda (opt name arg result)
141 (alist-cons 'user arg result)))
142 (option '(#\p "port") #t #f
143 (lambda (opt name arg result)
144 (alist-cons 'port (string->number* arg) result)))
145 (option '("listen") #t #f
146 (lambda (opt name arg result)
147 (match (getaddrinfo* arg)
148 ((info _ ...)
149 (alist-cons 'address (addrinfo:addr info)
150 result))
151 (()
152 (leave (G_ "lookup of host '~a' returned nothing")
153 name)))))
154 (option '(#\C "compression") #f #t
155 (lambda (opt name arg result)
156 (match (if arg (string->number* arg) 3)
157 (0
158 (alist-cons 'compression %no-compression result))
159 (level
160 (if (zlib-available?)
161 (alist-cons 'compression
162 (compression 'gzip level)
163 result)
164 (begin
165 (warning (G_ "zlib support is missing; \
166 compression disabled~%"))
167 result))))))
168 (option '(#\c "cache") #t #f
169 (lambda (opt name arg result)
170 (alist-cons 'cache arg result)))
171 (option '("workers") #t #f
172 (lambda (opt name arg result)
173 (alist-cons 'workers (string->number* arg)
174 result)))
175 (option '("ttl") #t #f
176 (lambda (opt name arg result)
177 (let ((duration (string->duration arg)))
178 (unless duration
179 (leave (G_ "~a: invalid duration~%") arg))
180 (alist-cons 'narinfo-ttl (time-second duration)
181 result))))
182 (option '("nar-path") #t #f
183 (lambda (opt name arg result)
184 (alist-cons 'nar-path arg result)))
185 (option '("public-key") #t #f
186 (lambda (opt name arg result)
187 (alist-cons 'public-key-file arg result)))
188 (option '("private-key" "secret-key") #t #f
189 (lambda (opt name arg result)
190 (alist-cons 'private-key-file arg result)))
191 (option '(#\r "repl") #f #t
192 (lambda (opt name arg result)
193 ;; If port unspecified, use default Guile REPL port.
194 (let ((port (and arg (string->number* arg))))
195 (alist-cons 'repl (or port 37146) result))))))
197 (define %default-options
198 `((port . 8080)
200 ;; By default, serve nars under "/nar".
201 (nar-path . "nar")
203 (public-key-file . ,%public-key-file)
204 (private-key-file . ,%private-key-file)
206 ;; Default to fast & low compression.
207 (compression . ,(if (zlib-available?)
208 %default-gzip-compression
209 %no-compression))
211 ;; Default number of workers when caching is enabled.
212 (workers . ,(current-processor-count))
214 (address . ,(make-socket-address AF_INET INADDR_ANY 0))
215 (repl . #f)))
217 ;; The key pair used to sign narinfos.
218 (define %private-key
219 (make-parameter #f))
220 (define %public-key
221 (make-parameter #f))
223 (define %nix-cache-info
224 `(("StoreDir" . ,%store-directory)
225 ("WantMassQuery" . 0)
226 ("Priority" . 100)))
228 (define (signed-string s)
229 "Sign the hash of the string S with the daemon's key."
230 (let* ((public-key (%public-key))
231 (hash (bytevector->hash-data (sha256 (string->utf8 s))
232 #:key-type (key-type public-key))))
233 (signature-sexp hash (%private-key) public-key)))
235 (define base64-encode-string
236 (compose base64-encode string->utf8))
238 (define* (narinfo-string store store-path key
239 #:key (compression %no-compression)
240 (nar-path "nar") file-size)
241 "Generate a narinfo key/value string for STORE-PATH; an exception is raised
242 if STORE-PATH is invalid. Produce a URL that corresponds to COMPRESSION. The
243 narinfo is signed with KEY. NAR-PATH specifies the prefix for nar URLs.
244 Optionally, FILE-SIZE can specify the size in bytes of the compressed NAR; it
245 informs the client of how much needs to be downloaded."
246 (let* ((path-info (query-path-info store store-path))
247 (compression (actual-compression store-path compression))
248 (url (encode-and-join-uri-path
249 `(,@(split-and-decode-uri-path nar-path)
250 ,@(match compression
251 (($ <compression> 'none)
252 '())
253 (($ <compression> type)
254 (list (symbol->string type))))
255 ,(basename store-path))))
256 (hash (bytevector->nix-base32-string
257 (path-info-hash path-info)))
258 (size (path-info-nar-size path-info))
259 (file-size (or file-size
260 (and (eq? compression %no-compression) size)))
261 (references (string-join
262 (map basename (path-info-references path-info))
263 " "))
264 (deriver (path-info-deriver path-info))
265 (base-info (format #f
266 "\
267 StorePath: ~a
268 URL: ~a
269 Compression: ~a
270 NarHash: sha256:~a
271 NarSize: ~d
272 References: ~a~%~a"
273 store-path url
274 (compression-type compression)
275 hash size references
276 (if file-size
277 (format #f "FileSize: ~a~%" file-size)
278 "")))
279 ;; Do not render a "Deriver" or "System" line if we are rendering
280 ;; info for a derivation.
281 (info (if (not deriver)
282 base-info
283 (catch 'system-error
284 (lambda ()
285 (let ((drv (read-derivation-from-file deriver)))
286 (format #f "~aSystem: ~a~%Deriver: ~a~%"
287 base-info (derivation-system drv)
288 (basename deriver))))
289 (lambda args
290 ;; DERIVER might be missing, but that's fine:
291 ;; it's only used for <substitutable> where it's
292 ;; optional. 'System' is currently unused.
293 (if (= ENOENT (system-error-errno args))
294 base-info
295 (apply throw args))))))
296 (signature (base64-encode-string
297 (canonical-sexp->string (signed-string info)))))
298 (format #f "~aSignature: 1;~a;~a~%" info (gethostname) signature)))
300 (define* (not-found request
301 #:key (phrase "Resource not found")
302 ttl)
303 "Render 404 response for REQUEST."
304 (values (build-response #:code 404
305 #:headers (if ttl
306 `((cache-control (max-age . ,ttl)))
307 '()))
308 (string-append phrase ": "
309 (uri-path (request-uri request)))))
311 (define (render-nix-cache-info)
312 "Render server information."
313 (values '((content-type . (text/plain)))
314 (lambda (port)
315 (for-each (match-lambda
316 ((key . value)
317 (format port "~a: ~a~%" key value)))
318 %nix-cache-info))))
320 (define* (render-narinfo store request hash
321 #:key ttl (compression %no-compression)
322 (nar-path "nar"))
323 "Render metadata for the store path corresponding to HASH. If TTL is true,
324 advertise it as the maximum validity period (in seconds) via the
325 'Cache-Control' header. This allows 'guix substitute' to cache it for an
326 appropriate duration. NAR-PATH specifies the prefix for nar URLs."
327 (let ((store-path (hash-part->path store hash)))
328 (if (string-null? store-path)
329 (not-found request #:phrase "")
330 (values `((content-type . (application/x-nix-narinfo))
331 ,@(if ttl
332 `((cache-control (max-age . ,ttl)))
333 '()))
334 (cut display
335 (narinfo-string store store-path (%private-key)
336 #:nar-path nar-path
337 #:compression compression)
338 <>)))))
340 (define* (nar-cache-file directory item
341 #:key (compression %no-compression))
342 (string-append directory "/"
343 (symbol->string (compression-type compression))
344 "/" (basename item) ".nar"))
346 (define* (narinfo-cache-file directory item
347 #:key (compression %no-compression))
348 (string-append directory "/"
349 (symbol->string (compression-type compression))
350 "/" (basename item)
351 ".narinfo"))
353 (define run-single-baker
354 (let ((baking (make-weak-value-hash-table))
355 (mutex (make-mutex)))
356 (lambda (item thunk)
357 "Run THUNK, which is supposed to bake ITEM, but make sure only one
358 thread is baking ITEM at a given time."
359 (define selected?
360 (with-mutex mutex
361 (and (not (hash-ref baking item))
362 (begin
363 (hash-set! baking item (current-thread))
364 #t))))
366 (when selected?
367 (dynamic-wind
368 (const #t)
369 thunk
370 (lambda ()
371 (with-mutex mutex
372 (hash-remove! baking item))))))))
374 (define-syntax-rule (single-baker item exp ...)
375 "Bake ITEM by evaluating EXP, but make sure there's only one baker for ITEM
376 at a time."
377 (run-single-baker item (lambda () exp ...)))
380 (define (narinfo-files cache)
381 "Return the list of .narinfo files under CACHE."
382 (if (file-is-directory? cache)
383 (find-files cache
384 (lambda (file stat)
385 (string-suffix? ".narinfo" file)))
386 '()))
388 (define (nar-expiration-time ttl)
389 "Return the narinfo expiration time (in seconds since the Epoch). The
390 expiration time is +inf.0 when passed an item that is still in the store; in
391 other cases, it is the last-access time of the item plus TTL.
393 This policy allows us to keep cached nars that correspond to valid store
394 items. Failing that, we could eventually have to recompute them and return
395 404 in the meantime."
396 (let ((expiration-time (file-expiration-time ttl)))
397 (lambda (file)
398 (let ((item (string-append (%store-prefix) "/"
399 (basename file ".narinfo"))))
400 ;; Note: We don't need to use 'valid-path?' here because FILE would
401 ;; not exist if ITEM were not valid in the first place.
402 (if (file-exists? item)
403 +inf.0
404 (expiration-time file))))))
406 (define* (render-narinfo/cached store request hash
407 #:key ttl (compression %no-compression)
408 (nar-path "nar")
409 cache pool)
410 "Respond to the narinfo request for REQUEST. If the narinfo is available in
411 CACHE, then send it; otherwise, return 404 and \"bake\" that nar and narinfo
412 requested using POOL."
413 (define (delete-entry narinfo)
414 ;; Delete NARINFO and the corresponding nar from CACHE.
415 (let ((nar (string-append (string-drop-right narinfo
416 (string-length ".narinfo"))
417 ".nar")))
418 (delete-file* narinfo)
419 (delete-file* nar)))
421 (let* ((item (hash-part->path store hash))
422 (compression (actual-compression item compression))
423 (cached (and (not (string-null? item))
424 (narinfo-cache-file cache item
425 #:compression compression))))
426 (cond ((string-null? item)
427 (not-found request))
428 ((file-exists? cached)
429 ;; Narinfo is in cache, send it.
430 (values `((content-type . (application/x-nix-narinfo))
431 ,@(if ttl
432 `((cache-control (max-age . ,ttl)))
433 '()))
434 (lambda (port)
435 (display (call-with-input-file cached
436 read-string)
437 port))))
438 ((and (file-exists? item) ;cheaper than the 'valid-path?' RPC
439 (valid-path? store item))
440 ;; Nothing in cache: bake the narinfo and nar in the background and
441 ;; return 404.
442 (eventually pool
443 (single-baker item
444 ;; Check whether CACHED has been produced in the meantime.
445 (unless (file-exists? cached)
446 ;; (format #t "baking ~s~%" item)
447 (bake-narinfo+nar cache item
448 #:ttl ttl
449 #:compression compression
450 #:nar-path nar-path)))
452 (when ttl
453 (single-baker 'cache-cleanup
454 (maybe-remove-expired-cache-entries cache
455 narinfo-files
456 #:entry-expiration
457 (nar-expiration-time ttl)
458 #:delete-entry delete-entry
459 #:cleanup-period ttl))))
460 (not-found request
461 #:phrase "We're baking it"
462 #:ttl 300)) ;should be available within 5m
463 (else
464 (not-found request #:phrase "")))))
466 (define* (bake-narinfo+nar cache item
467 #:key ttl (compression %no-compression)
468 (nar-path "/nar"))
469 "Write the narinfo and nar for ITEM to CACHE."
470 (let* ((compression (actual-compression item compression))
471 (nar (nar-cache-file cache item
472 #:compression compression))
473 (narinfo (narinfo-cache-file cache item
474 #:compression compression)))
476 (mkdir-p (dirname nar))
477 (match (compression-type compression)
478 ('gzip
479 ;; Note: the file port gets closed along with the gzip port.
480 (call-with-gzip-output-port (open-output-file (string-append nar ".tmp"))
481 (lambda (port)
482 (write-file item port))
483 #:level (compression-level compression)
484 #:buffer-size (* 128 1024))
485 (rename-file (string-append nar ".tmp") nar))
486 ('none
487 ;; Cache nars even when compression is disabled so that we can
488 ;; guarantee the TTL (see <https://bugs.gnu.org/28664>.)
489 (with-atomic-file-output nar
490 (lambda (port)
491 (write-file item port)))))
493 (mkdir-p (dirname narinfo))
494 (with-atomic-file-output narinfo
495 (lambda (port)
496 ;; Open a new connection to the store. We cannot reuse the main
497 ;; thread's connection to the store since we would end up sending
498 ;; stuff concurrently on the same channel.
499 (with-store store
500 (display (narinfo-string store item
501 (%private-key)
502 #:nar-path nar-path
503 #:compression compression
504 #:file-size (and=> (stat nar #f)
505 stat:size))
506 port))))))
508 ;; XXX: Declare the 'X-Nar-Compression' HTTP header, which is in fact for
509 ;; internal consumption: it allows us to pass the compression info to
510 ;; 'http-write', as part of the workaround to <http://bugs.gnu.org/21093>.
511 (declare-header! "X-Nar-Compression"
512 (lambda (str)
513 (match (call-with-input-string str read)
514 (('compression type level)
515 (compression type level))))
516 compression?
517 (lambda (compression port)
518 (match compression
519 (($ <compression> type level)
520 (write `(compression ,type ,level) port)))))
522 (define* (render-nar store request store-item
523 #:key (compression %no-compression))
524 "Render archive of the store path corresponding to STORE-ITEM."
525 (let ((store-path (string-append %store-directory "/" store-item)))
526 ;; The ISO-8859-1 charset *must* be used otherwise HTTP clients will
527 ;; interpret the byte stream as UTF-8 and arbitrarily change invalid byte
528 ;; sequences.
529 (if (valid-path? store store-path)
530 (values `((content-type . (application/x-nix-archive
531 (charset . "ISO-8859-1")))
532 (x-nar-compression . ,compression))
533 ;; XXX: We're not returning the actual contents, deferring
534 ;; instead to 'http-write'. This is a hack to work around
535 ;; <http://bugs.gnu.org/21093>.
536 store-path)
537 (not-found request))))
539 (define* (render-nar/cached store cache request store-item
540 #:key (compression %no-compression))
541 "Respond to REQUEST with a nar for STORE-ITEM. If the nar is in CACHE,
542 return it; otherwise, return 404."
543 (let ((cached (nar-cache-file cache store-item
544 #:compression compression)))
545 (if (file-exists? cached)
546 (values `((content-type . (application/octet-stream
547 (charset . "ISO-8859-1")))
548 ;; XXX: We're not returning the actual contents, deferring
549 ;; instead to 'http-write'. This is a hack to work around
550 ;; <http://bugs.gnu.org/21093>.
551 (x-raw-file . ,cached))
552 #f)
553 (not-found request))))
555 (define (render-content-addressed-file store request
556 name algo hash)
557 "Return the content of the result of the fixed-output derivation NAME that
558 has the given HASH of type ALGO."
559 ;; TODO: Support other hash algorithms.
560 (if (and (eq? algo 'sha256) (= 32 (bytevector-length hash)))
561 (let ((item (fixed-output-path name hash
562 #:hash-algo algo
563 #:recursive? #f)))
564 (if (valid-path? store item)
565 (values `((content-type . (application/octet-stream
566 (charset . "ISO-8859-1")))
567 ;; XXX: We're not returning the actual contents,
568 ;; deferring instead to 'http-write'. This is a hack to
569 ;; work around <http://bugs.gnu.org/21093>.
570 (x-raw-file . ,item))
571 #f)
572 (not-found request)))
573 (not-found request)))
575 (define (render-log-file store request name)
576 "Render the log file for NAME, the base name of a store item. Don't attempt
577 to compress or decompress the log file; just return it as-is."
578 (define (response-headers file)
579 ;; XXX: We're not returning the actual contents, deferring instead to
580 ;; 'http-write'. This is a hack to work around
581 ;; <http://bugs.gnu.org/21093>.
582 (cond ((string-suffix? ".gz" file)
583 `((content-type . (text/plain (charset . "UTF-8")))
584 (content-encoding . (gzip))
585 (x-raw-file . ,file)))
586 ((string-suffix? ".bz2" file)
587 `((content-type . (application/x-bzip2
588 (charset . "ISO-8859-1")))
589 (x-raw-file . ,file)))
590 (else ;uncompressed
591 `((content-type . (text/plain (charset . "UTF-8")))
592 (x-raw-file . ,file)))))
594 (let ((log (log-file store
595 (string-append (%store-prefix) "/" name))))
596 (if log
597 (values (response-headers log) log)
598 (not-found request))))
600 (define (render-home-page request)
601 "Render the home page."
602 (values `((content-type . (text/html (charset . "UTF-8"))))
603 (call-with-output-string
604 (lambda (port)
605 (sxml->xml '(html
606 (head (title "GNU Guix Substitute Server"))
607 (body
608 (h1 "GNU Guix Substitute Server")
609 (p "Hi, "
610 (a (@ (href
611 "https://gnu.org/s/guix/manual/html_node/Invoking-guix-publish.html"))
612 (tt "guix publish"))
613 " speaking. Welcome!")))
614 port)))))
616 (define (extract-narinfo-hash str)
617 "Return the hash within the narinfo resource string STR, or false if STR
618 is invalid."
619 (and (string-suffix? ".narinfo" str)
620 (let ((base (string-drop-right str 8)))
621 (and (string-every %nix-base32-charset base)
622 base))))
624 (define (get-request? request)
625 "Return #t if REQUEST uses the GET method."
626 (eq? (request-method request) 'GET))
628 (define (request-path-components request)
629 "Split the URI path of REQUEST into a list of component strings. For
630 example: \"/foo/bar\" yields '(\"foo\" \"bar\")."
631 (split-and-decode-uri-path (uri-path (request-uri request))))
633 \f
634 ;;;
635 ;;; Server.
636 ;;;
638 (define %http-write
639 (@@ (web server http) http-write))
641 (define (strip-headers response)
642 "Return RESPONSE's headers minus 'Content-Length' and our internal headers."
643 (fold alist-delete
644 (response-headers response)
645 '(content-length x-raw-file x-nar-compression)))
647 (define (sans-content-length response)
648 "Return RESPONSE without its 'content-length' header."
649 (set-field response (response-headers)
650 (strip-headers response)))
652 (define (with-content-length response length)
653 "Return RESPONSE with a 'content-length' header set to LENGTH."
654 (set-field response (response-headers)
655 (alist-cons 'content-length length
656 (strip-headers response))))
658 (define-syntax-rule (swallow-EPIPE exp ...)
659 "Swallow EPIPE errors raised by EXP..."
660 (catch 'system-error
661 (lambda ()
662 exp ...)
663 (lambda args
664 (if (= EPIPE (system-error-errno args))
665 (values)
666 (apply throw args)))))
668 (define-syntax-rule (swallow-zlib-error exp ...)
669 "Swallow 'zlib-error' exceptions raised by EXP..."
670 (catch 'zlib-error
671 (lambda ()
672 exp ...)
673 (const #f)))
675 (define (nar-response-port response compression)
676 "Return a port on which to write the body of RESPONSE, the response of a
677 /nar request, according to COMPRESSION."
678 (match compression
679 (($ <compression> 'gzip level)
680 ;; Note: We cannot used chunked encoding here because
681 ;; 'make-gzip-output-port' wants a file port.
682 (make-gzip-output-port (response-port response)
683 #:level level
684 #:buffer-size (* 64 1024)))
685 (($ <compression> 'none)
686 (response-port response))
687 (#f
688 (response-port response))))
690 (define (http-write server client response body)
691 "Write RESPONSE and BODY to CLIENT, possibly in a separate thread to avoid
692 blocking."
693 (match (response-content-type response)
694 (('application/x-nix-archive . _)
695 ;; Sending the the whole archive can take time so do it in a separate
696 ;; thread so that the main thread can keep working in the meantime.
697 (call-with-new-thread
698 (lambda ()
699 (set-thread-name "publish nar")
700 (let* ((compression (assoc-ref (response-headers response)
701 'x-nar-compression))
702 (response (write-response (sans-content-length response)
703 client))
704 (port (begin
705 (force-output client)
706 (nar-response-port response compression))))
707 ;; XXX: Given our ugly workaround for <http://bugs.gnu.org/21093> in
708 ;; 'render-nar', BODY here is just the file name of the store item.
709 ;; We call 'write-file' from here because we know that's the only
710 ;; way to avoid building the whole nar in memory, which could
711 ;; quickly become a real problem. As a bonus, we even do
712 ;; sendfile(2) directly from the store files to the socket.
713 (swallow-zlib-error
714 (swallow-EPIPE
715 (write-file (utf8->string body) port)))
716 (swallow-zlib-error
717 (close-port port))
718 (values)))))
719 (_
720 (match (assoc-ref (response-headers response) 'x-raw-file)
721 ((? string? file)
722 ;; Send a raw file in a separate thread.
723 (call-with-new-thread
724 (lambda ()
725 (set-thread-name "publish file")
726 (catch 'system-error
727 (lambda ()
728 (call-with-input-file file
729 (lambda (input)
730 (let* ((size (stat:size (stat input)))
731 (response (write-response (with-content-length response
732 size)
733 client))
734 (output (response-port response)))
735 (if (file-port? output)
736 (sendfile output input size)
737 (dump-port input output))
738 (close-port output)
739 (values)))))
740 (lambda args
741 ;; If the file was GC'd behind our back, that's fine. Likewise if
742 ;; the client closes the connection.
743 (unless (memv (system-error-errno args)
744 (list ENOENT EPIPE ECONNRESET))
745 (apply throw args))
746 (values))))))
747 (#f
748 ;; Handle other responses sequentially.
749 (%http-write server client response body))))))
751 (define-server-impl concurrent-http-server
752 ;; A variant of Guile's built-in HTTP server that offloads possibly long
753 ;; responses to a different thread.
754 (@@ (web server http) http-open)
755 (@@ (web server http) http-read)
756 http-write
757 (@@ (web server http) http-close))
759 (define* (make-request-handler store
760 #:key
761 cache pool
762 narinfo-ttl
763 (nar-path "nar")
764 (compression %no-compression))
765 (define nar-path?
766 (let ((expected (split-and-decode-uri-path nar-path)))
767 (cut equal? expected <>)))
769 (lambda (request body)
770 (format #t "~a ~a~%"
771 (request-method request)
772 (uri-path (request-uri request)))
773 (if (get-request? request) ;reject POST, PUT, etc.
774 (match (request-path-components request)
775 ;; /nix-cache-info
776 (("nix-cache-info")
777 (render-nix-cache-info))
778 ;; /
779 ((or () ("index.html"))
780 (render-home-page request))
781 ;; /<hash>.narinfo
782 (((= extract-narinfo-hash (? string? hash)))
783 ;; TODO: Register roots for HASH that will somehow remain for
784 ;; NARINFO-TTL.
785 (if cache
786 (render-narinfo/cached store request hash
787 #:cache cache
788 #:pool pool
789 #:ttl narinfo-ttl
790 #:nar-path nar-path
791 #:compression compression)
792 (render-narinfo store request hash
793 #:ttl narinfo-ttl
794 #:nar-path nar-path
795 #:compression compression)))
796 ;; /nar/file/NAME/sha256/HASH
797 (("file" name "sha256" hash)
798 (guard (c ((invalid-base32-character? c)
799 (not-found request)))
800 (let ((hash (nix-base32-string->bytevector hash)))
801 (render-content-addressed-file store request
802 name 'sha256 hash))))
804 ;; /log/OUTPUT
805 (("log" name)
806 (render-log-file store request name))
808 ;; Use different URLs depending on the compression type. This
809 ;; guarantees that /nar URLs remain valid even when 'guix publish'
810 ;; is restarted with different compression parameters.
812 ;; /nar/gzip/<store-item>
813 ((components ... "gzip" store-item)
814 (if (and (nar-path? components) (zlib-available?))
815 (let ((compression (match compression
816 (($ <compression> 'gzip)
817 compression)
818 (_
819 %default-gzip-compression))))
820 (if cache
821 (render-nar/cached store cache request store-item
822 #:compression compression)
823 (render-nar store request store-item
824 #:compression compression)))
825 (not-found request)))
827 ;; /nar/<store-item>
828 ((components ... store-item)
829 (if (nar-path? components)
830 (if cache
831 (render-nar/cached store cache request store-item
832 #:compression %no-compression)
833 (render-nar store request store-item
834 #:compression %no-compression))
835 (not-found request)))
837 (x (not-found request)))
838 (not-found request))))
840 (define* (run-publish-server socket store
841 #:key (compression %no-compression)
842 (nar-path "nar") narinfo-ttl
843 cache pool)
844 (run-server (make-request-handler store
845 #:cache cache
846 #:pool pool
847 #:nar-path nar-path
848 #:narinfo-ttl narinfo-ttl
849 #:compression compression)
850 concurrent-http-server
851 `(#:socket ,socket)))
853 (define (open-server-socket address)
854 "Return a TCP socket bound to ADDRESS, a socket address."
855 (let ((sock (socket (sockaddr:fam address) SOCK_STREAM 0)))
856 (setsockopt sock SOL_SOCKET SO_REUSEADDR 1)
857 (bind sock address)
858 sock))
860 (define (gather-user-privileges user)
861 "Switch to the identity of USER, a user name."
862 (catch 'misc-error
863 (lambda ()
864 (let ((user (getpw user)))
865 (setgroups #())
866 (setgid (passwd:gid user))
867 (setuid (passwd:uid user))))
868 (lambda (key proc message args . rest)
869 (leave (G_ "user '~a' not found: ~a~%")
870 user (apply format #f message args)))))
872 \f
873 ;;;
874 ;;; Entry point.
875 ;;;
877 (define (guix-publish . args)
878 (with-error-handling
879 (let* ((opts (args-fold* args %options
880 (lambda (opt name arg result)
881 (leave (G_ "~A: unrecognized option~%") name))
882 (lambda (arg result)
883 (leave (G_ "~A: extraneous argument~%") arg))
884 %default-options))
885 (user (assoc-ref opts 'user))
886 (port (assoc-ref opts 'port))
887 (ttl (assoc-ref opts 'narinfo-ttl))
888 (compression (assoc-ref opts 'compression))
889 (address (let ((addr (assoc-ref opts 'address)))
890 (make-socket-address (sockaddr:fam addr)
891 (sockaddr:addr addr)
892 port)))
893 (socket (open-server-socket address))
894 (nar-path (assoc-ref opts 'nar-path))
895 (repl-port (assoc-ref opts 'repl))
896 (cache (assoc-ref opts 'cache))
897 (workers (assoc-ref opts 'workers))
899 ;; Read the key right away so that (1) we fail early on if we can't
900 ;; access them, and (2) we can then drop privileges.
901 (public-key (read-file-sexp (assoc-ref opts 'public-key-file)))
902 (private-key (read-file-sexp (assoc-ref opts 'private-key-file))))
904 (when user
905 ;; Now that we've read the key material and opened the socket, we can
906 ;; drop privileges.
907 (gather-user-privileges user))
909 (when (zero? (getuid))
910 (warning (G_ "server running as root; \
911 consider using the '--user' option!~%")))
913 (parameterize ((%public-key public-key)
914 (%private-key private-key))
915 (format #t (G_ "publishing ~a on ~a, port ~d~%")
916 %store-directory
917 (inet-ntop (sockaddr:fam address) (sockaddr:addr address))
918 (sockaddr:port address))
919 (when repl-port
920 (repl:spawn-server (repl:make-tcp-server-socket #:port repl-port)))
922 ;; Set the name of the main thread.
923 (set-thread-name "guix publish")
925 (with-store store
926 (run-publish-server socket store
927 #:cache cache
928 #:pool (and cache (make-pool workers
929 #:thread-name
930 "publish worker"))
931 #:nar-path nar-path
932 #:compression compression
933 #:narinfo-ttl ttl))))))
935 ;;; Local Variables:
936 ;;; eval: (put 'single-baker 'scheme-indent-function 1)
937 ;;; End: