| blob | a236f3e45cba5bf50819771df95bd9623787aae4 |
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2015 David Thompson <davet@gnu.org>
3 ;;; Copyright © 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
4 ;;;
5 ;;; This file is part of GNU Guix.
6 ;;;
7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
8 ;;; under the terms of the GNU General Public License as published by
9 ;;; the Free Software Foundation; either version 3 of the License, or (at
10 ;;; your option) any later version.
11 ;;;
12 ;;; GNU Guix is distributed in the hope that it will be useful, but
13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ;;; GNU General Public License for more details.
16 ;;;
17 ;;; You should have received a copy of the GNU General Public License
18 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
20 (define-module (guix scripts publish)
21 #:use-module ((system repl server) #:prefix repl:)
22 #:use-module (ice-9 binary-ports)
23 #:use-module (ice-9 format)
24 #:use-module (ice-9 match)
25 #:use-module (ice-9 regex)
26 #:use-module (ice-9 rdelim)
27 #:use-module (ice-9 threads)
28 #:use-module (rnrs bytevectors)
29 #:use-module (srfi srfi-1)
30 #:use-module (srfi srfi-2)
31 #:use-module (srfi srfi-9)
32 #:use-module (srfi srfi-9 gnu)
33 #:use-module (srfi srfi-19)
34 #:use-module (srfi srfi-26)
35 #:use-module (srfi srfi-34)
36 #:use-module (srfi srfi-37)
37 #:use-module (web http)
38 #:use-module (web request)
39 #:use-module (web response)
40 #:use-module (web server)
41 #:use-module (web uri)
42 #:autoload (sxml simple) (sxml->xml)
43 #:use-module (guix base32)
44 #:use-module (guix base64)
45 #:use-module (guix config)
46 #:use-module (guix derivations)
47 #:use-module (gcrypt hash)
48 #:use-module (guix pki)
49 #:use-module (gcrypt pk-crypto)
50 #:use-module (guix workers)
51 #:use-module (guix store)
52 #:use-module ((guix serialization) #:select (write-file))
53 #:use-module (guix zlib)
54 #:use-module (guix cache)
55 #:use-module (guix ui)
56 #:use-module (guix scripts)
57 #:use-module ((guix utils)
58 #:select (with-atomic-file-output compressed-file?))
59 #:use-module ((guix build utils)
60 #:select (dump-port mkdir-p find-files))
61 #:use-module ((guix build syscalls) #:select (set-thread-name))
62 #:export (%public-key
63 %private-key
65 guix-publish))
67 (define (show-help)
68 (format #t (G_ "Usage: guix publish [OPTION]...
69 Publish ~a over HTTP.\n") %store-directory)
70 (display (G_ "
71 -p, --port=PORT listen on PORT"))
72 (display (G_ "
73 --listen=HOST listen on the network interface for HOST"))
74 (display (G_ "
75 -u, --user=USER change privileges to USER as soon as possible"))
76 (display (G_ "
77 -C, --compression[=LEVEL]
78 compress archives at LEVEL"))
79 (display (G_ "
80 -c, --cache=DIRECTORY cache published items to DIRECTORY"))
81 (display (G_ "
82 --workers=N use N workers to bake items"))
83 (display (G_ "
84 --ttl=TTL announce narinfos can be cached for TTL seconds"))
85 (display (G_ "
86 --nar-path=PATH use PATH as the prefix for nar URLs"))
87 (display (G_ "
88 --public-key=FILE use FILE as the public key for signatures"))
89 (display (G_ "
90 --private-key=FILE use FILE as the private key for signatures"))
91 (display (G_ "
92 -r, --repl[=PORT] spawn REPL server on PORT"))
93 (newline)
94 (display (G_ "
95 -h, --help display this help and exit"))
96 (display (G_ "
97 -V, --version display version information and exit"))
98 (newline)
99 (show-bug-report-information))
101 (define (getaddrinfo* host)
102 "Like 'getaddrinfo', but properly report errors."
103 (catch 'getaddrinfo-error
104 (lambda ()
105 (getaddrinfo host))
106 (lambda (key error)
107 (leave (G_ "lookup of host '~a' failed: ~a~%")
108 host (gai-strerror error)))))
110 ;; Nar compression parameters.
111 (define-record-type <compression>
112 (compression type level)
113 compression?
114 (type compression-type)
115 (level compression-level))
117 (define %no-compression
118 (compression 'none 0))
120 (define %default-gzip-compression
121 ;; Since we compress on the fly, default to fast compression.
122 (compression 'gzip 3))
124 (define (actual-compression item requested)
125 "Return the actual compression used for ITEM, which may be %NO-COMPRESSION
126 if ITEM is already compressed."
127 (if (compressed-file? item)
128 %no-compression
129 requested))
131 (define %options
132 (list (option '(#\h "help") #f #f
133 (lambda _
134 (show-help)
135 (exit 0)))
136 (option '(#\V "version") #f #f
137 (lambda _
138 (show-version-and-exit "guix publish")))
139 (option '(#\u "user") #t #f
140 (lambda (opt name arg result)
141 (alist-cons 'user arg result)))
142 (option '(#\p "port") #t #f
143 (lambda (opt name arg result)
144 (alist-cons 'port (string->number* arg) result)))
145 (option '("listen") #t #f
146 (lambda (opt name arg result)
147 (match (getaddrinfo* arg)
148 ((info _ ...)
149 (alist-cons 'address (addrinfo:addr info)
150 result))
151 (()
152 (leave (G_ "lookup of host '~a' returned nothing")
153 name)))))
154 (option '(#\C "compression") #f #t
155 (lambda (opt name arg result)
156 (match (if arg (string->number* arg) 3)
157 (0
158 (alist-cons 'compression %no-compression result))
159 (level
160 (if (zlib-available?)
161 (alist-cons 'compression
162 (compression 'gzip level)
163 result)
164 (begin
165 (warning (G_ "zlib support is missing; \
166 compression disabled~%"))
167 result))))))
168 (option '(#\c "cache") #t #f
169 (lambda (opt name arg result)
170 (alist-cons 'cache arg result)))
171 (option '("workers") #t #f
172 (lambda (opt name arg result)
173 (alist-cons 'workers (string->number* arg)
174 result)))
175 (option '("ttl") #t #f
176 (lambda (opt name arg result)
177 (let ((duration (string->duration arg)))
178 (unless duration
179 (leave (G_ "~a: invalid duration~%") arg))
180 (alist-cons 'narinfo-ttl (time-second duration)
181 result))))
182 (option '("nar-path") #t #f
183 (lambda (opt name arg result)
184 (alist-cons 'nar-path arg result)))
185 (option '("public-key") #t #f
186 (lambda (opt name arg result)
187 (alist-cons 'public-key-file arg result)))
188 (option '("private-key" "secret-key") #t #f
189 (lambda (opt name arg result)
190 (alist-cons 'private-key-file arg result)))
191 (option '(#\r "repl") #f #t
192 (lambda (opt name arg result)
193 ;; If port unspecified, use default Guile REPL port.
194 (let ((port (and arg (string->number* arg))))
195 (alist-cons 'repl (or port 37146) result))))))
197 (define %default-options
198 `((port . 8080)
200 ;; By default, serve nars under "/nar".
201 (nar-path . "nar")
203 (public-key-file . ,%public-key-file)
204 (private-key-file . ,%private-key-file)
206 ;; Default to fast & low compression.
207 (compression . ,(if (zlib-available?)
208 %default-gzip-compression
209 %no-compression))
211 ;; Default number of workers when caching is enabled.
212 (workers . ,(current-processor-count))
214 (address . ,(make-socket-address AF_INET INADDR_ANY 0))
215 (repl . #f)))
217 ;; The key pair used to sign narinfos.
218 (define %private-key
219 (make-parameter #f))
220 (define %public-key
221 (make-parameter #f))
223 (define %nix-cache-info
224 `(("StoreDir" . ,%store-directory)
225 ("WantMassQuery" . 0)
226 ("Priority" . 100)))
228 (define (signed-string s)
229 "Sign the hash of the string S with the daemon's key."
230 (let* ((public-key (%public-key))
231 (hash (bytevector->hash-data (sha256 (string->utf8 s))
232 #:key-type (key-type public-key))))
233 (signature-sexp hash (%private-key) public-key)))
235 (define base64-encode-string
236 (compose base64-encode string->utf8))
238 (define* (narinfo-string store store-path key
239 #:key (compression %no-compression)
240 (nar-path "nar") file-size)
241 "Generate a narinfo key/value string for STORE-PATH; an exception is raised
242 if STORE-PATH is invalid. Produce a URL that corresponds to COMPRESSION. The
243 narinfo is signed with KEY. NAR-PATH specifies the prefix for nar URLs.
244 Optionally, FILE-SIZE can specify the size in bytes of the compressed NAR; it
245 informs the client of how much needs to be downloaded."
246 (let* ((path-info (query-path-info store store-path))
247 (compression (actual-compression store-path compression))
248 (url (encode-and-join-uri-path
249 `(,@(split-and-decode-uri-path nar-path)
250 ,@(match compression
251 (($ <compression> 'none)
252 '())
253 (($ <compression> type)
254 (list (symbol->string type))))
255 ,(basename store-path))))
256 (hash (bytevector->nix-base32-string
257 (path-info-hash path-info)))
258 (size (path-info-nar-size path-info))
259 (file-size (or file-size
260 (and (eq? compression %no-compression) size)))
261 (references (string-join
262 (map basename (path-info-references path-info))
263 " "))
264 (deriver (path-info-deriver path-info))
265 (base-info (format #f
266 "\
267 StorePath: ~a
268 URL: ~a
269 Compression: ~a
270 NarHash: sha256:~a
271 NarSize: ~d
272 References: ~a~%~a"
273 store-path url
274 (compression-type compression)
275 hash size references
276 (if file-size
277 (format #f "FileSize: ~a~%" file-size)
278 "")))
279 ;; Do not render a "Deriver" or "System" line if we are rendering
280 ;; info for a derivation.
281 (info (if (not deriver)
282 base-info
283 (catch 'system-error
284 (lambda ()
285 (let ((drv (read-derivation-from-file deriver)))
286 (format #f "~aSystem: ~a~%Deriver: ~a~%"
287 base-info (derivation-system drv)
288 (basename deriver))))
289 (lambda args
290 ;; DERIVER might be missing, but that's fine:
291 ;; it's only used for <substitutable> where it's
292 ;; optional. 'System' is currently unused.
293 (if (= ENOENT (system-error-errno args))
294 base-info
295 (apply throw args))))))
296 (signature (base64-encode-string
297 (canonical-sexp->string (signed-string info)))))
298 (format #f "~aSignature: 1;~a;~a~%" info (gethostname) signature)))
300 (define* (not-found request
301 #:key (phrase "Resource not found")
302 ttl)
303 "Render 404 response for REQUEST."
304 (values (build-response #:code 404
305 #:headers (if ttl
306 `((cache-control (max-age . ,ttl)))
307 '()))
308 (string-append phrase ": "
309 (uri-path (request-uri request)))))
311 (define (render-nix-cache-info)
312 "Render server information."
313 (values '((content-type . (text/plain)))
314 (lambda (port)
315 (for-each (match-lambda
316 ((key . value)
317 (format port "~a: ~a~%" key value)))
318 %nix-cache-info))))
320 (define* (render-narinfo store request hash
321 #:key ttl (compression %no-compression)
322 (nar-path "nar"))
323 "Render metadata for the store path corresponding to HASH. If TTL is true,
324 advertise it as the maximum validity period (in seconds) via the
325 'Cache-Control' header. This allows 'guix substitute' to cache it for an
326 appropriate duration. NAR-PATH specifies the prefix for nar URLs."
327 (let ((store-path (hash-part->path store hash)))
328 (if (string-null? store-path)
329 (not-found request #:phrase "")
330 (values `((content-type . (application/x-nix-narinfo))
331 ,@(if ttl
332 `((cache-control (max-age . ,ttl)))
333 '()))
334 (cut display
335 (narinfo-string store store-path (%private-key)
336 #:nar-path nar-path
337 #:compression compression)
338 <>)))))
340 (define* (nar-cache-file directory item
341 #:key (compression %no-compression))
342 (string-append directory "/"
343 (symbol->string (compression-type compression))
344 "/" (basename item) ".nar"))
346 (define* (narinfo-cache-file directory item
347 #:key (compression %no-compression))
348 (string-append directory "/"
349 (symbol->string (compression-type compression))
350 "/" (basename item)
351 ".narinfo"))
353 (define run-single-baker
354 (let ((baking (make-weak-value-hash-table))
355 (mutex (make-mutex)))
356 (lambda (item thunk)
357 "Run THUNK, which is supposed to bake ITEM, but make sure only one
358 thread is baking ITEM at a given time."
359 (define selected?
360 (with-mutex mutex
361 (and (not (hash-ref baking item))
362 (begin
363 (hash-set! baking item (current-thread))
364 #t))))
366 (when selected?
367 (dynamic-wind
368 (const #t)
369 thunk
370 (lambda ()
371 (with-mutex mutex
372 (hash-remove! baking item))))))))
374 (define-syntax-rule (single-baker item exp ...)
375 "Bake ITEM by evaluating EXP, but make sure there's only one baker for ITEM
376 at a time."
377 (run-single-baker item (lambda () exp ...)))
380 (define (narinfo-files cache)
381 "Return the list of .narinfo files under CACHE."
382 (if (file-is-directory? cache)
383 (find-files cache
384 (lambda (file stat)
385 (string-suffix? ".narinfo" file)))
386 '()))
388 (define (nar-expiration-time ttl)
389 "Return the narinfo expiration time (in seconds since the Epoch). The
390 expiration time is +inf.0 when passed an item that is still in the store; in
391 other cases, it is the last-access time of the item plus TTL.
393 This policy allows us to keep cached nars that correspond to valid store
394 items. Failing that, we could eventually have to recompute them and return
395 404 in the meantime."
396 (let ((expiration-time (file-expiration-time ttl)))
397 (lambda (file)
398 (let ((item (string-append (%store-prefix) "/"
399 (basename file ".narinfo"))))
400 ;; Note: We don't need to use 'valid-path?' here because FILE would
401 ;; not exist if ITEM were not valid in the first place.
402 (if (file-exists? item)
403 +inf.0
404 (expiration-time file))))))
406 (define* (render-narinfo/cached store request hash
407 #:key ttl (compression %no-compression)
408 (nar-path "nar")
409 cache pool)
410 "Respond to the narinfo request for REQUEST. If the narinfo is available in
411 CACHE, then send it; otherwise, return 404 and \"bake\" that nar and narinfo
412 requested using POOL."
413 (define (delete-entry narinfo)
414 ;; Delete NARINFO and the corresponding nar from CACHE.
415 (let ((nar (string-append (string-drop-right narinfo
416 (string-length ".narinfo"))
417 ".nar")))
418 (delete-file* narinfo)
419 (delete-file* nar)))
421 (let* ((item (hash-part->path store hash))
422 (compression (actual-compression item compression))
423 (cached (and (not (string-null? item))
424 (narinfo-cache-file cache item
425 #:compression compression))))
426 (cond ((string-null? item)
427 (not-found request))
428 ((file-exists? cached)
429 ;; Narinfo is in cache, send it.
430 (values `((content-type . (application/x-nix-narinfo))
431 ,@(if ttl
432 `((cache-control (max-age . ,ttl)))
433 '()))
434 (lambda (port)
435 (display (call-with-input-file cached
436 read-string)
437 port))))
438 ((and (file-exists? item) ;cheaper than the 'valid-path?' RPC
439 (valid-path? store item))
440 ;; Nothing in cache: bake the narinfo and nar in the background and
441 ;; return 404.
442 (eventually pool
443 (single-baker item
444 ;; Check whether CACHED has been produced in the meantime.
445 (unless (file-exists? cached)
446 ;; (format #t "baking ~s~%" item)
447 (bake-narinfo+nar cache item
448 #:ttl ttl
449 #:compression compression
450 #:nar-path nar-path)))
452 (when ttl
453 (single-baker 'cache-cleanup
454 (maybe-remove-expired-cache-entries cache
455 narinfo-files
456 #:entry-expiration
457 (nar-expiration-time ttl)
458 #:delete-entry delete-entry
459 #:cleanup-period ttl))))
460 (not-found request
461 #:phrase "We're baking it"
462 #:ttl 300)) ;should be available within 5m
463 (else
464 (not-found request #:phrase "")))))
466 (define* (bake-narinfo+nar cache item
467 #:key ttl (compression %no-compression)
468 (nar-path "/nar"))
469 "Write the narinfo and nar for ITEM to CACHE."
470 (let* ((compression (actual-compression item compression))
471 (nar (nar-cache-file cache item
472 #:compression compression))
473 (narinfo (narinfo-cache-file cache item
474 #:compression compression)))
476 (mkdir-p (dirname nar))
477 (match (compression-type compression)
478 ('gzip
479 ;; Note: the file port gets closed along with the gzip port.
480 (call-with-gzip-output-port (open-output-file (string-append nar ".tmp"))
481 (lambda (port)
482 (write-file item port))
483 #:level (compression-level compression)
484 #:buffer-size (* 128 1024))
485 (rename-file (string-append nar ".tmp") nar))
486 ('none
487 ;; Cache nars even when compression is disabled so that we can
488 ;; guarantee the TTL (see <https://bugs.gnu.org/28664>.)
489 (with-atomic-file-output nar
490 (lambda (port)
491 (write-file item port)))))
493 (mkdir-p (dirname narinfo))
494 (with-atomic-file-output narinfo
495 (lambda (port)
496 ;; Open a new connection to the store. We cannot reuse the main
497 ;; thread's connection to the store since we would end up sending
498 ;; stuff concurrently on the same channel.
499 (with-store store
500 (display (narinfo-string store item
501 (%private-key)
502 #:nar-path nar-path
503 #:compression compression
504 #:file-size (and=> (stat nar #f)
505 stat:size))
506 port))))))
508 ;; XXX: Declare the 'X-Nar-Compression' HTTP header, which is in fact for
509 ;; internal consumption: it allows us to pass the compression info to
510 ;; 'http-write', as part of the workaround to <http://bugs.gnu.org/21093>.
511 (declare-header! "X-Nar-Compression"
512 (lambda (str)
513 (match (call-with-input-string str read)
514 (('compression type level)
515 (compression type level))))
516 compression?
517 (lambda (compression port)
518 (match compression
519 (($ <compression> type level)
520 (write `(compression ,type ,level) port)))))
522 (define* (render-nar store request store-item
523 #:key (compression %no-compression))
524 "Render archive of the store path corresponding to STORE-ITEM."
525 (let ((store-path (string-append %store-directory "/" store-item)))
526 ;; The ISO-8859-1 charset *must* be used otherwise HTTP clients will
527 ;; interpret the byte stream as UTF-8 and arbitrarily change invalid byte
528 ;; sequences.
529 (if (valid-path? store store-path)
530 (values `((content-type . (application/x-nix-archive
531 (charset . "ISO-8859-1")))
532 (x-nar-compression . ,compression))
533 ;; XXX: We're not returning the actual contents, deferring
534 ;; instead to 'http-write'. This is a hack to work around
535 ;; <http://bugs.gnu.org/21093>.
536 store-path)
537 (not-found request))))
539 (define* (render-nar/cached store cache request store-item
540 #:key ttl (compression %no-compression))
541 "Respond to REQUEST with a nar for STORE-ITEM. If the nar is in CACHE,
542 return it; otherwise, return 404. When TTL is true, use it as the
543 'Cache-Control' expiration time."
544 (let ((cached (nar-cache-file cache store-item
545 #:compression compression)))
546 (if (file-exists? cached)
547 (values `((content-type . (application/octet-stream
548 (charset . "ISO-8859-1")))
549 ,@(if ttl
550 `((cache-control (max-age . ,ttl)))
551 '())
553 ;; XXX: We're not returning the actual contents, deferring
554 ;; instead to 'http-write'. This is a hack to work around
555 ;; <http://bugs.gnu.org/21093>.
556 (x-raw-file . ,cached))
557 #f)
558 (not-found request))))
560 (define (render-content-addressed-file store request
561 name algo hash)
562 "Return the content of the result of the fixed-output derivation NAME that
563 has the given HASH of type ALGO."
564 ;; TODO: Support other hash algorithms.
565 (if (and (eq? algo 'sha256) (= 32 (bytevector-length hash)))
566 (let ((item (fixed-output-path name hash
567 #:hash-algo algo
568 #:recursive? #f)))
569 (if (valid-path? store item)
570 (values `((content-type . (application/octet-stream
571 (charset . "ISO-8859-1")))
572 ;; XXX: We're not returning the actual contents,
573 ;; deferring instead to 'http-write'. This is a hack to
574 ;; work around <http://bugs.gnu.org/21093>.
575 (x-raw-file . ,item))
576 #f)
577 (not-found request)))
578 (not-found request)))
580 (define (render-log-file store request name)
581 "Render the log file for NAME, the base name of a store item. Don't attempt
582 to compress or decompress the log file; just return it as-is."
583 (define (response-headers file)
584 ;; XXX: We're not returning the actual contents, deferring instead to
585 ;; 'http-write'. This is a hack to work around
586 ;; <http://bugs.gnu.org/21093>.
587 (cond ((string-suffix? ".gz" file)
588 `((content-type . (text/plain (charset . "UTF-8")))
589 (content-encoding . (gzip))
590 (x-raw-file . ,file)))
591 ((string-suffix? ".bz2" file)
592 `((content-type . (application/x-bzip2
593 (charset . "ISO-8859-1")))
594 (x-raw-file . ,file)))
595 (else ;uncompressed
596 `((content-type . (text/plain (charset . "UTF-8")))
597 (x-raw-file . ,file)))))
599 (let ((log (log-file store
600 (string-append (%store-prefix) "/" name))))
601 (if log
602 (values (response-headers log) log)
603 (not-found request))))
605 (define (render-home-page request)
606 "Render the home page."
607 (values `((content-type . (text/html (charset . "UTF-8"))))
608 (call-with-output-string
609 (lambda (port)
610 (sxml->xml '(html
611 (head (title "GNU Guix Substitute Server"))
612 (body
613 (h1 "GNU Guix Substitute Server")
614 (p "Hi, "
615 (a (@ (href
616 "https://gnu.org/s/guix/manual/html_node/Invoking-guix-publish.html"))
617 (tt "guix publish"))
618 " speaking. Welcome!")))
619 port)))))
621 (define (extract-narinfo-hash str)
622 "Return the hash within the narinfo resource string STR, or false if STR
623 is invalid."
624 (and (string-suffix? ".narinfo" str)
625 (let ((base (string-drop-right str 8)))
626 (and (string-every %nix-base32-charset base)
627 base))))
629 (define (get-request? request)
630 "Return #t if REQUEST uses the GET method."
631 (eq? (request-method request) 'GET))
633 (define (request-path-components request)
634 "Split the URI path of REQUEST into a list of component strings. For
635 example: \"/foo/bar\" yields '(\"foo\" \"bar\")."
636 (split-and-decode-uri-path (uri-path (request-uri request))))
638 \f
639 ;;;
640 ;;; Server.
641 ;;;
643 (define %http-write
644 (@@ (web server http) http-write))
646 (define (strip-headers response)
647 "Return RESPONSE's headers minus 'Content-Length' and our internal headers."
648 (fold alist-delete
649 (response-headers response)
650 '(content-length x-raw-file x-nar-compression)))
652 (define (sans-content-length response)
653 "Return RESPONSE without its 'content-length' header."
654 (set-field response (response-headers)
655 (strip-headers response)))
657 (define (with-content-length response length)
658 "Return RESPONSE with a 'content-length' header set to LENGTH."
659 (set-field response (response-headers)
660 (alist-cons 'content-length length
661 (strip-headers response))))
663 (define-syntax-rule (swallow-EPIPE exp ...)
664 "Swallow EPIPE errors raised by EXP..."
665 (catch 'system-error
666 (lambda ()
667 exp ...)
668 (lambda args
669 (if (= EPIPE (system-error-errno args))
670 (values)
671 (apply throw args)))))
673 (define-syntax-rule (swallow-zlib-error exp ...)
674 "Swallow 'zlib-error' exceptions raised by EXP..."
675 (catch 'zlib-error
676 (lambda ()
677 exp ...)
678 (const #f)))
680 (define (nar-response-port response compression)
681 "Return a port on which to write the body of RESPONSE, the response of a
682 /nar request, according to COMPRESSION."
683 (match compression
684 (($ <compression> 'gzip level)
685 ;; Note: We cannot used chunked encoding here because
686 ;; 'make-gzip-output-port' wants a file port.
687 (make-gzip-output-port (response-port response)
688 #:level level
689 #:buffer-size (* 64 1024)))
690 (($ <compression> 'none)
691 (response-port response))
692 (#f
693 (response-port response))))
695 (define (http-write server client response body)
696 "Write RESPONSE and BODY to CLIENT, possibly in a separate thread to avoid
697 blocking."
698 (match (response-content-type response)
699 (('application/x-nix-archive . _)
700 ;; Sending the the whole archive can take time so do it in a separate
701 ;; thread so that the main thread can keep working in the meantime.
702 (call-with-new-thread
703 (lambda ()
704 (set-thread-name "publish nar")
705 (let* ((compression (assoc-ref (response-headers response)
706 'x-nar-compression))
707 (response (write-response (sans-content-length response)
708 client))
709 (port (begin
710 (force-output client)
711 (nar-response-port response compression))))
712 ;; XXX: Given our ugly workaround for <http://bugs.gnu.org/21093> in
713 ;; 'render-nar', BODY here is just the file name of the store item.
714 ;; We call 'write-file' from here because we know that's the only
715 ;; way to avoid building the whole nar in memory, which could
716 ;; quickly become a real problem. As a bonus, we even do
717 ;; sendfile(2) directly from the store files to the socket.
718 (swallow-zlib-error
719 (swallow-EPIPE
720 (write-file (utf8->string body) port)))
721 (swallow-zlib-error
722 (close-port port))
723 (values)))))
724 (_
725 (match (assoc-ref (response-headers response) 'x-raw-file)
726 ((? string? file)
727 ;; Send a raw file in a separate thread.
728 (call-with-new-thread
729 (lambda ()
730 (set-thread-name "publish file")
731 (catch 'system-error
732 (lambda ()
733 (call-with-input-file file
734 (lambda (input)
735 (let* ((size (stat:size (stat input)))
736 (response (write-response (with-content-length response
737 size)
738 client))
739 (output (response-port response)))
740 (if (file-port? output)
741 (sendfile output input size)
742 (dump-port input output))
743 (close-port output)
744 (values)))))
745 (lambda args
746 ;; If the file was GC'd behind our back, that's fine. Likewise if
747 ;; the client closes the connection.
748 (unless (memv (system-error-errno args)
749 (list ENOENT EPIPE ECONNRESET))
750 (apply throw args))
751 (values))))))
752 (#f
753 ;; Handle other responses sequentially.
754 (%http-write server client response body))))))
756 (define-server-impl concurrent-http-server
757 ;; A variant of Guile's built-in HTTP server that offloads possibly long
758 ;; responses to a different thread.
759 (@@ (web server http) http-open)
760 (@@ (web server http) http-read)
761 http-write
762 (@@ (web server http) http-close))
764 (define* (make-request-handler store
765 #:key
766 cache pool
767 narinfo-ttl
768 (nar-path "nar")
769 (compression %no-compression))
770 (define nar-path?
771 (let ((expected (split-and-decode-uri-path nar-path)))
772 (cut equal? expected <>)))
774 (lambda (request body)
775 (format #t "~a ~a~%"
776 (request-method request)
777 (uri-path (request-uri request)))
778 (if (get-request? request) ;reject POST, PUT, etc.
779 (match (request-path-components request)
780 ;; /nix-cache-info
781 (("nix-cache-info")
782 (render-nix-cache-info))
783 ;; /
784 ((or () ("index.html"))
785 (render-home-page request))
786 ;; /<hash>.narinfo
787 (((= extract-narinfo-hash (? string? hash)))
788 ;; TODO: Register roots for HASH that will somehow remain for
789 ;; NARINFO-TTL.
790 (if cache
791 (render-narinfo/cached store request hash
792 #:cache cache
793 #:pool pool
794 #:ttl narinfo-ttl
795 #:nar-path nar-path
796 #:compression compression)
797 (render-narinfo store request hash
798 #:ttl narinfo-ttl
799 #:nar-path nar-path
800 #:compression compression)))
801 ;; /nar/file/NAME/sha256/HASH
802 (("file" name "sha256" hash)
803 (guard (c ((invalid-base32-character? c)
804 (not-found request)))
805 (let ((hash (nix-base32-string->bytevector hash)))
806 (render-content-addressed-file store request
807 name 'sha256 hash))))
809 ;; /log/OUTPUT
810 (("log" name)
811 (render-log-file store request name))
813 ;; Use different URLs depending on the compression type. This
814 ;; guarantees that /nar URLs remain valid even when 'guix publish'
815 ;; is restarted with different compression parameters.
817 ;; /nar/gzip/<store-item>
818 ((components ... "gzip" store-item)
819 (if (and (nar-path? components) (zlib-available?))
820 (let ((compression (match compression
821 (($ <compression> 'gzip)
822 compression)
823 (_
824 %default-gzip-compression))))
825 (if cache
826 (render-nar/cached store cache request store-item
827 #:ttl narinfo-ttl
828 #:compression compression)
829 (render-nar store request store-item
830 #:compression compression)))
831 (not-found request)))
833 ;; /nar/<store-item>
834 ((components ... store-item)
835 (if (nar-path? components)
836 (if cache
837 (render-nar/cached store cache request store-item
838 #:ttl narinfo-ttl
839 #:compression %no-compression)
840 (render-nar store request store-item
841 #:compression %no-compression))
842 (not-found request)))
844 (x (not-found request)))
845 (not-found request))))
847 (define* (run-publish-server socket store
848 #:key (compression %no-compression)
849 (nar-path "nar") narinfo-ttl
850 cache pool)
851 (run-server (make-request-handler store
852 #:cache cache
853 #:pool pool
854 #:nar-path nar-path
855 #:narinfo-ttl narinfo-ttl
856 #:compression compression)
857 concurrent-http-server
858 `(#:socket ,socket)))
860 (define (open-server-socket address)
861 "Return a TCP socket bound to ADDRESS, a socket address."
862 (let ((sock (socket (sockaddr:fam address) SOCK_STREAM 0)))
863 (setsockopt sock SOL_SOCKET SO_REUSEADDR 1)
864 (bind sock address)
865 sock))
867 (define (gather-user-privileges user)
868 "Switch to the identity of USER, a user name."
869 (catch 'misc-error
870 (lambda ()
871 (let ((user (getpw user)))
872 (setgroups #())
873 (setgid (passwd:gid user))
874 (setuid (passwd:uid user))))
875 (lambda (key proc message args . rest)
876 (leave (G_ "user '~a' not found: ~a~%")
877 user (apply format #f message args)))))
879 \f
880 ;;;
881 ;;; Entry point.
882 ;;;
884 (define (guix-publish . args)
885 (with-error-handling
886 (let* ((opts (args-fold* args %options
887 (lambda (opt name arg result)
888 (leave (G_ "~A: unrecognized option~%") name))
889 (lambda (arg result)
890 (leave (G_ "~A: extraneous argument~%") arg))
891 %default-options))
892 (user (assoc-ref opts 'user))
893 (port (assoc-ref opts 'port))
894 (ttl (assoc-ref opts 'narinfo-ttl))
895 (compression (assoc-ref opts 'compression))
896 (address (let ((addr (assoc-ref opts 'address)))
897 (make-socket-address (sockaddr:fam addr)
898 (sockaddr:addr addr)
899 port)))
900 (socket (open-server-socket address))
901 (nar-path (assoc-ref opts 'nar-path))
902 (repl-port (assoc-ref opts 'repl))
903 (cache (assoc-ref opts 'cache))
904 (workers (assoc-ref opts 'workers))
906 ;; Read the key right away so that (1) we fail early on if we can't
907 ;; access them, and (2) we can then drop privileges.
908 (public-key (read-file-sexp (assoc-ref opts 'public-key-file)))
909 (private-key (read-file-sexp (assoc-ref opts 'private-key-file))))
911 (when user
912 ;; Now that we've read the key material and opened the socket, we can
913 ;; drop privileges.
914 (gather-user-privileges user))
916 (when (zero? (getuid))
917 (warning (G_ "server running as root; \
918 consider using the '--user' option!~%")))
920 (parameterize ((%public-key public-key)
921 (%private-key private-key))
922 (format #t (G_ "publishing ~a on ~a, port ~d~%")
923 %store-directory
924 (inet-ntop (sockaddr:fam address) (sockaddr:addr address))
925 (sockaddr:port address))
926 (when repl-port
927 (repl:spawn-server (repl:make-tcp-server-socket #:port repl-port)))
929 ;; Set the name of the main thread.
930 (set-thread-name "guix publish")
932 (with-store store
933 (run-publish-server socket store
934 #:cache cache
935 #:pool (and cache (make-pool workers
936 #:thread-name
937 "publish worker"))
938 #:nar-path nar-path
939 #:compression compression
940 #:narinfo-ttl ttl))))))
942 ;;; Local Variables:
943 ;;; eval: (put 'single-baker 'scheme-indent-function 1)
944 ;;; End: