1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2015 David Thompson <davet@gnu.org>
3 ;;; Copyright © 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
5 ;;; This file is part of GNU Guix.
7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
8 ;;; under the terms of the GNU General Public License as published by
9 ;;; the Free Software Foundation; either version 3 of the License, or (at
10 ;;; your option) any later version.
12 ;;; GNU Guix is distributed in the hope that it will be useful, but
13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ;;; GNU General Public License for more details.
17 ;;; You should have received a copy of the GNU General Public License
18 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
20 ;; Avoid interference.
21 (unsetenv "http_proxy")
23 (define-module (test-publish)
24 #:use-module (guix scripts publish)
25 #:use-module (guix tests)
26 #:use-module (guix config)
27 #:use-module (guix utils)
28 #:use-module (gcrypt hash)
29 #:use-module (guix store)
30 #:use-module (guix derivations)
31 #:use-module (guix gexp)
32 #:use-module (guix base32)
33 #:use-module (guix base64)
34 #:use-module ((guix records) #:select (recutils->alist))
35 #:use-module ((guix serialization) #:select (restore-file))
36 #:use-module (gcrypt pk-crypto)
37 #:use-module ((guix pki) #:select (%public-key-file %private-key-file))
38 #:use-module (guix zlib)
39 #:use-module (guix lzlib)
40 #:use-module (web uri)
41 #:use-module (web client)
42 #:use-module (web response)
43 #:use-module (rnrs bytevectors)
44 #:use-module (ice-9 binary-ports)
45 #:use-module (srfi srfi-1)
46 #:use-module (srfi srfi-26)
47 #:use-module (srfi srfi-64)
48 #:use-module (ice-9 format)
49 #:use-module (ice-9 match)
50 #:use-module (ice-9 rdelim))
53 (open-connection-for-tests))
55 (define %reference (add-text-to-store %store "ref" "foo"))
57 (define %item (add-text-to-store %store "item" "bar" (list %reference)))
59 (define (http-get-body uri)
60 (call-with-values (lambda () (http-get uri))
61 (lambda (response body) body)))
63 (define (http-get-port uri)
64 (let ((socket (open-socket-for-uri uri)))
65 ;; Make sure to use an unbuffered port so that we can then peek at the
66 ;; underlying file descriptor via 'call-with-gzip-input-port'.
67 (setvbuf socket 'none)
70 (http-get uri #:port socket #:streaming? #t))
71 (lambda (response port)
72 ;; Don't (setvbuf port 'none) because of <http://bugs.gnu.org/19610>
73 ;; (PORT might be a custom binary input port).
76 (define (publish-uri route)
77 (string-append "http://localhost:6789" route))
79 (define-syntax-rule (with-separate-output-ports exp ...)
80 ;; Since ports aren't thread-safe in Guile 2.0, duplicate the output and
81 ;; error ports to make sure the two threads don't end up stepping on each
83 (with-output-to-port (duplicate-port (current-output-port) "w")
85 (with-error-to-port (duplicate-port (current-error-port) "w")
89 ;; Run a local publishing server in a separate thread.
90 (with-separate-output-ports
93 (guix-publish "--port=6789" "-C0")))) ;attempt to avoid port collision
95 (define (wait-until-ready port)
96 ;; Wait until the server is accepting connections.
97 (let ((conn (socket PF_INET SOCK_STREAM 0)))
99 (unless (false-if-exception
100 (connect conn AF_INET (inet-pton AF_INET "127.0.0.1") port))
103 (define (wait-for-file file)
104 ;; Wait until FILE shows up.
106 (cond ((file-exists? file)
109 (error "file didn't show up" file))
111 (pk 'wait-for-file file)
115 (define %gzip-magic-bytes
116 ;; Magic bytes of gzip file.
119 ;; Wait until the two servers are ready.
120 (wait-until-ready 6789)
122 ;; Initialize the public/private key SRFI-39 parameters.
123 (%public-key (read-file-sexp %public-key-file))
124 (%private-key (read-file-sexp %private-key-file))
127 (test-begin "publish")
129 (test-equal "/nix-cache-info"
130 (format #f "StoreDir: ~a\nWantMassQuery: 0\nPriority: 100\n"
132 (http-get-body (publish-uri "/nix-cache-info")))
134 (test-equal "/*.narinfo"
135 (let* ((info (query-path-info %store %item))
147 (path-info-nar-size info)
148 (bytevector->nix-base32-string
149 (path-info-hash info))
150 (path-info-nar-size info)
151 (basename (first (path-info-references info)))))
152 (signature (base64-encode
154 (canonical-sexp->string
155 ((@@ (guix scripts publish) signed-string)
157 (format #f "~aSignature: 1;~a;~a~%"
158 unsigned-info (gethostname) signature))
162 (string-append "/" (store-path-hash-part %item) ".narinfo")))))
164 (test-equal "/*.narinfo with properly encoded '+' sign"
165 ;; See <http://bugs.gnu.org/21888>.
166 (let* ((item (add-text-to-store %store "fake-gtk+" "Congrats!"))
167 (info (query-path-info %store item))
178 (uri-encode (basename item))
179 (path-info-nar-size info)
180 (bytevector->nix-base32-string
181 (path-info-hash info))
182 (path-info-nar-size info)))
183 (signature (base64-encode
185 (canonical-sexp->string
186 ((@@ (guix scripts publish) signed-string)
188 (format #f "~aSignature: 1;~a;~a~%"
189 unsigned-info (gethostname) signature))
191 (let ((item (add-text-to-store %store "fake-gtk+" "Congrats!")))
195 (string-append "/" (store-path-hash-part item) ".narinfo"))))))
199 (call-with-temporary-output-file
201 (let ((nar (utf8->string
204 (string-append "/nar/" (basename %item)))))))
205 (call-with-input-string nar (cut restore-file <> temp)))
206 (call-with-input-file temp read-string))))
208 (unless (zlib-available?)
210 (test-equal "/nar/gzip/*"
212 (call-with-temporary-output-file
214 (let ((nar (http-get-port
216 (string-append "/nar/gzip/" (basename %item))))))
217 (call-with-gzip-input-port nar
218 (cut restore-file <> temp)))
219 (call-with-input-file temp read-string))))
221 (unless (zlib-available?)
223 (test-equal "/nar/gzip/* is really gzip"
225 ;; Since 'gzdopen' (aka. 'call-with-gzip-input-port') transparently reads
226 ;; uncompressed gzip, the test above doesn't check whether it's actually
227 ;; gzip. This is what this test does. See <https://bugs.gnu.org/30184>.
228 (let ((nar (http-get-port
230 (string-append "/nar/gzip/" (basename %item))))))
231 (get-bytevector-n nar (bytevector-length %gzip-magic-bytes))))
233 (unless (lzlib-available?)
235 (test-equal "/nar/lzip/*"
237 (call-with-temporary-output-file
239 (let ((nar (http-get-port
241 (string-append "/nar/lzip/" (basename %item))))))
242 (call-with-lzip-input-port nar
243 (cut restore-file <> temp)))
244 (call-with-input-file temp read-string))))
246 (unless (zlib-available?)
248 (test-equal "/*.narinfo with compression"
249 `(("StorePath" . ,%item)
250 ("URL" . ,(string-append "nar/gzip/" (basename %item)))
251 ("Compression" . "gzip"))
252 (let ((thread (with-separate-output-ports
253 (call-with-new-thread
255 (guix-publish "--port=6799" "-C5"))))))
256 (wait-until-ready 6799)
257 (let* ((url (string-append "http://localhost:6799/"
258 (store-path-hash-part %item) ".narinfo"))
259 (body (http-get-port url)))
260 (filter (lambda (item)
262 (("Compression" . _) #t)
263 (("StorePath" . _) #t)
266 (recutils->alist body)))))
268 (unless (lzlib-available?)
270 (test-equal "/*.narinfo with lzip compression"
271 `(("StorePath" . ,%item)
272 ("URL" . ,(string-append "nar/lzip/" (basename %item)))
273 ("Compression" . "lzip"))
274 (let ((thread (with-separate-output-ports
275 (call-with-new-thread
277 (guix-publish "--port=6790" "-Clzip"))))))
278 (wait-until-ready 6790)
279 (let* ((url (string-append "http://localhost:6790/"
280 (store-path-hash-part %item) ".narinfo"))
281 (body (http-get-port url)))
282 (filter (lambda (item)
284 (("Compression" . _) #t)
285 (("StorePath" . _) #t)
288 (recutils->alist body)))))
290 (unless (zlib-available?)
292 (test-equal "/*.narinfo for a compressed file"
293 '("none" "nar") ;compression-less nar
294 ;; Assume 'guix publish -C' is already running on port 6799.
295 (let* ((item (add-text-to-store %store "fake.tar.gz"
296 "This is a fake compressed file."))
297 (url (string-append "http://localhost:6799/"
298 (store-path-hash-part item) ".narinfo"))
299 (body (http-get-port url))
300 (info (recutils->alist body)))
301 (list (assoc-ref info "Compression")
302 (dirname (assoc-ref info "URL")))))
304 (unless (and (zlib-available?) (lzlib-available?))
306 (test-equal "/*.narinfo with lzip + gzip"
307 `((("StorePath" . ,%item)
308 ("URL" . ,(string-append "nar/gzip/" (basename %item)))
309 ("Compression" . "gzip")
310 ("URL" . ,(string-append "nar/lzip/" (basename %item)))
311 ("Compression" . "lzip"))
314 (call-with-temporary-directory
316 (let ((thread (with-separate-output-ports
317 (call-with-new-thread
319 (guix-publish "--port=6793" "-Cgzip:2" "-Clzip:2"))))))
320 (wait-until-ready 6793)
321 (let* ((base "http://localhost:6793/")
322 (part (store-path-hash-part %item))
323 (url (string-append base part ".narinfo"))
324 (body (http-get-port url)))
325 (list (take (recutils->alist body) 5)
327 (http-get (string-append base "nar/gzip/"
330 (http-get (string-append base "nar/lzip/"
331 (basename %item))))))))))
333 (test-equal "custom nar path"
334 ;; Serve nars at /foo/bar/chbouib instead of /nar.
335 (list `(("StorePath" . ,%item)
336 ("URL" . ,(string-append "foo/bar/chbouib/" (basename %item)))
337 ("Compression" . "none"))
340 (let ((thread (with-separate-output-ports
341 (call-with-new-thread
343 (guix-publish "--port=6798" "-C0"
344 "--nar-path=///foo/bar//chbouib/"))))))
345 (wait-until-ready 6798)
346 (let* ((base "http://localhost:6798/")
347 (part (store-path-hash-part %item))
348 (url (string-append base part ".narinfo"))
349 (nar-url (string-append base "foo/bar/chbouib/"
351 (body (http-get-port url)))
352 (list (filter (lambda (item)
354 (("Compression" . _) #t)
355 (("StorePath" . _) #t)
358 (recutils->alist body))
359 (response-code (http-get nar-url))
361 (http-get (string-append base "nar/" (basename %item))))))))
363 (test-equal "/nar/ with properly encoded '+' sign"
365 (let ((item (add-text-to-store %store "fake-gtk+" "Congrats!")))
366 (call-with-temporary-output-file
368 (let ((nar (utf8->string
371 (string-append "/nar/" (uri-encode (basename item))))))))
372 (call-with-input-string nar (cut restore-file <> temp)))
373 (call-with-input-file temp read-string)))))
375 (test-equal "/nar/invalid"
378 (call-with-output-file (string-append (%store-prefix) "/invalid")
380 (display "This file is not a valid store item." port)))
381 (response-code (http-get (publish-uri (string-append "/nar/invalid"))))))
383 (test-equal "/file/NAME/sha256/HASH"
385 (let* ((data "Hello, Guix world!")
386 (hash (call-with-input-string data port-sha256))
387 (drv (run-with-store %store
388 (gexp->derivation "the-file.txt"
389 #~(call-with-output-file #$output
391 (display #$data port)))
394 (out (build-derivations %store (list drv))))
398 (string-append "/file/the-file.txt/sha256/"
399 (bytevector->nix-base32-string hash)))))))
401 (test-equal "/file/NAME/sha256/INVALID-NIX-BASE32-STRING"
403 (let ((uri (publish-uri
404 "/file/the-file.txt/sha256/not-a-nix-base32-string")))
405 (response-code (http-get uri))))
407 (test-equal "/file/NAME/sha256/INVALID-HASH"
409 (let ((uri (publish-uri
410 (string-append "/file/the-file.txt/sha256/"
411 (bytevector->nix-base32-string
412 (call-with-input-string "" port-sha256))))))
413 (response-code (http-get uri))))
415 (unless (zlib-available?)
417 (test-equal "with cache"
419 `(("StorePath" . ,%item)
420 ("URL" . ,(string-append "nar/gzip/" (basename %item)))
421 ("Compression" . "gzip"))
426 (call-with-temporary-directory
428 (let ((thread (with-separate-output-ports
429 (call-with-new-thread
431 (guix-publish "--port=6797" "-C2"
432 (string-append "--cache=" cache)))))))
433 (wait-until-ready 6797)
434 (let* ((base "http://localhost:6797/")
435 (part (store-path-hash-part %item))
436 (url (string-append base part ".narinfo"))
437 (nar-url (string-append base "nar/gzip/" (basename %item)))
438 (cached (string-append cache "/gzip/" (basename %item)
440 (nar (string-append cache "/gzip/"
441 (basename %item) ".nar"))
442 (response (http-get url)))
443 (and (= 404 (response-code response))
445 ;; We should get an explicitly short TTL for 404 in this case
446 ;; because it's going to become 200 shortly.
447 (match (assq-ref (response-headers response) 'cache-control)
451 (wait-for-file cached)
452 (let* ((body (http-get-port url))
453 (compressed (http-get nar-url))
454 (uncompressed (http-get (string-append base "nar/"
456 (narinfo (recutils->alist body)))
457 (list (file-exists? nar)
458 (filter (lambda (item)
460 (("Compression" . _) #t)
461 (("StorePath" . _) #t)
465 (response-code compressed)
466 (= (response-content-length compressed)
467 (stat:size (stat nar)))
469 (assoc-ref narinfo "FileSize"))
470 (stat:size (stat nar)))
471 (response-code uncompressed)))))))))
473 (unless (and (zlib-available?) (lzlib-available?))
475 (test-equal "with cache, lzip + gzip"
477 (call-with-temporary-directory
479 (let ((thread (with-separate-output-ports
480 (call-with-new-thread
482 (guix-publish "--port=6794" "-Cgzip:2" "-Clzip:2"
483 (string-append "--cache=" cache)))))))
484 (wait-until-ready 6794)
485 (let* ((base "http://localhost:6794/")
486 (part (store-path-hash-part %item))
487 (url (string-append base part ".narinfo"))
488 (nar-url (cute string-append "nar/" <> "/"
490 (cached (cute string-append cache "/" <> "/"
491 (basename %item) ".narinfo"))
492 (nar (cute string-append cache "/" <> "/"
493 (basename %item) ".nar"))
494 (response (http-get url)))
495 (wait-for-file (cached "gzip"))
496 (let* ((body (http-get-port url))
497 (narinfo (recutils->alist body))
498 (uncompressed (string-append base "nar/"
500 (and (file-exists? (nar "gzip"))
501 (file-exists? (nar "lzip"))
502 (equal? (take (pk 'narinfo/gzip+lzip narinfo) 7)
503 `(("StorePath" . ,%item)
504 ("URL" . ,(nar-url "gzip"))
505 ("Compression" . "gzip")
506 ("FileSize" . ,(number->string
507 (stat:size (stat (nar "gzip")))))
508 ("URL" . ,(nar-url "lzip"))
509 ("Compression" . "lzip")
510 ("FileSize" . ,(number->string
511 (stat:size (stat (nar "lzip")))))))
513 (http-get (string-append base (nar-url "gzip"))))
515 (http-get (string-append base (nar-url "lzip"))))
517 (http-get uncompressed))))))))))
519 (unless (zlib-available?)
521 (let ((item (add-text-to-store %store "fake-compressed-thing.tar.gz"
523 (test-equal "with cache, uncompressed"
525 (* 42 3600) ;TTL on narinfo
526 `(("StorePath" . ,item)
527 ("URL" . ,(string-append "nar/" (basename item)))
528 ("Compression" . "none"))
530 (* 42 3600) ;TTL on nar/…
532 (query-path-info %store item)) ;FileSize
534 (call-with-temporary-directory
536 (let ((thread (with-separate-output-ports
537 (call-with-new-thread
539 (guix-publish "--port=6796" "-C2" "--ttl=42h"
540 (string-append "--cache=" cache)))))))
541 (wait-until-ready 6796)
542 (let* ((base "http://localhost:6796/")
543 (part (store-path-hash-part item))
544 (url (string-append base part ".narinfo"))
545 (cached (string-append cache "/none/"
546 (basename item) ".narinfo"))
547 (nar (string-append cache "/none/"
548 (basename item) ".nar"))
549 (response (http-get url)))
550 (and (= 404 (response-code response))
552 (wait-for-file cached)
553 (let* ((response (http-get url))
554 (body (http-get-port url))
555 (compressed (http-get (string-append base "nar/gzip/"
557 (uncompressed (http-get (string-append base "nar/"
559 (narinfo (recutils->alist body)))
560 (list (file-exists? nar)
561 (match (assq-ref (response-headers response)
563 ((('max-age . ttl)) ttl)
566 (filter (lambda (item)
568 (("Compression" . _) #t)
569 (("StorePath" . _) #t)
573 (response-code uncompressed)
574 (match (assq-ref (response-headers uncompressed)
576 ((('max-age . ttl)) ttl)
580 (assoc-ref narinfo "FileSize"))
581 (response-code compressed))))))))))
583 (test-equal "with cache, vanishing item" ;<https://bugs.gnu.org/33897>
585 (call-with-temporary-directory
587 (let ((thread (with-separate-output-ports
588 (call-with-new-thread
590 (guix-publish "--port=6795"
591 (string-append "--cache=" cache)))))))
592 (wait-until-ready 6795)
594 ;; Make sure that, even if ITEM disappears, we're still able to fetch
596 (let* ((base "http://localhost:6795/")
597 (item (add-text-to-store %store "random" (random-text)))
598 (part (store-path-hash-part item))
599 (url (string-append base part ".narinfo"))
600 (cached (string-append cache
601 (if (zlib-available?)
605 (response (http-get url)))
606 (and (= 404 (response-code response))
607 (wait-for-file cached)
609 (delete-paths %store (list item))
610 (response-code (pk 'response (http-get url))))))))))
612 (test-equal "/log/NAME"
613 `(200 #t application/x-bzip2)
614 (let ((drv (run-with-store %store
615 (gexp->derivation "with-log"
616 #~(call-with-output-file #$output
618 (display "Hello, build log!"
619 (current-error-port))
620 (display #$(random-text) port)))))))
621 (build-derivations %store (list drv))
622 (let* ((response (http-get
623 (publish-uri (string-append "/log/"
624 (basename (derivation->output-path drv))))
626 (base (basename (derivation-file-name drv)))
627 (log (string-append (dirname %state-directory)
628 "/log/guix/drvs/" (string-take base 2)
629 "/" (string-drop base 2) ".bz2")))
630 (list (response-code response)
631 (= (response-content-length response) (stat:size (stat log)))
632 (first (response-content-type response))))))
634 (test-equal "/log/NAME not found"
636 (let ((uri (publish-uri "/log/does-not-exist")))
637 (response-code (http-get uri))))
639 (test-equal "non-GET query"
641 (let ((path (string-append "/" (store-path-hash-part %item)
644 (list (http-get (publish-uri path))
645 (http-post (publish-uri path))))))