2 ;;;; Copyright (c) 2008 Zachary Beane, All Rights Reserved
4 ;;;; Redistribution and use in source and binary forms, with or without
5 ;;;; modification, are permitted provided that the following conditions
8 ;;;; * Redistributions of source code must retain the above copyright
9 ;;;; notice, this list of conditions and the following disclaimer.
11 ;;;; * Redistributions in binary form must reproduce the above
12 ;;;; copyright notice, this list of conditions and the following
13 ;;;; disclaimer in the documentation and/or other materials
14 ;;;; provided with the distribution.
16 ;;;; THIS SOFTWARE IS PROVIDED BY THE AUTHOR 'AS IS' AND ANY EXPRESSED
17 ;;;; OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 ;;;; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ;;;; ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
20 ;;;; DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 ;;;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
22 ;;;; GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 ;;;; INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 ;;;; WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
25 ;;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26 ;;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 (defparameter *canned-access-policies
*
33 '((:private .
"private")
34 (:public-read .
"public-read")
35 (:public-read-write .
"public-read-write")
36 (:authenticated-read .
"authenticated-read")))
38 (defun canned-access-policy (access-policy)
39 (let ((value (assoc access-policy
*canned-access-policies
*)))
41 (error "~S is not a supported access policy.~%Supported policies are ~S"
43 (mapcar 'first
*canned-access-policies
*)))
44 (list (cons "acl" (cdr value
)))))
46 (defun access-policy-header (access-policy public
)
47 (cond ((and access-policy public
)
48 (error "Only one of ~S and ~S should be provided"
49 :public
:access-policy
))
51 (canned-access-policy :public-read
))
53 (canned-access-policy access-policy
))))
55 (defun head (&key bucket key parameters
56 ((:credentials
*credentials
*) *credentials
*))
57 "Return three values: the HTTP status, an alist of Drakma-style HTTP
58 headers, and the HTTP phrase, with the results of a HEAD request for
59 the object specified by the optional BUCKET and KEY arguments."
61 (submit-request (make-instance 'request
65 :parameters parameters
))))
67 (values (http-headers response
)
69 (http-phrase response
))))
71 ;;; Operations on buckets
73 (defun all-buckets (&key
((:credentials
*credentials
*) *credentials
*))
74 "Return a vector of all BUCKET objects associated with *CREDENTIALS*."
75 (let ((response (submit-request (make-instance 'request
79 (defun bucket-location (bucket &key
80 ((:credentials
*credentials
*) *credentials
*))
81 "If BUCKET was created with a LocationConstraint, return its
83 (let* ((request (make-instance 'request
85 :sub-resource
"location"
87 (response (submit-request request
))
88 (location (location response
)))
89 (when (plusp (length location
))
92 (defun query-bucket (bucket &key prefix marker max-keys delimiter
93 ((:credentials
*credentials
*) *credentials
*))
94 (submit-request (make-instance 'request
102 :delimiter delimiter
))))
104 (defun continue-bucket-query (response)
106 (let ((request (successive-request response
)))
108 (submit-request request
)))))
110 (defun all-keys (bucket &key prefix
111 ((:credentials
*credentials
*) *credentials
*))
112 "Reutrn a vector of all KEY objects in BUCKET."
113 (let ((response (query-bucket bucket
:prefix prefix
))
118 (push (keys response
) results
)
119 (setf response
(continue-bucket-query response
)))
120 (let ((combined (make-array (reduce #'+ results
:key
#'length
)))
122 (dolist (keys (nreverse results
) combined
)
123 (replace combined keys
:start1 start
)
124 (incf start
(length keys
))))))
126 (defun bucket-exists-p (bucket &key
127 ((:credentials
*credentials
*) *credentials
*))
128 (let ((code (nth-value 1 (head :bucket bucket
130 (parameters-alist :max-keys
0)))))
131 (not (<= 400 code
599))))
133 (defun create-bucket (name &key
137 ((:credentials
*credentials
*) *credentials
*))
138 (let ((policy-header (access-policy-header access-policy public
)))
139 (submit-request (make-instance 'request
142 :content
(and location
143 (location-constraint-xml
145 :amz-headers policy-header
))))
147 (defun delete-bucket (bucket &key
148 ((:credentials
*credentials
*) *credentials
*))
149 (let* ((request (make-instance 'request
152 (endpoint (endpoint request
))
153 (bucket (bucket request
)))
155 (submit-request request
)
156 (setf (redirected-endpoint endpoint bucket
) nil
))))
159 ;;; Getting objects as vectors, strings, or files
161 (defun check-request-success (response)
162 (let ((code (http-code response
)))
164 (throw 'not-modified
(values nil
(http-headers response
))))
165 ((not (<= 200 code
299))
166 (setf response
(specialize-response response
))
167 (maybe-signal-error response
)))))
169 (defun make-file-writer-handler (file &key
(if-exists :supersede
))
171 (check-request-success response
)
172 (with-open-stream (input (body response
))
173 (with-open-file (output file
:direction
:output
175 :element-type
'(unsigned-byte 8))
176 (copy-n-octets (content-length response
) input output
)))
177 (setf (body response
) (probe-file file
))
180 (defun vector-writer-handler (response)
181 (check-request-success response
)
182 (let ((buffer (make-octet-vector (content-length response
))))
183 (setf (body response
)
184 (with-open-stream (input (body response
))
185 (read-sequence buffer input
)
189 (defun make-string-writer-handler (external-format)
191 (setf response
(vector-writer-handler response
))
192 (setf (body response
)
193 (flexi-streams:octets-to-string
(body response
)
194 :external-format external-format
))
197 (defun get-object (bucket key
&key
199 unless-modified-since
204 (if-exists :supersede
)
205 (string-external-format :utf-8
)
206 ((:credentials
*credentials
*) *credentials
*))
207 (flet ((range-argument (start end
)
209 (format nil
"bytes=~D-~@[~D~]" start
(and end
(1- end
)))))
211 (and time
(http-date-string time
))))
212 (when (and end
(not start
))
214 (when (and start end
(<= end start
))
215 (error "START must be less than END."))
216 (let ((request (make-instance 'request
224 (maybe-date when-modified-since
)
226 (maybe-date unless-modified-since
)
227 :if-match when-etag-matches
228 :if-none-match unless-etag-matches
229 :range
(range-argument start end
))))
230 (handler (cond ((eql output
:vector
)
231 'vector-writer-handler
)
232 ((eql output
:string
)
233 (make-string-writer-handler string-external-format
))
234 ((or (stringp output
)
236 (make-file-writer-handler output
:if-exists if-exists
))
238 (error "Unknown ~S option ~S -- should be ~
239 :VECTOR, :STRING, or a pathname"
243 (let ((response (submit-request request
246 (values (body response
) (http-headers response
)))
247 (precondition-failed (c)
250 (http-headers (request-error-response c
))))))))))
252 (defun get-vector (bucket key
254 when-modified-since unless-modified-since
255 when-etag-matches unless-etag-matches
256 (if-exists :supersede
)
257 ((:credentials
*credentials
*) *credentials
*))
258 (get-object bucket key
262 :when-modified-since when-modified-since
263 :unless-modified-since unless-modified-since
264 :when-etag-matches when-etag-matches
265 :unless-etag-matches unless-etag-matches
266 :if-exists if-exists
))
268 (defun get-string (bucket key
270 (external-format :utf-8
)
271 when-modified-since unless-modified-since
272 when-etag-matches unless-etag-matches
273 (if-exists :supersede
)
274 ((:credentials
*credentials
*) *credentials
*))
275 (get-object bucket key
277 :string-external-format external-format
280 :when-modified-since when-modified-since
281 :unless-modified-since unless-modified-since
282 :when-etag-matches when-etag-matches
283 :unless-etag-matches unless-etag-matches
284 :if-exists if-exists
))
286 (defun get-file (bucket key file
288 when-modified-since unless-modified-since
289 when-etag-matches unless-etag-matches
290 (if-exists :supersede
)
291 ((:credentials
*credentials
*) *credentials
*))
292 (get-object bucket key
293 :output
(pathname file
)
296 :when-modified-since when-modified-since
297 :unless-modified-since unless-modified-since
298 :when-etag-matches when-etag-matches
299 :unless-etag-matches unless-etag-matches
300 :if-exists if-exists
))
306 (defun put-object (object bucket key
&key
310 (string-external-format :utf-8
)
316 ((:credentials
*credentials
*) *credentials
*))
320 (flexi-streams:string-to-octets object
322 string-external-format
))
323 ((or vector pathname
) object
)))
325 (policy-header (access-policy-header access-policy public
)))
326 (submit-request (make-instance 'request
331 :amz-headers policy-header
334 :cache-control cache-control
335 :content-encoding content-encoding
336 :content-disposition content-disposition
337 :expires
(and expires
338 (http-date-string expires
)))
339 :content-type content-type
340 :content-length content-length
344 (defun put-vector (vector bucket key
&key
352 (content-type "binary/octet-stream")
354 ((:credentials
*credentials
*) *credentials
*))
356 (setf vector
(subseq vector
(or start
0) end
)))
357 (put-object vector bucket key
358 :access-policy access-policy
361 :cache-control cache-control
362 :content-encoding content-encoding
363 :content-disposition content-disposition
364 :content-type content-type
367 (defun put-string (string bucket key
&key
372 (external-format :utf-8
)
376 (content-type "text/plain")
378 ((:credentials
*credentials
*) *credentials
*))
380 (setf string
(subseq string
(or start
0) end
)))
381 (put-object string bucket key
382 :access-policy access-policy
386 :content-disposition content-disposition
387 :content-encoding content-encoding
388 :content-type content-type
389 :cache-control cache-control
390 :string-external-format external-format
))
393 (defun put-file (file bucket key
&key
401 (content-type "binary/octet-stream")
403 ((:credentials
*credentials
*) *credentials
*))
405 (setf key
(file-namestring file
)))
406 (let ((content (pathname file
)))
408 ;;; FIXME: integrate with not-in-memory file uploading
409 (setf content
(file-subset-vector file start end
)))
410 (put-object content bucket key
411 :access-policy access-policy
414 :cache-control cache-control
415 :content-disposition content-disposition
416 :content-encoding content-encoding
417 :content-type content-type
420 (defun put-stream (stream bucket key
&key
428 (content-type "binary/octet-stream")
430 ((:credentials
*credentials
*) *credentials
*))
431 (let ((content (stream-subset-vector stream start end
)))
432 (put-object content bucket key
433 :access-policy access-policy
436 :cache-control cache-control
437 :content-disposition content-disposition
438 :content-encoding content-encoding
439 :content-type content-type
443 ;;; Delete & copy objects
445 (defun delete-object (bucket key
&key
446 ((:credentials
*credentials
*) *credentials
*))
447 "Delete one object from BUCKET identified by KEY."
448 (submit-request (make-instance 'request
453 (defun delete-objects (bucket keys
&key
454 ((:credentials
*credentials
*) *credentials
*))
455 "Delete the objects in BUCKET identified by KEYS."
458 (delete-object bucket key
))
462 (defun delete-all-objects (bucket &key
463 ((:credentials
*credentials
*) *credentials
*))
464 "Delete all objects in BUCKET."
465 (delete-objects bucket
(all-keys bucket
)))
467 (defun copy-object (&key
473 unless-modified-since
474 (metadata nil metadata-supplied-p
)
478 ((:credentials
*credentials
*) *credentials
*))
479 "Copy the object identified by FROM-BUCKET/FROM-KEY to
482 If TO-BUCKET is NIL, uses FROM-BUCKET as the target. If TO-KEY is NIL,
483 uses TO-KEY as the target.
485 If METADATA is provided, it should be an alist of metadata keys and
486 values to set on the new object. Otherwise, the source object's
489 Optional precondition variables are WHEN-ETAG-MATCHES,
490 UNLESS-ETAG-MATCHES, WHEN-MODIFIED-SINCE, UNLESS-MODIFIED-SINCE. The
491 etag variables use an etag as produced by the FILE-ETAG function,
492 i.e. a lowercase hex representation of the file's MD5 digest,
493 surrounded by quotes. The modified-since variables should use a
496 If PUBLIC is T, the new object is visible to all
497 users. Otherwise, a default ACL is present on the new object.
500 (error "FROM-BUCKET is required"))
502 (error "FROM-KEY is required"))
503 (unless (or to-bucket to-key
)
504 (error "Can't copy an object to itself."))
505 (setf to-bucket
(or to-bucket from-bucket
))
506 (setf to-key
(or to-key from-key
))
507 (handler-bind ((precondition-failed
509 (unless precondition-errors
510 (return-from copy-object
511 (values nil
(request-error-response condition
)))))))
513 (parameters-alist :copy-source
(format nil
"~A/~A"
514 (url-encode (name from-bucket
))
515 (url-encode (name from-key
)))
517 (if metadata-supplied-p
"REPLACE" "COPY")
518 :copy-source-if-match when-etag-matches
519 :copy-source-if-none-match unless-etag-matches
520 :copy-source-if-modified-since
521 (and when-modified-since
522 (http-date-string when-modified-since
))
523 :copy-source-if-unmodified-since
524 (and unless-modified-since
525 (http-date-string unless-modified-since
))))
526 (policy-header (access-policy-header access-policy public
)))
527 (submit-request (make-instance 'request
533 (nconc headers policy-header
))))))
536 (defun object-metadata (bucket key
&key
537 ((:credentials
*credentials
*) *credentials
*))
538 "Return the metadata headers as an alist, with keywords for the keys."
539 (let* ((prefix "X-AMZ-META-")
540 (plen (length prefix
)))
541 (flet ((metadata-symbol-p (k)
542 (and (< plen
(length (symbol-name k
)))
543 (string-equal k prefix
:end1 plen
)
544 (intern (subseq (symbol-name k
) plen
)
546 (let ((headers (head :bucket bucket
:key key
)))
547 (loop for
((k . value
)) on headers
548 for meta
= (metadata-symbol-p k
)
550 collect
(cons meta value
))))))
555 (defparameter *public-read-grant
*
556 (make-instance 'grant
558 :grantee
*all-users
*)
559 "This grant is added to or removed from an ACL to grant or revoke
560 read access for all users.")
562 (defun get-acl (&key bucket key
563 ((:credentials
*credentials
*) *credentials
*))
564 (let* ((request (make-instance 'request
568 :sub-resource
"acl"))
569 (response (submit-request request
))
570 (acl (acl response
)))
574 (defun put-acl (owner grants
&key bucket key
575 ((:credentials
*credentials
*) *credentials
*))
576 (let* ((acl (make-instance 'access-control-list
579 (request (make-instance 'request
584 :content
(acl-serialize acl
))))
585 (submit-request request
)))
588 (defun make-public (&key bucket key
589 ((:credentials
*credentials
*) *credentials
*))
590 (multiple-value-bind (owner grants
)
591 (get-acl :bucket bucket
:key key
)
593 (cons *public-read-grant
* grants
)
597 (defun make-private (&key bucket key
598 ((:credentials
*credentials
*) *credentials
*))
599 (multiple-value-bind (owner grants
)
600 (get-acl :bucket bucket
:key key
)
602 (remove *all-users
* grants
603 :test
#'acl-eqv
:key
#'grantee
))
604 (put-acl owner grants
:bucket bucket
:key key
)))
610 (defparameter *log-delivery-grants
*
611 (list (make-instance 'grant
613 :grantee
*log-delivery
*)
614 (make-instance 'grant
615 :permission
:read-acl
616 :grantee
*log-delivery
*))
617 "This list of grants is used to allow the Amazon log delivery group
618 to write logfile objects into a particular bucket.")
620 (defun enable-logging-to (bucket &key
621 ((:credentials
*credentials
*) *credentials
*))
622 "Configure the ACL of BUCKET to accept logfile objects."
623 (multiple-value-bind (owner grants
)
624 (get-acl :bucket bucket
)
625 (setf grants
(append *log-delivery-grants
* grants
))
626 (put-acl owner grants
:bucket bucket
)))
628 (defun disable-logging-to (bucket &key
629 ((:credentials
*credentials
*) *credentials
*))
630 "Configure the ACL of BUCKET to remove permissions for the log
632 (multiple-value-bind (owner grants
)
633 (get-acl :bucket bucket
)
634 (setf grants
(remove-if (lambda (grant)
635 (acl-eqv (grantee grant
) *log-delivery
*))
637 (put-acl owner grants
:bucket bucket
)))
639 (defun enable-logging (bucket target-bucket target-prefix
&key
641 ((:credentials
*credentials
*) *credentials
*))
642 "Enable logging of requests to BUCKET, putting logfile objects into
643 TARGET-BUCKET with a key prefix of TARGET-PREFIX."
644 (let* ((setup (make-instance 'logging-setup
645 :target-bucket target-bucket
646 :target-prefix target-prefix
647 :target-grants target-grants
))
648 (request (make-instance 'request
650 :sub-resource
"logging"
652 :content
(log-serialize setup
)))
656 (return (submit-request request
))
657 (invalid-logging-target (condition)
658 (when (starts-with "You must give the log-delivery group"
659 (message (request-error-response condition
)))
662 (enable-logging-to target-bucket
))))))))
665 (defparameter *empty-logging-setup
*
666 (log-serialize (make-instance 'logging-setup
))
667 "An empty logging setup; putting this into the logging setup of a
668 bucket effectively disables logging.")
670 (defun disable-logging (bucket &key
671 ((:credentials
*credentials
*) *credentials
*))
672 "Disable the creation of access logs for BUCKET."
673 (submit-request (make-instance 'request
675 :sub-resource
"logging"
677 :content
*empty-logging-setup
*)))
679 (defun logging-setup (bucket &key
680 ((:credentials
*credentials
*) *credentials
*))
682 (submit-request (make-instance 'request
684 :sub-resource
"logging")))))
685 (values (target-bucket setup
)
686 (target-prefix setup
)
687 (target-grants setup
))))
691 ;;; Creating unauthorized and authorized URLs for a resource
693 (defclass url-based-request
(request)
700 (defmethod date-string ((request url-based-request
))
701 (format nil
"~D" (expires request
)))
703 (defun resource-url (&key bucket key vhost ssl sub-resource
)
706 (format nil
"http~@[s~*~]://~A/~@[~A~]~@[?~A~]"
707 ssl bucket
(url-encode key
) sub-resource
))
709 (format nil
"http~@[s~*~]://~A.s3.amazonaws.com/~@[~A~]~@[?~A~]"
710 ssl bucket
(url-encode key
) sub-resource
))
712 (format nil
"http~@[s~*~]://s3.amazonaws.com/~@[~A/~]~@[~A~]~@[?~A~]"
713 ssl
(url-encode bucket
) (url-encode key
) sub-resource
))))
715 (defun authorized-url (&key bucket key vhost expires ssl sub-resource
716 ((:credentials
*credentials
*) *credentials
*))
717 (unless (and expires
(integerp expires
) (plusp expires
))
718 (error "~S option must be a positive integer" :expires
))
719 (let* ((request (make-instance 'url-based-request
722 :sub-resource sub-resource
724 :expires
(unix-time expires
)))
726 (alist-to-url-encoded-string
727 (list (cons "AWSAccessKeyId" (access-key *credentials
*))
728 (cons "Expires" (format nil
"~D" (expires request
)))
730 (signature request
))))))
733 (format nil
"http~@[s~*~]://~A/~@[~A~]?~@[~A&~]~A"
734 ssl bucket
(url-encode key
) sub-resource parameters
))
736 (format nil
"http~@[s~*~]://~A.s3.amazonaws.com/~@[~A~]?~@[~A&~]~A"
737 ssl bucket
(url-encode key
) sub-resource parameters
))
739 (format nil
"http~@[s~*~]://s3.amazonaws.com/~@[~A/~]~@[~A~]?~@[~A&~]~A"
740 ssl
(url-encode bucket
) (url-encode key
) sub-resource
744 ;;; Miscellaneous operations
746 (defparameter *me-cache
*
747 (make-hash-table :test
'equal
)
748 "A cache for the result of the ME function. Keys are Amazon access
752 ((:credentials
*credentials
*) *credentials
*))
753 "Return a PERSON object corresponding to the current credentials. Cached."
754 (or (gethash (access-key *credentials
*) *me-cache
*)
756 (gethash (access-key *credentials
*) *me-cache
*)
757 (let ((response (submit-request (make-instance 'request
))))
760 (defun make-post-policy (&key expires conditions
761 ((:credentials
*credentials
*) *credentials
*))
762 "Return an encoded HTTP POST policy string and policy signature as
765 (error "~S is required" :expires
))
766 (let ((policy (make-instance 'post-policy
768 :conditions conditions
)))
769 (values (policy-string64 policy
)
770 (policy-signature (secret-key *credentials
*) policy
))))