2 ;;;; Copyright (c) 2008 Zachary Beane, All Rights Reserved
4 ;;;; Redistribution and use in source and binary forms, with or without
5 ;;;; modification, are permitted provided that the following conditions
8 ;;;; * Redistributions of source code must retain the above copyright
9 ;;;; notice, this list of conditions and the following disclaimer.
11 ;;;; * Redistributions in binary form must reproduce the above
12 ;;;; copyright notice, this list of conditions and the following
13 ;;;; disclaimer in the documentation and/or other materials
14 ;;;; provided with the distribution.
16 ;;;; THIS SOFTWARE IS PROVIDED BY THE AUTHOR 'AS IS' AND ANY EXPRESSED
17 ;;;; OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 ;;;; WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 ;;;; ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
20 ;;;; DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 ;;;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
22 ;;;; GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 ;;;; INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
24 ;;;; WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
25 ;;;; NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
26 ;;;; SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 (defparameter *canned-access-policies
*
33 '((:private .
"private")
34 (:public-read .
"public-read")
35 (:public-read-write .
"public-read-write")
36 (:authenticated-read .
"authenticated-read")))
38 (defun canned-access-policy (access-policy)
39 (let ((value (assoc access-policy
*canned-access-policies
*)))
41 (error "~S is not a supported access policy.~%Supported policies are ~S"
43 (mapcar 'first
*canned-access-policies
*)))
44 (list (cons "acl" (cdr value
)))))
46 (defun access-policy-header (access-policy public
)
47 (cond ((and access-policy public
)
48 (error "Only one of ~S and ~S should be provided"
49 :public
:access-policy
))
51 (canned-access-policy :public-read
))
53 (canned-access-policy access-policy
))))
55 (defun head (&key bucket key parameters
56 ((:credentials
*credentials
*) *credentials
*))
57 "Return three values: the HTTP status, an alist of Drakma-style HTTP
58 headers, and the HTTP phrase, with the results of a HEAD request for
59 the object specified by the optional BUCKET and KEY arguments."
61 (submit-request (make-instance 'request
65 :parameters parameters
))))
67 (values (http-headers response
)
69 (http-phrase response
))))
71 ;;; Operations on buckets
73 (defun all-buckets (&key
((:credentials
*credentials
*) *credentials
*))
74 "Return a vector of all BUCKET objects associated with *CREDENTIALS*."
75 (let ((response (submit-request (make-instance 'request
79 (defun bucket-location (bucket &key
80 ((:credentials
*credentials
*) *credentials
*))
81 "If BUCKET was created with a LocationConstraint, return its
83 (let* ((request (make-instance 'request
85 :sub-resource
"location"
87 (response (submit-request request
))
88 (location (location response
)))
89 (when (plusp (length location
))
92 (defun query-bucket (bucket &key prefix marker max-keys delimiter
93 ((:credentials
*credentials
*) *credentials
*))
94 (submit-request (make-instance 'request
102 :delimiter delimiter
))))
104 (defun continue-bucket-query (response)
106 (let ((request (successive-request response
)))
108 (submit-request request
)))))
110 (defun all-keys (bucket &key prefix
111 ((:credentials
*credentials
*) *credentials
*))
112 "Reutrn a vector of all KEY objects in BUCKET."
113 (let ((response (query-bucket bucket
:prefix prefix
))
118 (push (keys response
) results
)
119 (setf response
(continue-bucket-query response
)))
120 (let ((combined (make-array (reduce #'+ results
:key
#'length
)))
122 (dolist (keys (nreverse results
) combined
)
123 (replace combined keys
:start1 start
)
124 (incf start
(length keys
))))))
126 (defun bucket-exists-p (bucket &key
127 ((:credentials
*credentials
*) *credentials
*))
128 (let ((code (nth-value 1 (head :bucket bucket
130 (parameters-alist :max-keys
0)))))
131 (not (<= 400 code
599))))
133 (defun create-bucket (name &key
137 ((:credentials
*credentials
*) *credentials
*))
138 (let ((policy-header (access-policy-header access-policy public
)))
139 (submit-request (make-instance 'request
142 :content
(and location
143 (location-constraint-xml
145 :amz-headers policy-header
))))
147 (defun delete-bucket (bucket &key
148 ((:credentials
*credentials
*) *credentials
*))
149 (let* ((request (make-instance 'request
152 (endpoint (endpoint request
))
153 (bucket (bucket request
)))
155 (submit-request request
)
156 (setf (redirected-endpoint endpoint bucket
) nil
))))
159 ;;; Getting objects as vectors, strings, or files
161 (defun check-request-success (response)
162 (let ((code (http-code response
)))
164 (throw 'not-modified
(values nil
(http-headers response
))))
165 ((not (<= 200 code
299))
166 (setf response
(specialize-response response
))
167 (maybe-signal-error response
)))))
169 (defun make-file-writer-handler (file &key
(if-exists :supersede
))
171 (check-request-success response
)
172 (with-open-stream (input (body response
))
173 (with-open-file (output file
:direction
:output
175 :element-type
'(unsigned-byte 8))
176 (copy-n-octets (content-length response
) input output
)))
177 (setf (body response
) (probe-file file
))
180 (defun vector-writer-handler (response)
181 (check-request-success response
)
182 (let ((buffer (make-octet-vector (content-length response
))))
183 (setf (body response
)
184 (with-open-stream (input (body response
))
185 (read-sequence buffer input
)
189 (defun stream-identity-handler (response)
190 (check-request-success response
)
193 (defun make-string-writer-handler (external-format)
195 (setf response
(vector-writer-handler response
))
196 (setf (body response
)
197 (flexi-streams:octets-to-string
(body response
)
198 :external-format external-format
))
203 (defun get-object (bucket key
&key
205 unless-modified-since
210 (if-exists :supersede
)
211 (string-external-format :utf-8
)
212 ((:credentials
*credentials
*) *credentials
*))
213 (flet ((range-argument (start end
)
215 (format nil
"bytes=~D-~@[~D~]" start
(and end
(1- end
)))))
217 (and time
(http-date-string time
))))
218 (when (and end
(not start
))
220 (when (and start end
(<= end start
))
221 (error "START must be less than END."))
222 (let ((request (make-instance 'request
230 (maybe-date when-modified-since
)
232 (maybe-date unless-modified-since
)
233 :if-match when-etag-matches
234 :if-none-match unless-etag-matches
235 :range
(range-argument start end
))))
236 (handler (cond ((eql output
:vector
)
237 'vector-writer-handler
)
238 ((eql output
:string
)
239 (make-string-writer-handler string-external-format
))
240 ((eql output
:stream
)
241 'stream-identity-handler
)
242 ((or (stringp output
)
244 (make-file-writer-handler output
:if-exists if-exists
))
246 (error "Unknown ~S option ~S -- should be ~
247 :VECTOR, :STRING, :STREAM, or a pathname"
251 (let ((response (submit-request request
252 :keep-stream
(eql output
:stream
)
255 (values (body response
) (http-headers response
)))
256 (precondition-failed (c)
259 (http-headers (request-error-response c
))))))))))
261 (defun get-vector (bucket key
263 when-modified-since unless-modified-since
264 when-etag-matches unless-etag-matches
265 (if-exists :supersede
)
266 ((:credentials
*credentials
*) *credentials
*))
267 (get-object bucket key
271 :when-modified-since when-modified-since
272 :unless-modified-since unless-modified-since
273 :when-etag-matches when-etag-matches
274 :unless-etag-matches unless-etag-matches
275 :if-exists if-exists
))
277 (defun get-string (bucket key
279 (external-format :utf-8
)
280 when-modified-since unless-modified-since
281 when-etag-matches unless-etag-matches
282 (if-exists :supersede
)
283 ((:credentials
*credentials
*) *credentials
*))
284 (get-object bucket key
286 :string-external-format external-format
289 :when-modified-since when-modified-since
290 :unless-modified-since unless-modified-since
291 :when-etag-matches when-etag-matches
292 :unless-etag-matches unless-etag-matches
293 :if-exists if-exists
))
295 (defun get-file (bucket key file
297 when-modified-since unless-modified-since
298 when-etag-matches unless-etag-matches
299 (if-exists :supersede
)
300 ((:credentials
*credentials
*) *credentials
*))
301 (get-object bucket key
302 :output
(pathname file
)
305 :when-modified-since when-modified-since
306 :unless-modified-since unless-modified-since
307 :when-etag-matches when-etag-matches
308 :unless-etag-matches unless-etag-matches
309 :if-exists if-exists
))
315 (defun put-object (object bucket key
&key
319 (string-external-format :utf-8
)
325 ((:credentials
*credentials
*) *credentials
*))
329 (flexi-streams:string-to-octets object
331 string-external-format
))
332 ((or vector pathname
) object
)))
334 (policy-header (access-policy-header access-policy public
)))
335 (submit-request (make-instance 'request
340 :amz-headers policy-header
343 :cache-control cache-control
344 :content-encoding content-encoding
345 :content-disposition content-disposition
346 :expires
(and expires
347 (http-date-string expires
)))
348 :content-type content-type
349 :content-length content-length
353 (defun put-vector (vector bucket key
&key
361 (content-type "binary/octet-stream")
363 ((:credentials
*credentials
*) *credentials
*))
365 (setf vector
(subseq vector
(or start
0) end
)))
366 (put-object vector bucket key
367 :access-policy access-policy
370 :cache-control cache-control
371 :content-encoding content-encoding
372 :content-disposition content-disposition
373 :content-type content-type
376 (defun put-string (string bucket key
&key
381 (external-format :utf-8
)
385 (content-type "text/plain")
387 ((:credentials
*credentials
*) *credentials
*))
389 (setf string
(subseq string
(or start
0) end
)))
390 (put-object string bucket key
391 :access-policy access-policy
395 :content-disposition content-disposition
396 :content-encoding content-encoding
397 :content-type content-type
398 :cache-control cache-control
399 :string-external-format external-format
))
402 (defun put-file (file bucket key
&key
410 (content-type "binary/octet-stream")
412 ((:credentials
*credentials
*) *credentials
*))
414 (setf key
(file-namestring file
)))
415 (let ((content (pathname file
)))
417 ;;; FIXME: integrate with not-in-memory file uploading
418 (setf content
(file-subset-vector file start end
)))
419 (put-object content bucket key
420 :access-policy access-policy
423 :cache-control cache-control
424 :content-disposition content-disposition
425 :content-encoding content-encoding
426 :content-type content-type
429 (defun put-stream (stream bucket key
&key
437 (content-type "binary/octet-stream")
439 ((:credentials
*credentials
*) *credentials
*))
440 (let ((content (stream-subset-vector stream start end
)))
441 (put-object content bucket key
442 :access-policy access-policy
445 :cache-control cache-control
446 :content-disposition content-disposition
447 :content-encoding content-encoding
448 :content-type content-type
452 ;;; Delete & copy objects
454 (defun delete-object (bucket key
&key
455 ((:credentials
*credentials
*) *credentials
*))
456 "Delete one object from BUCKET identified by KEY."
457 (submit-request (make-instance 'request
462 (defun delete-objects (bucket keys
&key
463 ((:credentials
*credentials
*) *credentials
*))
464 "Delete the objects in BUCKET identified by KEYS."
467 (delete-object bucket key
))
471 (defun delete-all-objects (bucket &key
472 ((:credentials
*credentials
*) *credentials
*))
473 "Delete all objects in BUCKET."
474 (delete-objects bucket
(all-keys bucket
)))
476 (defun copy-object (&key
482 unless-modified-since
483 (metadata nil metadata-supplied-p
)
487 ((:credentials
*credentials
*) *credentials
*))
488 "Copy the object identified by FROM-BUCKET/FROM-KEY to
491 If TO-BUCKET is NIL, uses FROM-BUCKET as the target. If TO-KEY is NIL,
492 uses TO-KEY as the target.
494 If METADATA is provided, it should be an alist of metadata keys and
495 values to set on the new object. Otherwise, the source object's
498 Optional precondition variables are WHEN-ETAG-MATCHES,
499 UNLESS-ETAG-MATCHES, WHEN-MODIFIED-SINCE, UNLESS-MODIFIED-SINCE. The
500 etag variables use an etag as produced by the FILE-ETAG function,
501 i.e. a lowercase hex representation of the file's MD5 digest,
502 surrounded by quotes. The modified-since variables should use a
505 If PUBLIC is T, the new object is visible to all
506 users. Otherwise, a default ACL is present on the new object.
509 (error "FROM-BUCKET is required"))
511 (error "FROM-KEY is required"))
512 (unless (or to-bucket to-key
)
513 (error "Can't copy an object to itself."))
514 (setf to-bucket
(or to-bucket from-bucket
))
515 (setf to-key
(or to-key from-key
))
516 (handler-bind ((precondition-failed
518 (unless precondition-errors
519 (return-from copy-object
520 (values nil
(request-error-response condition
)))))))
522 (parameters-alist :copy-source
(format nil
"~A/~A"
523 (url-encode (name from-bucket
))
524 (url-encode (name from-key
)))
526 (if metadata-supplied-p
"REPLACE" "COPY")
527 :copy-source-if-match when-etag-matches
528 :copy-source-if-none-match unless-etag-matches
529 :copy-source-if-modified-since
530 (and when-modified-since
531 (http-date-string when-modified-since
))
532 :copy-source-if-unmodified-since
533 (and unless-modified-since
534 (http-date-string unless-modified-since
))))
535 (policy-header (access-policy-header access-policy public
)))
536 (submit-request (make-instance 'request
542 (nconc headers policy-header
))))))
545 (defun object-metadata (bucket key
&key
546 ((:credentials
*credentials
*) *credentials
*))
547 "Return the metadata headers as an alist, with keywords for the keys."
548 (let* ((prefix "X-AMZ-META-")
549 (plen (length prefix
)))
550 (flet ((metadata-symbol-p (k)
551 (and (< plen
(length (symbol-name k
)))
552 (string-equal k prefix
:end1 plen
)
553 (intern (subseq (symbol-name k
) plen
)
555 (let ((headers (head :bucket bucket
:key key
)))
556 (loop for
((k . value
)) on headers
557 for meta
= (metadata-symbol-p k
)
559 collect
(cons meta value
))))))
564 (defparameter *public-read-grant
*
565 (make-instance 'grant
567 :grantee
*all-users
*)
568 "This grant is added to or removed from an ACL to grant or revoke
569 read access for all users.")
571 (defun get-acl (&key bucket key
572 ((:credentials
*credentials
*) *credentials
*))
573 (let* ((request (make-instance 'request
577 :sub-resource
"acl"))
578 (response (submit-request request
))
579 (acl (acl response
)))
583 (defun put-acl (owner grants
&key bucket key
584 ((:credentials
*credentials
*) *credentials
*))
585 (let* ((acl (make-instance 'access-control-list
588 (request (make-instance 'request
593 :content
(acl-serialize acl
))))
594 (submit-request request
)))
597 (defun make-public (&key bucket key
598 ((:credentials
*credentials
*) *credentials
*))
599 (multiple-value-bind (owner grants
)
600 (get-acl :bucket bucket
:key key
)
602 (cons *public-read-grant
* grants
)
606 (defun make-private (&key bucket key
607 ((:credentials
*credentials
*) *credentials
*))
608 (multiple-value-bind (owner grants
)
609 (get-acl :bucket bucket
:key key
)
611 (remove *all-users
* grants
612 :test
#'acl-eqv
:key
#'grantee
))
613 (put-acl owner grants
:bucket bucket
:key key
)))
619 (defparameter *log-delivery-grants
*
620 (list (make-instance 'grant
622 :grantee
*log-delivery
*)
623 (make-instance 'grant
624 :permission
:read-acl
625 :grantee
*log-delivery
*))
626 "This list of grants is used to allow the Amazon log delivery group
627 to write logfile objects into a particular bucket.")
629 (defun enable-logging-to (bucket &key
630 ((:credentials
*credentials
*) *credentials
*))
631 "Configure the ACL of BUCKET to accept logfile objects."
632 (multiple-value-bind (owner grants
)
633 (get-acl :bucket bucket
)
634 (setf grants
(append *log-delivery-grants
* grants
))
635 (put-acl owner grants
:bucket bucket
)))
637 (defun disable-logging-to (bucket &key
638 ((:credentials
*credentials
*) *credentials
*))
639 "Configure the ACL of BUCKET to remove permissions for the log
641 (multiple-value-bind (owner grants
)
642 (get-acl :bucket bucket
)
643 (setf grants
(remove-if (lambda (grant)
644 (acl-eqv (grantee grant
) *log-delivery
*))
646 (put-acl owner grants
:bucket bucket
)))
648 (defun enable-logging (bucket target-bucket target-prefix
&key
650 ((:credentials
*credentials
*) *credentials
*))
651 "Enable logging of requests to BUCKET, putting logfile objects into
652 TARGET-BUCKET with a key prefix of TARGET-PREFIX."
653 (let* ((setup (make-instance 'logging-setup
654 :target-bucket target-bucket
655 :target-prefix target-prefix
656 :target-grants target-grants
))
657 (request (make-instance 'request
659 :sub-resource
"logging"
661 :content
(log-serialize setup
)))
665 (return (submit-request request
))
666 (invalid-logging-target (condition)
667 (when (starts-with "You must give the log-delivery group"
668 (message (request-error-response condition
)))
671 (enable-logging-to target-bucket
))))))))
674 (defparameter *empty-logging-setup
*
675 (log-serialize (make-instance 'logging-setup
))
676 "An empty logging setup; putting this into the logging setup of a
677 bucket effectively disables logging.")
679 (defun disable-logging (bucket &key
680 ((:credentials
*credentials
*) *credentials
*))
681 "Disable the creation of access logs for BUCKET."
682 (submit-request (make-instance 'request
684 :sub-resource
"logging"
686 :content
*empty-logging-setup
*)))
688 (defun logging-setup (bucket &key
689 ((:credentials
*credentials
*) *credentials
*))
691 (submit-request (make-instance 'request
693 :sub-resource
"logging")))))
694 (values (target-bucket setup
)
695 (target-prefix setup
)
696 (target-grants setup
))))
700 ;;; Creating unauthorized and authorized URLs for a resource
702 (defclass url-based-request
(request)
709 (defmethod date-string ((request url-based-request
))
710 (format nil
"~D" (expires request
)))
712 (defun resource-url (&key bucket key vhost ssl sub-resource
)
715 (format nil
"http~@[s~*~]://~A/~@[~A~]~@[?~A~]"
716 ssl bucket
(url-encode key
) sub-resource
))
718 (format nil
"http~@[s~*~]://~A.s3.amazonaws.com/~@[~A~]~@[?~A~]"
719 ssl bucket
(url-encode key
) sub-resource
))
721 (format nil
"http~@[s~*~]://s3.amazonaws.com/~@[~A/~]~@[~A~]~@[?~A~]"
722 ssl
(url-encode bucket
) (url-encode key
) sub-resource
))))
724 (defun authorized-url (&key bucket key vhost expires ssl sub-resource
725 ((:credentials
*credentials
*) *credentials
*))
726 (unless (and expires
(integerp expires
) (plusp expires
))
727 (error "~S option must be a positive integer" :expires
))
728 (let* ((request (make-instance 'url-based-request
731 :sub-resource sub-resource
733 :expires
(unix-time expires
)))
735 (alist-to-url-encoded-string
736 (list (cons "AWSAccessKeyId" (access-key *credentials
*))
737 (cons "Expires" (format nil
"~D" (expires request
)))
739 (signature request
))))))
742 (format nil
"http~@[s~*~]://~A/~@[~A~]?~@[~A&~]~A"
743 ssl bucket
(url-encode key
) sub-resource parameters
))
745 (format nil
"http~@[s~*~]://~A.s3.amazonaws.com/~@[~A~]?~@[~A&~]~A"
746 ssl bucket
(url-encode key
) sub-resource parameters
))
748 (format nil
"http~@[s~*~]://s3.amazonaws.com/~@[~A/~]~@[~A~]?~@[~A&~]~A"
749 ssl
(url-encode bucket
) (url-encode key
) sub-resource
753 ;;; Miscellaneous operations
755 (defparameter *me-cache
*
756 (make-hash-table :test
'equal
)
757 "A cache for the result of the ME function. Keys are Amazon access
761 ((:credentials
*credentials
*) *credentials
*))
762 "Return a PERSON object corresponding to the current credentials. Cached."
763 (or (gethash (access-key *credentials
*) *me-cache
*)
765 (gethash (access-key *credentials
*) *me-cache
*)
766 (let ((response (submit-request (make-instance 'request
))))
769 (defun make-post-policy (&key expires conditions
770 ((:credentials
*credentials
*) *credentials
*))
771 "Return an encoded HTTP POST policy string and policy signature as
774 (error "~S is required" :expires
))
775 (let ((policy (make-instance 'post-policy
777 :conditions conditions
)))
778 (values (policy-string64 policy
)
779 (policy-signature (secret-key *credentials
*) policy
))))