| blob | 8c203c0eb8716b22891a54a05de91c4f855e2668 |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004, 2005, 2006, 2007, 2008,
4 ;; 2009, 2010 Free Software Foundation, Inc.
6 ;; Author: Bill Perry <wmperry@gnu.org>
7 ;; Keywords: comm, data, processes
9 ;; This file is part of GNU Emacs.
10 ;;
11 ;; GNU Emacs is free software: you can redistribute it and/or modify
12 ;; it under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation, either version 3 of the License, or
14 ;; (at your option) any later version.
16 ;; GNU Emacs is distributed in the hope that it will be useful,
17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 ;; GNU General Public License for more details.
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
24 ;;; Commentary:
26 ;;; Code:
54 "What version of HTTP we advertise, as a string.
55 Valid values are 1.1 and 1.0.
56 This is only useful when debugging the HTTP subsystem.
58 Setting this to 1.0 will tell servers not to send chunked encoding,
62 "Whether to use a single TCP connection multiple times in HTTP.
63 This is only useful when debugging the HTTP subsystem. Setting to
64 nil will explicitly close the connection to the server after every
67 ;(eval-when-compile
68 ;; These are all macros so that they are hidden from external sight
69 ;; when the file is byte-compiled.
70 ;;
71 ;; This allows us to expose just the entry points we want.
73 ;; These routines will allow us to implement persistent HTTP
74 ;; connections.
78 ;; The user hit C-g, honor it! Some things can get in an
79 ;; incredibly tight loop (chunked encoding)
92 url-http-open-connections)
93 proc)
103 url-http-open-connections))
104 nil)
123 host port
126 ;; `url-open-stream' needs a buffer in which to do things
127 ;; like authentication. But we use another buffer afterwards.
130 ;; url-open-stream might return nil.
132 ;; Drop the temp buffer link before killing the buffer.
134 proc)
137 ;; Building an HTTP request
142 ""
147 url-version
156 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
158 url-http-method url-http-data
159 url-http-extra-headers))
165 url-http-extra-headers))
167 nil
175 nil
178 proxy-info)
188 ;; Protection against stupid values in the referer
193 ;; We do not want to expose the referer if the user is paranoid.
199 ;; url-http-extra-headers contains an assoc-list of
200 ;; header/value pairs that we need to put into the request.
208 ;; This was done with a call to `format'. Concatting parts has
209 ;; the advantage of keeping the parts of each header together and
210 ;; allows us to elide null lines directly, at the cost of making
211 ;; the layout less clear.
213 ;; We used to concat directly, but if one of the strings happens
214 ;; to being multibyte (even if it only contains pure ASCII) then
215 ;; every string gets converted with `string-MAKE-multibyte' which
216 ;; turns the 127-255 codes into things like latin-1 accented chars
217 ;; (it would work right if it used `string-TO-multibyte' instead).
218 ;; So to avoid the problem we force every string to be unibyte.
220 ;; FIXME: Instead of `string-AS-unibyte' we'd want
221 ;; `string-to-unibyte', so as to properly signal an error if one
222 ;; of the strings contains a multibyte char.
223 'string-as-unibyte
226 ;; The request
230 ;; Version of MIME we speak
232 ;; (maybe) Try to keep the connection open
236 ;; HTTP extensions we support
240 ;; Who we want to talk to
247 ;; Who its from
251 ;; Encodings we understand
258 ;; Languages we understand
262 ;; Types we understand
264 ;; User agent
266 ;; Proxy Authorization
267 proxy-auth
268 ;; Authorization
269 auth
270 ;; Cookies
273 ;; If-modified-since
280 ;; Whence we came
283 extra-headers
284 ;; Length of data
290 ;; End request
292 ;; Any data
293 url-http-data))
296 request))
298 ;; Parsing routines
301 This allows us to use `mail-fetch-field', etc."
309 url-callback-function url-callback-arguments))
314 nil nil t))
320 url-http-proxy
321 url-current-object)))
323 ;; Cheating, but who cares? :)
324 'url-http-proxy-basic-auth-storage
326 auth
329 ;; find strongest supported auth
333 this-auth)))
342 type this-type
343 strength this-strength))))
350 " authentication. If you'd like to write it,"
356 type t args)))
360 url-http-extra-headers)
365 url-callback-arguments)))))))
368 "Parse just the response code."
370 url-http-response-version))
385 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
386 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
387 work correctly."
394 ;;; (while cookies2
395 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
396 )
397 )
400 "Parse and handle HTTP specific headers.
401 Return t if and only if the current buffer is still active and
402 should be shown to the user."
403 ;; The comments after each status code handled are taken from RFC
404 ;; 2616 (HTTP/1.1)
406 url-http-response-version
407 url-http-method url-http-data url-http-process
408 url-callback-function url-callback-arguments))
412 url-http-process)
421 ;;(narrow-to-region (point-min) url-http-end-of-headers)
423 ;; In HTTP 1.0, keep the connection only if there is a
424 ;; "Connection: keep-alive" header.
425 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
426 ;; "Connection: close" header
444 ;; Classes of response codes
445 ;;
446 ;; 5xx = Server Error
447 ;; 4xx = Client Error
448 ;; 3xx = Redirection
449 ;; 2xx = Successful
450 ;; 1xx = Informational
452 ;; 100 = Continue with request
453 ;; 101 = Switching protocols
454 ;; 102 = Processing (Added by DAV)
458 ;; 200 Ok
459 ;; 201 Created
460 ;; 202 Accepted
461 ;; 203 Non-authoritative information
462 ;; 204 No content
463 ;; 205 Reset content
464 ;; 206 Partial content
465 ;; 207 Multi-status (Added by DAV)
468 ;; No new data, just stay at the same document
472 ;; Generic success for all others. Store in the cache, and
473 ;; mark it as successful.
479 ;; 300 Multiple choices
480 ;; 301 Moved permanently
481 ;; 302 Found
482 ;; 303 See other
483 ;; 304 Not modified
484 ;; 305 Use proxy
485 ;; 307 Temporary redirect
490 ;; Quoth the spec (section 10.3.1)
491 ;; -------------------------------
492 ;; The requested resource corresponds to any one of a set of
493 ;; representations, each with its own specific location and
494 ;; agent-driven negotiation information is being provided so
495 ;; that the user can select a preferred representation and
496 ;; redirect its request to that location.
497 ;; [...]
498 ;; If the server has a preferred choice of representation, it
499 ;; SHOULD include the specific URI for that representation in
500 ;; the Location field; user agents MAY use the Location field
501 ;; value for automatic redirection.
502 ;; -------------------------------
503 ;; We do not support agent-driven negotiation, so we just
504 ;; redirect to the preferred URI if one is provided.
505 nil)
507 ;; If the 301|302 status code is received in response to a
508 ;; request other than GET or HEAD, the user agent MUST NOT
509 ;; automatically redirect the request unless it can be
510 ;; confirmed by the user, since this might change the
511 ;; conditions under which the request was issued.
513 ;; Automatic redirection is ok
514 nil
515 ;; It is just too big of a pain in the ass to get this
516 ;; prompt all the time. We will just silently lose our
517 ;; data and convert to a GET method.
519 url-http-method url-http-response-status)
521 url-http-data nil)))
523 ;; The response to the request can be found under a different
524 ;; URI and SHOULD be retrieved using a GET method on that
525 ;; resource.
527 url-http-data nil))
529 ;; The 304 response MUST NOT contain a message-body.
534 success t))
536 ;; The requested resource MUST be accessed through the
537 ;; proxy given by the Location field. The Location field
538 ;; gives the URI of the proxy. The recipient is expected
539 ;; to repeat this single request via the proxy. 305
540 ;; responses MUST only be generated by origin servers.
542 redirect-uri))
544 ;; Treat everything like '300'
545 nil))
547 ;; Clean off any whitespace and/or <...> cruft.
553 ;; Some stupid sites (like sourceforge) send a
554 ;; non-fully-qualified URL (ie: /), which royally confuses
555 ;; the URL library.
557 ;; Be careful to use the real target URL, otherwise we may
558 ;; compute the redirection relative to the URL of the proxy.
564 ;; Check existing number of redirects
575 ;; url-max-redirections hasn't been reached, so go
576 ;; ahead and redirect.
578 ;; Remember that the request was redirected.
582 ;; Put in the current buffer a forwarding pointer to the new
583 ;; destination buffer.
584 ;; FIXME: This is a hack to fix url-retrieve-synchronously
585 ;; without changing the API. Instead url-retrieve should
586 ;; either simply not return the "destination" buffer, or it
587 ;; should take an optional `dest-buf' argument.
590 redirect-uri url-callback-function
591 url-callback-arguments))
593 ;; We hit url-max-redirections, so issue an error and
594 ;; stop redirecting.
598 redirect-uri))
602 ;; 400 Bad Request
603 ;; 401 Unauthorized
604 ;; 402 Payment required
605 ;; 403 Forbidden
606 ;; 404 Not found
607 ;; 405 Method not allowed
608 ;; 406 Not acceptable
609 ;; 407 Proxy authentication required
610 ;; 408 Request time-out
611 ;; 409 Conflict
612 ;; 410 Gone
613 ;; 411 Length required
614 ;; 412 Precondition failed
615 ;; 413 Request entity too large
616 ;; 414 Request-URI too large
617 ;; 415 Unsupported media type
618 ;; 416 Requested range not satisfiable
619 ;; 417 Expectation failed
620 ;; 422 Unprocessable Entity (Added by DAV)
621 ;; 423 Locked
622 ;; 424 Failed Dependency
625 ;; The request requires user authentication. The response
626 ;; MUST include a WWW-Authenticate header field containing a
627 ;; challenge applicable to the requested resource. The
628 ;; client MAY repeat the request with a suitable
629 ;; Authorization header field.
632 ;; This code is reserved for future use
636 ;; The server understood the request, but is refusing to
637 ;; fulfill it. Authorization will not help and the request
638 ;; SHOULD NOT be repeated.
641 ;; Not found
644 ;; The method specified in the Request-Line is not allowed
645 ;; for the resource identified by the Request-URI. The
646 ;; response MUST include an Allow header containing a list of
647 ;; valid methods for the requested resource.
650 ;; The resource identified by the request is only capable of
651 ;; generating response entities which have content
652 ;; characteristics nota cceptable according to the accept
653 ;; headers sent in the request.
656 ;; This code is similar to 401 (Unauthorized), but indicates
657 ;; that the client must first authenticate itself with the
658 ;; proxy. The proxy MUST return a Proxy-Authenticate header
659 ;; field containing a challenge applicable to the proxy for
660 ;; the requested resource.
663 ;; The client did not produce a request within the time that
664 ;; the server was prepared to wait. The client MAY repeat
665 ;; the request without modifications at any later time.
668 ;; The request could not be completed due to a conflict with
669 ;; the current state of the resource. This code is only
670 ;; allowed in situations where it is expected that the user
671 ;; mioght be able to resolve the conflict and resubmit the
672 ;; request. The response body SHOULD include enough
673 ;; information for the user to recognize the source of the
674 ;; conflict.
677 ;; The requested resource is no longer available at the
678 ;; server and no forwarding address is known.
681 ;; The server refuses to accept the request without a defined
682 ;; Content-Length. The client MAY repeat the request if it
683 ;; adds a valid Content-Length header field containing the
684 ;; length of the message-body in the request message.
685 ;;
686 ;; NOTE - this will never happen because
687 ;; `url-http-create-request' automatically calculates the
688 ;; content-length.
691 ;; The precondition given in one or more of the
692 ;; request-header fields evaluated to false when it was
693 ;; tested on the server.
696 ;; The server is refusing to process a request because the
697 ;; request entity|URI is larger than the server is willing or
698 ;; able to process.
701 ;; The server is refusing to service the request because the
702 ;; entity of the request is in a format not supported by the
703 ;; requested resource for the requested method.
706 ;; A server SHOULD return a response with this status code if
707 ;; a request included a Range request-header field, and none
708 ;; of the range-specifier values in this field overlap the
709 ;; current extent of the selected resource, and the request
710 ;; did not include an If-Range request-header field.
713 ;; The expectation given in an Expect request-header field
714 ;; could not be met by this server, or, if the server is a
715 ;; proxy, the server has unambiguous evidence that the
716 ;; request could not be met by the next-hop server.
719 ;; The request could not be understood by the server due to
720 ;; malformed syntax. The client SHOULD NOT repeat the
721 ;; request without modifications.
723 ;; Tell the callback that an error occurred, and what the
724 ;; status code was.
730 ;; 500 Internal server error
731 ;; 501 Not implemented
732 ;; 502 Bad gateway
733 ;; 503 Service unavailable
734 ;; 504 Gateway time-out
735 ;; 505 HTTP version not supported
736 ;; 507 Insufficient storage
740 ;; The server does not support the functionality required to
741 ;; fulfill the request.
742 nil)
744 ;; The server, while acting as a gateway or proxy, received
745 ;; an invalid response from the upstream server it accessed
746 ;; in attempting to fulfill the request.
747 nil)
749 ;; The server is currently unable to handle the request due
750 ;; to a temporary overloading or maintenance of the server.
751 ;; The implication is that this is a temporary condition
752 ;; which will be alleviated after some delay. If known, the
753 ;; length of the delay MAY be indicated in a Retry-After
754 ;; header. If no Retry-After is given, the client SHOULD
755 ;; handle the response as it would for a 500 response.
756 nil)
758 ;; The server, while acting as a gateway or proxy, did not
759 ;; receive a timely response from the upstream server
760 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
761 ;; auxiliary server (e.g. DNS) it needed to access in
762 ;; attempting to complete the request.
763 nil)
765 ;; The server does not support, or refuses to support, the
766 ;; HTTP protocol version that was used in the request
767 ;; message.
768 nil)
770 ;; The method could not be performed on the resource
771 ;; because the server is unable to store the representation
772 ;; needed to successfully complete the request. This
773 ;; condition is considered to be temporary. If the request
774 ;; which received this status code was the result of a user
775 ;; action, the request MUST NOT be repeated until it is
776 ;; requested by a separate user action.
777 nil))
778 ;; Tell the callback that an error occurred, and what the
779 ;; status code was.
786 class url-http-response-status)))
791 success))
793 ;; Miscellaneous
795 "Activate callback specified when this buffer was created."
797 url-callback-function
798 url-callback-arguments))
801 url-http-process)
805 ;; )
807 ;; These unfortunately cannot be macros... please ignore them!
809 "Remove (now defunct) process PROC from the list of open connections."
813 url-http-open-connections))
816 ;; Sentinel used for old HTTP/0.9 or connections we know are going
817 ;; to die as the 'end of document' notifier.
824 ;; HTTP/0.9 just gets passed back no matter what
830 ;; Function used when we do NOT know how long the document is going to be
831 ;; Just _very_ simple 'downloaded %d' type of info.
836 "Function used when we DO know how long the document is going to be.
837 More sophisticated percentage downloaded, etc.
838 Also does minimal parsing of HTTP headers and will actually cause
839 the callback to be triggered."
841 url-http-end-of-headers
842 url-http-content-length
843 url-http-content-type
844 url-http-process))
847 "Reading [%s]... %s of %s (%d%%)"
849 url-http-content-length)
850 url-http-content-type
854 url-http-content-length))
856 "Reading... %s of %s (%d%%)"
858 url-http-content-length)
862 url-http-content-length)))
866 ;; Found the end of the document! Wheee!
873 "Function used when dealing with 'chunked' encoding.
874 Cannot give a sophisticated percentage, but we need a different
875 function to look for the special 0-length chunk that signifies
876 the end of the document."
878 url-http-end-of-headers
879 url-http-content-type
880 url-http-chunked-length
881 url-http-chunked-counter
882 url-http-process url-http-chunked-start))
889 ;; We need to loop thru looking for more chunks even within
890 ;; one after-change-function call.
895 "Reading [%s]... chunk #%d"
896 url-http-content-type url-http-chunked-counter)
898 "Reading... chunk #%d"
899 url-http-chunked-counter))
901 url-http-chunked-counter st nd length)
907 ;; We know how long the chunk is supposed to be, skip over
908 ;; leading crap if possible.
912 url-http-chunked-counter)
914 url-http-chunked-length)))
917 url-http-chunked-length)
918 nd))
925 ;; Must not have received the entirety of the chunk header,
926 ;; need to spin some more.
931 'end-open t
932 'chunked-encoding t
944 ; (if (not url-http-debug)
947 url-http-chunked-counter url-http-chunked-length
951 ;; Found the end of the document! Wheee!
955 ;; Every chunk, even the last 0-length one, is
956 ;; terminated by CRLF. Skip it.
966 ;; This will wait for the headers to arrive and then splice in the
967 ;; next appropriate after-change-function, etc.
969 url-http-end-of-headers
970 url-http-content-type
971 url-http-content-length
972 url-http-transfer-encoding
973 url-callback-function
974 url-callback-arguments
975 url-http-process
976 url-http-method
977 url-http-after-change-function
978 url-http-response-status))
988 ;; Not HTTP/x.y data, must be 0.9
989 ;; God, I wish this could die.
991 url-http-end-of-headers 0
992 old-http t)
994 ;; Saw the end of the headers
998 end-of-headers t)
1002 ;; Haven't seen the end of the headers yet, need to wait
1003 ;; for more data to arrive.
1004 nil
1010 ;;(narrow-to-region (point-min) url-http-end-of-headers)
1030 ;; A HEAD request is _ALWAYS_ terminated by the header
1031 ;; information, regardless of any entity headers,
1032 ;; according to section 4.4 of the HTTP/1.1 draft.
1038 ;; A CONNECT request is finished, but we cannot stick this
1039 ;; back on the free connectin list
1044 ;; Only allowed to have a header section. We have to handle
1045 ;; this here instead of in url-http-parse-headers because if
1046 ;; you have a cached copy of something without a known
1047 ;; content-length, and try to retrieve it from the cache, we'd
1048 ;; fall into the 'being dumb' section and wait for the
1049 ;; connection to terminate, which means we'd wait for 10
1050 ;; seconds for the keep-alives to time out on some servers.
1054 ;; HTTP/0.9 always signaled end-of-connection by closing the
1055 ;; connection.
1077 ;; We got a NULL body! Activate the callback
1078 ;; immediately!
1084 ;; Have some leftover data
1088 nd
1091 nil)))
1096 ;; We are still at the beginning of the buffer... must just be
1097 ;; waiting for a response.
1101 ;;;###autoload
1103 "Retrieve URL via HTTP asynchronously.
1104 URL must be a parsed URL. See `url-generic-parse-url' for details.
1105 When retrieval is completed, the function CALLBACK is executed with
1106 CBARGS as the arguments."
1109 url-http-end-of-headers
1110 url-http-content-type
1111 url-http-content-length
1112 url-http-transfer-encoding
1113 url-http-after-change-function
1114 url-callback-function
1115 url-callback-arguments
1116 url-http-method
1117 url-http-extra-headers
1118 url-http-data
1119 url-http-chunked-length
1120 url-http-chunked-start
1121 url-http-chunked-counter
1122 url-http-process))
1128 ;; Failed to open the connection for some reason
1139 url-http-content-type
1140 url-http-content-length
1141 url-http-transfer-encoding
1142 url-http-after-change-function
1143 url-http-response-version
1144 url-http-response-status
1145 url-http-chunked-length
1146 url-http-chunked-counter
1147 url-http-chunked-start
1148 url-callback-function
1149 url-callback-arguments
1150 url-http-process
1151 url-http-method
1152 url-http-extra-headers
1153 url-http-data
1154 url-http-target-url
1155 url-http-connection-opened
1156 url-http-proxy))
1160 url-http-extra-headers url-request-extra-headers
1161 url-http-data url-request-data
1162 url-http-process connection
1163 url-http-chunked-length nil
1164 url-http-chunked-start nil
1165 url-http-chunked-counter 0
1166 url-callback-function callback
1167 url-callback-arguments cbargs
1168 url-http-after-change-function 'url-http-wait-for-headers-change-function
1169 url-http-target-url url-current-object
1170 url-http-connection-opened nil
1171 url-http-proxy url-using-proxy)
1178 ;; Asynchronous connection
1181 ;; Asynchronous connection failed
1186 buffer))
1190 ;; We are performing an asynchronous connection, and a status change
1191 ;; has occurred.
1207 ;; Since Emacs 19/20 does not allow you to change the
1208 ;; `after-change-functions' hook in the midst of running them, we fake
1209 ;; an after change by hooking into the process filter and inserting
1210 ;; the data ourselves. This is slightly less efficient, but there
1211 ;; were tons of weird ways the after-change code was biting us in the
1212 ;; shorts.
1214 ;; Sometimes we get a zero-length data chunk after the process has
1215 ;; been changed to 'free', which means it has no buffer associated
1216 ;; with it. Do nothing if there is no buffer, or 0 length data.
1221 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1230 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1231 ;;; file-name-handler stuff from here on out
1232 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1235 'symbol-value-in-buffer
1237 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1240 unbound-value
1248 ;;;###autoload
1260 exists))
1262 ;;;###autoload
1270 nil ;dir / link / normal file
1273 nil nil nil ;atime ; mtime ; ctime
1282 ;;;###autoload
1288 ;;;###autoload
1290 "Return a property list describing options available for URL.
1291 This list is retrieved using the `OPTIONS' HTTP method.
1293 Property list members:
1295 methods
1296 A list of symbols specifying what HTTP methods the resource
1297 supports.
1299 dav
1300 A list of numbers specifying what DAV protocol/schema versions are
1301 supported.
1303 dasl
1304 A list of supported DASL search types supported (string form)
1306 ranges
1307 A list of the units available for use in partial document fetches.
1309 p3p
1310 The `Platform For Privacy Protection' description for the resource.
1311 Currently this is just the raw header contents. This is likely to
1312 change once P3P is formally supported by the URL package or
1313 Emacs/W3."
1321 ;; Only parse the options if we got a 2xx response code!
1327 ;; Figure out what methods are supported.
1330 options 'methods
1333 ;; Check for DAV
1336 options 'dav
1341 ;; Now for DASL
1344 options 'dasl
1347 ;; P3P - should get more detailed here. FIXME
1351 ;; Check for whether they accept byte-range requests.
1354 options 'ranges
1358 ))))
1360 options))
1362 ;; HTTPS. This used to be in url-https.el, but that file collides
1363 ;; with url-http.el on systems with 8-character file names.
1366 ;;;###autoload
1368 ;;;###autoload
1371 ;; FIXME what is the point of this alias being an autoload?
1372 ;; Trying to use it will not cause url-http to be loaded,
1373 ;; since the full alias just gets dumped into loaddefs.el.
1375 ;;;###autoload (autoload 'url-default-expander "url-expand")
1376 ;;;###autoload
1386 ;;;###autoload (autoload 'url-https "url-http")
1388 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1390 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1392 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1397 ;; arch-tag: ba7c59ae-c0f4-4a31-9617-d85f221732ee
1398 ;;; url-http.el ends here