| blob | d6e8181a94ea744e08e15111956395dd416812ef |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004, 2005, 2006, 2007,
4 ;; 2008 Free Software Foundation, Inc.
6 ;; Author: Bill Perry <wmperry@gnu.org>
7 ;; Keywords: comm, data, processes
9 ;; This file is part of GNU Emacs.
10 ;;
11 ;; GNU Emacs is free software: you can redistribute it and/or modify
12 ;; it under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation, either version 3 of the License, or
14 ;; (at your option) any later version.
16 ;; GNU Emacs is distributed in the hope that it will be useful,
17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 ;; GNU General Public License for more details.
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
24 ;;; Commentary:
26 ;;; Code:
54 "What version of HTTP we advertise, as a string.
55 Valid values are 1.1 and 1.0.
56 This is only useful when debugging the HTTP subsystem.
58 Setting this to 1.0 will tell servers not to send chunked encoding,
62 "Whether to use a single TCP connection multiple times in HTTP.
63 This is only useful when debugging the HTTP subsystem. Setting to
64 nil will explicitly close the connection to the server after every
67 ;(eval-when-compile
68 ;; These are all macros so that they are hidden from external sight
69 ;; when the file is byte-compiled.
70 ;;
71 ;; This allows us to expose just the entry points we want.
73 ;; These routines will allow us to implement persistent HTTP
74 ;; connections.
78 ;; The user hit C-g, honor it! Some things can get in an
79 ;; incredibly tight loop (chunked encoding)
92 url-http-open-connections)
93 proc)
103 url-http-open-connections))
104 nil)
123 host port
126 ;; `url-open-stream' needs a buffer in which to do things
127 ;; like authentication. But we use another buffer afterwards.
130 ;; url-open-stream might return nil.
132 ;; Drop the temp buffer link before killing the buffer.
134 proc)
137 ;; Building an HTTP request
142 ""
147 url-version
156 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
158 url-http-method url-http-data
159 url-http-extra-headers))
165 url-http-extra-headers))
167 nil
175 nil
178 proxy-info)
188 ;; Protection against stupid values in the referer
193 ;; We do not want to expose the referer if the user is paranoid.
199 ;; url-http-extra-headers contains an assoc-list of
200 ;; header/value pairs that we need to put into the request.
208 ;; This was done with a call to `format'. Concatting parts has
209 ;; the advantage of keeping the parts of each header together and
210 ;; allows us to elide null lines directly, at the cost of making
211 ;; the layout less clear.
213 ;; We used to concat directly, but if one of the strings happens
214 ;; to being multibyte (even if it only contains pure ASCII) then
215 ;; every string gets converted with `string-MAKE-multibyte' which
216 ;; turns the 127-255 codes into things like latin-1 accented chars
217 ;; (it would work right if it used `string-TO-multibyte' instead).
218 ;; So to avoid the problem we force every string to be unibyte.
220 ;; FIXME: Instead of `string-AS-unibyte' we'd want
221 ;; `string-to-unibyte', so as to properly signal an error if one
222 ;; of the strings contains a multibyte char.
223 'string-as-unibyte
226 ;; The request
230 ;; Version of MIME we speak
232 ;; (maybe) Try to keep the connection open
236 ;; HTTP extensions we support
240 ;; Who we want to talk to
247 ;; Who its from
251 ;; Encodings we understand
258 ;; Languages we understand
262 ;; Types we understand
264 ;; User agent
266 ;; Proxy Authorization
267 proxy-auth
268 ;; Authorization
269 auth
270 ;; Cookies
273 ;; If-modified-since
280 ;; Whence we came
283 extra-headers
284 ;; Length of data
290 ;; End request
292 ;; Any data
293 url-http-data))
296 request))
298 ;; Parsing routines
301 This allows us to use `mail-fetch-field', etc."
309 url-callback-function url-callback-arguments))
314 nil nil t))
319 auth
321 ;; Cheating, but who cares? :)
325 ;; find strongest supported auth
329 this-auth)))
338 type this-type
339 strength this-strength))))
346 " authentication. If you'd like to write it,"
351 type t args)))
355 url-http-extra-headers)
360 url-callback-arguments)))))))
363 "Parse just the response code."
365 url-http-response-version))
380 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
381 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
382 work correctly."
389 ;;; (while cookies2
390 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
391 )
392 )
395 "Parse and handle HTTP specific headers.
396 Return t if and only if the current buffer is still active and
397 should be shown to the user."
398 ;; The comments after each status code handled are taken from RFC
399 ;; 2616 (HTTP/1.1)
401 url-http-response-version
402 url-http-method url-http-data url-http-process
403 url-callback-function url-callback-arguments))
407 url-http-process)
416 ;;(narrow-to-region (point-min) url-http-end-of-headers)
418 ;; In HTTP 1.0, keep the connection only if there is a
419 ;; "Connection: keep-alive" header.
420 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
421 ;; "Connection: close" header
439 ;; Classes of response codes
440 ;;
441 ;; 5xx = Server Error
442 ;; 4xx = Client Error
443 ;; 3xx = Redirection
444 ;; 2xx = Successful
445 ;; 1xx = Informational
447 ;; 100 = Continue with request
448 ;; 101 = Switching protocols
449 ;; 102 = Processing (Added by DAV)
453 ;; 200 Ok
454 ;; 201 Created
455 ;; 202 Accepted
456 ;; 203 Non-authoritative information
457 ;; 204 No content
458 ;; 205 Reset content
459 ;; 206 Partial content
460 ;; 207 Multi-status (Added by DAV)
463 ;; No new data, just stay at the same document
467 ;; Generic success for all others. Store in the cache, and
468 ;; mark it as successful.
474 ;; 300 Multiple choices
475 ;; 301 Moved permanently
476 ;; 302 Found
477 ;; 303 See other
478 ;; 304 Not modified
479 ;; 305 Use proxy
480 ;; 307 Temporary redirect
485 ;; Quoth the spec (section 10.3.1)
486 ;; -------------------------------
487 ;; The requested resource corresponds to any one of a set of
488 ;; representations, each with its own specific location and
489 ;; agent-driven negotiation information is being provided so
490 ;; that the user can select a preferred representation and
491 ;; redirect its request to that location.
492 ;; [...]
493 ;; If the server has a preferred choice of representation, it
494 ;; SHOULD include the specific URI for that representation in
495 ;; the Location field; user agents MAY use the Location field
496 ;; value for automatic redirection.
497 ;; -------------------------------
498 ;; We do not support agent-driven negotiation, so we just
499 ;; redirect to the preferred URI if one is provided.
500 nil)
502 ;; If the 301|302 status code is received in response to a
503 ;; request other than GET or HEAD, the user agent MUST NOT
504 ;; automatically redirect the request unless it can be
505 ;; confirmed by the user, since this might change the
506 ;; conditions under which the request was issued.
508 ;; Automatic redirection is ok
509 nil
510 ;; It is just too big of a pain in the ass to get this
511 ;; prompt all the time. We will just silently lose our
512 ;; data and convert to a GET method.
514 url-http-method url-http-response-status)
516 url-http-data nil)))
518 ;; The response to the request can be found under a different
519 ;; URI and SHOULD be retrieved using a GET method on that
520 ;; resource.
522 url-http-data nil))
524 ;; The 304 response MUST NOT contain a message-body.
529 success t))
531 ;; The requested resource MUST be accessed through the
532 ;; proxy given by the Location field. The Location field
533 ;; gives the URI of the proxy. The recipient is expected
534 ;; to repeat this single request via the proxy. 305
535 ;; responses MUST only be generated by origin servers.
537 redirect-uri))
539 ;; Treat everything like '300'
540 nil))
542 ;; Clean off any whitespace and/or <...> cruft.
548 ;; Some stupid sites (like sourceforge) send a
549 ;; non-fully-qualified URL (ie: /), which royally confuses
550 ;; the URL library.
552 ;; Be careful to use the real target URL, otherwise we may
553 ;; compute the redirection relative to the URL of the proxy.
559 ;; Check existing number of redirects
570 ;; url-max-redirections hasn't been reached, so go
571 ;; ahead and redirect.
573 ;; Remember that the request was redirected.
577 ;; Put in the current buffer a forwarding pointer to the new
578 ;; destination buffer.
579 ;; FIXME: This is a hack to fix url-retrieve-synchronously
580 ;; without changing the API. Instead url-retrieve should
581 ;; either simply not return the "destination" buffer, or it
582 ;; should take an optional `dest-buf' argument.
585 redirect-uri url-callback-function
586 url-callback-arguments))
588 ;; We hit url-max-redirections, so issue an error and
589 ;; stop redirecting.
593 redirect-uri))
597 ;; 400 Bad Request
598 ;; 401 Unauthorized
599 ;; 402 Payment required
600 ;; 403 Forbidden
601 ;; 404 Not found
602 ;; 405 Method not allowed
603 ;; 406 Not acceptable
604 ;; 407 Proxy authentication required
605 ;; 408 Request time-out
606 ;; 409 Conflict
607 ;; 410 Gone
608 ;; 411 Length required
609 ;; 412 Precondition failed
610 ;; 413 Request entity too large
611 ;; 414 Request-URI too large
612 ;; 415 Unsupported media type
613 ;; 416 Requested range not satisfiable
614 ;; 417 Expectation failed
615 ;; 422 Unprocessable Entity (Added by DAV)
616 ;; 423 Locked
617 ;; 424 Failed Dependency
620 ;; The request requires user authentication. The response
621 ;; MUST include a WWW-Authenticate header field containing a
622 ;; challenge applicable to the requested resource. The
623 ;; client MAY repeat the request with a suitable
624 ;; Authorization header field.
627 ;; This code is reserved for future use
631 ;; The server understood the request, but is refusing to
632 ;; fulfill it. Authorization will not help and the request
633 ;; SHOULD NOT be repeated.
636 ;; Not found
639 ;; The method specified in the Request-Line is not allowed
640 ;; for the resource identified by the Request-URI. The
641 ;; response MUST include an Allow header containing a list of
642 ;; valid methods for the requested resource.
645 ;; The resource identified by the request is only capable of
646 ;; generating response entities which have content
647 ;; characteristics nota cceptable according to the accept
648 ;; headers sent in the request.
651 ;; This code is similar to 401 (Unauthorized), but indicates
652 ;; that the client must first authenticate itself with the
653 ;; proxy. The proxy MUST return a Proxy-Authenticate header
654 ;; field containing a challenge applicable to the proxy for
655 ;; the requested resource.
658 ;; The client did not produce a request within the time that
659 ;; the server was prepared to wait. The client MAY repeat
660 ;; the request without modifications at any later time.
663 ;; The request could not be completed due to a conflict with
664 ;; the current state of the resource. This code is only
665 ;; allowed in situations where it is expected that the user
666 ;; mioght be able to resolve the conflict and resubmit the
667 ;; request. The response body SHOULD include enough
668 ;; information for the user to recognize the source of the
669 ;; conflict.
672 ;; The requested resource is no longer available at the
673 ;; server and no forwarding address is known.
676 ;; The server refuses to accept the request without a defined
677 ;; Content-Length. The client MAY repeat the request if it
678 ;; adds a valid Content-Length header field containing the
679 ;; length of the message-body in the request message.
680 ;;
681 ;; NOTE - this will never happen because
682 ;; `url-http-create-request' automatically calculates the
683 ;; content-length.
686 ;; The precondition given in one or more of the
687 ;; request-header fields evaluated to false when it was
688 ;; tested on the server.
691 ;; The server is refusing to process a request because the
692 ;; request entity|URI is larger than the server is willing or
693 ;; able to process.
696 ;; The server is refusing to service the request because the
697 ;; entity of the request is in a format not supported by the
698 ;; requested resource for the requested method.
701 ;; A server SHOULD return a response with this status code if
702 ;; a request included a Range request-header field, and none
703 ;; of the range-specifier values in this field overlap the
704 ;; current extent of the selected resource, and the request
705 ;; did not include an If-Range request-header field.
708 ;; The expectation given in an Expect request-header field
709 ;; could not be met by this server, or, if the server is a
710 ;; proxy, the server has unambiguous evidence that the
711 ;; request could not be met by the next-hop server.
714 ;; The request could not be understood by the server due to
715 ;; malformed syntax. The client SHOULD NOT repeat the
716 ;; request without modifications.
718 ;; Tell the callback that an error occurred, and what the
719 ;; status code was.
725 ;; 500 Internal server error
726 ;; 501 Not implemented
727 ;; 502 Bad gateway
728 ;; 503 Service unavailable
729 ;; 504 Gateway time-out
730 ;; 505 HTTP version not supported
731 ;; 507 Insufficient storage
735 ;; The server does not support the functionality required to
736 ;; fulfill the request.
737 nil)
739 ;; The server, while acting as a gateway or proxy, received
740 ;; an invalid response from the upstream server it accessed
741 ;; in attempting to fulfill the request.
742 nil)
744 ;; The server is currently unable to handle the request due
745 ;; to a temporary overloading or maintenance of the server.
746 ;; The implication is that this is a temporary condition
747 ;; which will be alleviated after some delay. If known, the
748 ;; length of the delay MAY be indicated in a Retry-After
749 ;; header. If no Retry-After is given, the client SHOULD
750 ;; handle the response as it would for a 500 response.
751 nil)
753 ;; The server, while acting as a gateway or proxy, did not
754 ;; receive a timely response from the upstream server
755 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
756 ;; auxiliary server (e.g. DNS) it needed to access in
757 ;; attempting to complete the request.
758 nil)
760 ;; The server does not support, or refuses to support, the
761 ;; HTTP protocol version that was used in the request
762 ;; message.
763 nil)
765 ;; The method could not be performed on the resource
766 ;; because the server is unable to store the representation
767 ;; needed to successfully complete the request. This
768 ;; condition is considered to be temporary. If the request
769 ;; which received this status code was the result of a user
770 ;; action, the request MUST NOT be repeated until it is
771 ;; requested by a separate user action.
772 nil))
773 ;; Tell the callback that an error occurred, and what the
774 ;; status code was.
781 class url-http-response-status)))
786 success))
788 ;; Miscellaneous
790 "Activate callback specified when this buffer was created."
792 url-callback-function
793 url-callback-arguments))
796 url-http-process)
800 ;; )
802 ;; These unfortunately cannot be macros... please ignore them!
804 "Remove (now defunct) process PROC from the list of open connections."
808 url-http-open-connections))
811 ;; Sentinel used for old HTTP/0.9 or connections we know are going
812 ;; to die as the 'end of document' notifier.
819 ;; HTTP/0.9 just gets passed back no matter what
825 ;; Function used when we do NOT know how long the document is going to be
826 ;; Just _very_ simple 'downloaded %d' type of info.
831 "Function used when we DO know how long the document is going to be.
832 More sophisticated percentage downloaded, etc.
833 Also does minimal parsing of HTTP headers and will actually cause
834 the callback to be triggered."
836 url-http-end-of-headers
837 url-http-content-length
838 url-http-content-type
839 url-http-process))
842 "Reading [%s]... %s of %s (%d%%)"
844 url-http-content-length)
845 url-http-content-type
849 url-http-content-length))
851 "Reading... %s of %s (%d%%)"
853 url-http-content-length)
857 url-http-content-length)))
861 ;; Found the end of the document! Wheee!
868 "Function used when dealing with 'chunked' encoding.
869 Cannot give a sophisticated percentage, but we need a different
870 function to look for the special 0-length chunk that signifies
871 the end of the document."
873 url-http-end-of-headers
874 url-http-content-type
875 url-http-chunked-length
876 url-http-chunked-counter
877 url-http-process url-http-chunked-start))
884 ;; We need to loop thru looking for more chunks even within
885 ;; one after-change-function call.
890 "Reading [%s]... chunk #%d"
891 url-http-content-type url-http-chunked-counter)
893 "Reading... chunk #%d"
894 url-http-chunked-counter))
896 url-http-chunked-counter st nd length)
902 ;; We know how long the chunk is supposed to be, skip over
903 ;; leading crap if possible.
907 url-http-chunked-counter)
909 url-http-chunked-length)))
912 url-http-chunked-length)
913 nd))
920 ;; Must not have received the entirety of the chunk header,
921 ;; need to spin some more.
926 'end-open t
927 'chunked-encoding t
939 ; (if (not url-http-debug)
942 url-http-chunked-counter url-http-chunked-length
946 ;; Found the end of the document! Wheee!
950 ;; Every chunk, even the last 0-length one, is
951 ;; terminated by CRLF. Skip it.
961 ;; This will wait for the headers to arrive and then splice in the
962 ;; next appropriate after-change-function, etc.
964 url-http-end-of-headers
965 url-http-content-type
966 url-http-content-length
967 url-http-transfer-encoding
968 url-callback-function
969 url-callback-arguments
970 url-http-process
971 url-http-method
972 url-http-after-change-function
973 url-http-response-status))
983 ;; Not HTTP/x.y data, must be 0.9
984 ;; God, I wish this could die.
986 url-http-end-of-headers 0
987 old-http t)
989 ;; Saw the end of the headers
993 end-of-headers t)
997 ;; Haven't seen the end of the headers yet, need to wait
998 ;; for more data to arrive.
999 nil
1005 ;;(narrow-to-region (point-min) url-http-end-of-headers)
1025 ;; A HEAD request is _ALWAYS_ terminated by the header
1026 ;; information, regardless of any entity headers,
1027 ;; according to section 4.4 of the HTTP/1.1 draft.
1033 ;; A CONNECT request is finished, but we cannot stick this
1034 ;; back on the free connectin list
1039 ;; Only allowed to have a header section. We have to handle
1040 ;; this here instead of in url-http-parse-headers because if
1041 ;; you have a cached copy of something without a known
1042 ;; content-length, and try to retrieve it from the cache, we'd
1043 ;; fall into the 'being dumb' section and wait for the
1044 ;; connection to terminate, which means we'd wait for 10
1045 ;; seconds for the keep-alives to time out on some servers.
1049 ;; HTTP/0.9 always signaled end-of-connection by closing the
1050 ;; connection.
1072 ;; We got a NULL body! Activate the callback
1073 ;; immediately!
1079 ;; Have some leftover data
1083 nd
1086 nil)))
1091 ;; We are still at the beginning of the buffer... must just be
1092 ;; waiting for a response.
1096 ;;;###autoload
1098 "Retrieve URL via HTTP asynchronously.
1099 URL must be a parsed URL. See `url-generic-parse-url' for details.
1100 When retrieval is completed, the function CALLBACK is executed with
1101 CBARGS as the arguments."
1104 url-http-end-of-headers
1105 url-http-content-type
1106 url-http-content-length
1107 url-http-transfer-encoding
1108 url-http-after-change-function
1109 url-callback-function
1110 url-callback-arguments
1111 url-http-method
1112 url-http-extra-headers
1113 url-http-data
1114 url-http-chunked-length
1115 url-http-chunked-start
1116 url-http-chunked-counter
1117 url-http-process))
1123 ;; Failed to open the connection for some reason
1134 url-http-content-type
1135 url-http-content-length
1136 url-http-transfer-encoding
1137 url-http-after-change-function
1138 url-http-response-version
1139 url-http-response-status
1140 url-http-chunked-length
1141 url-http-chunked-counter
1142 url-http-chunked-start
1143 url-callback-function
1144 url-callback-arguments
1145 url-http-process
1146 url-http-method
1147 url-http-extra-headers
1148 url-http-data
1149 url-http-target-url
1150 url-http-connection-opened
1151 url-http-proxy))
1155 url-http-extra-headers url-request-extra-headers
1156 url-http-data url-request-data
1157 url-http-process connection
1158 url-http-chunked-length nil
1159 url-http-chunked-start nil
1160 url-http-chunked-counter 0
1161 url-callback-function callback
1162 url-callback-arguments cbargs
1163 url-http-after-change-function 'url-http-wait-for-headers-change-function
1164 url-http-target-url url-current-object
1165 url-http-connection-opened nil
1166 url-http-proxy url-using-proxy)
1173 ;; Asynchronous connection
1176 ;; Asynchronous connection failed
1181 buffer))
1185 ;; We are performing an asynchronous connection, and a status change
1186 ;; has occurred.
1202 ;; Since Emacs 19/20 does not allow you to change the
1203 ;; `after-change-functions' hook in the midst of running them, we fake
1204 ;; an after change by hooking into the process filter and inserting
1205 ;; the data ourselves. This is slightly less efficient, but there
1206 ;; were tons of weird ways the after-change code was biting us in the
1207 ;; shorts.
1209 ;; Sometimes we get a zero-length data chunk after the process has
1210 ;; been changed to 'free', which means it has no buffer associated
1211 ;; with it. Do nothing if there is no buffer, or 0 length data.
1216 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1225 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1226 ;;; file-name-handler stuff from here on out
1227 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1230 'symbol-value-in-buffer
1232 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1235 unbound-value
1243 ;;;###autoload
1255 exists))
1257 ;;;###autoload
1265 nil ;dir / link / normal file
1268 nil nil nil ;atime ; mtime ; ctime
1277 ;;;###autoload
1283 ;;;###autoload
1285 "Return a property list describing options available for URL.
1286 This list is retrieved using the `OPTIONS' HTTP method.
1288 Property list members:
1290 methods
1291 A list of symbols specifying what HTTP methods the resource
1292 supports.
1294 dav
1295 A list of numbers specifying what DAV protocol/schema versions are
1296 supported.
1298 dasl
1299 A list of supported DASL search types supported (string form)
1301 ranges
1302 A list of the units available for use in partial document fetches.
1304 p3p
1305 The `Platform For Privacy Protection' description for the resource.
1306 Currently this is just the raw header contents. This is likely to
1307 change once P3P is formally supported by the URL package or
1308 Emacs/W3."
1316 ;; Only parse the options if we got a 2xx response code!
1322 ;; Figure out what methods are supported.
1325 options 'methods
1328 ;; Check for DAV
1331 options 'dav
1336 ;; Now for DASL
1339 options 'dasl
1342 ;; P3P - should get more detailed here. FIXME
1346 ;; Check for whether they accept byte-range requests.
1349 options 'ranges
1353 ))))
1355 options))
1357 ;; HTTPS. This used to be in url-https.el, but that file collides
1358 ;; with url-http.el on systems with 8-character file names.
1361 ;;;###autoload
1363 ;;;###autoload
1366 ;; FIXME what is the point of this alias being an autoload?
1367 ;; Trying to use it will not cause url-http to be loaded,
1368 ;; since the full alias just gets dumped into loaddefs.el.
1370 ;;;###autoload (autoload 'url-default-expander "url-expand")
1371 ;;;###autoload
1381 ;;;###autoload (autoload 'url-https "url-http")
1383 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1385 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1387 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1392 ;; arch-tag: ba7c59ae-c0f4-4a31-9617-d85f221732ee
1393 ;;; url-http.el ends here