| blob | c5931c7d877e146df1dac3ffc9b315627abcea08 |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
5 ;; Author: Bill Perry <wmperry@gnu.org>
6 ;; Keywords: comm, data, processes
8 ;; This file is part of GNU Emacs.
9 ;;
10 ;; GNU Emacs is free software; you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation; either version 3, or (at your option)
13 ;; any later version.
14 ;;
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
19 ;;
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs; see the file COPYING. If not, write to the
22 ;; Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
23 ;; Boston, MA 02110-1301, USA.
25 ;;; Commentary:
27 ;;; Code:
55 "What version of HTTP we advertise, as a string.
56 Valid values are 1.1 and 1.0.
57 This is only useful when debugging the HTTP subsystem.
59 Setting this to 1.0 will tell servers not to send chunked encoding,
63 "Whether to use a single TCP connection multiple times in HTTP.
64 This is only useful when debugging the HTTP subsystem. Setting to
65 nil will explicitly close the connection to the server after every
68 ;(eval-when-compile
69 ;; These are all macros so that they are hidden from external sight
70 ;; when the file is byte-compiled.
71 ;;
72 ;; This allows us to expose just the entry points we want.
74 ;; These routines will allow us to implement persistent HTTP
75 ;; connections.
79 ;; The user hit C-g, honor it! Some things can get in an
80 ;; incredibly tight loop (chunked encoding)
93 url-http-open-connections)
94 proc)
104 url-http-open-connections))
105 nil)
124 host port
127 ;; `url-open-stream' needs a buffer in which to do things
128 ;; like authentication. But we use another buffer afterwards.
131 ;; url-open-stream might return nil.
133 ;; Drop the temp buffer link before killing the buffer.
135 proc)
138 ;; Building an HTTP request
143 ""
148 url-version
157 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
159 url-http-method url-http-data
160 url-http-extra-headers))
166 url-http-extra-headers))
168 nil
176 nil
179 proxy-info)
189 ;; Protection against stupid values in the referer
194 ;; We do not want to expose the referer if the user is paranoid.
200 ;; url-http-extra-headers contains an assoc-list of
201 ;; header/value pairs that we need to put into the request.
209 ;; This was done with a call to `format'. Concatting parts has
210 ;; the advantage of keeping the parts of each header together and
211 ;; allows us to elide null lines directly, at the cost of making
212 ;; the layout less clear.
214 ;; We used to concat directly, but if one of the strings happens
215 ;; to being multibyte (even if it only contains pure ASCII) then
216 ;; every string gets converted with `string-MAKE-multibyte' which
217 ;; turns the 127-255 codes into things like latin-1 accented chars
218 ;; (it would work right if it used `string-TO-multibyte' instead).
219 ;; So to avoid the problem we force every string to be unibyte.
221 ;; FIXME: Instead of `string-AS-unibyte' we'd want
222 ;; `string-to-unibyte', so as to properly signal an error if one
223 ;; of the strings contains a multibyte char.
224 'string-as-unibyte
227 ;; The request
231 ;; Version of MIME we speak
233 ;; (maybe) Try to keep the connection open
237 ;; HTTP extensions we support
241 ;; Who we want to talk to
248 ;; Who its from
252 ;; Encodings we understand
259 ;; Languages we understand
263 ;; Types we understand
265 ;; User agent
267 ;; Proxy Authorization
268 proxy-auth
269 ;; Authorization
270 auth
271 ;; Cookies
274 ;; If-modified-since
281 ;; Whence we came
284 extra-headers
285 ;; Length of data
291 ;; End request
293 ;; Any data
294 url-http-data))
297 request))
299 ;; Parsing routines
302 This allows us to use `mail-fetch-field', etc."
310 url-callback-function url-callback-arguments))
315 nil nil t))
320 auth
322 ;; Cheating, but who cares? :)
326 ;; find strongest supported auth
330 this-auth)))
339 type this-type
340 strength this-strength))))
347 " authentication. If you'd like to write it,"
352 type t args)))
356 url-http-extra-headers)
361 url-callback-arguments)))))))
364 "Parse just the response code."
366 url-http-response-version))
381 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
382 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
383 work correctly."
390 ;;; (while cookies2
391 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
392 )
393 )
396 "Parse and handle HTTP specific headers.
397 Return t if and only if the current buffer is still active and
398 should be shown to the user."
399 ;; The comments after each status code handled are taken from RFC
400 ;; 2616 (HTTP/1.1)
402 url-http-response-version
403 url-http-method url-http-data url-http-process
404 url-callback-function url-callback-arguments))
408 url-http-process)
417 ;;(narrow-to-region (point-min) url-http-end-of-headers)
419 ;; In HTTP 1.0, keep the connection only if there is a
420 ;; "Connection: keep-alive" header.
421 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
422 ;; "Connection: close" header
440 ;; Classes of response codes
441 ;;
442 ;; 5xx = Server Error
443 ;; 4xx = Client Error
444 ;; 3xx = Redirection
445 ;; 2xx = Successful
446 ;; 1xx = Informational
448 ;; 100 = Continue with request
449 ;; 101 = Switching protocols
450 ;; 102 = Processing (Added by DAV)
454 ;; 200 Ok
455 ;; 201 Created
456 ;; 202 Accepted
457 ;; 203 Non-authoritative information
458 ;; 204 No content
459 ;; 205 Reset content
460 ;; 206 Partial content
461 ;; 207 Multi-status (Added by DAV)
464 ;; No new data, just stay at the same document
468 ;; Generic success for all others. Store in the cache, and
469 ;; mark it as successful.
475 ;; 300 Multiple choices
476 ;; 301 Moved permanently
477 ;; 302 Found
478 ;; 303 See other
479 ;; 304 Not modified
480 ;; 305 Use proxy
481 ;; 307 Temporary redirect
486 ;; Quoth the spec (section 10.3.1)
487 ;; -------------------------------
488 ;; The requested resource corresponds to any one of a set of
489 ;; representations, each with its own specific location and
490 ;; agent-driven negotiation information is being provided so
491 ;; that the user can select a preferred representation and
492 ;; redirect its request to that location.
493 ;; [...]
494 ;; If the server has a preferred choice of representation, it
495 ;; SHOULD include the specific URI for that representation in
496 ;; the Location field; user agents MAY use the Location field
497 ;; value for automatic redirection.
498 ;; -------------------------------
499 ;; We do not support agent-driven negotiation, so we just
500 ;; redirect to the preferred URI if one is provided.
501 nil)
503 ;; If the 301|302 status code is received in response to a
504 ;; request other than GET or HEAD, the user agent MUST NOT
505 ;; automatically redirect the request unless it can be
506 ;; confirmed by the user, since this might change the
507 ;; conditions under which the request was issued.
509 ;; Automatic redirection is ok
510 nil
511 ;; It is just too big of a pain in the ass to get this
512 ;; prompt all the time. We will just silently lose our
513 ;; data and convert to a GET method.
515 url-http-method url-http-response-status)
517 url-http-data nil)))
519 ;; The response to the request can be found under a different
520 ;; URI and SHOULD be retrieved using a GET method on that
521 ;; resource.
523 url-http-data nil))
525 ;; The 304 response MUST NOT contain a message-body.
530 success t))
532 ;; The requested resource MUST be accessed through the
533 ;; proxy given by the Location field. The Location field
534 ;; gives the URI of the proxy. The recipient is expected
535 ;; to repeat this single request via the proxy. 305
536 ;; responses MUST only be generated by origin servers.
538 redirect-uri))
540 ;; Treat everything like '300'
541 nil))
543 ;; Clean off any whitespace and/or <...> cruft.
549 ;; Some stupid sites (like sourceforge) send a
550 ;; non-fully-qualified URL (ie: /), which royally confuses
551 ;; the URL library.
553 ;; Be careful to use the real target URL, otherwise we may
554 ;; compute the redirection relative to the URL of the proxy.
560 ;; Check existing number of redirects
571 ;; url-max-redirections hasn't been reached, so go
572 ;; ahead and redirect.
574 ;; Remember that the request was redirected.
578 ;; Put in the current buffer a forwarding pointer to the new
579 ;; destination buffer.
580 ;; FIXME: This is a hack to fix url-retrieve-synchronously
581 ;; without changing the API. Instead url-retrieve should
582 ;; either simply not return the "destination" buffer, or it
583 ;; should take an optional `dest-buf' argument.
586 redirect-uri url-callback-function
587 url-callback-arguments))
589 ;; We hit url-max-redirections, so issue an error and
590 ;; stop redirecting.
594 redirect-uri))
598 ;; 400 Bad Request
599 ;; 401 Unauthorized
600 ;; 402 Payment required
601 ;; 403 Forbidden
602 ;; 404 Not found
603 ;; 405 Method not allowed
604 ;; 406 Not acceptable
605 ;; 407 Proxy authentication required
606 ;; 408 Request time-out
607 ;; 409 Conflict
608 ;; 410 Gone
609 ;; 411 Length required
610 ;; 412 Precondition failed
611 ;; 413 Request entity too large
612 ;; 414 Request-URI too large
613 ;; 415 Unsupported media type
614 ;; 416 Requested range not satisfiable
615 ;; 417 Expectation failed
616 ;; 422 Unprocessable Entity (Added by DAV)
617 ;; 423 Locked
618 ;; 424 Failed Dependency
621 ;; The request requires user authentication. The response
622 ;; MUST include a WWW-Authenticate header field containing a
623 ;; challenge applicable to the requested resource. The
624 ;; client MAY repeat the request with a suitable
625 ;; Authorization header field.
628 ;; This code is reserved for future use
632 ;; The server understood the request, but is refusing to
633 ;; fulfill it. Authorization will not help and the request
634 ;; SHOULD NOT be repeated.
637 ;; Not found
640 ;; The method specified in the Request-Line is not allowed
641 ;; for the resource identified by the Request-URI. The
642 ;; response MUST include an Allow header containing a list of
643 ;; valid methods for the requested resource.
646 ;; The resource identified by the request is only capable of
647 ;; generating response entities which have content
648 ;; characteristics nota cceptable according to the accept
649 ;; headers sent in the request.
652 ;; This code is similar to 401 (Unauthorized), but indicates
653 ;; that the client must first authenticate itself with the
654 ;; proxy. The proxy MUST return a Proxy-Authenticate header
655 ;; field containing a challenge applicable to the proxy for
656 ;; the requested resource.
659 ;; The client did not produce a request within the time that
660 ;; the server was prepared to wait. The client MAY repeat
661 ;; the request without modifications at any later time.
664 ;; The request could not be completed due to a conflict with
665 ;; the current state of the resource. This code is only
666 ;; allowed in situations where it is expected that the user
667 ;; mioght be able to resolve the conflict and resubmit the
668 ;; request. The response body SHOULD include enough
669 ;; information for the user to recognize the source of the
670 ;; conflict.
673 ;; The requested resource is no longer available at the
674 ;; server and no forwarding address is known.
677 ;; The server refuses to accept the request without a defined
678 ;; Content-Length. The client MAY repeat the request if it
679 ;; adds a valid Content-Length header field containing the
680 ;; length of the message-body in the request message.
681 ;;
682 ;; NOTE - this will never happen because
683 ;; `url-http-create-request' automatically calculates the
684 ;; content-length.
687 ;; The precondition given in one or more of the
688 ;; request-header fields evaluated to false when it was
689 ;; tested on the server.
692 ;; The server is refusing to process a request because the
693 ;; request entity|URI is larger than the server is willing or
694 ;; able to process.
697 ;; The server is refusing to service the request because the
698 ;; entity of the request is in a format not supported by the
699 ;; requested resource for the requested method.
702 ;; A server SHOULD return a response with this status code if
703 ;; a request included a Range request-header field, and none
704 ;; of the range-specifier values in this field overlap the
705 ;; current extent of the selected resource, and the request
706 ;; did not include an If-Range request-header field.
709 ;; The expectation given in an Expect request-header field
710 ;; could not be met by this server, or, if the server is a
711 ;; proxy, the server has unambiguous evidence that the
712 ;; request could not be met by the next-hop server.
715 ;; The request could not be understood by the server due to
716 ;; malformed syntax. The client SHOULD NOT repeat the
717 ;; request without modifications.
719 ;; Tell the callback that an error occurred, and what the
720 ;; status code was.
726 ;; 500 Internal server error
727 ;; 501 Not implemented
728 ;; 502 Bad gateway
729 ;; 503 Service unavailable
730 ;; 504 Gateway time-out
731 ;; 505 HTTP version not supported
732 ;; 507 Insufficient storage
736 ;; The server does not support the functionality required to
737 ;; fulfill the request.
738 nil)
740 ;; The server, while acting as a gateway or proxy, received
741 ;; an invalid response from the upstream server it accessed
742 ;; in attempting to fulfill the request.
743 nil)
745 ;; The server is currently unable to handle the request due
746 ;; to a temporary overloading or maintenance of the server.
747 ;; The implication is that this is a temporary condition
748 ;; which will be alleviated after some delay. If known, the
749 ;; length of the delay MAY be indicated in a Retry-After
750 ;; header. If no Retry-After is given, the client SHOULD
751 ;; handle the response as it would for a 500 response.
752 nil)
754 ;; The server, while acting as a gateway or proxy, did not
755 ;; receive a timely response from the upstream server
756 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
757 ;; auxiliary server (e.g. DNS) it needed to access in
758 ;; attempting to complete the request.
759 nil)
761 ;; The server does not support, or refuses to support, the
762 ;; HTTP protocol version that was used in the request
763 ;; message.
764 nil)
766 ;; The method could not be performed on the resource
767 ;; because the server is unable to store the representation
768 ;; needed to successfully complete the request. This
769 ;; condition is considered to be temporary. If the request
770 ;; which received this status code was the result of a user
771 ;; action, the request MUST NOT be repeated until it is
772 ;; requested by a separate user action.
773 nil))
774 ;; Tell the callback that an error occurred, and what the
775 ;; status code was.
782 class url-http-response-status)))
787 success))
789 ;; Miscellaneous
791 "Activate callback specified when this buffer was created."
793 url-callback-function
794 url-callback-arguments))
797 url-http-process)
801 ;; )
803 ;; These unfortunately cannot be macros... please ignore them!
805 "Remove this (now defunct) process PROC from the list of open connections."
809 url-http-open-connections))
812 ;; Sentinel used for old HTTP/0.9 or connections we know are going
813 ;; to die as the 'end of document' notifier.
820 ;; HTTP/0.9 just gets passed back no matter what
826 ;; Function used when we do NOT know how long the document is going to be
827 ;; Just _very_ simple 'downloaded %d' type of info.
832 "Function used when we DO know how long the document is going to be.
833 More sophisticated percentage downloaded, etc.
834 Also does minimal parsing of HTTP headers and will actually cause
835 the callback to be triggered."
837 url-http-end-of-headers
838 url-http-content-length
839 url-http-content-type
840 url-http-process))
843 "Reading [%s]... %s of %s (%d%%)"
845 url-http-content-length)
846 url-http-content-type
850 url-http-content-length))
852 "Reading... %s of %s (%d%%)"
854 url-http-content-length)
858 url-http-content-length)))
862 ;; Found the end of the document! Wheee!
869 "Function used when dealing with 'chunked' encoding.
870 Cannot give a sophisticated percentage, but we need a different
871 function to look for the special 0-length chunk that signifies
872 the end of the document."
874 url-http-end-of-headers
875 url-http-content-type
876 url-http-chunked-length
877 url-http-chunked-counter
878 url-http-process url-http-chunked-start))
885 ;; We need to loop thru looking for more chunks even within
886 ;; one after-change-function call.
891 "Reading [%s]... chunk #%d"
892 url-http-content-type url-http-chunked-counter)
894 "Reading... chunk #%d"
895 url-http-chunked-counter))
897 url-http-chunked-counter st nd length)
903 ;; We know how long the chunk is supposed to be, skip over
904 ;; leading crap if possible.
908 url-http-chunked-counter)
910 url-http-chunked-length)))
913 url-http-chunked-length)
914 nd))
921 ;; Must not have received the entirety of the chunk header,
922 ;; need to spin some more.
927 'end-open t
928 'chunked-encoding t
940 ; (if (not url-http-debug)
943 url-http-chunked-counter url-http-chunked-length
947 ;; Found the end of the document! Wheee!
958 ;; This will wait for the headers to arrive and then splice in the
959 ;; next appropriate after-change-function, etc.
961 url-http-end-of-headers
962 url-http-content-type
963 url-http-content-length
964 url-http-transfer-encoding
965 url-callback-function
966 url-callback-arguments
967 url-http-process
968 url-http-method
969 url-http-after-change-function
970 url-http-response-status))
980 ;; Not HTTP/x.y data, must be 0.9
981 ;; God, I wish this could die.
983 url-http-end-of-headers 0
984 old-http t)
986 ;; Saw the end of the headers
990 end-of-headers t)
994 ;; Haven't seen the end of the headers yet, need to wait
995 ;; for more data to arrive.
996 nil
1002 ;;(narrow-to-region (point-min) url-http-end-of-headers)
1022 ;; A HEAD request is _ALWAYS_ terminated by the header
1023 ;; information, regardless of any entity headers,
1024 ;; according to section 4.4 of the HTTP/1.1 draft.
1030 ;; A CONNECT request is finished, but we cannot stick this
1031 ;; back on the free connectin list
1036 ;; Only allowed to have a header section. We have to handle
1037 ;; this here instead of in url-http-parse-headers because if
1038 ;; you have a cached copy of something without a known
1039 ;; content-length, and try to retrieve it from the cache, we'd
1040 ;; fall into the 'being dumb' section and wait for the
1041 ;; connection to terminate, which means we'd wait for 10
1042 ;; seconds for the keep-alives to time out on some servers.
1046 ;; HTTP/0.9 always signaled end-of-connection by closing the
1047 ;; connection.
1069 ;; We got a NULL body! Activate the callback
1070 ;; immediately!
1076 ;; Have some leftover data
1080 nd
1083 nil)))
1088 ;; We are still at the beginning of the buffer... must just be
1089 ;; waiting for a response.
1093 ;;;###autoload
1095 "Retrieve URL via HTTP asynchronously.
1096 URL must be a parsed URL. See `url-generic-parse-url' for details.
1097 When retrieval is completed, the function CALLBACK is executed with
1098 CBARGS as the arguments."
1101 url-http-end-of-headers
1102 url-http-content-type
1103 url-http-content-length
1104 url-http-transfer-encoding
1105 url-http-after-change-function
1106 url-callback-function
1107 url-callback-arguments
1108 url-http-method
1109 url-http-extra-headers
1110 url-http-data
1111 url-http-chunked-length
1112 url-http-chunked-start
1113 url-http-chunked-counter
1114 url-http-process))
1120 ;; Failed to open the connection for some reason
1131 url-http-content-type
1132 url-http-content-length
1133 url-http-transfer-encoding
1134 url-http-after-change-function
1135 url-http-response-version
1136 url-http-response-status
1137 url-http-chunked-length
1138 url-http-chunked-counter
1139 url-http-chunked-start
1140 url-callback-function
1141 url-callback-arguments
1142 url-http-process
1143 url-http-method
1144 url-http-extra-headers
1145 url-http-data
1146 url-http-target-url
1147 url-http-connection-opened
1148 url-http-proxy))
1152 url-http-extra-headers url-request-extra-headers
1153 url-http-data url-request-data
1154 url-http-process connection
1155 url-http-chunked-length nil
1156 url-http-chunked-start nil
1157 url-http-chunked-counter 0
1158 url-callback-function callback
1159 url-callback-arguments cbargs
1160 url-http-after-change-function 'url-http-wait-for-headers-change-function
1161 url-http-target-url url-current-object
1162 url-http-connection-opened nil
1163 url-http-proxy url-using-proxy)
1170 ;; Asynchronous connection
1173 ;; Asynchronous connection failed
1178 buffer))
1182 ;; We are performing an asynchronous connection, and a status change
1183 ;; has occurred.
1199 ;; Since Emacs 19/20 does not allow you to change the
1200 ;; `after-change-functions' hook in the midst of running them, we fake
1201 ;; an after change by hooking into the process filter and inserting
1202 ;; the data ourselves. This is slightly less efficient, but there
1203 ;; were tons of weird ways the after-change code was biting us in the
1204 ;; shorts.
1206 ;; Sometimes we get a zero-length data chunk after the process has
1207 ;; been changed to 'free', which means it has no buffer associated
1208 ;; with it. Do nothing if there is no buffer, or 0 length data.
1213 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1222 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1223 ;;; file-name-handler stuff from here on out
1224 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1227 'symbol-value-in-buffer
1229 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1232 unbound-value
1240 ;;;###autoload
1252 exists))
1254 ;;;###autoload
1262 nil ;dir / link / normal file
1265 nil nil nil ;atime ; mtime ; ctime
1272 ;;;###autoload
1278 ;;;###autoload
1280 "Return a property list describing options available for URL.
1281 This list is retrieved using the `OPTIONS' HTTP method.
1283 Property list members:
1285 methods
1286 A list of symbols specifying what HTTP methods the resource
1287 supports.
1289 dav
1290 A list of numbers specifying what DAV protocol/schema versions are
1291 supported.
1293 dasl
1294 A list of supported DASL search types supported (string form)
1296 ranges
1297 A list of the units available for use in partial document fetches.
1299 p3p
1300 The `Platform For Privacy Protection' description for the resource.
1301 Currently this is just the raw header contents. This is likely to
1302 change once P3P is formally supported by the URL package or
1303 Emacs/W3."
1311 ;; Only parse the options if we got a 2xx response code!
1317 ;; Figure out what methods are supported.
1320 options 'methods
1323 ;; Check for DAV
1326 options 'dav
1331 ;; Now for DASL
1334 options 'dasl
1337 ;; P3P - should get more detailed here. FIXME
1341 ;; Check for whether they accept byte-range requests.
1344 options 'ranges
1348 ))))
1350 options))
1352 ;; HTTPS. This used to be in url-https.el, but that file collides
1353 ;; with url-http.el on systems with 8-character file names.
1356 ;;;###autoload
1358 ;;;###autoload
1360 ;;;###autoload
1370 ;;;###autoload (autoload 'url-https "url-http")
1372 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1374 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1376 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1381 ;; arch-tag: ba7c59ae-c0f4-4a31-9617-d85f221732ee
1382 ;;; url-http.el ends here