| blob | 9f988beaf0a343e778c1a3b39443c274f1769d6f |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004, 2005, 2006, 2007, 2008,
4 ;; 2009, 2010 Free Software Foundation, Inc.
6 ;; Author: Bill Perry <wmperry@gnu.org>
7 ;; Keywords: comm, data, processes
9 ;; This file is part of GNU Emacs.
10 ;;
11 ;; GNU Emacs is free software: you can redistribute it and/or modify
12 ;; it under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation, either version 3 of the License, or
14 ;; (at your option) any later version.
16 ;; GNU Emacs is distributed in the hope that it will be useful,
17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 ;; GNU General Public License for more details.
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
24 ;;; Commentary:
26 ;;; Code:
54 "What version of HTTP we advertise, as a string.
55 Valid values are 1.1 and 1.0.
56 This is only useful when debugging the HTTP subsystem.
58 Setting this to 1.0 will tell servers not to send chunked encoding,
62 "Whether to use a single TCP connection multiple times in HTTP.
63 This is only useful when debugging the HTTP subsystem. Setting to
64 nil will explicitly close the connection to the server after every
116 ;(eval-when-compile
117 ;; These are all macros so that they are hidden from external sight
118 ;; when the file is byte-compiled.
119 ;;
120 ;; This allows us to expose just the entry points we want.
122 ;; These routines will allow us to implement persistent HTTP
123 ;; connections.
127 ;; The user hit C-g, honor it! Some things can get in an
128 ;; incredibly tight loop (chunked encoding)
141 url-http-open-connections)
142 proc)
152 url-http-open-connections))
153 nil)
172 host port
175 ;; `url-open-stream' needs a buffer in which to do things
176 ;; like authentication. But we use another buffer afterwards.
179 ;; url-open-stream might return nil.
181 ;; Drop the temp buffer link before killing the buffer.
183 proc)
186 ;; Building an HTTP request
191 ""
196 url-version
205 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
207 url-http-method url-http-data
208 url-http-extra-headers))
214 url-http-extra-headers))
216 nil
224 nil
227 proxy-info)
237 ;; Protection against stupid values in the referer
242 ;; We do not want to expose the referer if the user is paranoid.
248 ;; url-http-extra-headers contains an assoc-list of
249 ;; header/value pairs that we need to put into the request.
257 ;; This was done with a call to `format'. Concatting parts has
258 ;; the advantage of keeping the parts of each header together and
259 ;; allows us to elide null lines directly, at the cost of making
260 ;; the layout less clear.
262 ;; We used to concat directly, but if one of the strings happens
263 ;; to being multibyte (even if it only contains pure ASCII) then
264 ;; every string gets converted with `string-MAKE-multibyte' which
265 ;; turns the 127-255 codes into things like latin-1 accented chars
266 ;; (it would work right if it used `string-TO-multibyte' instead).
267 ;; So to avoid the problem we force every string to be unibyte.
269 ;; FIXME: Instead of `string-AS-unibyte' we'd want
270 ;; `string-to-unibyte', so as to properly signal an error if one
271 ;; of the strings contains a multibyte char.
272 'string-as-unibyte
275 ;; The request
279 ;; Version of MIME we speak
281 ;; (maybe) Try to keep the connection open
285 ;; HTTP extensions we support
289 ;; Who we want to talk to
296 ;; Who its from
300 ;; Encodings we understand
307 ;; Languages we understand
311 ;; Types we understand
313 ;; User agent
315 ;; Proxy Authorization
316 proxy-auth
317 ;; Authorization
318 auth
319 ;; Cookies
322 ;; If-modified-since
329 ;; Whence we came
332 extra-headers
333 ;; Length of data
339 ;; End request
341 ;; Any data
342 url-http-data))
345 request))
347 ;; Parsing routines
350 This allows us to use `mail-fetch-field', etc."
358 url-callback-function url-callback-arguments))
363 nil nil t))
369 url-http-proxy
370 url-current-object)))
372 ;; Cheating, but who cares? :)
373 'url-http-proxy-basic-auth-storage
375 auth
378 ;; find strongest supported auth
382 this-auth)))
391 type this-type
392 strength this-strength))))
399 " authentication. If you'd like to write it,"
405 type t args)))
409 url-http-extra-headers)
414 url-callback-arguments)))))))
417 "Parse just the response code."
419 url-http-response-version))
434 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
435 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
436 work correctly."
443 ;;; (while cookies2
444 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
445 )
446 )
449 "Parse and handle HTTP specific headers.
450 Return t if and only if the current buffer is still active and
451 should be shown to the user."
452 ;; The comments after each status code handled are taken from RFC
453 ;; 2616 (HTTP/1.1)
455 url-http-response-version
456 url-http-method url-http-data url-http-process
457 url-callback-function url-callback-arguments))
461 url-http-process)
470 ;;(narrow-to-region (point-min) url-http-end-of-headers)
472 ;; In HTTP 1.0, keep the connection only if there is a
473 ;; "Connection: keep-alive" header.
474 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
475 ;; "Connection: close" header
488 ;; other status symbols: jewelry and luxury cars
495 ;; Classes of response codes
496 ;;
497 ;; 5xx = Server Error
498 ;; 4xx = Client Error
499 ;; 3xx = Redirection
500 ;; 2xx = Successful
501 ;; 1xx = Informational
503 ;; 100 = Continue with request
504 ;; 101 = Switching protocols
505 ;; 102 = Processing (Added by DAV)
509 ;; 200 Ok
510 ;; 201 Created
511 ;; 202 Accepted
512 ;; 203 Non-authoritative information
513 ;; 204 No content
514 ;; 205 Reset content
515 ;; 206 Partial content
516 ;; 207 Multi-status (Added by DAV)
519 ;; No new data, just stay at the same document
523 ;; Generic success for all others. Store in the cache, and
524 ;; mark it as successful.
530 ;; 300 Multiple choices
531 ;; 301 Moved permanently
532 ;; 302 Found
533 ;; 303 See other
534 ;; 304 Not modified
535 ;; 305 Use proxy
536 ;; 307 Temporary redirect
541 ;; Quoth the spec (section 10.3.1)
542 ;; -------------------------------
543 ;; The requested resource corresponds to any one of a set of
544 ;; representations, each with its own specific location and
545 ;; agent-driven negotiation information is being provided so
546 ;; that the user can select a preferred representation and
547 ;; redirect its request to that location.
548 ;; [...]
549 ;; If the server has a preferred choice of representation, it
550 ;; SHOULD include the specific URI for that representation in
551 ;; the Location field; user agents MAY use the Location field
552 ;; value for automatic redirection.
553 ;; -------------------------------
554 ;; We do not support agent-driven negotiation, so we just
555 ;; redirect to the preferred URI if one is provided.
556 nil)
558 ;; If the 301|302 status code is received in response to a
559 ;; request other than GET or HEAD, the user agent MUST NOT
560 ;; automatically redirect the request unless it can be
561 ;; confirmed by the user, since this might change the
562 ;; conditions under which the request was issued.
564 ;; Automatic redirection is ok
565 nil
566 ;; It is just too big of a pain in the ass to get this
567 ;; prompt all the time. We will just silently lose our
568 ;; data and convert to a GET method.
570 url-http-method url-http-response-status)
572 url-http-data nil)))
574 ;; The response to the request can be found under a different
575 ;; URI and SHOULD be retrieved using a GET method on that
576 ;; resource.
578 url-http-data nil))
580 ;; The 304 response MUST NOT contain a message-body.
585 success t))
587 ;; The requested resource MUST be accessed through the
588 ;; proxy given by the Location field. The Location field
589 ;; gives the URI of the proxy. The recipient is expected
590 ;; to repeat this single request via the proxy. 305
591 ;; responses MUST only be generated by origin servers.
593 redirect-uri))
595 ;; Treat everything like '300'
596 nil))
598 ;; Clean off any whitespace and/or <...> cruft.
604 ;; Some stupid sites (like sourceforge) send a
605 ;; non-fully-qualified URL (ie: /), which royally confuses
606 ;; the URL library.
608 ;; Be careful to use the real target URL, otherwise we may
609 ;; compute the redirection relative to the URL of the proxy.
615 ;; Check existing number of redirects
626 ;; url-max-redirections hasn't been reached, so go
627 ;; ahead and redirect.
629 ;; Remember that the request was redirected.
633 ;; Put in the current buffer a forwarding pointer to the new
634 ;; destination buffer.
635 ;; FIXME: This is a hack to fix url-retrieve-synchronously
636 ;; without changing the API. Instead url-retrieve should
637 ;; either simply not return the "destination" buffer, or it
638 ;; should take an optional `dest-buf' argument.
641 redirect-uri url-callback-function
642 url-callback-arguments))
644 ;; We hit url-max-redirections, so issue an error and
645 ;; stop redirecting.
649 redirect-uri))
653 ;; 400 Bad Request
654 ;; 401 Unauthorized
655 ;; 402 Payment required
656 ;; 403 Forbidden
657 ;; 404 Not found
658 ;; 405 Method not allowed
659 ;; 406 Not acceptable
660 ;; 407 Proxy authentication required
661 ;; 408 Request time-out
662 ;; 409 Conflict
663 ;; 410 Gone
664 ;; 411 Length required
665 ;; 412 Precondition failed
666 ;; 413 Request entity too large
667 ;; 414 Request-URI too large
668 ;; 415 Unsupported media type
669 ;; 416 Requested range not satisfiable
670 ;; 417 Expectation failed
671 ;; 422 Unprocessable Entity (Added by DAV)
672 ;; 423 Locked
673 ;; 424 Failed Dependency
676 ;; The request requires user authentication. The response
677 ;; MUST include a WWW-Authenticate header field containing a
678 ;; challenge applicable to the requested resource. The
679 ;; client MAY repeat the request with a suitable
680 ;; Authorization header field.
683 ;; This code is reserved for future use
687 ;; The server understood the request, but is refusing to
688 ;; fulfill it. Authorization will not help and the request
689 ;; SHOULD NOT be repeated.
692 ;; Not found
695 ;; The method specified in the Request-Line is not allowed
696 ;; for the resource identified by the Request-URI. The
697 ;; response MUST include an Allow header containing a list of
698 ;; valid methods for the requested resource.
701 ;; The resource identified by the request is only capable of
702 ;; generating response entities which have content
703 ;; characteristics nota cceptable according to the accept
704 ;; headers sent in the request.
707 ;; This code is similar to 401 (Unauthorized), but indicates
708 ;; that the client must first authenticate itself with the
709 ;; proxy. The proxy MUST return a Proxy-Authenticate header
710 ;; field containing a challenge applicable to the proxy for
711 ;; the requested resource.
714 ;; The client did not produce a request within the time that
715 ;; the server was prepared to wait. The client MAY repeat
716 ;; the request without modifications at any later time.
719 ;; The request could not be completed due to a conflict with
720 ;; the current state of the resource. This code is only
721 ;; allowed in situations where it is expected that the user
722 ;; mioght be able to resolve the conflict and resubmit the
723 ;; request. The response body SHOULD include enough
724 ;; information for the user to recognize the source of the
725 ;; conflict.
728 ;; The requested resource is no longer available at the
729 ;; server and no forwarding address is known.
732 ;; The server refuses to accept the request without a defined
733 ;; Content-Length. The client MAY repeat the request if it
734 ;; adds a valid Content-Length header field containing the
735 ;; length of the message-body in the request message.
736 ;;
737 ;; NOTE - this will never happen because
738 ;; `url-http-create-request' automatically calculates the
739 ;; content-length.
742 ;; The precondition given in one or more of the
743 ;; request-header fields evaluated to false when it was
744 ;; tested on the server.
747 ;; The server is refusing to process a request because the
748 ;; request entity|URI is larger than the server is willing or
749 ;; able to process.
752 ;; The server is refusing to service the request because the
753 ;; entity of the request is in a format not supported by the
754 ;; requested resource for the requested method.
757 ;; A server SHOULD return a response with this status code if
758 ;; a request included a Range request-header field, and none
759 ;; of the range-specifier values in this field overlap the
760 ;; current extent of the selected resource, and the request
761 ;; did not include an If-Range request-header field.
764 ;; The expectation given in an Expect request-header field
765 ;; could not be met by this server, or, if the server is a
766 ;; proxy, the server has unambiguous evidence that the
767 ;; request could not be met by the next-hop server.
770 ;; The request could not be understood by the server due to
771 ;; malformed syntax. The client SHOULD NOT repeat the
772 ;; request without modifications.
774 ;; Tell the callback that an error occurred, and what the
775 ;; status code was.
781 ;; 500 Internal server error
782 ;; 501 Not implemented
783 ;; 502 Bad gateway
784 ;; 503 Service unavailable
785 ;; 504 Gateway time-out
786 ;; 505 HTTP version not supported
787 ;; 507 Insufficient storage
791 ;; The server does not support the functionality required to
792 ;; fulfill the request.
793 nil)
795 ;; The server, while acting as a gateway or proxy, received
796 ;; an invalid response from the upstream server it accessed
797 ;; in attempting to fulfill the request.
798 nil)
800 ;; The server is currently unable to handle the request due
801 ;; to a temporary overloading or maintenance of the server.
802 ;; The implication is that this is a temporary condition
803 ;; which will be alleviated after some delay. If known, the
804 ;; length of the delay MAY be indicated in a Retry-After
805 ;; header. If no Retry-After is given, the client SHOULD
806 ;; handle the response as it would for a 500 response.
807 nil)
809 ;; The server, while acting as a gateway or proxy, did not
810 ;; receive a timely response from the upstream server
811 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
812 ;; auxiliary server (e.g. DNS) it needed to access in
813 ;; attempting to complete the request.
814 nil)
816 ;; The server does not support, or refuses to support, the
817 ;; HTTP protocol version that was used in the request
818 ;; message.
819 nil)
821 ;; The method could not be performed on the resource
822 ;; because the server is unable to store the representation
823 ;; needed to successfully complete the request. This
824 ;; condition is considered to be temporary. If the request
825 ;; which received this status code was the result of a user
826 ;; action, the request MUST NOT be repeated until it is
827 ;; requested by a separate user action.
828 nil))
829 ;; Tell the callback that an error occurred, and what the
830 ;; status code was.
837 class url-http-response-status)))
842 success))
844 ;; Miscellaneous
846 "Activate callback specified when this buffer was created."
848 url-callback-function
849 url-callback-arguments))
852 url-http-process)
856 ;; )
858 ;; These unfortunately cannot be macros... please ignore them!
860 "Remove (now defunct) process PROC from the list of open connections."
864 url-http-open-connections))
867 ;; Sentinel used for old HTTP/0.9 or connections we know are going
868 ;; to die as the 'end of document' notifier.
875 ;; HTTP/0.9 just gets passed back no matter what
881 ;; Function used when we do NOT know how long the document is going to be
882 ;; Just _very_ simple 'downloaded %d' type of info.
887 "Function used when we DO know how long the document is going to be.
888 More sophisticated percentage downloaded, etc.
889 Also does minimal parsing of HTTP headers and will actually cause
890 the callback to be triggered."
892 url-http-end-of-headers
893 url-http-content-length
894 url-http-content-type
895 url-http-process))
898 "Reading [%s]... %s of %s (%d%%)"
900 url-http-content-length)
901 url-http-content-type
905 url-http-content-length))
907 "Reading... %s of %s (%d%%)"
909 url-http-content-length)
913 url-http-content-length)))
917 ;; Found the end of the document! Wheee!
924 "Function used when dealing with 'chunked' encoding.
925 Cannot give a sophisticated percentage, but we need a different
926 function to look for the special 0-length chunk that signifies
927 the end of the document."
929 url-http-end-of-headers
930 url-http-content-type
931 url-http-chunked-length
932 url-http-chunked-counter
933 url-http-process url-http-chunked-start))
940 ;; We need to loop thru looking for more chunks even within
941 ;; one after-change-function call.
946 "Reading [%s]... chunk #%d"
947 url-http-content-type url-http-chunked-counter)
949 "Reading... chunk #%d"
950 url-http-chunked-counter))
952 url-http-chunked-counter st nd length)
958 ;; We know how long the chunk is supposed to be, skip over
959 ;; leading crap if possible.
963 url-http-chunked-counter)
965 url-http-chunked-length)))
968 url-http-chunked-length)
969 nd))
976 ;; Must not have received the entirety of the chunk header,
977 ;; need to spin some more.
982 'end-open t
983 'chunked-encoding t
995 ; (if (not url-http-debug)
998 url-http-chunked-counter url-http-chunked-length
1002 ;; Found the end of the document! Wheee!
1006 ;; Every chunk, even the last 0-length one, is
1007 ;; terminated by CRLF. Skip it.
1017 ;; This will wait for the headers to arrive and then splice in the
1018 ;; next appropriate after-change-function, etc.
1020 url-http-end-of-headers
1021 url-http-content-type
1022 url-http-content-length
1023 url-http-transfer-encoding
1024 url-callback-function
1025 url-callback-arguments
1026 url-http-process
1027 url-http-method
1028 url-http-after-change-function
1029 url-http-response-status))
1039 ;; Not HTTP/x.y data, must be 0.9
1040 ;; God, I wish this could die.
1042 url-http-end-of-headers 0
1043 old-http t)
1045 ;; Saw the end of the headers
1049 end-of-headers t)
1053 ;; Haven't seen the end of the headers yet, need to wait
1054 ;; for more data to arrive.
1055 nil
1061 ;;(narrow-to-region (point-min) url-http-end-of-headers)
1081 ;; A HEAD request is _ALWAYS_ terminated by the header
1082 ;; information, regardless of any entity headers,
1083 ;; according to section 4.4 of the HTTP/1.1 draft.
1089 ;; A CONNECT request is finished, but we cannot stick this
1090 ;; back on the free connectin list
1095 ;; Only allowed to have a header section. We have to handle
1096 ;; this here instead of in url-http-parse-headers because if
1097 ;; you have a cached copy of something without a known
1098 ;; content-length, and try to retrieve it from the cache, we'd
1099 ;; fall into the 'being dumb' section and wait for the
1100 ;; connection to terminate, which means we'd wait for 10
1101 ;; seconds for the keep-alives to time out on some servers.
1105 ;; HTTP/0.9 always signaled end-of-connection by closing the
1106 ;; connection.
1128 ;; We got a NULL body! Activate the callback
1129 ;; immediately!
1135 ;; Have some leftover data
1139 nd
1142 nil)))
1147 ;; We are still at the beginning of the buffer... must just be
1148 ;; waiting for a response.
1152 ;;;###autoload
1154 "Retrieve URL via HTTP asynchronously.
1155 URL must be a parsed URL. See `url-generic-parse-url' for details.
1156 When retrieval is completed, the function CALLBACK is executed with
1157 CBARGS as the arguments."
1160 url-http-end-of-headers
1161 url-http-content-type
1162 url-http-content-length
1163 url-http-transfer-encoding
1164 url-http-after-change-function
1165 url-callback-function
1166 url-callback-arguments
1167 url-http-method
1168 url-http-extra-headers
1169 url-http-data
1170 url-http-chunked-length
1171 url-http-chunked-start
1172 url-http-chunked-counter
1173 url-http-process))
1179 ;; Failed to open the connection for some reason
1190 url-http-content-type
1191 url-http-content-length
1192 url-http-transfer-encoding
1193 url-http-after-change-function
1194 url-http-response-version
1195 url-http-response-status
1196 url-http-chunked-length
1197 url-http-chunked-counter
1198 url-http-chunked-start
1199 url-callback-function
1200 url-callback-arguments
1201 url-http-process
1202 url-http-method
1203 url-http-extra-headers
1204 url-http-data
1205 url-http-target-url
1206 url-http-connection-opened
1207 url-http-proxy))
1211 url-http-extra-headers url-request-extra-headers
1212 url-http-data url-request-data
1213 url-http-process connection
1214 url-http-chunked-length nil
1215 url-http-chunked-start nil
1216 url-http-chunked-counter 0
1217 url-callback-function callback
1218 url-callback-arguments cbargs
1219 url-http-after-change-function 'url-http-wait-for-headers-change-function
1220 url-http-target-url url-current-object
1221 url-http-connection-opened nil
1222 url-http-proxy url-using-proxy)
1229 ;; Asynchronous connection
1232 ;; Asynchronous connection failed
1237 buffer))
1241 ;; We are performing an asynchronous connection, and a status change
1242 ;; has occurred.
1258 ;; Since Emacs 19/20 does not allow you to change the
1259 ;; `after-change-functions' hook in the midst of running them, we fake
1260 ;; an after change by hooking into the process filter and inserting
1261 ;; the data ourselves. This is slightly less efficient, but there
1262 ;; were tons of weird ways the after-change code was biting us in the
1263 ;; shorts.
1265 ;; Sometimes we get a zero-length data chunk after the process has
1266 ;; been changed to 'free', which means it has no buffer associated
1267 ;; with it. Do nothing if there is no buffer, or 0 length data.
1272 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1281 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1282 ;;; file-name-handler stuff from here on out
1283 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1286 'symbol-value-in-buffer
1288 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1291 unbound-value
1299 ;;;###autoload
1311 exists))
1313 ;;;###autoload
1321 nil ;dir / link / normal file
1324 nil nil nil ;atime ; mtime ; ctime
1333 ;;;###autoload
1339 ;;;###autoload
1341 "Return a property list describing options available for URL.
1342 This list is retrieved using the `OPTIONS' HTTP method.
1344 Property list members:
1346 methods
1347 A list of symbols specifying what HTTP methods the resource
1348 supports.
1350 dav
1351 A list of numbers specifying what DAV protocol/schema versions are
1352 supported.
1354 dasl
1355 A list of supported DASL search types supported (string form)
1357 ranges
1358 A list of the units available for use in partial document fetches.
1360 p3p
1361 The `Platform For Privacy Protection' description for the resource.
1362 Currently this is just the raw header contents. This is likely to
1363 change once P3P is formally supported by the URL package or
1364 Emacs/W3."
1372 ;; Only parse the options if we got a 2xx response code!
1378 ;; Figure out what methods are supported.
1381 options 'methods
1384 ;; Check for DAV
1387 options 'dav
1392 ;; Now for DASL
1395 options 'dasl
1398 ;; P3P - should get more detailed here. FIXME
1402 ;; Check for whether they accept byte-range requests.
1405 options 'ranges
1409 ))))
1411 options))
1413 ;; HTTPS. This used to be in url-https.el, but that file collides
1414 ;; with url-http.el on systems with 8-character file names.
1417 ;;;###autoload
1419 ;;;###autoload
1422 ;; FIXME what is the point of this alias being an autoload?
1423 ;; Trying to use it will not cause url-http to be loaded,
1424 ;; since the full alias just gets dumped into loaddefs.el.
1426 ;;;###autoload (autoload 'url-default-expander "url-expand")
1427 ;;;###autoload
1437 ;;;###autoload (autoload 'url-https "url-http")
1439 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1441 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1443 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1448 ;; arch-tag: ba7c59ae-c0f4-4a31-9617-d85f221732ee
1449 ;;; url-http.el ends here