| blob | a21aed21436ffd3aaf9a5b3d4b190d820897bf55 |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004-2011 Free Software Foundation, Inc.
5 ;; Author: Bill Perry <wmperry@gnu.org>
6 ;; Keywords: comm, data, processes
8 ;; This file is part of GNU Emacs.
9 ;;
10 ;; GNU Emacs is free software: you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation, either version 3 of the License, or
13 ;; (at your option) any later version.
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
23 ;;; Commentary:
25 ;;; Code:
53 "What version of HTTP we advertise, as a string.
54 Valid values are 1.1 and 1.0.
55 This is only useful when debugging the HTTP subsystem.
57 Setting this to 1.0 will tell servers not to send chunked encoding,
61 "Whether to use a single TCP connection multiple times in HTTP.
62 This is only useful when debugging the HTTP subsystem. Setting to
63 nil will explicitly close the connection to the server after every
115 ;(eval-when-compile
116 ;; These are all macros so that they are hidden from external sight
117 ;; when the file is byte-compiled.
118 ;;
119 ;; This allows us to expose just the entry points we want.
121 ;; These routines will allow us to implement persistent HTTP
122 ;; connections.
126 ;; The user hit C-g, honor it! Some things can get in an
127 ;; incredibly tight loop (chunked encoding)
140 url-http-open-connections)
141 proc)
151 url-http-open-connections))
152 nil)
171 host port
174 ;; `url-open-stream' needs a buffer in which to do things
175 ;; like authentication. But we use another buffer afterwards.
178 ;; url-open-stream might return nil.
180 ;; Drop the temp buffer link before killing the buffer.
182 proc)
185 ;; Building an HTTP request
190 ""
195 url-version
204 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
206 url-http-method url-http-data
207 url-http-extra-headers))
213 url-http-extra-headers))
215 nil
223 nil
226 proxy-info)
236 ;; Protection against stupid values in the referer
241 ;; We do not want to expose the referer if the user is paranoid.
247 ;; url-http-extra-headers contains an assoc-list of
248 ;; header/value pairs that we need to put into the request.
256 ;; This was done with a call to `format'. Concatting parts has
257 ;; the advantage of keeping the parts of each header together and
258 ;; allows us to elide null lines directly, at the cost of making
259 ;; the layout less clear.
261 ;; We used to concat directly, but if one of the strings happens
262 ;; to being multibyte (even if it only contains pure ASCII) then
263 ;; every string gets converted with `string-MAKE-multibyte' which
264 ;; turns the 127-255 codes into things like latin-1 accented chars
265 ;; (it would work right if it used `string-TO-multibyte' instead).
266 ;; So to avoid the problem we force every string to be unibyte.
268 ;; FIXME: Instead of `string-AS-unibyte' we'd want
269 ;; `string-to-unibyte', so as to properly signal an error if one
270 ;; of the strings contains a multibyte char.
271 'string-as-unibyte
274 ;; The request
278 ;; Version of MIME we speak
280 ;; (maybe) Try to keep the connection open
284 ;; HTTP extensions we support
288 ;; Who we want to talk to
295 ;; Who its from
299 ;; Encodings we understand
306 ;; Languages we understand
310 ;; Types we understand
312 ;; User agent
314 ;; Proxy Authorization
315 proxy-auth
316 ;; Authorization
317 auth
318 ;; Cookies
321 ;; If-modified-since
328 ;; Whence we came
331 extra-headers
332 ;; Length of data
338 ;; End request
340 ;; Any data
341 url-http-data))
344 request))
346 ;; Parsing routines
349 This allows us to use `mail-fetch-field', etc."
357 url-callback-function url-callback-arguments))
362 nil nil t))
368 url-http-proxy
369 url-current-object)))
371 ;; Cheating, but who cares? :)
372 'url-http-proxy-basic-auth-storage
374 auth
377 ;; find strongest supported auth
381 this-auth)))
390 type this-type
391 strength this-strength))))
398 " authentication. If you'd like to write it,"
404 type t args)))
408 url-http-extra-headers)
413 url-callback-arguments)))))))
416 "Parse just the response code."
418 url-http-response-version))
433 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
434 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
435 work correctly."
442 ;;; (while cookies2
443 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
444 )
445 )
448 "Parse and handle HTTP specific headers.
449 Return t if and only if the current buffer is still active and
450 should be shown to the user."
451 ;; The comments after each status code handled are taken from RFC
452 ;; 2616 (HTTP/1.1)
454 url-http-response-version
455 url-http-method url-http-data url-http-process
456 url-callback-function url-callback-arguments))
460 url-http-process)
469 ;;(narrow-to-region (point-min) url-http-end-of-headers)
471 ;; In HTTP 1.0, keep the connection only if there is a
472 ;; "Connection: keep-alive" header.
473 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
474 ;; "Connection: close" header
487 ;; other status symbols: jewelry and luxury cars
489 ;; The filename part of a URL could be in remote file syntax,
490 ;; see Bug#6717 for an example. We disable file name
491 ;; handlers, therefore.
498 ;; Classes of response codes
499 ;;
500 ;; 5xx = Server Error
501 ;; 4xx = Client Error
502 ;; 3xx = Redirection
503 ;; 2xx = Successful
504 ;; 1xx = Informational
506 ;; 100 = Continue with request
507 ;; 101 = Switching protocols
508 ;; 102 = Processing (Added by DAV)
512 ;; 200 Ok
513 ;; 201 Created
514 ;; 202 Accepted
515 ;; 203 Non-authoritative information
516 ;; 204 No content
517 ;; 205 Reset content
518 ;; 206 Partial content
519 ;; 207 Multi-status (Added by DAV)
522 ;; No new data, just stay at the same document
526 ;; Generic success for all others. Store in the cache, and
527 ;; mark it as successful.
533 ;; 300 Multiple choices
534 ;; 301 Moved permanently
535 ;; 302 Found
536 ;; 303 See other
537 ;; 304 Not modified
538 ;; 305 Use proxy
539 ;; 307 Temporary redirect
544 ;; Quoth the spec (section 10.3.1)
545 ;; -------------------------------
546 ;; The requested resource corresponds to any one of a set of
547 ;; representations, each with its own specific location and
548 ;; agent-driven negotiation information is being provided so
549 ;; that the user can select a preferred representation and
550 ;; redirect its request to that location.
551 ;; [...]
552 ;; If the server has a preferred choice of representation, it
553 ;; SHOULD include the specific URI for that representation in
554 ;; the Location field; user agents MAY use the Location field
555 ;; value for automatic redirection.
556 ;; -------------------------------
557 ;; We do not support agent-driven negotiation, so we just
558 ;; redirect to the preferred URI if one is provided.
559 nil)
561 ;; If the 301|302 status code is received in response to a
562 ;; request other than GET or HEAD, the user agent MUST NOT
563 ;; automatically redirect the request unless it can be
564 ;; confirmed by the user, since this might change the
565 ;; conditions under which the request was issued.
569 ;; The response to the request can be found under a different
570 ;; URI and SHOULD be retrieved using a GET method on that
571 ;; resource.
573 url-http-data nil))
575 ;; The 304 response MUST NOT contain a message-body.
580 success t))
582 ;; The requested resource MUST be accessed through the
583 ;; proxy given by the Location field. The Location field
584 ;; gives the URI of the proxy. The recipient is expected
585 ;; to repeat this single request via the proxy. 305
586 ;; responses MUST only be generated by origin servers.
588 redirect-uri))
590 ;; Treat everything like '300'
591 nil))
593 ;; Clean off any whitespace and/or <...> cruft.
599 ;; Some stupid sites (like sourceforge) send a
600 ;; non-fully-qualified URL (ie: /), which royally confuses
601 ;; the URL library.
603 ;; Be careful to use the real target URL, otherwise we may
604 ;; compute the redirection relative to the URL of the proxy.
610 ;; Check existing number of redirects
621 ;; url-max-redirections hasn't been reached, so go
622 ;; ahead and redirect.
624 ;; Remember that the request was redirected.
628 ;; Put in the current buffer a forwarding pointer to the new
629 ;; destination buffer.
630 ;; FIXME: This is a hack to fix url-retrieve-synchronously
631 ;; without changing the API. Instead url-retrieve should
632 ;; either simply not return the "destination" buffer, or it
633 ;; should take an optional `dest-buf' argument.
636 redirect-uri url-callback-function
637 url-callback-arguments
640 ;; We hit url-max-redirections, so issue an error and
641 ;; stop redirecting.
645 redirect-uri))
649 ;; 400 Bad Request
650 ;; 401 Unauthorized
651 ;; 402 Payment required
652 ;; 403 Forbidden
653 ;; 404 Not found
654 ;; 405 Method not allowed
655 ;; 406 Not acceptable
656 ;; 407 Proxy authentication required
657 ;; 408 Request time-out
658 ;; 409 Conflict
659 ;; 410 Gone
660 ;; 411 Length required
661 ;; 412 Precondition failed
662 ;; 413 Request entity too large
663 ;; 414 Request-URI too large
664 ;; 415 Unsupported media type
665 ;; 416 Requested range not satisfiable
666 ;; 417 Expectation failed
667 ;; 422 Unprocessable Entity (Added by DAV)
668 ;; 423 Locked
669 ;; 424 Failed Dependency
672 ;; The request requires user authentication. The response
673 ;; MUST include a WWW-Authenticate header field containing a
674 ;; challenge applicable to the requested resource. The
675 ;; client MAY repeat the request with a suitable
676 ;; Authorization header field.
679 ;; This code is reserved for future use
683 ;; The server understood the request, but is refusing to
684 ;; fulfill it. Authorization will not help and the request
685 ;; SHOULD NOT be repeated.
688 ;; Not found
691 ;; The method specified in the Request-Line is not allowed
692 ;; for the resource identified by the Request-URI. The
693 ;; response MUST include an Allow header containing a list of
694 ;; valid methods for the requested resource.
697 ;; The resource identified by the request is only capable of
698 ;; generating response entities which have content
699 ;; characteristics nota cceptable according to the accept
700 ;; headers sent in the request.
703 ;; This code is similar to 401 (Unauthorized), but indicates
704 ;; that the client must first authenticate itself with the
705 ;; proxy. The proxy MUST return a Proxy-Authenticate header
706 ;; field containing a challenge applicable to the proxy for
707 ;; the requested resource.
710 ;; The client did not produce a request within the time that
711 ;; the server was prepared to wait. The client MAY repeat
712 ;; the request without modifications at any later time.
715 ;; The request could not be completed due to a conflict with
716 ;; the current state of the resource. This code is only
717 ;; allowed in situations where it is expected that the user
718 ;; mioght be able to resolve the conflict and resubmit the
719 ;; request. The response body SHOULD include enough
720 ;; information for the user to recognize the source of the
721 ;; conflict.
724 ;; The requested resource is no longer available at the
725 ;; server and no forwarding address is known.
728 ;; The server refuses to accept the request without a defined
729 ;; Content-Length. The client MAY repeat the request if it
730 ;; adds a valid Content-Length header field containing the
731 ;; length of the message-body in the request message.
732 ;;
733 ;; NOTE - this will never happen because
734 ;; `url-http-create-request' automatically calculates the
735 ;; content-length.
738 ;; The precondition given in one or more of the
739 ;; request-header fields evaluated to false when it was
740 ;; tested on the server.
743 ;; The server is refusing to process a request because the
744 ;; request entity|URI is larger than the server is willing or
745 ;; able to process.
748 ;; The server is refusing to service the request because the
749 ;; entity of the request is in a format not supported by the
750 ;; requested resource for the requested method.
753 ;; A server SHOULD return a response with this status code if
754 ;; a request included a Range request-header field, and none
755 ;; of the range-specifier values in this field overlap the
756 ;; current extent of the selected resource, and the request
757 ;; did not include an If-Range request-header field.
760 ;; The expectation given in an Expect request-header field
761 ;; could not be met by this server, or, if the server is a
762 ;; proxy, the server has unambiguous evidence that the
763 ;; request could not be met by the next-hop server.
766 ;; The request could not be understood by the server due to
767 ;; malformed syntax. The client SHOULD NOT repeat the
768 ;; request without modifications.
770 ;; Tell the callback that an error occurred, and what the
771 ;; status code was.
777 ;; 500 Internal server error
778 ;; 501 Not implemented
779 ;; 502 Bad gateway
780 ;; 503 Service unavailable
781 ;; 504 Gateway time-out
782 ;; 505 HTTP version not supported
783 ;; 507 Insufficient storage
787 ;; The server does not support the functionality required to
788 ;; fulfill the request.
789 nil)
791 ;; The server, while acting as a gateway or proxy, received
792 ;; an invalid response from the upstream server it accessed
793 ;; in attempting to fulfill the request.
794 nil)
796 ;; The server is currently unable to handle the request due
797 ;; to a temporary overloading or maintenance of the server.
798 ;; The implication is that this is a temporary condition
799 ;; which will be alleviated after some delay. If known, the
800 ;; length of the delay MAY be indicated in a Retry-After
801 ;; header. If no Retry-After is given, the client SHOULD
802 ;; handle the response as it would for a 500 response.
803 nil)
805 ;; The server, while acting as a gateway or proxy, did not
806 ;; receive a timely response from the upstream server
807 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
808 ;; auxiliary server (e.g. DNS) it needed to access in
809 ;; attempting to complete the request.
810 nil)
812 ;; The server does not support, or refuses to support, the
813 ;; HTTP protocol version that was used in the request
814 ;; message.
815 nil)
817 ;; The method could not be performed on the resource
818 ;; because the server is unable to store the representation
819 ;; needed to successfully complete the request. This
820 ;; condition is considered to be temporary. If the request
821 ;; which received this status code was the result of a user
822 ;; action, the request MUST NOT be repeated until it is
823 ;; requested by a separate user action.
824 nil))
825 ;; Tell the callback that an error occurred, and what the
826 ;; status code was.
833 class url-http-response-status)))
838 success))
840 ;; Miscellaneous
842 "Activate callback specified when this buffer was created."
844 url-callback-function
845 url-callback-arguments))
848 url-http-process)
852 ;; )
854 ;; These unfortunately cannot be macros... please ignore them!
856 "Remove (now defunct) process PROC from the list of open connections."
860 url-http-open-connections))
863 ;; Sentinel used for old HTTP/0.9 or connections we know are going
864 ;; to die as the 'end of document' notifier.
872 ;; HTTP/0.9 just gets passed back no matter what
878 ;; Function used when we do NOT know how long the document is going to be
879 ;; Just _very_ simple 'downloaded %d' type of info.
884 "Function used when we DO know how long the document is going to be.
885 More sophisticated percentage downloaded, etc.
886 Also does minimal parsing of HTTP headers and will actually cause
887 the callback to be triggered."
889 url-http-end-of-headers
890 url-http-content-length
891 url-http-content-type
892 url-http-process))
895 "Reading [%s]... %s of %s (%d%%)"
897 url-http-content-length)
898 url-http-content-type
902 url-http-content-length))
904 "Reading... %s of %s (%d%%)"
906 url-http-content-length)
910 url-http-content-length)))
914 ;; Found the end of the document! Wheee!
921 "Function used when dealing with 'chunked' encoding.
922 Cannot give a sophisticated percentage, but we need a different
923 function to look for the special 0-length chunk that signifies
924 the end of the document."
926 url-http-end-of-headers
927 url-http-content-type
928 url-http-chunked-length
929 url-http-chunked-counter
930 url-http-process url-http-chunked-start))
937 ;; We need to loop thru looking for more chunks even within
938 ;; one after-change-function call.
943 "Reading [%s]... chunk #%d"
944 url-http-content-type url-http-chunked-counter)
946 "Reading... chunk #%d"
947 url-http-chunked-counter))
949 url-http-chunked-counter st nd length)
955 ;; We know how long the chunk is supposed to be, skip over
956 ;; leading crap if possible.
960 url-http-chunked-counter)
962 url-http-chunked-length)))
965 url-http-chunked-length)
966 nd))
973 ;; Must not have received the entirety of the chunk header,
974 ;; need to spin some more.
979 'end-open t
980 'chunked-encoding t
992 ; (if (not url-http-debug)
995 url-http-chunked-counter url-http-chunked-length
999 ;; Found the end of the document! Wheee!
1003 ;; Every chunk, even the last 0-length one, is
1004 ;; terminated by CRLF. Skip it.
1014 ;; This will wait for the headers to arrive and then splice in the
1015 ;; next appropriate after-change-function, etc.
1017 url-http-end-of-headers
1018 url-http-content-type
1019 url-http-content-length
1020 url-http-transfer-encoding
1021 url-callback-function
1022 url-callback-arguments
1023 url-http-process
1024 url-http-method
1025 url-http-after-change-function
1026 url-http-response-status))
1037 ;; Not HTTP/x.y data, must be 0.9
1038 ;; God, I wish this could die.
1040 url-http-end-of-headers 0
1041 old-http t)
1043 ;; Saw the end of the headers
1047 end-of-headers t)
1051 ;; Haven't seen the end of the headers yet, need to wait
1052 ;; for more data to arrive.
1053 nil
1070 ;; We got back a headerless malformed response from the
1071 ;; server.
1080 ;; A HEAD request is _ALWAYS_ terminated by the header
1081 ;; information, regardless of any entity headers,
1082 ;; according to section 4.4 of the HTTP/1.1 draft.
1088 ;; A CONNECT request is finished, but we cannot stick this
1089 ;; back on the free connectin list
1094 ;; Only allowed to have a header section. We have to handle
1095 ;; this here instead of in url-http-parse-headers because if
1096 ;; you have a cached copy of something without a known
1097 ;; content-length, and try to retrieve it from the cache, we'd
1098 ;; fall into the 'being dumb' section and wait for the
1099 ;; connection to terminate, which means we'd wait for 10
1100 ;; seconds for the keep-alives to time out on some servers.
1104 ;; HTTP/0.9 always signaled end-of-connection by closing the
1105 ;; connection.
1127 ;; We got a NULL body! Activate the callback
1128 ;; immediately!
1134 ;; Have some leftover data
1138 nd
1141 nil)))
1146 ;; We are still at the beginning of the buffer... must just be
1147 ;; waiting for a response.
1152 ;;;###autoload
1154 "Retrieve URL via HTTP asynchronously.
1155 URL must be a parsed URL. See `url-generic-parse-url' for details.
1156 When retrieval is completed, the function CALLBACK is executed with
1157 CBARGS as the arguments."
1160 url-http-end-of-headers
1161 url-http-content-type
1162 url-http-content-length
1163 url-http-transfer-encoding
1164 url-http-after-change-function
1165 url-callback-function
1166 url-callback-arguments
1167 url-show-status
1168 url-http-method
1169 url-http-extra-headers
1170 url-http-data
1171 url-http-chunked-length
1172 url-http-chunked-start
1173 url-http-chunked-counter
1174 url-http-process))
1180 ;; Failed to open the connection for some reason
1191 url-http-content-type
1192 url-http-content-length
1193 url-http-transfer-encoding
1194 url-http-after-change-function
1195 url-http-response-version
1196 url-http-response-status
1197 url-http-chunked-length
1198 url-http-chunked-counter
1199 url-http-chunked-start
1200 url-callback-function
1201 url-callback-arguments
1202 url-show-status
1203 url-http-process
1204 url-http-method
1205 url-http-extra-headers
1206 url-http-data
1207 url-http-target-url
1208 url-http-connection-opened
1209 url-http-proxy))
1213 url-http-extra-headers url-request-extra-headers
1214 url-http-data url-request-data
1215 url-http-process connection
1216 url-http-chunked-length nil
1217 url-http-chunked-start nil
1218 url-http-chunked-counter 0
1219 url-callback-function callback
1220 url-callback-arguments cbargs
1221 url-http-after-change-function 'url-http-wait-for-headers-change-function
1222 url-http-target-url url-current-object
1223 url-http-connection-opened nil
1224 url-http-proxy url-using-proxy)
1231 ;; Asynchronous connection
1234 ;; Asynchronous connection failed
1239 buffer))
1243 ;; We are performing an asynchronous connection, and a status change
1244 ;; has occurred.
1261 ;; Since Emacs 19/20 does not allow you to change the
1262 ;; `after-change-functions' hook in the midst of running them, we fake
1263 ;; an after change by hooking into the process filter and inserting
1264 ;; the data ourselves. This is slightly less efficient, but there
1265 ;; were tons of weird ways the after-change code was biting us in the
1266 ;; shorts.
1267 ;; FIXME this can probably be simplified since the above is no longer true.
1269 ;; Sometimes we get a zero-length data chunk after the process has
1270 ;; been changed to 'free', which means it has no buffer associated
1271 ;; with it. Do nothing if there is no buffer, or 0 length data.
1276 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1285 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1286 ;;; file-name-handler stuff from here on out
1287 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1290 'symbol-value-in-buffer
1292 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1295 unbound-value
1303 ;;;###autoload
1315 exists))
1317 ;;;###autoload
1325 nil ;dir / link / normal file
1328 nil nil nil ;atime ; mtime ; ctime
1337 ;;;###autoload
1343 ;;;###autoload
1345 "Return a property list describing options available for URL.
1346 This list is retrieved using the `OPTIONS' HTTP method.
1348 Property list members:
1350 methods
1351 A list of symbols specifying what HTTP methods the resource
1352 supports.
1354 dav
1355 A list of numbers specifying what DAV protocol/schema versions are
1356 supported.
1358 dasl
1359 A list of supported DASL search types supported (string form)
1361 ranges
1362 A list of the units available for use in partial document fetches.
1364 p3p
1365 The `Platform For Privacy Protection' description for the resource.
1366 Currently this is just the raw header contents. This is likely to
1367 change once P3P is formally supported by the URL package or
1368 Emacs/W3."
1376 ;; Only parse the options if we got a 2xx response code!
1382 ;; Figure out what methods are supported.
1385 options 'methods
1388 ;; Check for DAV
1391 options 'dav
1396 ;; Now for DASL
1399 options 'dasl
1402 ;; P3P - should get more detailed here. FIXME
1406 ;; Check for whether they accept byte-range requests.
1409 options 'ranges
1413 ))))
1415 options))
1417 ;; HTTPS. This used to be in url-https.el, but that file collides
1418 ;; with url-http.el on systems with 8-character file names.
1421 ;;;###autoload
1423 ;;;###autoload
1426 ;; FIXME what is the point of this alias being an autoload?
1427 ;; Trying to use it will not cause url-http to be loaded,
1428 ;; since the full alias just gets dumped into loaddefs.el.
1430 ;;;###autoload (autoload 'url-default-expander "url-expand")
1431 ;;;###autoload
1441 ;;;###autoload (autoload 'url-https "url-http")
1443 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1445 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1447 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1452 ;;; url-http.el ends here