| blob | b0a3b688a55d751bd568f30134ed0e98b2bbdae0 |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004-2014 Free Software Foundation, Inc.
5 ;; Author: Bill Perry <wmperry@gnu.org>
6 ;; Keywords: comm, data, processes
8 ;; This file is part of GNU Emacs.
9 ;;
10 ;; GNU Emacs is free software: you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation, either version 3 of the License, or
13 ;; (at your option) any later version.
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
23 ;;; Commentary:
25 ;;; Code:
72 "What version of HTTP we advertise, as a string.
73 Valid values are 1.1 and 1.0.
74 This is only useful when debugging the HTTP subsystem.
76 Setting this to 1.0 will tell servers not to send chunked encoding,
80 "Whether to use a single TCP connection multiple times in HTTP.
81 This is only useful when debugging the HTTP subsystem. Setting to
82 nil will explicitly close the connection to the server after every
134 ;(eval-when-compile
135 ;; These are all macros so that they are hidden from external sight
136 ;; when the file is byte-compiled.
137 ;;
138 ;; This allows us to expose just the entry points we want.
140 ;; These routines will allow us to implement persistent HTTP
141 ;; connections.
145 ;; The user hit C-g, honor it! Some things can get in an
146 ;; incredibly tight loop (chunked encoding)
159 url-http-open-connections)
160 proc)
170 url-http-open-connections))
171 nil)
192 ;; `url-open-stream' needs a buffer in which to do things
193 ;; like authentication. But we use another buffer afterwards.
196 ;; url-open-stream might return nil.
198 ;; Drop the temp buffer link before killing the buffer.
201 ;; If there was an error on connect, make sure we don't
202 ;; get queried.
210 ;; Building an HTTP request
215 ""
220 url-version)))
223 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
229 url-http-extra-headers))
231 nil
238 nil
241 proxy-info)
251 ;; Protection against stupid values in the referrer
256 ;; We do not want to expose the referrer if the user is paranoid.
262 ;; url-http-extra-headers contains an assoc-list of
263 ;; header/value pairs that we need to put into the request.
271 ;; This was done with a call to `format'. Concatenating parts has
272 ;; the advantage of keeping the parts of each header together and
273 ;; allows us to elide null lines directly, at the cost of making
274 ;; the layout less clear.
276 ;; We used to concat directly, but if one of the strings happens
277 ;; to being multibyte (even if it only contains pure ASCII) then
278 ;; every string gets converted with `string-MAKE-multibyte' which
279 ;; turns the 127-255 codes into things like latin-1 accented chars
280 ;; (it would work right if it used `string-TO-multibyte' instead).
281 ;; So to avoid the problem we force every string to be unibyte.
283 ;; FIXME: Instead of `string-AS-unibyte' we'd want
284 ;; `string-to-unibyte', so as to properly signal an error if one
285 ;; of the strings contains a multibyte char.
286 'string-as-unibyte
289 ;; The request
293 ;; Version of MIME we speak
295 ;; (maybe) Try to keep the connection open
299 ;; HTTP extensions we support
303 ;; Who we want to talk to
310 ;; Who its from
314 ;; Encodings we understand
321 ;; Languages we understand
325 ;; Types we understand
327 ;; User agent
329 ;; Proxy Authorization
330 proxy-auth
331 ;; Authorization
332 auth
333 ;; Cookies
336 host real-fname
338 ;; If-modified-since
345 ;; Whence we came
348 extra-headers
349 ;; Length of data
355 ;; End request
357 ;; Any data
358 url-http-data))
361 request))
363 ;; Parsing routines
366 This allows us to use `mail-fetch-field', etc.
367 Return the number of characters removed."
379 nil nil t))
385 url-http-proxy
386 url-current-object)))
388 ;; Cheating, but who cares? :)
389 'url-http-proxy-basic-auth-storage
391 auth
394 ;; find strongest supported auth
398 this-auth)))
402 this-auth)))
407 type this-type
408 strength this-strength))))
415 " authentication. If you'd like to write it,"
417 ;; We used to set a `status' var (declared "special") but I can't
418 ;; find the corresponding let-binding, so it's probably an error.
419 ;; FIXME: Maybe it was supposed to set `success', i.e. to return t?
420 ;; (setq status t)
426 type t args)))
428 t ;Success.
430 url-http-extra-headers)
435 url-callback-arguments))
439 "Parse just the response code."
454 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
455 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
456 work correctly."
463 ;;; (while cookies2
464 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
465 )
466 )
469 "Parse and handle HTTP specific headers.
470 Return t if and only if the current buffer is still active and
471 should be shown to the user."
472 ;; The comments after each status code handled are taken from RFC
473 ;; 2616 (HTTP/1.1)
476 url-http-process)
485 ;;(narrow-to-region (point-min) url-http-end-of-headers)
487 ;; In HTTP 1.0, keep the connection only if there is a
488 ;; "Connection: keep-alive" header.
489 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
490 ;; "Connection: close" header
503 ;; other status symbols: jewelry and luxury cars
506 class url-http-response-status)
511 ;; Classes of response codes
512 ;;
513 ;; 5xx = Server Error
514 ;; 4xx = Client Error
515 ;; 3xx = Redirection
516 ;; 2xx = Successful
517 ;; 1xx = Informational
519 ;; 100 = Continue with request
520 ;; 101 = Switching protocols
521 ;; 102 = Processing (Added by DAV)
524 url-http-response-status))
526 ;; 200 Ok
527 ;; 201 Created
528 ;; 202 Accepted
529 ;; 203 Non-authoritative information
530 ;; 204 No content
531 ;; 205 Reset content
532 ;; 206 Partial content
533 ;; 207 Multi-status (Added by DAV)
536 ;; No new data, just stay at the same document
539 ;; Generic success for all others. Store in the cache, and
540 ;; mark it as successful.
546 ;; 300 Multiple choices
547 ;; 301 Moved permanently
548 ;; 302 Found
549 ;; 303 See other
550 ;; 304 Not modified
551 ;; 305 Use proxy
552 ;; 307 Temporary redirect
557 ;; Quoth the spec (section 10.3.1)
558 ;; -------------------------------
559 ;; The requested resource corresponds to any one of a set of
560 ;; representations, each with its own specific location and
561 ;; agent-driven negotiation information is being provided so
562 ;; that the user can select a preferred representation and
563 ;; redirect its request to that location.
564 ;; [...]
565 ;; If the server has a preferred choice of representation, it
566 ;; SHOULD include the specific URI for that representation in
567 ;; the Location field; user agents MAY use the Location field
568 ;; value for automatic redirection.
569 ;; -------------------------------
570 ;; We do not support agent-driven negotiation, so we just
571 ;; redirect to the preferred URI if one is provided.
572 nil)
574 ;; If the 301|302 status code is received in response to a
575 ;; request other than GET or HEAD, the user agent MUST NOT
576 ;; automatically redirect the request unless it can be
577 ;; confirmed by the user, since this might change the
578 ;; conditions under which the request was issued.
582 ;; The response to the request can be found under a different
583 ;; URI and SHOULD be retrieved using a GET method on that
584 ;; resource.
586 url-http-data nil))
588 ;; The 304 response MUST NOT contain a message-body.
593 success t))
595 ;; The requested resource MUST be accessed through the
596 ;; proxy given by the Location field. The Location field
597 ;; gives the URI of the proxy. The recipient is expected
598 ;; to repeat this single request via the proxy. 305
599 ;; responses MUST only be generated by origin servers.
601 redirect-uri))
603 ;; Treat everything like '300'
604 nil))
606 ;; Clean off any whitespace and/or <...> cruft.
612 ;; Some stupid sites (like sourceforge) send a
613 ;; non-fully-qualified URL (ie: /), which royally confuses
614 ;; the URL library.
616 ;; Be careful to use the real target URL, otherwise we may
617 ;; compute the redirection relative to the URL of the proxy.
623 ;; Check existing number of redirects
634 ;; url-max-redirections hasn't been reached, so go
635 ;; ahead and redirect.
637 ;; Remember that the request was redirected.
641 ;; Put in the current buffer a forwarding pointer to the new
642 ;; destination buffer.
643 ;; FIXME: This is a hack to fix url-retrieve-synchronously
644 ;; without changing the API. Instead url-retrieve should
645 ;; either simply not return the "destination" buffer, or it
646 ;; should take an optional `dest-buf' argument.
649 redirect-uri url-callback-function
650 url-callback-arguments
654 ;; We hit url-max-redirections, so issue an error and
655 ;; stop redirecting.
659 redirect-uri))
663 ;; 400 Bad Request
664 ;; 401 Unauthorized
665 ;; 402 Payment required
666 ;; 403 Forbidden
667 ;; 404 Not found
668 ;; 405 Method not allowed
669 ;; 406 Not acceptable
670 ;; 407 Proxy authentication required
671 ;; 408 Request time-out
672 ;; 409 Conflict
673 ;; 410 Gone
674 ;; 411 Length required
675 ;; 412 Precondition failed
676 ;; 413 Request entity too large
677 ;; 414 Request-URI too large
678 ;; 415 Unsupported media type
679 ;; 416 Requested range not satisfiable
680 ;; 417 Expectation failed
681 ;; 422 Unprocessable Entity (Added by DAV)
682 ;; 423 Locked
683 ;; 424 Failed Dependency
687 ;; The request requires user authentication. The response
688 ;; MUST include a WWW-Authenticate header field containing a
689 ;; challenge applicable to the requested resource. The
690 ;; client MAY repeat the request with a suitable
691 ;; Authorization header field.
694 ;; This code is reserved for future use
698 ;; The server understood the request, but is refusing to
699 ;; fulfill it. Authorization will not help and the request
700 ;; SHOULD NOT be repeated.
701 t)
703 ;; Not found
704 t)
706 ;; The method specified in the Request-Line is not allowed
707 ;; for the resource identified by the Request-URI. The
708 ;; response MUST include an Allow header containing a list of
709 ;; valid methods for the requested resource.
710 t)
712 ;; The resource identified by the request is only capable of
713 ;; generating response entities which have content
714 ;; characteristics not acceptable according to the accept
715 ;; headers sent in the request.
716 t)
718 ;; This code is similar to 401 (Unauthorized), but indicates
719 ;; that the client must first authenticate itself with the
720 ;; proxy. The proxy MUST return a Proxy-Authenticate header
721 ;; field containing a challenge applicable to the proxy for
722 ;; the requested resource.
725 ;; The client did not produce a request within the time that
726 ;; the server was prepared to wait. The client MAY repeat
727 ;; the request without modifications at any later time.
728 t)
730 ;; The request could not be completed due to a conflict with
731 ;; the current state of the resource. This code is only
732 ;; allowed in situations where it is expected that the user
733 ;; might be able to resolve the conflict and resubmit the
734 ;; request. The response body SHOULD include enough
735 ;; information for the user to recognize the source of the
736 ;; conflict.
737 t)
739 ;; The requested resource is no longer available at the
740 ;; server and no forwarding address is known.
741 t)
743 ;; The server refuses to accept the request without a defined
744 ;; Content-Length. The client MAY repeat the request if it
745 ;; adds a valid Content-Length header field containing the
746 ;; length of the message-body in the request message.
747 ;;
748 ;; NOTE - this will never happen because
749 ;; `url-http-create-request' automatically calculates the
750 ;; content-length.
751 t)
753 ;; The precondition given in one or more of the
754 ;; request-header fields evaluated to false when it was
755 ;; tested on the server.
756 t)
758 ;; The server is refusing to process a request because the
759 ;; request entity|URI is larger than the server is willing or
760 ;; able to process.
761 t)
763 ;; The server is refusing to service the request because the
764 ;; entity of the request is in a format not supported by the
765 ;; requested resource for the requested method.
766 t)
768 ;; A server SHOULD return a response with this status code if
769 ;; a request included a Range request-header field, and none
770 ;; of the range-specifier values in this field overlap the
771 ;; current extent of the selected resource, and the request
772 ;; did not include an If-Range request-header field.
773 t)
775 ;; The expectation given in an Expect request-header field
776 ;; could not be met by this server, or, if the server is a
777 ;; proxy, the server has unambiguous evidence that the
778 ;; request could not be met by the next-hop server.
779 t)
781 ;; The request could not be understood by the server due to
782 ;; malformed syntax. The client SHOULD NOT repeat the
783 ;; request without modifications.
784 t)))
785 ;; Tell the callback that an error occurred, and what the
786 ;; status code was.
792 ;; 500 Internal server error
793 ;; 501 Not implemented
794 ;; 502 Bad gateway
795 ;; 503 Service unavailable
796 ;; 504 Gateway time-out
797 ;; 505 HTTP version not supported
798 ;; 507 Insufficient storage
802 ;; The server does not support the functionality required to
803 ;; fulfill the request.
804 nil)
806 ;; The server, while acting as a gateway or proxy, received
807 ;; an invalid response from the upstream server it accessed
808 ;; in attempting to fulfill the request.
809 nil)
811 ;; The server is currently unable to handle the request due
812 ;; to a temporary overloading or maintenance of the server.
813 ;; The implication is that this is a temporary condition
814 ;; which will be alleviated after some delay. If known, the
815 ;; length of the delay MAY be indicated in a Retry-After
816 ;; header. If no Retry-After is given, the client SHOULD
817 ;; handle the response as it would for a 500 response.
818 nil)
820 ;; The server, while acting as a gateway or proxy, did not
821 ;; receive a timely response from the upstream server
822 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
823 ;; auxiliary server (e.g. DNS) it needed to access in
824 ;; attempting to complete the request.
825 nil)
827 ;; The server does not support, or refuses to support, the
828 ;; HTTP protocol version that was used in the request
829 ;; message.
830 nil)
832 ;; The method could not be performed on the resource
833 ;; because the server is unable to store the representation
834 ;; needed to successfully complete the request. This
835 ;; condition is considered to be temporary. If the request
836 ;; which received this status code was the result of a user
837 ;; action, the request MUST NOT be repeated until it is
838 ;; requested by a separate user action.
839 nil))
840 ;; Tell the callback that an error occurred, and what the
841 ;; status code was.
848 class url-http-response-status)))
855 success))
871 ;; Miscellaneous
873 "Activate callback specified when this buffer was created."
876 url-http-process)
880 ;; )
882 ;; These unfortunately cannot be macros... please ignore them!
884 "Remove (now defunct) process PROC from the list of open connections."
888 url-http-open-connections))
891 ;; Sentinel used to handle (i) terminated old HTTP/0.9 connections,
892 ;; and (ii) closed connection due to reusing a HTTP connection which
893 ;; we believed was still alive, but which the server closed on us.
894 ;; We handle case (ii) by calling `url-http' again.
903 ;; HTTP/0.9 just gets passed back no matter what
905 ;; Call `url-http' again if our connection expired.
916 ;; Function used when we do NOT know how long the document is going to be
917 ;; Just _very_ simple 'downloaded %d' type of info.
921 "Function used when we DO know how long the document is going to be.
922 More sophisticated percentage downloaded, etc.
923 Also does minimal parsing of HTTP headers and will actually cause
924 the callback to be triggered."
927 "Reading [%s]... %s of %s (%d%%)"
929 url-http-content-length)
930 url-http-content-type
934 url-http-content-length))
936 "Reading... %s of %s (%d%%)"
938 url-http-content-length)
942 url-http-content-length)))
946 ;; Found the end of the document! Wheee!
953 "Function used when dealing with 'chunked' encoding.
954 Cannot give a sophisticated percentage, but we need a different
955 function to look for the special 0-length chunk that signifies
956 the end of the document."
963 ;; We need to loop thru looking for more chunks even within
964 ;; one after-change-function call.
969 "Reading [%s]... chunk #%d"
970 url-http-content-type url-http-chunked-counter)
972 "Reading... chunk #%d"
973 url-http-chunked-counter))
975 url-http-chunked-counter st nd length)
981 ;; We know how long the chunk is supposed to be, skip over
982 ;; leading crap if possible.
986 url-http-chunked-counter)
988 url-http-chunked-length)))
991 url-http-chunked-length)
992 nd))
999 ;; Must not have received the entirety of the chunk header,
1000 ;; need to spin some more.
1005 'end-open t
1006 'chunked-encoding t
1018 ; (if (not url-http-debug)
1021 url-http-chunked-counter url-http-chunked-length
1025 ;; Found the end of the document! Wheee!
1029 ;; Every chunk, even the last 0-length one, is
1030 ;; terminated by CRLF. Skip it.
1040 ;; This will wait for the headers to arrive and then splice in the
1041 ;; next appropriate after-change-function, etc.
1052 ;; Not HTTP/x.y data, must be 0.9
1053 ;; God, I wish this could die.
1055 url-http-end-of-headers 0
1056 old-http t)
1057 ;; Blank line at end of headers.
1060 ;; Saw the end of the headers
1064 end-of-headers t)
1068 ;; Haven't seen the end of the headers yet, need to wait
1069 ;; for more data to arrive.
1070 nil
1087 ;; We got back a headerless malformed response from the
1088 ;; server.
1097 ;; A HEAD request is _ALWAYS_ terminated by the header
1098 ;; information, regardless of any entity headers,
1099 ;; according to section 4.4 of the HTTP/1.1 draft.
1105 ;; A CONNECT request is finished, but we cannot stick this
1106 ;; back on the free connection list
1111 ;; Only allowed to have a header section. We have to handle
1112 ;; this here instead of in url-http-parse-headers because if
1113 ;; you have a cached copy of something without a known
1114 ;; content-length, and try to retrieve it from the cache, we'd
1115 ;; fall into the 'being dumb' section and wait for the
1116 ;; connection to terminate, which means we'd wait for 10
1117 ;; seconds for the keep-alives to time out on some servers.
1121 ;; HTTP/0.9 always signaled end-of-connection by closing the
1122 ;; connection.
1144 ;; We got a NULL body! Activate the callback
1145 ;; immediately!
1151 ;; Have some leftover data
1155 nd
1158 nil)))
1163 ;; We are still at the beginning of the buffer... must just be
1164 ;; waiting for a response.
1170 "Retrieve URL via HTTP asynchronously.
1171 URL must be a parsed URL. See `url-generic-parse-url' for details.
1173 When retrieval is completed, execute the function CALLBACK, passing it
1174 an updated value of CBARGS as arguments. The first element in CBARGS
1175 should be a plist describing what has happened so far during the
1176 request, as described in the docstring of `url-retrieve' (if in
1177 doubt, specify nil).
1179 Optional arg RETRY-BUFFER, if non-nil, specifies the buffer of a
1180 previous `url-http' call, which is being re-attempted."
1189 ;; Failed to open the connection for some reason
1200 url-http-content-type
1201 url-http-content-length
1202 url-http-transfer-encoding
1203 url-http-after-change-function
1204 url-http-response-version
1205 url-http-response-status
1206 url-http-chunked-length
1207 url-http-chunked-counter
1208 url-http-chunked-start
1209 url-callback-function
1210 url-callback-arguments
1211 url-show-status
1212 url-http-process
1213 url-http-method
1214 url-http-extra-headers
1215 url-http-data
1216 url-http-target-url
1217 url-http-no-retry
1218 url-http-connection-opened
1219 url-http-proxy))
1223 url-http-extra-headers url-request-extra-headers
1224 url-http-data url-request-data
1225 url-http-process connection
1226 url-http-chunked-length nil
1227 url-http-chunked-start nil
1228 url-http-chunked-counter 0
1229 url-callback-function callback
1230 url-callback-arguments cbargs
1231 url-http-after-change-function 'url-http-wait-for-headers-change-function
1232 url-http-target-url url-current-object
1233 url-http-no-retry retry-buffer
1234 url-http-connection-opened nil
1235 url-http-proxy url-using-proxy)
1240 (`connect
1241 ;; Asynchronous connection
1243 (`failed
1244 ;; Asynchronous connection failed
1250 buffer))
1253 ;; We are performing an asynchronous connection, and a status change
1254 ;; has occurred.
1276 ;; Since Emacs 19/20 does not allow you to change the
1277 ;; `after-change-functions' hook in the midst of running them, we fake
1278 ;; an after change by hooking into the process filter and inserting
1279 ;; the data ourselves. This is slightly less efficient, but there
1280 ;; were tons of weird ways the after-change code was biting us in the
1281 ;; shorts.
1282 ;; FIXME this can probably be simplified since the above is no longer true.
1284 ;; Sometimes we get a zero-length data chunk after the process has
1285 ;; been changed to 'free', which means it has no buffer associated
1286 ;; with it. Do nothing if there is no buffer, or 0 length data.
1290 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1299 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1300 ;;; file-name-handler stuff from here on out
1301 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1304 'symbol-value-in-buffer
1306 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1309 unbound-value
1334 nil ;dir / link / normal file
1337 nil nil nil ;atime ; mtime ; ctime
1352 "Return a property list describing options available for URL.
1353 This list is retrieved using the `OPTIONS' HTTP method.
1355 Property list members:
1357 methods
1358 A list of symbols specifying what HTTP methods the resource
1359 supports.
1361 dav
1362 A list of numbers specifying what DAV protocol/schema versions are
1363 supported.
1365 dasl
1366 A list of supported DASL search types supported (string form)
1368 ranges
1369 A list of the units available for use in partial document fetches.
1371 p3p
1372 The `Platform For Privacy Protection' description for the resource.
1373 Currently this is just the raw header contents. This is likely to
1374 change once P3P is formally supported by the URL package or
1375 Emacs/W3."
1383 ;; Only parse the options if we got a 2xx response code!
1389 ;; Figure out what methods are supported.
1392 options 'methods
1395 ;; Check for DAV
1398 options 'dav
1403 ;; Now for DASL
1406 options 'dasl
1409 ;; P3P - should get more detailed here. FIXME
1413 ;; Check for whether they accept byte-range requests.
1416 options 'ranges
1420 ))))
1422 options))
1424 ;; HTTPS. This used to be in url-https.el, but that file collides
1425 ;; with url-http.el on systems with 8-character file names.
1431 ;; FIXME what is the point of this alias being an autoload?
1432 ;; Trying to use it will not cause url-http to be loaded,
1433 ;; since the full alias just gets dumped into loaddefs.el.
1435 ;;;###autoload (autoload 'url-default-expander "url-expand")
1436 ;;;###autoload
1446 ;;;###autoload (autoload 'url-https "url-http")
1448 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1450 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1452 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1457 ;;; url-http.el ends here