| blob | 5832e92c5a36a554edc18736db533fb5991ebc9f |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004-2016 Free Software Foundation, Inc.
5 ;; Author: Bill Perry <wmperry@gnu.org>
6 ;; Maintainer: emacs-devel@gnu.org
7 ;; Keywords: comm, data, processes
9 ;; This file is part of GNU Emacs.
10 ;;
11 ;; GNU Emacs is free software: you can redistribute it and/or modify
12 ;; it under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation, either version 3 of the License, or
14 ;; (at your option) any later version.
16 ;; GNU Emacs is distributed in the hope that it will be useful,
17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 ;; GNU General Public License for more details.
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
24 ;;; Commentary:
26 ;;; Code:
76 "What version of HTTP we advertise, as a string.
77 Valid values are 1.1 and 1.0.
78 This is only useful when debugging the HTTP subsystem.
80 Setting this to 1.0 will tell servers not to send chunked encoding,
84 "Whether to use a single TCP connection multiple times in HTTP.
85 This is only useful when debugging the HTTP subsystem. Setting to
86 nil will explicitly close the connection to the server after every
138 ;(eval-when-compile
139 ;; These are all macros so that they are hidden from external sight
140 ;; when the file is byte-compiled.
141 ;;
142 ;; This allows us to expose just the entry points we want.
144 ;; These routines will allow us to implement persistent HTTP
145 ;; connections.
149 ;; The user hit C-g, honor it! Some things can get in an
150 ;; incredibly tight loop (chunked encoding)
163 url-http-open-connections)
164 proc)
174 url-http-open-connections))
175 nil)
196 ;; `url-open-stream' needs a buffer in which to do things
197 ;; like authentication. But we use another buffer afterwards.
200 ;; url-open-stream might return nil.
202 ;; Drop the temp buffer link before killing the buffer.
205 ;; If there was an error on connect, make sure we don't
206 ;; get queried.
214 ;; Building an HTTP request
219 ""
222 url-user-agent)))
225 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
231 url-http-extra-headers))
233 nil
240 nil
243 proxy-info)
253 ;; Protection against stupid values in the referrer
258 ;; We do not want to expose the referrer if the user is paranoid.
264 ;; url-http-extra-headers contains an assoc-list of
265 ;; header/value pairs that we need to put into the request.
273 ;; This was done with a call to `format'. Concatenating parts has
274 ;; the advantage of keeping the parts of each header together and
275 ;; allows us to elide null lines directly, at the cost of making
276 ;; the layout less clear.
278 ;; We used to concat directly, but if one of the strings happens
279 ;; to being multibyte (even if it only contains pure ASCII) then
280 ;; every string gets converted with `string-MAKE-multibyte' which
281 ;; turns the 127-255 codes into things like latin-1 accented chars
282 ;; (it would work right if it used `string-TO-multibyte' instead).
283 ;; So to avoid the problem we force every string to be unibyte.
285 ;; FIXME: Instead of `string-AS-unibyte' we'd want
286 ;; `string-to-unibyte', so as to properly signal an error if one
287 ;; of the strings contains a multibyte char.
288 'string-as-unibyte
291 ;; The request
295 ;; Version of MIME we speak
297 ;; (maybe) Try to keep the connection open
301 ;; HTTP extensions we support
305 ;; Who we want to talk to
312 ;; Who its from
316 ;; Encodings we understand
318 ;; MS-Windows loads zlib dynamically, so recheck
319 ;; in case they made it available since
320 ;; initialization in url-vars.el.
330 ;; Languages we understand
334 ;; Types we understand
336 ;; User agent
338 ;; Proxy Authorization
339 proxy-auth
340 ;; Authorization
341 auth
342 ;; Cookies
345 host real-fname
347 ;; If-modified-since
354 ;; Whence we came
357 extra-headers
358 ;; Length of data
364 ;; End request
366 ;; Any data
367 url-http-data))
370 request))
372 ;; Parsing routines
375 This allows us to use `mail-fetch-field', etc.
376 Return the number of characters removed."
388 nil nil t))
394 url-http-proxy
395 url-current-object)))
397 ;; Cheating, but who cares? :)
398 'url-http-proxy-basic-auth-storage
400 auth
403 ;; find strongest supported auth
407 this-auth)))
411 this-auth)))
416 type this-type
417 strength this-strength))))
424 " authentication. If you'd like to write it,"
426 ;; We used to set a `status' var (declared "special") but I can't
427 ;; find the corresponding let-binding, so it's probably an error.
428 ;; FIXME: Maybe it was supposed to set `success', i.e. to return t?
429 ;; (setq status t)
435 type t args)))
437 t ;Success.
439 url-http-extra-headers)
444 url-callback-arguments))
448 "Parse just the response code."
463 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
464 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
465 work correctly."
472 ;;; (while cookies2
473 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
474 )
475 )
480 "Parse and handle HTTP specific headers.
481 Return t if and only if the current buffer is still active and
482 should be shown to the user."
483 ;; The comments after each status code handled are taken from RFC
484 ;; 2616 (HTTP/1.1)
487 url-http-process)
488 ;; Pass the https certificate on to the caller.
503 ;;(narrow-to-region (point-min) url-http-end-of-headers)
505 ;; In HTTP 1.0, keep the connection only if there is a
506 ;; "Connection: keep-alive" header.
507 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
508 ;; "Connection: close" header
521 ;; other status symbols: jewelry and luxury cars
524 class url-http-response-status)
529 ;; Classes of response codes
530 ;;
531 ;; 5xx = Server Error
532 ;; 4xx = Client Error
533 ;; 3xx = Redirection
534 ;; 2xx = Successful
535 ;; 1xx = Informational
537 ;; 100 = Continue with request
538 ;; 101 = Switching protocols
539 ;; 102 = Processing (Added by DAV)
542 url-http-response-status))
544 ;; 200 Ok
545 ;; 201 Created
546 ;; 202 Accepted
547 ;; 203 Non-authoritative information
548 ;; 204 No content
549 ;; 205 Reset content
550 ;; 206 Partial content
551 ;; 207 Multi-status (Added by DAV)
554 ;; No new data, just stay at the same document
557 ;; Generic success for all others. Store in the cache, and
558 ;; mark it as successful.
564 ;; 300 Multiple choices
565 ;; 301 Moved permanently
566 ;; 302 Found
567 ;; 303 See other
568 ;; 304 Not modified
569 ;; 305 Use proxy
570 ;; 307 Temporary redirect
575 ;; Quoth the spec (section 10.3.1)
576 ;; -------------------------------
577 ;; The requested resource corresponds to any one of a set of
578 ;; representations, each with its own specific location and
579 ;; agent-driven negotiation information is being provided so
580 ;; that the user can select a preferred representation and
581 ;; redirect its request to that location.
582 ;; [...]
583 ;; If the server has a preferred choice of representation, it
584 ;; SHOULD include the specific URI for that representation in
585 ;; the Location field; user agents MAY use the Location field
586 ;; value for automatic redirection.
587 ;; -------------------------------
588 ;; We do not support agent-driven negotiation, so we just
589 ;; redirect to the preferred URI if one is provided.
590 nil)
592 ;; If the 301|302 status code is received in response to a
593 ;; request other than GET or HEAD, the user agent MUST NOT
594 ;; automatically redirect the request unless it can be
595 ;; confirmed by the user, since this might change the
596 ;; conditions under which the request was issued.
600 ;; The response to the request can be found under a different
601 ;; URI and SHOULD be retrieved using a GET method on that
602 ;; resource.
604 url-http-data nil))
606 ;; The 304 response MUST NOT contain a message-body.
611 success t))
613 ;; The requested resource MUST be accessed through the
614 ;; proxy given by the Location field. The Location field
615 ;; gives the URI of the proxy. The recipient is expected
616 ;; to repeat this single request via the proxy. 305
617 ;; responses MUST only be generated by origin servers.
619 redirect-uri))
621 ;; Treat everything like '300'
622 nil))
624 ;; Clean off any whitespace and/or <...> cruft.
630 ;; Some stupid sites (like sourceforge) send a
631 ;; non-fully-qualified URL (ie: /), which royally confuses
632 ;; the URL library.
634 ;; Be careful to use the real target URL, otherwise we may
635 ;; compute the redirection relative to the URL of the proxy.
638 ;; Do not automatically include an authorization header in the
639 ;; redirect. If needed it will be regenerated by the relevant
640 ;; auth scheme when the new request happens.
647 ;; Check existing number of redirects
658 ;; url-max-redirections hasn't been reached, so go
659 ;; ahead and redirect.
661 ;; Remember that the request was redirected.
665 ;; Put in the current buffer a forwarding pointer to the new
666 ;; destination buffer.
667 ;; FIXME: This is a hack to fix url-retrieve-synchronously
668 ;; without changing the API. Instead url-retrieve should
669 ;; either simply not return the "destination" buffer, or it
670 ;; should take an optional `dest-buf' argument.
673 redirect-uri url-callback-function
674 url-callback-arguments
678 ;; We hit url-max-redirections, so issue an error and
679 ;; stop redirecting.
683 redirect-uri))
687 ;; 400 Bad Request
688 ;; 401 Unauthorized
689 ;; 402 Payment required
690 ;; 403 Forbidden
691 ;; 404 Not found
692 ;; 405 Method not allowed
693 ;; 406 Not acceptable
694 ;; 407 Proxy authentication required
695 ;; 408 Request time-out
696 ;; 409 Conflict
697 ;; 410 Gone
698 ;; 411 Length required
699 ;; 412 Precondition failed
700 ;; 413 Request entity too large
701 ;; 414 Request-URI too large
702 ;; 415 Unsupported media type
703 ;; 416 Requested range not satisfiable
704 ;; 417 Expectation failed
705 ;; 422 Unprocessable Entity (Added by DAV)
706 ;; 423 Locked
707 ;; 424 Failed Dependency
711 ;; The request requires user authentication. The response
712 ;; MUST include a WWW-Authenticate header field containing a
713 ;; challenge applicable to the requested resource. The
714 ;; client MAY repeat the request with a suitable
715 ;; Authorization header field.
718 ;; This code is reserved for future use
722 ;; The server understood the request, but is refusing to
723 ;; fulfill it. Authorization will not help and the request
724 ;; SHOULD NOT be repeated.
725 t)
727 ;; Not found
728 t)
730 ;; The method specified in the Request-Line is not allowed
731 ;; for the resource identified by the Request-URI. The
732 ;; response MUST include an Allow header containing a list of
733 ;; valid methods for the requested resource.
734 t)
736 ;; The resource identified by the request is only capable of
737 ;; generating response entities which have content
738 ;; characteristics not acceptable according to the accept
739 ;; headers sent in the request.
740 t)
742 ;; This code is similar to 401 (Unauthorized), but indicates
743 ;; that the client must first authenticate itself with the
744 ;; proxy. The proxy MUST return a Proxy-Authenticate header
745 ;; field containing a challenge applicable to the proxy for
746 ;; the requested resource.
749 ;; The client did not produce a request within the time that
750 ;; the server was prepared to wait. The client MAY repeat
751 ;; the request without modifications at any later time.
752 t)
754 ;; The request could not be completed due to a conflict with
755 ;; the current state of the resource. This code is only
756 ;; allowed in situations where it is expected that the user
757 ;; might be able to resolve the conflict and resubmit the
758 ;; request. The response body SHOULD include enough
759 ;; information for the user to recognize the source of the
760 ;; conflict.
761 t)
763 ;; The requested resource is no longer available at the
764 ;; server and no forwarding address is known.
765 t)
767 ;; The server refuses to accept the request without a defined
768 ;; Content-Length. The client MAY repeat the request if it
769 ;; adds a valid Content-Length header field containing the
770 ;; length of the message-body in the request message.
771 ;;
772 ;; NOTE - this will never happen because
773 ;; `url-http-create-request' automatically calculates the
774 ;; content-length.
775 t)
777 ;; The precondition given in one or more of the
778 ;; request-header fields evaluated to false when it was
779 ;; tested on the server.
780 t)
782 ;; The server is refusing to process a request because the
783 ;; request entity|URI is larger than the server is willing or
784 ;; able to process.
785 t)
787 ;; The server is refusing to service the request because the
788 ;; entity of the request is in a format not supported by the
789 ;; requested resource for the requested method.
790 t)
792 ;; A server SHOULD return a response with this status code if
793 ;; a request included a Range request-header field, and none
794 ;; of the range-specifier values in this field overlap the
795 ;; current extent of the selected resource, and the request
796 ;; did not include an If-Range request-header field.
797 t)
799 ;; The expectation given in an Expect request-header field
800 ;; could not be met by this server, or, if the server is a
801 ;; proxy, the server has unambiguous evidence that the
802 ;; request could not be met by the next-hop server.
803 t)
805 ;; The request could not be understood by the server due to
806 ;; malformed syntax. The client SHOULD NOT repeat the
807 ;; request without modifications.
808 t)))
809 ;; Tell the callback that an error occurred, and what the
810 ;; status code was.
816 ;; 500 Internal server error
817 ;; 501 Not implemented
818 ;; 502 Bad gateway
819 ;; 503 Service unavailable
820 ;; 504 Gateway time-out
821 ;; 505 HTTP version not supported
822 ;; 507 Insufficient storage
826 ;; The server does not support the functionality required to
827 ;; fulfill the request.
828 nil)
830 ;; The server, while acting as a gateway or proxy, received
831 ;; an invalid response from the upstream server it accessed
832 ;; in attempting to fulfill the request.
833 nil)
835 ;; The server is currently unable to handle the request due
836 ;; to a temporary overloading or maintenance of the server.
837 ;; The implication is that this is a temporary condition
838 ;; which will be alleviated after some delay. If known, the
839 ;; length of the delay MAY be indicated in a Retry-After
840 ;; header. If no Retry-After is given, the client SHOULD
841 ;; handle the response as it would for a 500 response.
842 nil)
844 ;; The server, while acting as a gateway or proxy, did not
845 ;; receive a timely response from the upstream server
846 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
847 ;; auxiliary server (e.g. DNS) it needed to access in
848 ;; attempting to complete the request.
849 nil)
851 ;; The server does not support, or refuses to support, the
852 ;; HTTP protocol version that was used in the request
853 ;; message.
854 nil)
856 ;; The method could not be performed on the resource
857 ;; because the server is unable to store the representation
858 ;; needed to successfully complete the request. This
859 ;; condition is considered to be temporary. If the request
860 ;; which received this status code was the result of a user
861 ;; action, the request MUST NOT be repeated until it is
862 ;; requested by a separate user action.
863 nil))
864 ;; Tell the callback that an error occurred, and what the
865 ;; status code was.
872 class url-http-response-status)))
879 success))
895 ;; Miscellaneous
897 "Activate callback specified when this buffer was created."
900 url-http-process)
905 ;; )
907 ;; These unfortunately cannot be macros... please ignore them!
909 "Remove (now defunct) process PROC from the list of open connections."
913 url-http-open-connections))
916 ;; Sentinel used to handle (i) terminated old HTTP/0.9 connections,
917 ;; and (ii) closed connection due to reusing a HTTP connection which
918 ;; we believed was still alive, but which the server closed on us.
919 ;; We handle case (ii) by calling `url-http' again.
928 ;; HTTP/0.9 just gets passed back no matter what
930 ;; Call `url-http' again if our connection expired.
941 ;; Function used when we do NOT know how long the document is going to be
942 ;; Just _very_ simple 'downloaded %d' type of info.
946 "Function used when we DO know how long the document is going to be.
947 More sophisticated percentage downloaded, etc.
948 Also does minimal parsing of HTTP headers and will actually cause
949 the callback to be triggered."
952 "Reading [%s]... %s of %s (%d%%)"
954 url-http-content-length)
955 url-http-content-type
959 url-http-content-length))
961 "Reading... %s of %s (%d%%)"
963 url-http-content-length)
967 url-http-content-length)))
971 ;; Found the end of the document! Wheee!
978 "Function used when dealing with chunked encoding.
979 Cannot give a sophisticated percentage, but we need a different
980 function to look for the special 0-length chunk that signifies
981 the end of the document."
988 ;; We need to loop thru looking for more chunks even within
989 ;; one after-change-function call.
994 "Reading [%s]... chunk #%d"
995 url-http-content-type url-http-chunked-counter)
997 "Reading... chunk #%d"
998 url-http-chunked-counter))
1000 url-http-chunked-counter st nd length)
1006 ;; We know how long the chunk is supposed to be, skip over
1007 ;; leading crap if possible.
1011 url-http-chunked-counter)
1013 url-http-chunked-length)))
1016 url-http-chunked-length)
1017 nd))
1024 ;; Must not have received the entirety of the chunk header,
1025 ;; need to spin some more.
1030 'end-open t
1031 'chunked-encoding t
1043 ; (if (not url-http-debug)
1046 url-http-chunked-counter url-http-chunked-length
1050 ;; Found the end of the document! Wheee!
1054 ;; Every chunk, even the last 0-length one, is
1055 ;; terminated by CRLF. Skip it.
1065 ;; This will wait for the headers to arrive and then splice in the
1066 ;; next appropriate after-change-function, etc.
1077 ;; Not HTTP/x.y data, must be 0.9
1078 ;; God, I wish this could die.
1080 url-http-end-of-headers 0
1081 old-http t)
1082 ;; Blank line at end of headers.
1085 ;; Saw the end of the headers
1089 end-of-headers t)
1093 ;; Haven't seen the end of the headers yet, need to wait
1094 ;; for more data to arrive.
1095 nil
1112 ;; We got back a headerless malformed response from the
1113 ;; server.
1122 ;; A HEAD request is _ALWAYS_ terminated by the header
1123 ;; information, regardless of any entity headers,
1124 ;; according to section 4.4 of the HTTP/1.1 draft.
1130 ;; A CONNECT request is finished, but we cannot stick this
1131 ;; back on the free connection list
1136 ;; Only allowed to have a header section. We have to handle
1137 ;; this here instead of in url-http-parse-headers because if
1138 ;; you have a cached copy of something without a known
1139 ;; content-length, and try to retrieve it from the cache, we'd
1140 ;; fall into the 'being dumb' section and wait for the
1141 ;; connection to terminate, which means we'd wait for 10
1142 ;; seconds for the keep-alives to time out on some servers.
1146 ;; HTTP/0.9 always signaled end-of-connection by closing the
1147 ;; connection.
1169 ;; We got a NULL body! Activate the callback
1170 ;; immediately!
1176 ;; Have some leftover data
1180 nd
1183 nil)))
1188 ;; We are still at the beginning of the buffer... must just be
1189 ;; waiting for a response.
1195 "Retrieve URL via HTTP asynchronously.
1196 URL must be a parsed URL. See `url-generic-parse-url' for details.
1198 When retrieval is completed, execute the function CALLBACK, passing it
1199 an updated value of CBARGS as arguments. The first element in CBARGS
1200 should be a plist describing what has happened so far during the
1201 request, as described in the docstring of `url-retrieve' (if in
1202 doubt, specify nil).
1204 Optional arg RETRY-BUFFER, if non-nil, specifies the buffer of a
1205 previous `url-http' call, which is being re-attempted.
1207 Optional arg GATEWAY-METHOD specifies the gateway to be used,
1208 overriding the value of `url-gateway-method'."
1214 url-http-noninteractive)))
1221 ;; Failed to open the connection for some reason
1232 url-http-content-type
1233 url-http-content-length
1234 url-http-transfer-encoding
1235 url-http-after-change-function
1236 url-http-response-version
1237 url-http-response-status
1238 url-http-chunked-length
1239 url-http-chunked-counter
1240 url-http-chunked-start
1241 url-callback-function
1242 url-callback-arguments
1243 url-show-status
1244 url-http-process
1245 url-http-method
1246 url-http-extra-headers
1247 url-http-noninteractive
1248 url-http-data
1249 url-http-target-url
1250 url-http-no-retry
1251 url-http-connection-opened
1252 url-mime-accept-string
1253 url-http-proxy))
1257 url-http-extra-headers url-request-extra-headers
1258 url-http-noninteractive url-request-noninteractive
1259 url-http-data url-request-data
1260 url-http-process connection
1261 url-http-chunked-length nil
1262 url-http-chunked-start nil
1263 url-http-chunked-counter 0
1264 url-callback-function callback
1265 url-callback-arguments cbargs
1266 url-http-after-change-function 'url-http-wait-for-headers-change-function
1267 url-http-target-url url-current-object
1268 url-http-no-retry retry-buffer
1269 url-http-connection-opened nil
1270 url-mime-accept-string mime-accept-string
1271 url-http-proxy url-using-proxy)
1276 (`connect
1277 ;; Asynchronous connection
1279 (`failed
1280 ;; Asynchronous connection failed
1286 buffer))
1289 ;; We are performing an asynchronous connection, and a status change
1290 ;; has occurred.
1312 ;; Since Emacs 19/20 does not allow you to change the
1313 ;; `after-change-functions' hook in the midst of running them, we fake
1314 ;; an after change by hooking into the process filter and inserting
1315 ;; the data ourselves. This is slightly less efficient, but there
1316 ;; were tons of weird ways the after-change code was biting us in the
1317 ;; shorts.
1318 ;; FIXME this can probably be simplified since the above is no longer true.
1320 ;; Sometimes we get a zero-length data chunk after the process has
1321 ;; been changed to 'free', which means it has no buffer associated
1322 ;; with it. Do nothing if there is no buffer, or 0 length data.
1326 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1335 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1336 ;;; file-name-handler stuff from here on out
1337 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1340 'symbol-value-in-buffer
1342 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1345 unbound-value
1370 nil ;dir / link / normal file
1373 nil nil nil ;atime ; mtime ; ctime
1388 "Return a property list describing options available for URL.
1389 This list is retrieved using the `OPTIONS' HTTP method.
1391 Property list members:
1393 methods
1394 A list of symbols specifying what HTTP methods the resource
1395 supports.
1397 dav
1398 A list of numbers specifying what DAV protocol/schema versions are
1399 supported.
1401 dasl
1402 A list of supported DASL search types supported (string form)
1404 ranges
1405 A list of the units available for use in partial document fetches.
1407 p3p
1408 The `Platform For Privacy Protection' description for the resource.
1409 Currently this is just the raw header contents. This is likely to
1410 change once P3P is formally supported by the URL package or
1411 Emacs/W3."
1419 ;; Only parse the options if we got a 2xx response code!
1425 ;; Figure out what methods are supported.
1428 options 'methods
1431 ;; Check for DAV
1434 options 'dav
1439 ;; Now for DASL
1442 options 'dasl
1445 ;; P3P - should get more detailed here. FIXME
1449 ;; Check for whether they accept byte-range requests.
1452 options 'ranges
1456 ))))
1458 options))
1460 ;; HTTPS. This used to be in url-https.el, but that file collides
1461 ;; with url-http.el on systems with 8-character file names.
1467 ;; FIXME what is the point of this alias being an autoload?
1468 ;; Trying to use it will not cause url-http to be loaded,
1469 ;; since the full alias just gets dumped into loaddefs.el.
1471 ;;;###autoload (autoload 'url-default-expander "url-expand")
1472 ;;;###autoload
1481 ;;;###autoload (autoload 'url-https "url-http")
1483 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1485 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1487 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1492 ;;; url-http.el ends here