| blob | aed0efab01a22c3ae73c5658969042860d6ea020 |
1 ;;; url-http.el --- HTTP retrieval routines -*- lexical-binding:t -*-
3 ;; Copyright (C) 1999, 2001, 2004-2018 Free Software Foundation, Inc.
5 ;; Author: Bill Perry <wmperry@gnu.org>
6 ;; Maintainer: emacs-devel@gnu.org
7 ;; Keywords: comm, data, processes
9 ;; This file is part of GNU Emacs.
10 ;;
11 ;; GNU Emacs is free software: you can redistribute it and/or modify
12 ;; it under the terms of the GNU General Public License as published by
13 ;; the Free Software Foundation, either version 3 of the License, or
14 ;; (at your option) any later version.
16 ;; GNU Emacs is distributed in the hope that it will be useful,
17 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
18 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 ;; GNU General Public License for more details.
21 ;; You should have received a copy of the GNU General Public License
22 ;; along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>.
24 ;;; Commentary:
26 ;;; Code:
78 "What version of HTTP we advertise, as a string.
79 Valid values are 1.1 and 1.0.
80 This is only useful when debugging the HTTP subsystem.
82 Setting this to 1.0 will tell servers not to send chunked encoding,
86 "Whether to use a single TCP connection multiple times in HTTP.
87 This is only useful when debugging the HTTP subsystem. Setting to
88 nil will explicitly close the connection to the server after every
143 ;(eval-when-compile
144 ;; These are all macros so that they are hidden from external sight
145 ;; when the file is byte-compiled.
146 ;;
147 ;; This allows us to expose just the entry points we want.
149 ;; These routines will allow us to implement persistent HTTP
150 ;; connections.
154 ;; The user hit C-g, honor it! Some things can get in an
155 ;; incredibly tight loop (chunked encoding)
168 url-http-open-connections)
169 proc)
179 url-http-open-connections))
180 nil)
201 ;; `url-open-stream' needs a buffer in which to do things
202 ;; like authentication. But we use another buffer afterwards.
207 host)
210 port)
211 gateway-method)))
212 ;; url-open-stream might return nil.
214 ;; Drop the temp buffer link before killing the buffer.
217 ;; If there was an error on connect, make sure we don't
218 ;; get queried.
227 "Compute a default User-Agent string based on `url-privacy-level'."
238 emacs-info os-info))
241 ;; Building an HTTP request
243 "Compute a User-Agent string.
244 The string is based on `url-privacy-level' and `url-user-agent'."
258 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
264 url-http-extra-headers))
266 nil
273 nil
276 proxy-info)
286 ;; Protection against stupid values in the referrer
291 ;; We do not want to expose the referrer if the user is paranoid.
297 ;; url-http-extra-headers contains an assoc-list of
298 ;; header/value pairs that we need to put into the request.
306 ;; This was done with a call to `format'. Concatenating parts has
307 ;; the advantage of keeping the parts of each header together and
308 ;; allows us to elide null lines directly, at the cost of making
309 ;; the layout less clear.
312 ;; The request
317 ;; Version of MIME we speak
319 ;; (maybe) Try to keep the connection open
323 ;; HTTP extensions we support
327 ;; Who we want to talk to
335 ;; Who its from
339 ;; Encodings we understand
341 ;; MS-Windows loads zlib dynamically, so recheck
342 ;; in case they made it available since
343 ;; initialization in url-vars.el.
352 "Accept-charset: "
355 ;; Languages we understand
359 ;; Types we understand
361 ;; User agent
363 ;; Proxy Authorization
364 proxy-auth
365 ;; Authorization
366 auth
367 ;; Cookies
371 host real-fname
373 ;; If-modified-since
380 ;; Whence we came
383 extra-headers
384 ;; Length of data
390 ;; End request
392 ;; Any data
393 url-http-data))
394 ;; Bug#23750
399 request))
404 s))
406 ;; Parsing routines
409 This allows us to use `mail-fetch-field', etc.
410 Return the number of characters removed."
422 nil nil t))
428 url-http-proxy
429 url-current-object)))
431 ;; Cheating, but who cares? :)
432 'url-http-proxy-basic-auth-storage
434 auth
437 ;; find strongest supported auth
441 this-auth)))
445 this-auth)))
450 type this-type
451 strength this-strength))))
458 " authentication. If you'd like to write it,"
460 ;; We used to set a `status' var (declared "special") but I can't
461 ;; find the corresponding let-binding, so it's probably an error.
462 ;; FIXME: Maybe it was supposed to set `success', i.e. to return t?
463 ;; (setq status t)
469 type t args)))
471 t ;Success.
473 url-http-extra-headers)
478 url-callback-arguments))
482 "Parse just the response code."
497 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
498 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
499 work correctly."
506 ;;; (while cookies2
507 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
508 )
509 )
515 "Parse and handle HTTP specific headers.
516 Return t if and only if the current buffer is still active and
517 should be shown to the user."
518 ;; The comments after each status code handled are taken from RFC
519 ;; 2616 (HTTP/1.1)
522 url-http-process)
523 ;; Pass the https certificate on to the caller.
538 ;;(narrow-to-region (point-min) url-http-end-of-headers)
540 ;; In HTTP 1.0, keep the connection only if there is a
541 ;; "Connection: keep-alive" header.
542 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
543 ;; "Connection: close" header
556 ;; other status symbols: jewelry and luxury cars
559 class url-http-response-status)
564 ;; Classes of response codes
565 ;;
566 ;; 5xx = Server Error
567 ;; 4xx = Client Error
568 ;; 3xx = Redirection
569 ;; 2xx = Successful
570 ;; 1xx = Informational
572 ;; 100 = Continue with request
573 ;; 101 = Switching protocols
574 ;; 102 = Processing (Added by DAV)
577 url-http-response-status))
579 ;; 200 Ok
580 ;; 201 Created
581 ;; 202 Accepted
582 ;; 203 Non-authoritative information
583 ;; 204 No content
584 ;; 205 Reset content
585 ;; 206 Partial content
586 ;; 207 Multi-status (Added by DAV)
589 ;; No new data, just stay at the same document
592 ;; Generic success for all others. Store in the cache, and
593 ;; mark it as successful.
599 ;; 300 Multiple choices
600 ;; 301 Moved permanently
601 ;; 302 Found
602 ;; 303 See other
603 ;; 304 Not modified
604 ;; 305 Use proxy
605 ;; 307 Temporary redirect
610 ;; Quoth the spec (section 10.3.1)
611 ;; -------------------------------
612 ;; The requested resource corresponds to any one of a set of
613 ;; representations, each with its own specific location and
614 ;; agent-driven negotiation information is being provided so
615 ;; that the user can select a preferred representation and
616 ;; redirect its request to that location.
617 ;; [...]
618 ;; If the server has a preferred choice of representation, it
619 ;; SHOULD include the specific URI for that representation in
620 ;; the Location field; user agents MAY use the Location field
621 ;; value for automatic redirection.
622 ;; -------------------------------
623 ;; We do not support agent-driven negotiation, so we just
624 ;; redirect to the preferred URI if one is provided.
625 nil)
627 ;; The response to the request can be found under a different
628 ;; URI and SHOULD be retrieved using a GET method on that
629 ;; resource.
631 url-http-data nil))
633 ;; The 304 response MUST NOT contain a message-body.
638 success t))
640 ;; The requested resource MUST be accessed through the
641 ;; proxy given by the Location field. The Location field
642 ;; gives the URI of the proxy. The recipient is expected
643 ;; to repeat this single request via the proxy. 305
644 ;; responses MUST only be generated by origin servers.
646 redirect-uri))
648 ;; Treat everything like '300'
649 nil))
651 ;; Clean off any whitespace and/or <...> cruft.
657 ;; Some stupid sites (like sourceforge) send a
658 ;; non-fully-qualified URL (ie: /), which royally confuses
659 ;; the URL library.
661 ;; Be careful to use the real target URL, otherwise we may
662 ;; compute the redirection relative to the URL of the proxy.
665 ;; Do not automatically include an authorization header in the
666 ;; redirect. If needed it will be regenerated by the relevant
667 ;; auth scheme when the new request happens.
674 ;; Check existing number of redirects
685 ;; url-max-redirections hasn't been reached, so go
686 ;; ahead and redirect.
688 ;; Remember that the request was redirected.
692 ;; Put in the current buffer a forwarding pointer to the new
693 ;; destination buffer.
694 ;; FIXME: This is a hack to fix url-retrieve-synchronously
695 ;; without changing the API. Instead url-retrieve should
696 ;; either simply not return the "destination" buffer, or it
697 ;; should take an optional `dest-buf' argument.
700 redirect-uri url-callback-function
701 url-callback-arguments
705 ;; We hit url-max-redirections, so issue an error and
706 ;; stop redirecting.
710 redirect-uri))
714 ;; 400 Bad Request
715 ;; 401 Unauthorized
716 ;; 402 Payment required
717 ;; 403 Forbidden
718 ;; 404 Not found
719 ;; 405 Method not allowed
720 ;; 406 Not acceptable
721 ;; 407 Proxy authentication required
722 ;; 408 Request time-out
723 ;; 409 Conflict
724 ;; 410 Gone
725 ;; 411 Length required
726 ;; 412 Precondition failed
727 ;; 413 Request entity too large
728 ;; 414 Request-URI too large
729 ;; 415 Unsupported media type
730 ;; 416 Requested range not satisfiable
731 ;; 417 Expectation failed
732 ;; 422 Unprocessable Entity (Added by DAV)
733 ;; 423 Locked
734 ;; 424 Failed Dependency
738 ;; The request requires user authentication. The response
739 ;; MUST include a WWW-Authenticate header field containing a
740 ;; challenge applicable to the requested resource. The
741 ;; client MAY repeat the request with a suitable
742 ;; Authorization header field.
745 ;; This code is reserved for future use
749 ;; The server understood the request, but is refusing to
750 ;; fulfill it. Authorization will not help and the request
751 ;; SHOULD NOT be repeated.
752 t)
754 ;; Not found
755 t)
757 ;; The method specified in the Request-Line is not allowed
758 ;; for the resource identified by the Request-URI. The
759 ;; response MUST include an Allow header containing a list of
760 ;; valid methods for the requested resource.
761 t)
763 ;; The resource identified by the request is only capable of
764 ;; generating response entities which have content
765 ;; characteristics not acceptable according to the accept
766 ;; headers sent in the request.
767 t)
769 ;; This code is similar to 401 (Unauthorized), but indicates
770 ;; that the client must first authenticate itself with the
771 ;; proxy. The proxy MUST return a Proxy-Authenticate header
772 ;; field containing a challenge applicable to the proxy for
773 ;; the requested resource.
776 ;; The client did not produce a request within the time that
777 ;; the server was prepared to wait. The client MAY repeat
778 ;; the request without modifications at any later time.
779 t)
781 ;; The request could not be completed due to a conflict with
782 ;; the current state of the resource. This code is only
783 ;; allowed in situations where it is expected that the user
784 ;; might be able to resolve the conflict and resubmit the
785 ;; request. The response body SHOULD include enough
786 ;; information for the user to recognize the source of the
787 ;; conflict.
788 t)
790 ;; The requested resource is no longer available at the
791 ;; server and no forwarding address is known.
792 t)
794 ;; The server refuses to accept the request without a defined
795 ;; Content-Length. The client MAY repeat the request if it
796 ;; adds a valid Content-Length header field containing the
797 ;; length of the message-body in the request message.
798 ;;
799 ;; NOTE - this will never happen because
800 ;; `url-http-create-request' automatically calculates the
801 ;; content-length.
802 t)
804 ;; The precondition given in one or more of the
805 ;; request-header fields evaluated to false when it was
806 ;; tested on the server.
807 t)
809 ;; The server is refusing to process a request because the
810 ;; request entity|URI is larger than the server is willing or
811 ;; able to process.
812 t)
814 ;; The server is refusing to service the request because the
815 ;; entity of the request is in a format not supported by the
816 ;; requested resource for the requested method.
817 t)
819 ;; A server SHOULD return a response with this status code if
820 ;; a request included a Range request-header field, and none
821 ;; of the range-specifier values in this field overlap the
822 ;; current extent of the selected resource, and the request
823 ;; did not include an If-Range request-header field.
824 t)
826 ;; The expectation given in an Expect request-header field
827 ;; could not be met by this server, or, if the server is a
828 ;; proxy, the server has unambiguous evidence that the
829 ;; request could not be met by the next-hop server.
830 t)
832 ;; The request could not be understood by the server due to
833 ;; malformed syntax. The client SHOULD NOT repeat the
834 ;; request without modifications.
835 t)))
836 ;; Tell the callback that an error occurred, and what the
837 ;; status code was.
843 ;; 500 Internal server error
844 ;; 501 Not implemented
845 ;; 502 Bad gateway
846 ;; 503 Service unavailable
847 ;; 504 Gateway time-out
848 ;; 505 HTTP version not supported
849 ;; 507 Insufficient storage
853 ;; The server does not support the functionality required to
854 ;; fulfill the request.
855 nil)
857 ;; The server, while acting as a gateway or proxy, received
858 ;; an invalid response from the upstream server it accessed
859 ;; in attempting to fulfill the request.
860 nil)
862 ;; The server is currently unable to handle the request due
863 ;; to a temporary overloading or maintenance of the server.
864 ;; The implication is that this is a temporary condition
865 ;; which will be alleviated after some delay. If known, the
866 ;; length of the delay MAY be indicated in a Retry-After
867 ;; header. If no Retry-After is given, the client SHOULD
868 ;; handle the response as it would for a 500 response.
869 nil)
871 ;; The server, while acting as a gateway or proxy, did not
872 ;; receive a timely response from the upstream server
873 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
874 ;; auxiliary server (e.g. DNS) it needed to access in
875 ;; attempting to complete the request.
876 nil)
878 ;; The server does not support, or refuses to support, the
879 ;; HTTP protocol version that was used in the request
880 ;; message.
881 nil)
883 ;; The method could not be performed on the resource
884 ;; because the server is unable to store the representation
885 ;; needed to successfully complete the request. This
886 ;; condition is considered to be temporary. If the request
887 ;; which received this status code was the result of a user
888 ;; action, the request MUST NOT be repeated until it is
889 ;; requested by a separate user action.
890 nil))
891 ;; Tell the callback that an error occurred, and what the
892 ;; status code was.
899 class url-http-response-status)))
906 success))
922 ;; Miscellaneous
924 "Activate callback specified when this buffer was created."
927 url-http-process)
932 ;; )
934 ;; These unfortunately cannot be macros... please ignore them!
936 "Remove (now defunct) process PROC from the list of open connections."
940 url-http-open-connections))
943 ;; Sentinel used to handle (i) terminated old HTTP/0.9 connections,
944 ;; and (ii) closed connection due to reusing a HTTP connection which
945 ;; we believed was still alive, but which the server closed on us.
946 ;; We handle case (ii) by calling `url-http' again.
955 ;; HTTP/0.9 just gets passed back no matter what
957 ;; Call `url-http' again if our connection expired.
963 url-current-object
974 ;; Function used when we do NOT know how long the document is going to be
975 ;; Just _very_ simple 'downloaded %d' type of info.
979 "Function used when we DO know how long the document is going to be.
980 More sophisticated percentage downloaded, etc.
981 Also does minimal parsing of HTTP headers and will actually cause
982 the callback to be triggered."
985 "Reading [%s]... %s of %s (%d%%)"
987 url-http-content-length)
988 url-http-content-type
992 url-http-content-length))
994 "Reading... %s of %s (%d%%)"
996 url-http-content-length)
1000 url-http-content-length)))
1004 ;; Found the end of the document! Wheee!
1011 "Function used when dealing with chunked encoding.
1012 Cannot give a sophisticated percentage, but we need a different
1013 function to look for the special 0-length chunk that signifies
1014 the end of the document."
1021 ;; We need to loop thru looking for more chunks even within
1022 ;; one after-change-function call.
1027 "Reading [%s]... chunk #%d"
1028 url-http-content-type url-http-chunked-counter)
1030 "Reading... chunk #%d"
1031 url-http-chunked-counter))
1033 url-http-chunked-counter st nd length)
1039 ;; We know how long the chunk is supposed to be, skip over
1040 ;; leading crap if possible.
1044 url-http-chunked-counter)
1046 url-http-chunked-length)))
1049 url-http-chunked-length)
1050 nd))
1057 ;; Must not have received the entirety of the chunk header,
1058 ;; need to spin some more.
1063 'end-open t
1064 'chunked-encoding t
1076 ; (if (not url-http-debug)
1079 url-http-chunked-counter url-http-chunked-length
1083 ;; Found the end of the document! Wheee!
1087 ;; Every chunk, even the last 0-length one, is
1088 ;; terminated by CRLF. Skip it.
1098 ;; This will wait for the headers to arrive and then splice in the
1099 ;; next appropriate after-change-function, etc.
1105 ;; (content-length nil)
1106 )
1111 ;; Not HTTP/x.y data, must be 0.9
1112 ;; God, I wish this could die.
1114 url-http-end-of-headers 0
1115 old-http t)
1116 ;; Blank line at end of headers.
1119 ;; Saw the end of the headers
1123 end-of-headers t)
1127 ;; Haven't seen the end of the headers yet, need to wait
1128 ;; for more data to arrive.
1129 nil
1146 ;; We got back a headerless malformed response from the
1147 ;; server.
1156 ;; A HEAD request is _ALWAYS_ terminated by the header
1157 ;; information, regardless of any entity headers,
1158 ;; according to section 4.4 of the HTTP/1.1 draft.
1164 ;; A CONNECT request is finished, but we cannot stick this
1165 ;; back on the free connection list
1170 ;; Only allowed to have a header section. We have to handle
1171 ;; this here instead of in url-http-parse-headers because if
1172 ;; you have a cached copy of something without a known
1173 ;; content-length, and try to retrieve it from the cache, we'd
1174 ;; fall into the 'being dumb' section and wait for the
1175 ;; connection to terminate, which means we'd wait for 10
1176 ;; seconds for the keep-alives to time out on some servers.
1180 ;; HTTP/0.9 always signaled end-of-connection by closing the
1181 ;; connection.
1203 ;; We got a NULL body! Activate the callback
1204 ;; immediately!
1210 ;; Have some leftover data
1214 nd
1217 nil)))
1222 ;; We are still at the beginning of the buffer... must just be
1223 ;; waiting for a response.
1229 "Retrieve URL via HTTP asynchronously.
1230 URL must be a parsed URL. See `url-generic-parse-url' for details.
1232 When retrieval is completed, execute the function CALLBACK,
1233 passing it an updated value of CBARGS as arguments. The first
1234 element in CBARGS should be a plist describing what has happened
1235 so far during the request, as described in the docstring of
1236 `url-retrieve' (if in doubt, specify nil). The current buffer
1237 then CALLBACK is executed is the retrieval buffer.
1239 Optional arg RETRY-BUFFER, if non-nil, specifies the buffer of a
1240 previous `url-http' call, which is being re-attempted.
1242 Optional arg GATEWAY-METHOD specifies the gateway to be used,
1243 overriding the value of `url-gateway-method'.
1245 The return value of this function is the retrieval buffer."
1248 ;; (port (url-port (or url-using-proxy url)))
1251 url-http-noninteractive)))
1252 ;; The following binding is needed in url-open-stream, which
1253 ;; is called from url-http-find-free-connection.
1257 gateway-method))
1263 ;; Failed to open the connection for some reason
1275 url-http-content-type
1276 url-http-content-length
1277 url-http-transfer-encoding
1278 url-http-after-change-function
1279 url-http-response-version
1280 url-http-response-status
1281 url-http-chunked-length
1282 url-http-chunked-counter
1283 url-http-chunked-start
1284 url-callback-function
1285 url-callback-arguments
1286 url-show-status
1287 url-http-process
1288 url-http-method
1289 url-http-extra-headers
1290 url-http-noninteractive
1291 url-http-data
1292 url-http-target-url
1293 url-http-no-retry
1294 url-http-connection-opened
1295 url-mime-accept-string
1296 url-http-proxy))
1300 url-http-extra-headers url-request-extra-headers
1301 url-http-noninteractive url-request-noninteractive
1302 url-http-data url-request-data
1303 url-http-process connection
1304 url-http-chunked-length nil
1305 url-http-chunked-start nil
1306 url-http-chunked-counter 0
1307 url-callback-function callback
1308 url-callback-arguments cbargs
1309 url-http-after-change-function 'url-http-wait-for-headers-change-function
1310 url-http-target-url url-current-object
1311 url-http-no-retry retry-buffer
1312 url-http-connection-opened nil
1313 url-mime-accept-string mime-accept-string
1314 url-http-proxy url-using-proxy)
1319 (`connect
1320 ;; Asynchronous connection
1322 (`failed
1323 ;; Asynchronous connection failed
1333 buffer))
1342 url-https-default-port)
1351 ;; Saw the end of the headers
1356 ;; We got back a headerless malformed response from the
1357 ;; server.
1364 :process proc
1367 ;; check certificate validity
1391 ;; We are performing an asynchronous connection, and a status change
1392 ;; has occurred.
1416 ;; Since Emacs 19/20 does not allow you to change the
1417 ;; `after-change-functions' hook in the midst of running them, we fake
1418 ;; an after change by hooking into the process filter and inserting
1419 ;; the data ourselves. This is slightly less efficient, but there
1420 ;; were tons of weird ways the after-change code was biting us in the
1421 ;; shorts.
1422 ;; FIXME this can probably be simplified since the above is no longer true.
1424 ;; Sometimes we get a zero-length data chunk after the process has
1425 ;; been changed to 'free', which means it has no buffer associated
1426 ;; with it. Do nothing if there is no buffer, or 0 length data.
1430 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1439 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1440 ;;; file-name-handler stuff from here on out
1441 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1444 'symbol-value-in-buffer
1446 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1449 unbound-value
1474 nil ;dir / link / normal file
1477 nil nil nil ;atime ; mtime ; ctime
1492 "Return a property list describing options available for URL.
1493 This list is retrieved using the `OPTIONS' HTTP method.
1495 Property list members:
1497 methods
1498 A list of symbols specifying what HTTP methods the resource
1499 supports.
1501 dav
1502 A list of numbers specifying what DAV protocol/schema versions are
1503 supported.
1505 dasl
1506 A list of supported DASL search types supported (string form)
1508 ranges
1509 A list of the units available for use in partial document fetches.
1511 p3p
1512 The `Platform For Privacy Protection' description for the resource.
1513 Currently this is just the raw header contents. This is likely to
1514 change once P3P is formally supported by the URL package or
1515 Emacs/W3."
1523 ;; Only parse the options if we got a 2xx response code!
1529 ;; Figure out what methods are supported.
1532 options 'methods
1535 ;; Check for DAV
1538 options 'dav
1543 ;; Now for DASL
1546 options 'dasl
1549 ;; P3P - should get more detailed here. FIXME
1553 ;; Check for whether they accept byte-range requests.
1556 options 'ranges
1560 ))))
1562 options))
1564 ;; HTTPS. This used to be in url-https.el, but that file collides
1565 ;; with url-http.el on systems with 8-character file names.
1570 ;; FIXME what is the point of this alias being an autoload?
1571 ;; Trying to use it will not cause url-http to be loaded,
1572 ;; since the full alias just gets dumped into loaddefs.el.
1574 ;;;###autoload (autoload 'url-default-expander "url-expand")
1575 ;;;###autoload
1584 ;;;###autoload (autoload 'url-https "url-http")
1586 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1588 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1590 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1595 ;;; url-http.el ends here