| blob | 0c911260ca58b665cb2a170114d56fb617f68b46 |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004-2012 Free Software Foundation, Inc.
5 ;; Author: Bill Perry <wmperry@gnu.org>
6 ;; Keywords: comm, data, processes
8 ;; This file is part of GNU Emacs.
9 ;;
10 ;; GNU Emacs is free software: you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation, either version 3 of the License, or
13 ;; (at your option) any later version.
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
23 ;;; Commentary:
25 ;;; Code:
53 "What version of HTTP we advertise, as a string.
54 Valid values are 1.1 and 1.0.
55 This is only useful when debugging the HTTP subsystem.
57 Setting this to 1.0 will tell servers not to send chunked encoding,
61 "Whether to use a single TCP connection multiple times in HTTP.
62 This is only useful when debugging the HTTP subsystem. Setting to
63 nil will explicitly close the connection to the server after every
115 ;(eval-when-compile
116 ;; These are all macros so that they are hidden from external sight
117 ;; when the file is byte-compiled.
118 ;;
119 ;; This allows us to expose just the entry points we want.
121 ;; These routines will allow us to implement persistent HTTP
122 ;; connections.
126 ;; The user hit C-g, honor it! Some things can get in an
127 ;; incredibly tight loop (chunked encoding)
140 url-http-open-connections)
141 proc)
151 url-http-open-connections))
152 nil)
171 host port
174 ;; `url-open-stream' needs a buffer in which to do things
175 ;; like authentication. But we use another buffer afterwards.
178 ;; url-open-stream might return nil.
180 ;; Drop the temp buffer link before killing the buffer.
182 proc)
183 ;; If there was an error on connect, make sure we don't
184 ;; get queried.
189 ;; Building an HTTP request
194 ""
199 url-version
208 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
210 url-http-method url-http-data
211 url-http-extra-headers))
217 url-http-extra-headers))
219 nil
227 nil
230 proxy-info)
240 ;; Protection against stupid values in the referrer
245 ;; We do not want to expose the referrer if the user is paranoid.
251 ;; url-http-extra-headers contains an assoc-list of
252 ;; header/value pairs that we need to put into the request.
260 ;; This was done with a call to `format'. Concatenating parts has
261 ;; the advantage of keeping the parts of each header together and
262 ;; allows us to elide null lines directly, at the cost of making
263 ;; the layout less clear.
265 ;; We used to concat directly, but if one of the strings happens
266 ;; to being multibyte (even if it only contains pure ASCII) then
267 ;; every string gets converted with `string-MAKE-multibyte' which
268 ;; turns the 127-255 codes into things like latin-1 accented chars
269 ;; (it would work right if it used `string-TO-multibyte' instead).
270 ;; So to avoid the problem we force every string to be unibyte.
272 ;; FIXME: Instead of `string-AS-unibyte' we'd want
273 ;; `string-to-unibyte', so as to properly signal an error if one
274 ;; of the strings contains a multibyte char.
275 'string-as-unibyte
278 ;; The request
282 ;; Version of MIME we speak
284 ;; (maybe) Try to keep the connection open
288 ;; HTTP extensions we support
292 ;; Who we want to talk to
299 ;; Who its from
303 ;; Encodings we understand
310 ;; Languages we understand
314 ;; Types we understand
316 ;; User agent
318 ;; Proxy Authorization
319 proxy-auth
320 ;; Authorization
321 auth
322 ;; Cookies
325 host real-fname
327 ;; If-modified-since
334 ;; Whence we came
337 extra-headers
338 ;; Length of data
344 ;; End request
346 ;; Any data
347 url-http-data
348 ;; If `url-http-data' is nil, avoid two CRLFs (Bug#8931).
352 request))
354 ;; Parsing routines
357 This allows us to use `mail-fetch-field', etc.
358 Return the number of characters removed."
368 url-callback-function url-callback-arguments))
373 nil nil t))
379 url-http-proxy
380 url-current-object)))
382 ;; Cheating, but who cares? :)
383 'url-http-proxy-basic-auth-storage
385 auth
388 ;; find strongest supported auth
392 this-auth)))
401 type this-type
402 strength this-strength))))
409 " authentication. If you'd like to write it,"
415 type t args)))
419 url-http-extra-headers)
424 url-callback-arguments)))))))
427 "Parse just the response code."
429 url-http-response-version))
444 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
445 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
446 work correctly."
453 ;;; (while cookies2
454 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
455 )
456 )
459 "Parse and handle HTTP specific headers.
460 Return t if and only if the current buffer is still active and
461 should be shown to the user."
462 ;; The comments after each status code handled are taken from RFC
463 ;; 2616 (HTTP/1.1)
465 url-http-response-version
466 url-http-method url-http-data url-http-process
467 url-callback-function url-callback-arguments))
471 url-http-process)
480 ;;(narrow-to-region (point-min) url-http-end-of-headers)
482 ;; In HTTP 1.0, keep the connection only if there is a
483 ;; "Connection: keep-alive" header.
484 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
485 ;; "Connection: close" header
498 ;; other status symbols: jewelry and luxury cars
500 ;; The filename part of a URL could be in remote file syntax,
501 ;; see Bug#6717 for an example. We disable file name
502 ;; handlers, therefore.
510 ;; Classes of response codes
511 ;;
512 ;; 5xx = Server Error
513 ;; 4xx = Client Error
514 ;; 3xx = Redirection
515 ;; 2xx = Successful
516 ;; 1xx = Informational
518 ;; 100 = Continue with request
519 ;; 101 = Switching protocols
520 ;; 102 = Processing (Added by DAV)
524 ;; 200 Ok
525 ;; 201 Created
526 ;; 202 Accepted
527 ;; 203 Non-authoritative information
528 ;; 204 No content
529 ;; 205 Reset content
530 ;; 206 Partial content
531 ;; 207 Multi-status (Added by DAV)
534 ;; No new data, just stay at the same document
538 ;; Generic success for all others. Store in the cache, and
539 ;; mark it as successful.
545 ;; 300 Multiple choices
546 ;; 301 Moved permanently
547 ;; 302 Found
548 ;; 303 See other
549 ;; 304 Not modified
550 ;; 305 Use proxy
551 ;; 307 Temporary redirect
556 ;; Quoth the spec (section 10.3.1)
557 ;; -------------------------------
558 ;; The requested resource corresponds to any one of a set of
559 ;; representations, each with its own specific location and
560 ;; agent-driven negotiation information is being provided so
561 ;; that the user can select a preferred representation and
562 ;; redirect its request to that location.
563 ;; [...]
564 ;; If the server has a preferred choice of representation, it
565 ;; SHOULD include the specific URI for that representation in
566 ;; the Location field; user agents MAY use the Location field
567 ;; value for automatic redirection.
568 ;; -------------------------------
569 ;; We do not support agent-driven negotiation, so we just
570 ;; redirect to the preferred URI if one is provided.
571 nil)
573 ;; If the 301|302 status code is received in response to a
574 ;; request other than GET or HEAD, the user agent MUST NOT
575 ;; automatically redirect the request unless it can be
576 ;; confirmed by the user, since this might change the
577 ;; conditions under which the request was issued.
581 ;; The response to the request can be found under a different
582 ;; URI and SHOULD be retrieved using a GET method on that
583 ;; resource.
585 url-http-data nil))
587 ;; The 304 response MUST NOT contain a message-body.
592 success t))
594 ;; The requested resource MUST be accessed through the
595 ;; proxy given by the Location field. The Location field
596 ;; gives the URI of the proxy. The recipient is expected
597 ;; to repeat this single request via the proxy. 305
598 ;; responses MUST only be generated by origin servers.
600 redirect-uri))
602 ;; Treat everything like '300'
603 nil))
605 ;; Clean off any whitespace and/or <...> cruft.
611 ;; Some stupid sites (like sourceforge) send a
612 ;; non-fully-qualified URL (ie: /), which royally confuses
613 ;; the URL library.
615 ;; Be careful to use the real target URL, otherwise we may
616 ;; compute the redirection relative to the URL of the proxy.
622 ;; Check existing number of redirects
633 ;; url-max-redirections hasn't been reached, so go
634 ;; ahead and redirect.
636 ;; Remember that the request was redirected.
640 ;; Put in the current buffer a forwarding pointer to the new
641 ;; destination buffer.
642 ;; FIXME: This is a hack to fix url-retrieve-synchronously
643 ;; without changing the API. Instead url-retrieve should
644 ;; either simply not return the "destination" buffer, or it
645 ;; should take an optional `dest-buf' argument.
648 redirect-uri url-callback-function
649 url-callback-arguments
653 ;; We hit url-max-redirections, so issue an error and
654 ;; stop redirecting.
658 redirect-uri))
662 ;; 400 Bad Request
663 ;; 401 Unauthorized
664 ;; 402 Payment required
665 ;; 403 Forbidden
666 ;; 404 Not found
667 ;; 405 Method not allowed
668 ;; 406 Not acceptable
669 ;; 407 Proxy authentication required
670 ;; 408 Request time-out
671 ;; 409 Conflict
672 ;; 410 Gone
673 ;; 411 Length required
674 ;; 412 Precondition failed
675 ;; 413 Request entity too large
676 ;; 414 Request-URI too large
677 ;; 415 Unsupported media type
678 ;; 416 Requested range not satisfiable
679 ;; 417 Expectation failed
680 ;; 422 Unprocessable Entity (Added by DAV)
681 ;; 423 Locked
682 ;; 424 Failed Dependency
685 ;; The request requires user authentication. The response
686 ;; MUST include a WWW-Authenticate header field containing a
687 ;; challenge applicable to the requested resource. The
688 ;; client MAY repeat the request with a suitable
689 ;; Authorization header field.
692 ;; This code is reserved for future use
696 ;; The server understood the request, but is refusing to
697 ;; fulfill it. Authorization will not help and the request
698 ;; SHOULD NOT be repeated.
701 ;; Not found
704 ;; The method specified in the Request-Line is not allowed
705 ;; for the resource identified by the Request-URI. The
706 ;; response MUST include an Allow header containing a list of
707 ;; valid methods for the requested resource.
710 ;; The resource identified by the request is only capable of
711 ;; generating response entities which have content
712 ;; characteristics not acceptable according to the accept
713 ;; headers sent in the request.
716 ;; This code is similar to 401 (Unauthorized), but indicates
717 ;; that the client must first authenticate itself with the
718 ;; proxy. The proxy MUST return a Proxy-Authenticate header
719 ;; field containing a challenge applicable to the proxy for
720 ;; the requested resource.
723 ;; The client did not produce a request within the time that
724 ;; the server was prepared to wait. The client MAY repeat
725 ;; the request without modifications at any later time.
728 ;; The request could not be completed due to a conflict with
729 ;; the current state of the resource. This code is only
730 ;; allowed in situations where it is expected that the user
731 ;; might be able to resolve the conflict and resubmit the
732 ;; request. The response body SHOULD include enough
733 ;; information for the user to recognize the source of the
734 ;; conflict.
737 ;; The requested resource is no longer available at the
738 ;; server and no forwarding address is known.
741 ;; The server refuses to accept the request without a defined
742 ;; Content-Length. The client MAY repeat the request if it
743 ;; adds a valid Content-Length header field containing the
744 ;; length of the message-body in the request message.
745 ;;
746 ;; NOTE - this will never happen because
747 ;; `url-http-create-request' automatically calculates the
748 ;; content-length.
751 ;; The precondition given in one or more of the
752 ;; request-header fields evaluated to false when it was
753 ;; tested on the server.
756 ;; The server is refusing to process a request because the
757 ;; request entity|URI is larger than the server is willing or
758 ;; able to process.
761 ;; The server is refusing to service the request because the
762 ;; entity of the request is in a format not supported by the
763 ;; requested resource for the requested method.
766 ;; A server SHOULD return a response with this status code if
767 ;; a request included a Range request-header field, and none
768 ;; of the range-specifier values in this field overlap the
769 ;; current extent of the selected resource, and the request
770 ;; did not include an If-Range request-header field.
773 ;; The expectation given in an Expect request-header field
774 ;; could not be met by this server, or, if the server is a
775 ;; proxy, the server has unambiguous evidence that the
776 ;; request could not be met by the next-hop server.
779 ;; The request could not be understood by the server due to
780 ;; malformed syntax. The client SHOULD NOT repeat the
781 ;; request without modifications.
783 ;; Tell the callback that an error occurred, and what the
784 ;; status code was.
790 ;; 500 Internal server error
791 ;; 501 Not implemented
792 ;; 502 Bad gateway
793 ;; 503 Service unavailable
794 ;; 504 Gateway time-out
795 ;; 505 HTTP version not supported
796 ;; 507 Insufficient storage
800 ;; The server does not support the functionality required to
801 ;; fulfill the request.
802 nil)
804 ;; The server, while acting as a gateway or proxy, received
805 ;; an invalid response from the upstream server it accessed
806 ;; in attempting to fulfill the request.
807 nil)
809 ;; The server is currently unable to handle the request due
810 ;; to a temporary overloading or maintenance of the server.
811 ;; The implication is that this is a temporary condition
812 ;; which will be alleviated after some delay. If known, the
813 ;; length of the delay MAY be indicated in a Retry-After
814 ;; header. If no Retry-After is given, the client SHOULD
815 ;; handle the response as it would for a 500 response.
816 nil)
818 ;; The server, while acting as a gateway or proxy, did not
819 ;; receive a timely response from the upstream server
820 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
821 ;; auxiliary server (e.g. DNS) it needed to access in
822 ;; attempting to complete the request.
823 nil)
825 ;; The server does not support, or refuses to support, the
826 ;; HTTP protocol version that was used in the request
827 ;; message.
828 nil)
830 ;; The method could not be performed on the resource
831 ;; because the server is unable to store the representation
832 ;; needed to successfully complete the request. This
833 ;; condition is considered to be temporary. If the request
834 ;; which received this status code was the result of a user
835 ;; action, the request MUST NOT be repeated until it is
836 ;; requested by a separate user action.
837 nil))
838 ;; Tell the callback that an error occurred, and what the
839 ;; status code was.
846 class url-http-response-status)))
851 success))
853 ;; Miscellaneous
855 "Activate callback specified when this buffer was created."
857 url-callback-function
858 url-callback-arguments))
861 url-http-process)
865 ;; )
867 ;; These unfortunately cannot be macros... please ignore them!
869 "Remove (now defunct) process PROC from the list of open connections."
873 url-http-open-connections))
876 ;; Sentinel used for old HTTP/0.9 or connections we know are going
877 ;; to die as the 'end of document' notifier.
885 ;; HTTP/0.9 just gets passed back no matter what
891 ;; Function used when we do NOT know how long the document is going to be
892 ;; Just _very_ simple 'downloaded %d' type of info.
897 "Function used when we DO know how long the document is going to be.
898 More sophisticated percentage downloaded, etc.
899 Also does minimal parsing of HTTP headers and will actually cause
900 the callback to be triggered."
902 url-http-end-of-headers
903 url-http-content-length
904 url-http-content-type
905 url-http-process))
908 "Reading [%s]... %s of %s (%d%%)"
910 url-http-content-length)
911 url-http-content-type
915 url-http-content-length))
917 "Reading... %s of %s (%d%%)"
919 url-http-content-length)
923 url-http-content-length)))
927 ;; Found the end of the document! Wheee!
934 "Function used when dealing with 'chunked' encoding.
935 Cannot give a sophisticated percentage, but we need a different
936 function to look for the special 0-length chunk that signifies
937 the end of the document."
939 url-http-end-of-headers
940 url-http-content-type
941 url-http-chunked-length
942 url-http-chunked-counter
943 url-http-process url-http-chunked-start))
950 ;; We need to loop thru looking for more chunks even within
951 ;; one after-change-function call.
956 "Reading [%s]... chunk #%d"
957 url-http-content-type url-http-chunked-counter)
959 "Reading... chunk #%d"
960 url-http-chunked-counter))
962 url-http-chunked-counter st nd length)
968 ;; We know how long the chunk is supposed to be, skip over
969 ;; leading crap if possible.
973 url-http-chunked-counter)
975 url-http-chunked-length)))
978 url-http-chunked-length)
979 nd))
986 ;; Must not have received the entirety of the chunk header,
987 ;; need to spin some more.
992 'end-open t
993 'chunked-encoding t
1005 ; (if (not url-http-debug)
1008 url-http-chunked-counter url-http-chunked-length
1012 ;; Found the end of the document! Wheee!
1016 ;; Every chunk, even the last 0-length one, is
1017 ;; terminated by CRLF. Skip it.
1027 ;; This will wait for the headers to arrive and then splice in the
1028 ;; next appropriate after-change-function, etc.
1030 url-http-end-of-headers
1031 url-http-content-type
1032 url-http-content-length
1033 url-http-transfer-encoding
1034 url-callback-function
1035 url-callback-arguments
1036 url-http-process
1037 url-http-method
1038 url-http-after-change-function
1039 url-http-response-status))
1050 ;; Not HTTP/x.y data, must be 0.9
1051 ;; God, I wish this could die.
1053 url-http-end-of-headers 0
1054 old-http t)
1056 ;; Saw the end of the headers
1060 end-of-headers t)
1064 ;; Haven't seen the end of the headers yet, need to wait
1065 ;; for more data to arrive.
1066 nil
1083 ;; We got back a headerless malformed response from the
1084 ;; server.
1093 ;; A HEAD request is _ALWAYS_ terminated by the header
1094 ;; information, regardless of any entity headers,
1095 ;; according to section 4.4 of the HTTP/1.1 draft.
1101 ;; A CONNECT request is finished, but we cannot stick this
1102 ;; back on the free connection list
1107 ;; Only allowed to have a header section. We have to handle
1108 ;; this here instead of in url-http-parse-headers because if
1109 ;; you have a cached copy of something without a known
1110 ;; content-length, and try to retrieve it from the cache, we'd
1111 ;; fall into the 'being dumb' section and wait for the
1112 ;; connection to terminate, which means we'd wait for 10
1113 ;; seconds for the keep-alives to time out on some servers.
1117 ;; HTTP/0.9 always signaled end-of-connection by closing the
1118 ;; connection.
1140 ;; We got a NULL body! Activate the callback
1141 ;; immediately!
1147 ;; Have some leftover data
1151 nd
1154 nil)))
1159 ;; We are still at the beginning of the buffer... must just be
1160 ;; waiting for a response.
1165 ;;;###autoload
1167 "Retrieve URL via HTTP asynchronously.
1168 URL must be a parsed URL. See `url-generic-parse-url' for details.
1169 When retrieval is completed, the function CALLBACK is executed with
1170 CBARGS as the arguments."
1173 url-http-end-of-headers
1174 url-http-content-type
1175 url-http-content-length
1176 url-http-transfer-encoding
1177 url-http-after-change-function
1178 url-callback-function
1179 url-callback-arguments
1180 url-show-status
1181 url-http-method
1182 url-http-extra-headers
1183 url-http-data
1184 url-http-chunked-length
1185 url-http-chunked-start
1186 url-http-chunked-counter
1187 url-http-process))
1193 ;; Failed to open the connection for some reason
1204 url-http-content-type
1205 url-http-content-length
1206 url-http-transfer-encoding
1207 url-http-after-change-function
1208 url-http-response-version
1209 url-http-response-status
1210 url-http-chunked-length
1211 url-http-chunked-counter
1212 url-http-chunked-start
1213 url-callback-function
1214 url-callback-arguments
1215 url-show-status
1216 url-http-process
1217 url-http-method
1218 url-http-extra-headers
1219 url-http-data
1220 url-http-target-url
1221 url-http-connection-opened
1222 url-http-proxy))
1226 url-http-extra-headers url-request-extra-headers
1227 url-http-data url-request-data
1228 url-http-process connection
1229 url-http-chunked-length nil
1230 url-http-chunked-start nil
1231 url-http-chunked-counter 0
1232 url-callback-function callback
1233 url-callback-arguments cbargs
1234 url-http-after-change-function 'url-http-wait-for-headers-change-function
1235 url-http-target-url url-current-object
1236 url-http-connection-opened nil
1237 url-http-proxy url-using-proxy)
1244 ;; Asynchronous connection
1247 ;; Asynchronous connection failed
1252 buffer))
1256 ;; We are performing an asynchronous connection, and a status change
1257 ;; has occurred.
1278 ;; Since Emacs 19/20 does not allow you to change the
1279 ;; `after-change-functions' hook in the midst of running them, we fake
1280 ;; an after change by hooking into the process filter and inserting
1281 ;; the data ourselves. This is slightly less efficient, but there
1282 ;; were tons of weird ways the after-change code was biting us in the
1283 ;; shorts.
1284 ;; FIXME this can probably be simplified since the above is no longer true.
1286 ;; Sometimes we get a zero-length data chunk after the process has
1287 ;; been changed to 'free', which means it has no buffer associated
1288 ;; with it. Do nothing if there is no buffer, or 0 length data.
1293 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1302 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1303 ;;; file-name-handler stuff from here on out
1304 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1307 'symbol-value-in-buffer
1309 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1312 unbound-value
1320 ;;;###autoload
1332 exists))
1334 ;;;###autoload
1342 nil ;dir / link / normal file
1345 nil nil nil ;atime ; mtime ; ctime
1354 ;;;###autoload
1360 ;;;###autoload
1362 "Return a property list describing options available for URL.
1363 This list is retrieved using the `OPTIONS' HTTP method.
1365 Property list members:
1367 methods
1368 A list of symbols specifying what HTTP methods the resource
1369 supports.
1371 dav
1372 A list of numbers specifying what DAV protocol/schema versions are
1373 supported.
1375 dasl
1376 A list of supported DASL search types supported (string form)
1378 ranges
1379 A list of the units available for use in partial document fetches.
1381 p3p
1382 The `Platform For Privacy Protection' description for the resource.
1383 Currently this is just the raw header contents. This is likely to
1384 change once P3P is formally supported by the URL package or
1385 Emacs/W3."
1393 ;; Only parse the options if we got a 2xx response code!
1399 ;; Figure out what methods are supported.
1402 options 'methods
1405 ;; Check for DAV
1408 options 'dav
1413 ;; Now for DASL
1416 options 'dasl
1419 ;; P3P - should get more detailed here. FIXME
1423 ;; Check for whether they accept byte-range requests.
1426 options 'ranges
1430 ))))
1432 options))
1434 ;; HTTPS. This used to be in url-https.el, but that file collides
1435 ;; with url-http.el on systems with 8-character file names.
1438 ;;;###autoload
1440 ;;;###autoload
1443 ;; FIXME what is the point of this alias being an autoload?
1444 ;; Trying to use it will not cause url-http to be loaded,
1445 ;; since the full alias just gets dumped into loaddefs.el.
1447 ;;;###autoload (autoload 'url-default-expander "url-expand")
1448 ;;;###autoload
1458 ;;;###autoload (autoload 'url-https "url-http")
1460 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1462 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1464 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1469 ;;; url-http.el ends here