| blob | a9ff042d681a993a0985c0bde4ee10c2e981206a |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004-2011 Free Software Foundation, Inc.
5 ;; Author: Bill Perry <wmperry@gnu.org>
6 ;; Keywords: comm, data, processes
8 ;; This file is part of GNU Emacs.
9 ;;
10 ;; GNU Emacs is free software: you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation, either version 3 of the License, or
13 ;; (at your option) any later version.
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
23 ;;; Commentary:
25 ;;; Code:
53 "What version of HTTP we advertise, as a string.
54 Valid values are 1.1 and 1.0.
55 This is only useful when debugging the HTTP subsystem.
57 Setting this to 1.0 will tell servers not to send chunked encoding,
61 "Whether to use a single TCP connection multiple times in HTTP.
62 This is only useful when debugging the HTTP subsystem. Setting to
63 nil will explicitly close the connection to the server after every
115 ;(eval-when-compile
116 ;; These are all macros so that they are hidden from external sight
117 ;; when the file is byte-compiled.
118 ;;
119 ;; This allows us to expose just the entry points we want.
121 ;; These routines will allow us to implement persistent HTTP
122 ;; connections.
126 ;; The user hit C-g, honor it! Some things can get in an
127 ;; incredibly tight loop (chunked encoding)
140 url-http-open-connections)
141 proc)
151 url-http-open-connections))
152 nil)
171 host port
174 ;; `url-open-stream' needs a buffer in which to do things
175 ;; like authentication. But we use another buffer afterwards.
178 ;; url-open-stream might return nil.
180 ;; Drop the temp buffer link before killing the buffer.
182 proc)
183 ;; If there was an error on connect, make sure we don't
184 ;; get queried.
189 ;; Building an HTTP request
194 ""
199 url-version
208 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
210 url-http-method url-http-data
211 url-http-extra-headers))
217 url-http-extra-headers))
219 nil
227 nil
230 proxy-info)
240 ;; Protection against stupid values in the referer
245 ;; We do not want to expose the referer if the user is paranoid.
251 ;; url-http-extra-headers contains an assoc-list of
252 ;; header/value pairs that we need to put into the request.
260 ;; This was done with a call to `format'. Concatenating parts has
261 ;; the advantage of keeping the parts of each header together and
262 ;; allows us to elide null lines directly, at the cost of making
263 ;; the layout less clear.
265 ;; We used to concat directly, but if one of the strings happens
266 ;; to being multibyte (even if it only contains pure ASCII) then
267 ;; every string gets converted with `string-MAKE-multibyte' which
268 ;; turns the 127-255 codes into things like latin-1 accented chars
269 ;; (it would work right if it used `string-TO-multibyte' instead).
270 ;; So to avoid the problem we force every string to be unibyte.
272 ;; FIXME: Instead of `string-AS-unibyte' we'd want
273 ;; `string-to-unibyte', so as to properly signal an error if one
274 ;; of the strings contains a multibyte char.
275 'string-as-unibyte
278 ;; The request
282 ;; Version of MIME we speak
284 ;; (maybe) Try to keep the connection open
288 ;; HTTP extensions we support
292 ;; Who we want to talk to
299 ;; Who its from
303 ;; Encodings we understand
310 ;; Languages we understand
314 ;; Types we understand
316 ;; User agent
318 ;; Proxy Authorization
319 proxy-auth
320 ;; Authorization
321 auth
322 ;; Cookies
325 ;; If-modified-since
332 ;; Whence we came
335 extra-headers
336 ;; Length of data
342 ;; End request
344 ;; Any data
345 url-http-data
346 ;; If `url-http-data' is nil, avoid two CRLFs (Bug#8931).
350 request))
352 ;; Parsing routines
355 This allows us to use `mail-fetch-field', etc."
363 url-callback-function url-callback-arguments))
368 nil nil t))
374 url-http-proxy
375 url-current-object)))
377 ;; Cheating, but who cares? :)
378 'url-http-proxy-basic-auth-storage
380 auth
383 ;; find strongest supported auth
387 this-auth)))
396 type this-type
397 strength this-strength))))
404 " authentication. If you'd like to write it,"
410 type t args)))
414 url-http-extra-headers)
419 url-callback-arguments)))))))
422 "Parse just the response code."
424 url-http-response-version))
439 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
440 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
441 work correctly."
448 ;;; (while cookies2
449 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
450 )
451 )
454 "Parse and handle HTTP specific headers.
455 Return t if and only if the current buffer is still active and
456 should be shown to the user."
457 ;; The comments after each status code handled are taken from RFC
458 ;; 2616 (HTTP/1.1)
460 url-http-response-version
461 url-http-method url-http-data url-http-process
462 url-callback-function url-callback-arguments))
466 url-http-process)
475 ;;(narrow-to-region (point-min) url-http-end-of-headers)
477 ;; In HTTP 1.0, keep the connection only if there is a
478 ;; "Connection: keep-alive" header.
479 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
480 ;; "Connection: close" header
493 ;; other status symbols: jewelry and luxury cars
495 ;; The filename part of a URL could be in remote file syntax,
496 ;; see Bug#6717 for an example. We disable file name
497 ;; handlers, therefore.
504 ;; Classes of response codes
505 ;;
506 ;; 5xx = Server Error
507 ;; 4xx = Client Error
508 ;; 3xx = Redirection
509 ;; 2xx = Successful
510 ;; 1xx = Informational
512 ;; 100 = Continue with request
513 ;; 101 = Switching protocols
514 ;; 102 = Processing (Added by DAV)
518 ;; 200 Ok
519 ;; 201 Created
520 ;; 202 Accepted
521 ;; 203 Non-authoritative information
522 ;; 204 No content
523 ;; 205 Reset content
524 ;; 206 Partial content
525 ;; 207 Multi-status (Added by DAV)
528 ;; No new data, just stay at the same document
532 ;; Generic success for all others. Store in the cache, and
533 ;; mark it as successful.
539 ;; 300 Multiple choices
540 ;; 301 Moved permanently
541 ;; 302 Found
542 ;; 303 See other
543 ;; 304 Not modified
544 ;; 305 Use proxy
545 ;; 307 Temporary redirect
550 ;; Quoth the spec (section 10.3.1)
551 ;; -------------------------------
552 ;; The requested resource corresponds to any one of a set of
553 ;; representations, each with its own specific location and
554 ;; agent-driven negotiation information is being provided so
555 ;; that the user can select a preferred representation and
556 ;; redirect its request to that location.
557 ;; [...]
558 ;; If the server has a preferred choice of representation, it
559 ;; SHOULD include the specific URI for that representation in
560 ;; the Location field; user agents MAY use the Location field
561 ;; value for automatic redirection.
562 ;; -------------------------------
563 ;; We do not support agent-driven negotiation, so we just
564 ;; redirect to the preferred URI if one is provided.
565 nil)
567 ;; If the 301|302 status code is received in response to a
568 ;; request other than GET or HEAD, the user agent MUST NOT
569 ;; automatically redirect the request unless it can be
570 ;; confirmed by the user, since this might change the
571 ;; conditions under which the request was issued.
575 ;; The response to the request can be found under a different
576 ;; URI and SHOULD be retrieved using a GET method on that
577 ;; resource.
579 url-http-data nil))
581 ;; The 304 response MUST NOT contain a message-body.
586 success t))
588 ;; The requested resource MUST be accessed through the
589 ;; proxy given by the Location field. The Location field
590 ;; gives the URI of the proxy. The recipient is expected
591 ;; to repeat this single request via the proxy. 305
592 ;; responses MUST only be generated by origin servers.
594 redirect-uri))
596 ;; Treat everything like '300'
597 nil))
599 ;; Clean off any whitespace and/or <...> cruft.
605 ;; Some stupid sites (like sourceforge) send a
606 ;; non-fully-qualified URL (ie: /), which royally confuses
607 ;; the URL library.
609 ;; Be careful to use the real target URL, otherwise we may
610 ;; compute the redirection relative to the URL of the proxy.
616 ;; Check existing number of redirects
627 ;; url-max-redirections hasn't been reached, so go
628 ;; ahead and redirect.
630 ;; Remember that the request was redirected.
634 ;; Put in the current buffer a forwarding pointer to the new
635 ;; destination buffer.
636 ;; FIXME: This is a hack to fix url-retrieve-synchronously
637 ;; without changing the API. Instead url-retrieve should
638 ;; either simply not return the "destination" buffer, or it
639 ;; should take an optional `dest-buf' argument.
642 redirect-uri url-callback-function
643 url-callback-arguments
646 ;; We hit url-max-redirections, so issue an error and
647 ;; stop redirecting.
651 redirect-uri))
655 ;; 400 Bad Request
656 ;; 401 Unauthorized
657 ;; 402 Payment required
658 ;; 403 Forbidden
659 ;; 404 Not found
660 ;; 405 Method not allowed
661 ;; 406 Not acceptable
662 ;; 407 Proxy authentication required
663 ;; 408 Request time-out
664 ;; 409 Conflict
665 ;; 410 Gone
666 ;; 411 Length required
667 ;; 412 Precondition failed
668 ;; 413 Request entity too large
669 ;; 414 Request-URI too large
670 ;; 415 Unsupported media type
671 ;; 416 Requested range not satisfiable
672 ;; 417 Expectation failed
673 ;; 422 Unprocessable Entity (Added by DAV)
674 ;; 423 Locked
675 ;; 424 Failed Dependency
678 ;; The request requires user authentication. The response
679 ;; MUST include a WWW-Authenticate header field containing a
680 ;; challenge applicable to the requested resource. The
681 ;; client MAY repeat the request with a suitable
682 ;; Authorization header field.
685 ;; This code is reserved for future use
689 ;; The server understood the request, but is refusing to
690 ;; fulfill it. Authorization will not help and the request
691 ;; SHOULD NOT be repeated.
694 ;; Not found
697 ;; The method specified in the Request-Line is not allowed
698 ;; for the resource identified by the Request-URI. The
699 ;; response MUST include an Allow header containing a list of
700 ;; valid methods for the requested resource.
703 ;; The resource identified by the request is only capable of
704 ;; generating response entities which have content
705 ;; characteristics not acceptable according to the accept
706 ;; headers sent in the request.
709 ;; This code is similar to 401 (Unauthorized), but indicates
710 ;; that the client must first authenticate itself with the
711 ;; proxy. The proxy MUST return a Proxy-Authenticate header
712 ;; field containing a challenge applicable to the proxy for
713 ;; the requested resource.
716 ;; The client did not produce a request within the time that
717 ;; the server was prepared to wait. The client MAY repeat
718 ;; the request without modifications at any later time.
721 ;; The request could not be completed due to a conflict with
722 ;; the current state of the resource. This code is only
723 ;; allowed in situations where it is expected that the user
724 ;; might be able to resolve the conflict and resubmit the
725 ;; request. The response body SHOULD include enough
726 ;; information for the user to recognize the source of the
727 ;; conflict.
730 ;; The requested resource is no longer available at the
731 ;; server and no forwarding address is known.
734 ;; The server refuses to accept the request without a defined
735 ;; Content-Length. The client MAY repeat the request if it
736 ;; adds a valid Content-Length header field containing the
737 ;; length of the message-body in the request message.
738 ;;
739 ;; NOTE - this will never happen because
740 ;; `url-http-create-request' automatically calculates the
741 ;; content-length.
744 ;; The precondition given in one or more of the
745 ;; request-header fields evaluated to false when it was
746 ;; tested on the server.
749 ;; The server is refusing to process a request because the
750 ;; request entity|URI is larger than the server is willing or
751 ;; able to process.
754 ;; The server is refusing to service the request because the
755 ;; entity of the request is in a format not supported by the
756 ;; requested resource for the requested method.
759 ;; A server SHOULD return a response with this status code if
760 ;; a request included a Range request-header field, and none
761 ;; of the range-specifier values in this field overlap the
762 ;; current extent of the selected resource, and the request
763 ;; did not include an If-Range request-header field.
766 ;; The expectation given in an Expect request-header field
767 ;; could not be met by this server, or, if the server is a
768 ;; proxy, the server has unambiguous evidence that the
769 ;; request could not be met by the next-hop server.
772 ;; The request could not be understood by the server due to
773 ;; malformed syntax. The client SHOULD NOT repeat the
774 ;; request without modifications.
776 ;; Tell the callback that an error occurred, and what the
777 ;; status code was.
783 ;; 500 Internal server error
784 ;; 501 Not implemented
785 ;; 502 Bad gateway
786 ;; 503 Service unavailable
787 ;; 504 Gateway time-out
788 ;; 505 HTTP version not supported
789 ;; 507 Insufficient storage
793 ;; The server does not support the functionality required to
794 ;; fulfill the request.
795 nil)
797 ;; The server, while acting as a gateway or proxy, received
798 ;; an invalid response from the upstream server it accessed
799 ;; in attempting to fulfill the request.
800 nil)
802 ;; The server is currently unable to handle the request due
803 ;; to a temporary overloading or maintenance of the server.
804 ;; The implication is that this is a temporary condition
805 ;; which will be alleviated after some delay. If known, the
806 ;; length of the delay MAY be indicated in a Retry-After
807 ;; header. If no Retry-After is given, the client SHOULD
808 ;; handle the response as it would for a 500 response.
809 nil)
811 ;; The server, while acting as a gateway or proxy, did not
812 ;; receive a timely response from the upstream server
813 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
814 ;; auxiliary server (e.g. DNS) it needed to access in
815 ;; attempting to complete the request.
816 nil)
818 ;; The server does not support, or refuses to support, the
819 ;; HTTP protocol version that was used in the request
820 ;; message.
821 nil)
823 ;; The method could not be performed on the resource
824 ;; because the server is unable to store the representation
825 ;; needed to successfully complete the request. This
826 ;; condition is considered to be temporary. If the request
827 ;; which received this status code was the result of a user
828 ;; action, the request MUST NOT be repeated until it is
829 ;; requested by a separate user action.
830 nil))
831 ;; Tell the callback that an error occurred, and what the
832 ;; status code was.
839 class url-http-response-status)))
844 success))
846 ;; Miscellaneous
848 "Activate callback specified when this buffer was created."
850 url-callback-function
851 url-callback-arguments))
854 url-http-process)
858 ;; )
860 ;; These unfortunately cannot be macros... please ignore them!
862 "Remove (now defunct) process PROC from the list of open connections."
866 url-http-open-connections))
869 ;; Sentinel used for old HTTP/0.9 or connections we know are going
870 ;; to die as the 'end of document' notifier.
878 ;; HTTP/0.9 just gets passed back no matter what
884 ;; Function used when we do NOT know how long the document is going to be
885 ;; Just _very_ simple 'downloaded %d' type of info.
890 "Function used when we DO know how long the document is going to be.
891 More sophisticated percentage downloaded, etc.
892 Also does minimal parsing of HTTP headers and will actually cause
893 the callback to be triggered."
895 url-http-end-of-headers
896 url-http-content-length
897 url-http-content-type
898 url-http-process))
901 "Reading [%s]... %s of %s (%d%%)"
903 url-http-content-length)
904 url-http-content-type
908 url-http-content-length))
910 "Reading... %s of %s (%d%%)"
912 url-http-content-length)
916 url-http-content-length)))
920 ;; Found the end of the document! Wheee!
927 "Function used when dealing with 'chunked' encoding.
928 Cannot give a sophisticated percentage, but we need a different
929 function to look for the special 0-length chunk that signifies
930 the end of the document."
932 url-http-end-of-headers
933 url-http-content-type
934 url-http-chunked-length
935 url-http-chunked-counter
936 url-http-process url-http-chunked-start))
943 ;; We need to loop thru looking for more chunks even within
944 ;; one after-change-function call.
949 "Reading [%s]... chunk #%d"
950 url-http-content-type url-http-chunked-counter)
952 "Reading... chunk #%d"
953 url-http-chunked-counter))
955 url-http-chunked-counter st nd length)
961 ;; We know how long the chunk is supposed to be, skip over
962 ;; leading crap if possible.
966 url-http-chunked-counter)
968 url-http-chunked-length)))
971 url-http-chunked-length)
972 nd))
979 ;; Must not have received the entirety of the chunk header,
980 ;; need to spin some more.
985 'end-open t
986 'chunked-encoding t
998 ; (if (not url-http-debug)
1001 url-http-chunked-counter url-http-chunked-length
1005 ;; Found the end of the document! Wheee!
1009 ;; Every chunk, even the last 0-length one, is
1010 ;; terminated by CRLF. Skip it.
1020 ;; This will wait for the headers to arrive and then splice in the
1021 ;; next appropriate after-change-function, etc.
1023 url-http-end-of-headers
1024 url-http-content-type
1025 url-http-content-length
1026 url-http-transfer-encoding
1027 url-callback-function
1028 url-callback-arguments
1029 url-http-process
1030 url-http-method
1031 url-http-after-change-function
1032 url-http-response-status))
1043 ;; Not HTTP/x.y data, must be 0.9
1044 ;; God, I wish this could die.
1046 url-http-end-of-headers 0
1047 old-http t)
1049 ;; Saw the end of the headers
1053 end-of-headers t)
1057 ;; Haven't seen the end of the headers yet, need to wait
1058 ;; for more data to arrive.
1059 nil
1076 ;; We got back a headerless malformed response from the
1077 ;; server.
1086 ;; A HEAD request is _ALWAYS_ terminated by the header
1087 ;; information, regardless of any entity headers,
1088 ;; according to section 4.4 of the HTTP/1.1 draft.
1094 ;; A CONNECT request is finished, but we cannot stick this
1095 ;; back on the free connection list
1100 ;; Only allowed to have a header section. We have to handle
1101 ;; this here instead of in url-http-parse-headers because if
1102 ;; you have a cached copy of something without a known
1103 ;; content-length, and try to retrieve it from the cache, we'd
1104 ;; fall into the 'being dumb' section and wait for the
1105 ;; connection to terminate, which means we'd wait for 10
1106 ;; seconds for the keep-alives to time out on some servers.
1110 ;; HTTP/0.9 always signaled end-of-connection by closing the
1111 ;; connection.
1133 ;; We got a NULL body! Activate the callback
1134 ;; immediately!
1140 ;; Have some leftover data
1144 nd
1147 nil)))
1152 ;; We are still at the beginning of the buffer... must just be
1153 ;; waiting for a response.
1158 ;;;###autoload
1160 "Retrieve URL via HTTP asynchronously.
1161 URL must be a parsed URL. See `url-generic-parse-url' for details.
1162 When retrieval is completed, the function CALLBACK is executed with
1163 CBARGS as the arguments."
1166 url-http-end-of-headers
1167 url-http-content-type
1168 url-http-content-length
1169 url-http-transfer-encoding
1170 url-http-after-change-function
1171 url-callback-function
1172 url-callback-arguments
1173 url-show-status
1174 url-http-method
1175 url-http-extra-headers
1176 url-http-data
1177 url-http-chunked-length
1178 url-http-chunked-start
1179 url-http-chunked-counter
1180 url-http-process))
1186 ;; Failed to open the connection for some reason
1197 url-http-content-type
1198 url-http-content-length
1199 url-http-transfer-encoding
1200 url-http-after-change-function
1201 url-http-response-version
1202 url-http-response-status
1203 url-http-chunked-length
1204 url-http-chunked-counter
1205 url-http-chunked-start
1206 url-callback-function
1207 url-callback-arguments
1208 url-show-status
1209 url-http-process
1210 url-http-method
1211 url-http-extra-headers
1212 url-http-data
1213 url-http-target-url
1214 url-http-connection-opened
1215 url-http-proxy))
1219 url-http-extra-headers url-request-extra-headers
1220 url-http-data url-request-data
1221 url-http-process connection
1222 url-http-chunked-length nil
1223 url-http-chunked-start nil
1224 url-http-chunked-counter 0
1225 url-callback-function callback
1226 url-callback-arguments cbargs
1227 url-http-after-change-function 'url-http-wait-for-headers-change-function
1228 url-http-target-url url-current-object
1229 url-http-connection-opened nil
1230 url-http-proxy url-using-proxy)
1237 ;; Asynchronous connection
1240 ;; Asynchronous connection failed
1245 buffer))
1249 ;; We are performing an asynchronous connection, and a status change
1250 ;; has occurred.
1271 ;; Since Emacs 19/20 does not allow you to change the
1272 ;; `after-change-functions' hook in the midst of running them, we fake
1273 ;; an after change by hooking into the process filter and inserting
1274 ;; the data ourselves. This is slightly less efficient, but there
1275 ;; were tons of weird ways the after-change code was biting us in the
1276 ;; shorts.
1277 ;; FIXME this can probably be simplified since the above is no longer true.
1279 ;; Sometimes we get a zero-length data chunk after the process has
1280 ;; been changed to 'free', which means it has no buffer associated
1281 ;; with it. Do nothing if there is no buffer, or 0 length data.
1286 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1295 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1296 ;;; file-name-handler stuff from here on out
1297 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1300 'symbol-value-in-buffer
1302 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1305 unbound-value
1313 ;;;###autoload
1325 exists))
1327 ;;;###autoload
1335 nil ;dir / link / normal file
1338 nil nil nil ;atime ; mtime ; ctime
1347 ;;;###autoload
1353 ;;;###autoload
1355 "Return a property list describing options available for URL.
1356 This list is retrieved using the `OPTIONS' HTTP method.
1358 Property list members:
1360 methods
1361 A list of symbols specifying what HTTP methods the resource
1362 supports.
1364 dav
1365 A list of numbers specifying what DAV protocol/schema versions are
1366 supported.
1368 dasl
1369 A list of supported DASL search types supported (string form)
1371 ranges
1372 A list of the units available for use in partial document fetches.
1374 p3p
1375 The `Platform For Privacy Protection' description for the resource.
1376 Currently this is just the raw header contents. This is likely to
1377 change once P3P is formally supported by the URL package or
1378 Emacs/W3."
1386 ;; Only parse the options if we got a 2xx response code!
1392 ;; Figure out what methods are supported.
1395 options 'methods
1398 ;; Check for DAV
1401 options 'dav
1406 ;; Now for DASL
1409 options 'dasl
1412 ;; P3P - should get more detailed here. FIXME
1416 ;; Check for whether they accept byte-range requests.
1419 options 'ranges
1423 ))))
1425 options))
1427 ;; HTTPS. This used to be in url-https.el, but that file collides
1428 ;; with url-http.el on systems with 8-character file names.
1431 ;;;###autoload
1433 ;;;###autoload
1436 ;; FIXME what is the point of this alias being an autoload?
1437 ;; Trying to use it will not cause url-http to be loaded,
1438 ;; since the full alias just gets dumped into loaddefs.el.
1440 ;;;###autoload (autoload 'url-default-expander "url-expand")
1441 ;;;###autoload
1451 ;;;###autoload (autoload 'url-https "url-http")
1453 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1455 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1457 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1462 ;;; url-http.el ends here