| blob | bd3d3cb075ab6afa2b3fffd2131e0038231378e8 |
1 ;;; url-http.el --- HTTP retrieval routines
3 ;; Copyright (C) 1999, 2001, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
5 ;; Author: Bill Perry <wmperry@gnu.org>
6 ;; Keywords: comm, data, processes
8 ;; This file is part of GNU Emacs.
9 ;;
10 ;; GNU Emacs is free software: you can redistribute it and/or modify
11 ;; it under the terms of the GNU General Public License as published by
12 ;; the Free Software Foundation, either version 3 of the License, or
13 ;; (at your option) any later version.
15 ;; GNU Emacs is distributed in the hope that it will be useful,
16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;; GNU General Public License for more details.
20 ;; You should have received a copy of the GNU General Public License
21 ;; along with GNU Emacs. If not, see <http://www.gnu.org/licenses/>.
23 ;;; Commentary:
25 ;;; Code:
53 "What version of HTTP we advertise, as a string.
54 Valid values are 1.1 and 1.0.
55 This is only useful when debugging the HTTP subsystem.
57 Setting this to 1.0 will tell servers not to send chunked encoding,
61 "Whether to use a single TCP connection multiple times in HTTP.
62 This is only useful when debugging the HTTP subsystem. Setting to
63 nil will explicitly close the connection to the server after every
66 ;(eval-when-compile
67 ;; These are all macros so that they are hidden from external sight
68 ;; when the file is byte-compiled.
69 ;;
70 ;; This allows us to expose just the entry points we want.
72 ;; These routines will allow us to implement persistent HTTP
73 ;; connections.
77 ;; The user hit C-g, honor it! Some things can get in an
78 ;; incredibly tight loop (chunked encoding)
91 url-http-open-connections)
92 proc)
102 url-http-open-connections))
103 nil)
122 host port
125 ;; `url-open-stream' needs a buffer in which to do things
126 ;; like authentication. But we use another buffer afterwards.
129 ;; url-open-stream might return nil.
131 ;; Drop the temp buffer link before killing the buffer.
133 proc)
136 ;; Building an HTTP request
141 ""
146 url-version
155 "Create an HTTP request for `url-http-target-url', referred to by REF-URL."
157 url-http-method url-http-data
158 url-http-extra-headers))
164 url-http-extra-headers))
166 nil
174 nil
177 proxy-info)
187 ;; Protection against stupid values in the referer
192 ;; We do not want to expose the referer if the user is paranoid.
198 ;; url-http-extra-headers contains an assoc-list of
199 ;; header/value pairs that we need to put into the request.
207 ;; This was done with a call to `format'. Concatting parts has
208 ;; the advantage of keeping the parts of each header together and
209 ;; allows us to elide null lines directly, at the cost of making
210 ;; the layout less clear.
212 ;; We used to concat directly, but if one of the strings happens
213 ;; to being multibyte (even if it only contains pure ASCII) then
214 ;; every string gets converted with `string-MAKE-multibyte' which
215 ;; turns the 127-255 codes into things like latin-1 accented chars
216 ;; (it would work right if it used `string-TO-multibyte' instead).
217 ;; So to avoid the problem we force every string to be unibyte.
219 ;; FIXME: Instead of `string-AS-unibyte' we'd want
220 ;; `string-to-unibyte', so as to properly signal an error if one
221 ;; of the strings contains a multibyte char.
222 'string-as-unibyte
225 ;; The request
229 ;; Version of MIME we speak
231 ;; (maybe) Try to keep the connection open
235 ;; HTTP extensions we support
239 ;; Who we want to talk to
246 ;; Who its from
250 ;; Encodings we understand
257 ;; Languages we understand
261 ;; Types we understand
263 ;; User agent
265 ;; Proxy Authorization
266 proxy-auth
267 ;; Authorization
268 auth
269 ;; Cookies
272 ;; If-modified-since
279 ;; Whence we came
282 extra-headers
283 ;; Length of data
289 ;; End request
291 ;; Any data
292 url-http-data))
295 request))
297 ;; Parsing routines
300 This allows us to use `mail-fetch-field', etc."
308 url-callback-function url-callback-arguments))
313 nil nil t))
318 auth
320 ;; Cheating, but who cares? :)
324 ;; find strongest supported auth
328 this-auth)))
337 type this-type
338 strength this-strength))))
345 " authentication. If you'd like to write it,"
350 type t args)))
354 url-http-extra-headers)
359 url-callback-arguments)))))))
362 "Parse just the response code."
364 url-http-response-version))
379 "Handle all set-cookie / set-cookie2 headers in an HTTP response.
380 The buffer must already be narrowed to the headers, so `mail-fetch-field' will
381 work correctly."
388 ;;; (while cookies2
389 ;;; (url-cookie-handle-set-cookie2 (pop cookies)))
390 )
391 )
394 "Parse and handle HTTP specific headers.
395 Return t if and only if the current buffer is still active and
396 should be shown to the user."
397 ;; The comments after each status code handled are taken from RFC
398 ;; 2616 (HTTP/1.1)
400 url-http-response-version
401 url-http-method url-http-data url-http-process
402 url-callback-function url-callback-arguments))
406 url-http-process)
415 ;;(narrow-to-region (point-min) url-http-end-of-headers)
417 ;; In HTTP 1.0, keep the connection only if there is a
418 ;; "Connection: keep-alive" header.
419 ;; In HTTP 1.1 (and greater), keep the connection unless there is a
420 ;; "Connection: close" header
438 ;; Classes of response codes
439 ;;
440 ;; 5xx = Server Error
441 ;; 4xx = Client Error
442 ;; 3xx = Redirection
443 ;; 2xx = Successful
444 ;; 1xx = Informational
446 ;; 100 = Continue with request
447 ;; 101 = Switching protocols
448 ;; 102 = Processing (Added by DAV)
452 ;; 200 Ok
453 ;; 201 Created
454 ;; 202 Accepted
455 ;; 203 Non-authoritative information
456 ;; 204 No content
457 ;; 205 Reset content
458 ;; 206 Partial content
459 ;; 207 Multi-status (Added by DAV)
462 ;; No new data, just stay at the same document
466 ;; Generic success for all others. Store in the cache, and
467 ;; mark it as successful.
473 ;; 300 Multiple choices
474 ;; 301 Moved permanently
475 ;; 302 Found
476 ;; 303 See other
477 ;; 304 Not modified
478 ;; 305 Use proxy
479 ;; 307 Temporary redirect
484 ;; Quoth the spec (section 10.3.1)
485 ;; -------------------------------
486 ;; The requested resource corresponds to any one of a set of
487 ;; representations, each with its own specific location and
488 ;; agent-driven negotiation information is being provided so
489 ;; that the user can select a preferred representation and
490 ;; redirect its request to that location.
491 ;; [...]
492 ;; If the server has a preferred choice of representation, it
493 ;; SHOULD include the specific URI for that representation in
494 ;; the Location field; user agents MAY use the Location field
495 ;; value for automatic redirection.
496 ;; -------------------------------
497 ;; We do not support agent-driven negotiation, so we just
498 ;; redirect to the preferred URI if one is provided.
499 nil)
501 ;; If the 301|302 status code is received in response to a
502 ;; request other than GET or HEAD, the user agent MUST NOT
503 ;; automatically redirect the request unless it can be
504 ;; confirmed by the user, since this might change the
505 ;; conditions under which the request was issued.
507 ;; Automatic redirection is ok
508 nil
509 ;; It is just too big of a pain in the ass to get this
510 ;; prompt all the time. We will just silently lose our
511 ;; data and convert to a GET method.
513 url-http-method url-http-response-status)
515 url-http-data nil)))
517 ;; The response to the request can be found under a different
518 ;; URI and SHOULD be retrieved using a GET method on that
519 ;; resource.
521 url-http-data nil))
523 ;; The 304 response MUST NOT contain a message-body.
528 success t))
530 ;; The requested resource MUST be accessed through the
531 ;; proxy given by the Location field. The Location field
532 ;; gives the URI of the proxy. The recipient is expected
533 ;; to repeat this single request via the proxy. 305
534 ;; responses MUST only be generated by origin servers.
536 redirect-uri))
538 ;; Treat everything like '300'
539 nil))
541 ;; Clean off any whitespace and/or <...> cruft.
547 ;; Some stupid sites (like sourceforge) send a
548 ;; non-fully-qualified URL (ie: /), which royally confuses
549 ;; the URL library.
551 ;; Be careful to use the real target URL, otherwise we may
552 ;; compute the redirection relative to the URL of the proxy.
558 ;; Check existing number of redirects
569 ;; url-max-redirections hasn't been reached, so go
570 ;; ahead and redirect.
572 ;; Remember that the request was redirected.
576 ;; Put in the current buffer a forwarding pointer to the new
577 ;; destination buffer.
578 ;; FIXME: This is a hack to fix url-retrieve-synchronously
579 ;; without changing the API. Instead url-retrieve should
580 ;; either simply not return the "destination" buffer, or it
581 ;; should take an optional `dest-buf' argument.
584 redirect-uri url-callback-function
585 url-callback-arguments))
587 ;; We hit url-max-redirections, so issue an error and
588 ;; stop redirecting.
592 redirect-uri))
596 ;; 400 Bad Request
597 ;; 401 Unauthorized
598 ;; 402 Payment required
599 ;; 403 Forbidden
600 ;; 404 Not found
601 ;; 405 Method not allowed
602 ;; 406 Not acceptable
603 ;; 407 Proxy authentication required
604 ;; 408 Request time-out
605 ;; 409 Conflict
606 ;; 410 Gone
607 ;; 411 Length required
608 ;; 412 Precondition failed
609 ;; 413 Request entity too large
610 ;; 414 Request-URI too large
611 ;; 415 Unsupported media type
612 ;; 416 Requested range not satisfiable
613 ;; 417 Expectation failed
614 ;; 422 Unprocessable Entity (Added by DAV)
615 ;; 423 Locked
616 ;; 424 Failed Dependency
619 ;; The request requires user authentication. The response
620 ;; MUST include a WWW-Authenticate header field containing a
621 ;; challenge applicable to the requested resource. The
622 ;; client MAY repeat the request with a suitable
623 ;; Authorization header field.
626 ;; This code is reserved for future use
630 ;; The server understood the request, but is refusing to
631 ;; fulfill it. Authorization will not help and the request
632 ;; SHOULD NOT be repeated.
635 ;; Not found
638 ;; The method specified in the Request-Line is not allowed
639 ;; for the resource identified by the Request-URI. The
640 ;; response MUST include an Allow header containing a list of
641 ;; valid methods for the requested resource.
644 ;; The resource identified by the request is only capable of
645 ;; generating response entities which have content
646 ;; characteristics nota cceptable according to the accept
647 ;; headers sent in the request.
650 ;; This code is similar to 401 (Unauthorized), but indicates
651 ;; that the client must first authenticate itself with the
652 ;; proxy. The proxy MUST return a Proxy-Authenticate header
653 ;; field containing a challenge applicable to the proxy for
654 ;; the requested resource.
657 ;; The client did not produce a request within the time that
658 ;; the server was prepared to wait. The client MAY repeat
659 ;; the request without modifications at any later time.
662 ;; The request could not be completed due to a conflict with
663 ;; the current state of the resource. This code is only
664 ;; allowed in situations where it is expected that the user
665 ;; mioght be able to resolve the conflict and resubmit the
666 ;; request. The response body SHOULD include enough
667 ;; information for the user to recognize the source of the
668 ;; conflict.
671 ;; The requested resource is no longer available at the
672 ;; server and no forwarding address is known.
675 ;; The server refuses to accept the request without a defined
676 ;; Content-Length. The client MAY repeat the request if it
677 ;; adds a valid Content-Length header field containing the
678 ;; length of the message-body in the request message.
679 ;;
680 ;; NOTE - this will never happen because
681 ;; `url-http-create-request' automatically calculates the
682 ;; content-length.
685 ;; The precondition given in one or more of the
686 ;; request-header fields evaluated to false when it was
687 ;; tested on the server.
690 ;; The server is refusing to process a request because the
691 ;; request entity|URI is larger than the server is willing or
692 ;; able to process.
695 ;; The server is refusing to service the request because the
696 ;; entity of the request is in a format not supported by the
697 ;; requested resource for the requested method.
700 ;; A server SHOULD return a response with this status code if
701 ;; a request included a Range request-header field, and none
702 ;; of the range-specifier values in this field overlap the
703 ;; current extent of the selected resource, and the request
704 ;; did not include an If-Range request-header field.
707 ;; The expectation given in an Expect request-header field
708 ;; could not be met by this server, or, if the server is a
709 ;; proxy, the server has unambiguous evidence that the
710 ;; request could not be met by the next-hop server.
713 ;; The request could not be understood by the server due to
714 ;; malformed syntax. The client SHOULD NOT repeat the
715 ;; request without modifications.
717 ;; Tell the callback that an error occurred, and what the
718 ;; status code was.
724 ;; 500 Internal server error
725 ;; 501 Not implemented
726 ;; 502 Bad gateway
727 ;; 503 Service unavailable
728 ;; 504 Gateway time-out
729 ;; 505 HTTP version not supported
730 ;; 507 Insufficient storage
734 ;; The server does not support the functionality required to
735 ;; fulfill the request.
736 nil)
738 ;; The server, while acting as a gateway or proxy, received
739 ;; an invalid response from the upstream server it accessed
740 ;; in attempting to fulfill the request.
741 nil)
743 ;; The server is currently unable to handle the request due
744 ;; to a temporary overloading or maintenance of the server.
745 ;; The implication is that this is a temporary condition
746 ;; which will be alleviated after some delay. If known, the
747 ;; length of the delay MAY be indicated in a Retry-After
748 ;; header. If no Retry-After is given, the client SHOULD
749 ;; handle the response as it would for a 500 response.
750 nil)
752 ;; The server, while acting as a gateway or proxy, did not
753 ;; receive a timely response from the upstream server
754 ;; specified by the URI (e.g. HTTP, FTP, LDAP) or some other
755 ;; auxiliary server (e.g. DNS) it needed to access in
756 ;; attempting to complete the request.
757 nil)
759 ;; The server does not support, or refuses to support, the
760 ;; HTTP protocol version that was used in the request
761 ;; message.
762 nil)
764 ;; The method could not be performed on the resource
765 ;; because the server is unable to store the representation
766 ;; needed to successfully complete the request. This
767 ;; condition is considered to be temporary. If the request
768 ;; which received this status code was the result of a user
769 ;; action, the request MUST NOT be repeated until it is
770 ;; requested by a separate user action.
771 nil))
772 ;; Tell the callback that an error occurred, and what the
773 ;; status code was.
780 class url-http-response-status)))
785 success))
787 ;; Miscellaneous
789 "Activate callback specified when this buffer was created."
791 url-callback-function
792 url-callback-arguments))
795 url-http-process)
799 ;; )
801 ;; These unfortunately cannot be macros... please ignore them!
803 "Remove this (now defunct) process PROC from the list of open connections."
807 url-http-open-connections))
810 ;; Sentinel used for old HTTP/0.9 or connections we know are going
811 ;; to die as the 'end of document' notifier.
818 ;; HTTP/0.9 just gets passed back no matter what
824 ;; Function used when we do NOT know how long the document is going to be
825 ;; Just _very_ simple 'downloaded %d' type of info.
830 "Function used when we DO know how long the document is going to be.
831 More sophisticated percentage downloaded, etc.
832 Also does minimal parsing of HTTP headers and will actually cause
833 the callback to be triggered."
835 url-http-end-of-headers
836 url-http-content-length
837 url-http-content-type
838 url-http-process))
841 "Reading [%s]... %s of %s (%d%%)"
843 url-http-content-length)
844 url-http-content-type
848 url-http-content-length))
850 "Reading... %s of %s (%d%%)"
852 url-http-content-length)
856 url-http-content-length)))
860 ;; Found the end of the document! Wheee!
867 "Function used when dealing with 'chunked' encoding.
868 Cannot give a sophisticated percentage, but we need a different
869 function to look for the special 0-length chunk that signifies
870 the end of the document."
872 url-http-end-of-headers
873 url-http-content-type
874 url-http-chunked-length
875 url-http-chunked-counter
876 url-http-process url-http-chunked-start))
883 ;; We need to loop thru looking for more chunks even within
884 ;; one after-change-function call.
889 "Reading [%s]... chunk #%d"
890 url-http-content-type url-http-chunked-counter)
892 "Reading... chunk #%d"
893 url-http-chunked-counter))
895 url-http-chunked-counter st nd length)
901 ;; We know how long the chunk is supposed to be, skip over
902 ;; leading crap if possible.
906 url-http-chunked-counter)
908 url-http-chunked-length)))
911 url-http-chunked-length)
912 nd))
919 ;; Must not have received the entirety of the chunk header,
920 ;; need to spin some more.
925 'end-open t
926 'chunked-encoding t
938 ; (if (not url-http-debug)
941 url-http-chunked-counter url-http-chunked-length
945 ;; Found the end of the document! Wheee!
949 ;; Every chunk, even the last 0-length one, is
950 ;; terminated by CRLF. Skip it.
960 ;; This will wait for the headers to arrive and then splice in the
961 ;; next appropriate after-change-function, etc.
963 url-http-end-of-headers
964 url-http-content-type
965 url-http-content-length
966 url-http-transfer-encoding
967 url-callback-function
968 url-callback-arguments
969 url-http-process
970 url-http-method
971 url-http-after-change-function
972 url-http-response-status))
982 ;; Not HTTP/x.y data, must be 0.9
983 ;; God, I wish this could die.
985 url-http-end-of-headers 0
986 old-http t)
988 ;; Saw the end of the headers
992 end-of-headers t)
996 ;; Haven't seen the end of the headers yet, need to wait
997 ;; for more data to arrive.
998 nil
1004 ;;(narrow-to-region (point-min) url-http-end-of-headers)
1024 ;; A HEAD request is _ALWAYS_ terminated by the header
1025 ;; information, regardless of any entity headers,
1026 ;; according to section 4.4 of the HTTP/1.1 draft.
1032 ;; A CONNECT request is finished, but we cannot stick this
1033 ;; back on the free connectin list
1038 ;; Only allowed to have a header section. We have to handle
1039 ;; this here instead of in url-http-parse-headers because if
1040 ;; you have a cached copy of something without a known
1041 ;; content-length, and try to retrieve it from the cache, we'd
1042 ;; fall into the 'being dumb' section and wait for the
1043 ;; connection to terminate, which means we'd wait for 10
1044 ;; seconds for the keep-alives to time out on some servers.
1048 ;; HTTP/0.9 always signaled end-of-connection by closing the
1049 ;; connection.
1071 ;; We got a NULL body! Activate the callback
1072 ;; immediately!
1078 ;; Have some leftover data
1082 nd
1085 nil)))
1090 ;; We are still at the beginning of the buffer... must just be
1091 ;; waiting for a response.
1095 ;;;###autoload
1097 "Retrieve URL via HTTP asynchronously.
1098 URL must be a parsed URL. See `url-generic-parse-url' for details.
1099 When retrieval is completed, the function CALLBACK is executed with
1100 CBARGS as the arguments."
1103 url-http-end-of-headers
1104 url-http-content-type
1105 url-http-content-length
1106 url-http-transfer-encoding
1107 url-http-after-change-function
1108 url-callback-function
1109 url-callback-arguments
1110 url-http-method
1111 url-http-extra-headers
1112 url-http-data
1113 url-http-chunked-length
1114 url-http-chunked-start
1115 url-http-chunked-counter
1116 url-http-process))
1122 ;; Failed to open the connection for some reason
1133 url-http-content-type
1134 url-http-content-length
1135 url-http-transfer-encoding
1136 url-http-after-change-function
1137 url-http-response-version
1138 url-http-response-status
1139 url-http-chunked-length
1140 url-http-chunked-counter
1141 url-http-chunked-start
1142 url-callback-function
1143 url-callback-arguments
1144 url-http-process
1145 url-http-method
1146 url-http-extra-headers
1147 url-http-data
1148 url-http-target-url
1149 url-http-connection-opened
1150 url-http-proxy))
1154 url-http-extra-headers url-request-extra-headers
1155 url-http-data url-request-data
1156 url-http-process connection
1157 url-http-chunked-length nil
1158 url-http-chunked-start nil
1159 url-http-chunked-counter 0
1160 url-callback-function callback
1161 url-callback-arguments cbargs
1162 url-http-after-change-function 'url-http-wait-for-headers-change-function
1163 url-http-target-url url-current-object
1164 url-http-connection-opened nil
1165 url-http-proxy url-using-proxy)
1172 ;; Asynchronous connection
1175 ;; Asynchronous connection failed
1180 buffer))
1184 ;; We are performing an asynchronous connection, and a status change
1185 ;; has occurred.
1201 ;; Since Emacs 19/20 does not allow you to change the
1202 ;; `after-change-functions' hook in the midst of running them, we fake
1203 ;; an after change by hooking into the process filter and inserting
1204 ;; the data ourselves. This is slightly less efficient, but there
1205 ;; were tons of weird ways the after-change code was biting us in the
1206 ;; shorts.
1208 ;; Sometimes we get a zero-length data chunk after the process has
1209 ;; been changed to 'free', which means it has no buffer associated
1210 ;; with it. Do nothing if there is no buffer, or 0 length data.
1215 (url-http-debug "Calling after change function `%s' for `%S'" url-http-after-change-function proc)
1224 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1225 ;;; file-name-handler stuff from here on out
1226 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1229 'symbol-value-in-buffer
1231 "Return the value of SYMBOL in BUFFER, or UNBOUND-VALUE if it is unbound."
1234 unbound-value
1242 ;;;###autoload
1254 exists))
1256 ;;;###autoload
1264 nil ;dir / link / normal file
1267 nil nil nil ;atime ; mtime ; ctime
1276 ;;;###autoload
1282 ;;;###autoload
1284 "Return a property list describing options available for URL.
1285 This list is retrieved using the `OPTIONS' HTTP method.
1287 Property list members:
1289 methods
1290 A list of symbols specifying what HTTP methods the resource
1291 supports.
1293 dav
1294 A list of numbers specifying what DAV protocol/schema versions are
1295 supported.
1297 dasl
1298 A list of supported DASL search types supported (string form)
1300 ranges
1301 A list of the units available for use in partial document fetches.
1303 p3p
1304 The `Platform For Privacy Protection' description for the resource.
1305 Currently this is just the raw header contents. This is likely to
1306 change once P3P is formally supported by the URL package or
1307 Emacs/W3."
1315 ;; Only parse the options if we got a 2xx response code!
1321 ;; Figure out what methods are supported.
1324 options 'methods
1327 ;; Check for DAV
1330 options 'dav
1335 ;; Now for DASL
1338 options 'dasl
1341 ;; P3P - should get more detailed here. FIXME
1345 ;; Check for whether they accept byte-range requests.
1348 options 'ranges
1352 ))))
1354 options))
1356 ;; HTTPS. This used to be in url-https.el, but that file collides
1357 ;; with url-http.el on systems with 8-character file names.
1360 ;;;###autoload
1362 ;;;###autoload
1364 ;;;###autoload
1374 ;;;###autoload (autoload 'url-https "url-http")
1376 ;;;###autoload (autoload 'url-https-file-exists-p "url-http")
1378 ;;;###autoload (autoload 'url-https-file-readable-p "url-http")
1380 ;;;###autoload (autoload 'url-https-file-attributes "url-http")
1385 ;; arch-tag: ba7c59ae-c0f4-4a31-9617-d85f221732ee
1386 ;;; url-http.el ends here