lisp/url/url.el

   1 ;;; url.el --- Uniform Resource Locator retrieval tool
   2
   3 ;; Copyright (C) 1996-1999, 2001, 2004-2012  Free Software Foundation, Inc.
   4
   5 ;; Author: Bill Perry <wmperry@gnu.org>
   6 ;; Keywords: comm, data, processes, hypermedia
   7
   8 ;; This file is part of GNU Emacs.
   9 ;;
  10 ;; GNU Emacs is free software: you can redistribute it and/or modify
  11 ;; it under the terms of the GNU General Public License as published by
  12 ;; the Free Software Foundation, either version 3 of the License, or
  13 ;; (at your option) any later version.
  14
  15 ;; GNU Emacs is distributed in the hope that it will be useful,
  16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 ;; GNU General Public License for more details.
  19
  20 ;; You should have received a copy of the GNU General Public License
  21 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 ;;; Commentary:
  24
  25 ;; Registered URI schemes: http://www.iana.org/assignments/uri-schemes
  26
  27 ;;; Code:
  28
  29 (eval-when-compile (require 'cl))
  30
  31 (require 'mailcap)
  32
  33 (eval-when-compile
  34   (require 'mm-decode)
  35   (require 'mm-view))
  36
  37 (require 'url-vars)
  38 (require 'url-cookie)
  39 (require 'url-history)
  40 (require 'url-expand)
  41 (require 'url-privacy)
  42 (require 'url-methods)
  43 (require 'url-proxy)
  44 (require 'url-parse)
  45 (require 'url-util)
  46
  47
  48 (defcustom url-configuration-directory
  49   (locate-user-emacs-file "url/" ".url/")
  50   "Directory used by the URL package for cookies, history, etc."
  51   :type 'directory
  52   :group 'url)
  53
  54 (defun url-do-setup ()
  55   "Setup the URL package.
  56 This is to avoid conflict with user settings if URL is dumped with
  57 Emacs."
  58   (unless url-setup-done
  59
  60     ;; Make OS/2 happy
  61     ;;(push '("http" "80") tcp-binary-process-input-services)
  62
  63     (mailcap-parse-mailcaps)
  64     (mailcap-parse-mimetypes)
  65
  66     ;; Register all the authentication schemes we can handle
  67     (url-register-auth-scheme "basic" nil 4)
  68     (url-register-auth-scheme "digest" nil 7)
  69
  70     (setq url-cookie-file
  71           (or url-cookie-file
  72               (expand-file-name "cookies" url-configuration-directory)))
  73
  74     (setq url-history-file
  75           (or url-history-file
  76               (expand-file-name "history" url-configuration-directory)))
  77
  78     ;; Parse the global history file if it exists, so that it can be used
  79     ;; for URL completion, etc.
  80     (url-history-parse-history)
  81     (url-history-setup-save-timer)
  82
  83     ;; Ditto for cookies
  84     (url-cookie-setup-save-timer)
  85     (url-cookie-parse-file url-cookie-file)
  86
  87     ;; Read in proxy gateways
  88     (let ((noproxy (and (not (assoc "no_proxy" url-proxy-services))
  89                         (or (getenv "NO_PROXY")
  90                             (getenv "no_PROXY")
  91                             (getenv "no_proxy")))))
  92       (if noproxy
  93           (setq url-proxy-services
  94                 (cons (cons "no_proxy"
  95                             (concat "\\("
  96                                     (mapconcat
  97                                      (lambda (x)
  98                                        (cond
  99                                         ((= x ?,) "\\|")
 100                                         ((= x ? ) "")
 101                                         ((= x ?.) (regexp-quote "."))
 102                                         ((= x ?*) ".*")
 103                                         ((= x ??) ".")
 104                                         (t (char-to-string x))))
 105                                      noproxy "") "\\)"))
 106                       url-proxy-services))))
 107
 108     (url-setup-privacy-info)
 109     (run-hooks 'url-load-hook)
 110     (setq url-setup-done t)))
 111
 112 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 113 ;;; Retrieval functions
 114 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 115
 116 (defvar url-redirect-buffer nil
 117   "New buffer into which the retrieval will take place.
 118 Sometimes while retrieving a URL, the URL library needs to use another buffer
 119 than the one returned initially by `url-retrieve'.  In this case, it sets this
 120 variable in the original buffer as a forwarding pointer.")
 121
 122 (defvar url-retrieve-number-of-calls 0)
 123 (autoload 'url-cache-prune-cache "url-cache")
 124
 125 ;;;###autoload
 126 (defun url-retrieve (url callback &optional cbargs silent inhibit-cookies)
 127   "Retrieve URL asynchronously and call CALLBACK with CBARGS when finished.
 128 URL is either a string or a parsed URL.
 129
 130 CALLBACK is called when the object has been completely retrieved, with
 131 the current buffer containing the object, and any MIME headers associated
 132 with it.  It is called as (apply CALLBACK STATUS CBARGS).
 133 STATUS is a list with an even number of elements representing
 134 what happened during the request, with most recent events first,
 135 or an empty list if no events have occurred.  Each pair is one of:
 136
 137 \(:redirect REDIRECTED-TO) - the request was redirected to this URL
 138 \(:error (ERROR-SYMBOL . DATA)) - an error occurred.  The error can be
 139 signaled with (signal ERROR-SYMBOL DATA).
 140
 141 Return the buffer URL will load into, or nil if the process has
 142 already completed (i.e. URL was a mailto URL or similar; in this case
 143 the callback is not called).
 144
 145 The variables `url-request-data', `url-request-method' and
 146 `url-request-extra-headers' can be dynamically bound around the
 147 request; dynamic binding of other variables doesn't necessarily
 148 take effect.
 149
 150 If SILENT, then don't message progress reports and the like.
 151 If INHIBIT-COOKIES, cookies will neither be stored nor sent to
 152 the server."
 153 ;;; XXX: There is code in Emacs that does dynamic binding
 154 ;;; of the following variables around url-retrieve:
 155 ;;; url-standalone-mode, url-gateway-unplugged, w3-honor-stylesheets,
 156 ;;; url-confirmation-func, url-cookie-multiple-line,
 157 ;;; url-cookie-{{,secure-}storage,confirmation}
 158 ;;; url-standalone-mode and url-gateway-unplugged should work as
 159 ;;; usual.  url-confirmation-func is only used in nnwarchive.el and
 160 ;;; webmail.el; the latter should be updated.  Is
 161 ;;; url-cookie-multiple-line needed anymore?  The other url-cookie-*
 162 ;;; are (for now) only used in synchronous retrievals.
 163   (url-retrieve-internal url callback (cons nil cbargs) silent
 164                          inhibit-cookies))
 165
 166 (defun url-retrieve-internal (url callback cbargs &optional silent
 167                                   inhibit-cookies)
 168   "Internal function; external interface is `url-retrieve'.
 169 CBARGS is what the callback will actually receive - the first item is
 170 the list of events, as described in the docstring of `url-retrieve'.
 171
 172 If SILENT, don't message progress reports and the like.
 173 If INHIBIT-COOKIES, cookies will neither be stored nor sent to
 174 the server."
 175   (url-do-setup)
 176   (url-gc-dead-buffers)
 177   (if (stringp url)
 178        (set-text-properties 0 (length url) nil url))
 179   (if (not (vectorp url))
 180       (setq url (url-generic-parse-url url)))
 181   (if (not (functionp callback))
 182       (error "Must provide a callback function to url-retrieve"))
 183   (unless (url-type url)
 184     (error "Bad url: %s" (url-recreate-url url)))
 185   (setf (url-silent url) silent)
 186   (setf (url-use-cookies url) (not inhibit-cookies))
 187   ;; Once in a while, remove old entries from the URL cache.
 188   (when (zerop (% url-retrieve-number-of-calls 1000))
 189     (condition-case error
 190         (url-cache-prune-cache)
 191       (file-error
 192        (message "Error when expiring the cache: %s" error))))
 193   (setq url-retrieve-number-of-calls (1+ url-retrieve-number-of-calls))
 194   (let ((loader (url-scheme-get-property (url-type url) 'loader))
 195         (url-using-proxy (if (url-host url)
 196                              (url-find-proxy-for-url url (url-host url))))
 197         (buffer nil)
 198         (asynch (url-scheme-get-property (url-type url) 'asynchronous-p)))
 199     (if url-using-proxy
 200         (setq asynch t
 201               loader 'url-proxy))
 202     (if asynch
 203         (let ((url-current-object url))
 204           (setq buffer (funcall loader url callback cbargs)))
 205       (setq buffer (funcall loader url))
 206       (if buffer
 207           (with-current-buffer buffer
 208             (apply callback cbargs))))
 209     (if url-history-track
 210         (url-history-update-url url (current-time)))
 211     buffer))
 212
 213 ;;;###autoload
 214 (defun url-retrieve-synchronously (url)
 215   "Retrieve URL synchronously.
 216 Return the buffer containing the data, or nil if there are no data
 217 associated with it (the case for dired, info, or mailto URLs that need
 218 no further processing).  URL is either a string or a parsed URL."
 219   (url-do-setup)
 220
 221   (lexical-let ((retrieval-done nil)
 222                 (asynch-buffer nil))
 223     (setq asynch-buffer
 224           (url-retrieve url (lambda (&rest ignored)
 225                               (url-debug 'retrieval "Synchronous fetching done (%S)" (current-buffer))
 226                               (setq retrieval-done t
 227                                     asynch-buffer (current-buffer)))))
 228     (if (null asynch-buffer)
 229         ;; We do not need to do anything, it was a mailto or something
 230         ;; similar that takes processing completely outside of the URL
 231         ;; package.
 232         nil
 233       (let ((proc (get-buffer-process asynch-buffer)))
 234         ;; If the access method was synchronous, `retrieval-done' should
 235         ;; hopefully already be set to t.  If it is nil, and `proc' is also
 236         ;; nil, it implies that the async process is not running in
 237         ;; asynch-buffer.  This happens e.g. for FTP files.  In such a case
 238         ;; url-file.el should probably set something like a `url-process'
 239         ;; buffer-local variable so we can find the exact process that we
 240         ;; should be waiting for.  In the mean time, we'll just wait for any
 241         ;; process output.
 242         (while (not retrieval-done)
 243           (url-debug 'retrieval
 244                      "Spinning in url-retrieve-synchronously: %S (%S)"
 245                      retrieval-done asynch-buffer)
 246           (if (buffer-local-value 'url-redirect-buffer asynch-buffer)
 247               (setq proc (get-buffer-process
 248                           (setq asynch-buffer
 249                                 (buffer-local-value 'url-redirect-buffer
 250                                                     asynch-buffer))))
 251             (if (and proc (memq (process-status proc)
 252                                 '(closed exit signal failed))
 253                      ;; Make sure another process hasn't been started.
 254                      (eq proc (or (get-buffer-process asynch-buffer) proc)))
 255                 ;; FIXME: It's not clear whether url-retrieve's callback is
 256                 ;; guaranteed to be called or not.  It seems that url-http
 257                 ;; decides sometimes consciously not to call it, so it's not
 258                 ;; clear that it's a bug, but even then we need to decide how
 259                 ;; url-http can then warn us that the download has completed.
 260                 ;; In the mean time, we use this here workaround.
 261                 ;; XXX: The callback must always be called.  Any
 262                 ;; exception is a bug that should be fixed, not worked
 263                 ;; around.
 264                 (progn ;; Call delete-process so we run any sentinel now.
 265                   (delete-process proc)
 266                   (setq retrieval-done t)))
 267             ;; We used to use `sit-for' here, but in some cases it wouldn't
 268             ;; work because apparently pending keyboard input would always
 269             ;; interrupt it before it got a chance to handle process input.
 270             ;; `sleep-for' was tried but it lead to other forms of
 271             ;; hanging.  --Stef
 272             (unless (or (with-local-quit
 273                           (accept-process-output proc))
 274                         (null proc))
 275               ;; accept-process-output returned nil, maybe because the process
 276               ;; exited (and may have been replaced with another).  If we got
 277               ;; a quit, just stop.
 278               (when quit-flag
 279                 (delete-process proc))
 280               (setq proc (and (not quit-flag)
 281                               (get-buffer-process asynch-buffer)))))))
 282       asynch-buffer)))
 283
 284 (defun url-mm-callback (&rest ignored)
 285   (let ((handle (mm-dissect-buffer t)))
 286     (url-mark-buffer-as-dead (current-buffer))
 287     (with-current-buffer
 288         (generate-new-buffer (url-recreate-url url-current-object))
 289       (if (eq (mm-display-part handle) 'external)
 290           (progn
 291             (set-process-sentinel
 292              ;; Fixme: this shouldn't have to know the form of the
 293              ;; undisplayer produced by `mm-display-part'.
 294              (get-buffer-process (cdr (mm-handle-undisplayer handle)))
 295              `(lambda (proc event)
 296                 (mm-destroy-parts (quote ,handle))))
 297             (message "Viewing externally")
 298             (kill-buffer (current-buffer)))
 299         (display-buffer (current-buffer))
 300         (add-hook 'kill-buffer-hook
 301                   `(lambda () (mm-destroy-parts ',handle))
 302                   nil
 303                   t)))))
 304
 305 (defun url-mm-url (url)
 306   "Retrieve URL and pass to the appropriate viewing application."
 307   ;; These requires could advantageously be moved to url-mm-callback or
 308   ;; turned into autoloads, but I suspect that it would introduce some bugs
 309   ;; because loading those files from a process sentinel or filter may
 310   ;; result in some undesirable corner cases.
 311   (require 'mm-decode)
 312   (require 'mm-view)
 313   (url-retrieve url 'url-mm-callback nil))
 314
 315 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 316 ;;; Miscellaneous
 317 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 318 (defvar url-dead-buffer-list nil)
 319
 320 (defun url-mark-buffer-as-dead (buff)
 321   (push buff url-dead-buffer-list))
 322
 323 (defun url-gc-dead-buffers ()
 324   (let ((buff))
 325     (while (setq buff (pop url-dead-buffer-list))
 326       (if (buffer-live-p buff)
 327           (kill-buffer buff)))))
 328
 329 (cond
 330  ((fboundp 'display-warning)
 331   (defalias 'url-warn 'display-warning))
 332  ((fboundp 'warn)
 333   (defun url-warn (class message &optional level)
 334     (warn "(%s/%s) %s" class (or level 'warning) message)))
 335  (t
 336   (defun url-warn (class message &optional level)
 337     (with-current-buffer (get-buffer-create "*URL-WARNINGS*")
 338       (goto-char (point-max))
 339       (save-excursion
 340         (insert (format "(%s/%s) %s\n" class (or level 'warning) message)))
 341       (display-buffer (current-buffer))))))
 342
 343 (provide 'url)
 344
 345 ;;; url.el ends here