lisp/net/network-stream.el

   1 ;;; network-stream.el --- open network processes, possibly with encryption -*- lexical-binding: t -*-
   2
   3 ;; Copyright (C) 2010-2018 Free Software Foundation, Inc.
   4
   5 ;; Author: Lars Magne Ingebrigtsen <larsi@gnus.org>
   6 ;; Keywords: network
   7
   8 ;; This file is part of GNU Emacs.
   9
  10 ;; GNU Emacs is free software: you can redistribute it and/or modify
  11 ;; it under the terms of the GNU General Public License as published by
  12 ;; the Free Software Foundation, either version 3 of the License, or
  13 ;; (at your option) any later version.
  14
  15 ;; GNU Emacs is distributed in the hope that it will be useful,
  16 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  17 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 ;; GNU General Public License for more details.
  19
  20 ;; You should have received a copy of the GNU General Public License
  21 ;; along with GNU Emacs.  If not, see <https://www.gnu.org/licenses/>.
  22
  23 ;;; Commentary:
  24
  25 ;; This library provides the function `open-network-stream', which provides a
  26 ;; higher-level interface for opening TCP network processes than the built-in
  27 ;; function `make-network-process'.  In addition to plain connections, it
  28 ;; supports TLS/SSL and STARTTLS connections.
  29
  30 ;; Usage example:
  31
  32 ;; (open-network-stream
  33 ;;  "*nnimap*" buffer address port
  34 ;;  :type 'network
  35 ;;  :capability-command "1 CAPABILITY\r\n"
  36 ;;  :success " OK "
  37 ;;  :starttls-function
  38 ;;  (lambda (capabilities)
  39 ;;    (if (not (string-match "STARTTLS" capabilities))
  40 ;;        nil
  41 ;;      "1 STARTTLS\r\n")))
  42
  43 ;;; Code:
  44
  45 (require 'tls)
  46 (require 'starttls)
  47 (require 'auth-source)
  48 (require 'nsm)
  49 (require 'puny)
  50
  51 (autoload 'gnutls-negotiate "gnutls")
  52 (autoload 'open-gnutls-stream "gnutls")
  53
  54 ;;;###autoload
  55 (defun open-network-stream (name buffer host service &rest parameters)
  56   "Open a TCP connection to HOST, optionally with encryption.
  57 Normally, return a network process object; with a non-nil
  58 :return-list parameter, return a list instead (see below).
  59 Input and output work as for subprocesses; `delete-process'
  60 closes it.
  61
  62 NAME is the name for the process.  It is modified if necessary to
  63  make it unique.
  64 BUFFER is a buffer or buffer name to associate with the process.
  65  Process output goes at end of that buffer.  BUFFER may be nil,
  66  meaning that the process is not associated with any buffer.
  67 HOST is the name or IP address of the host to connect to.
  68 SERVICE is the name of the service desired, or an integer or
  69  integer string specifying a port number to connect to.
  70
  71 The remaining PARAMETERS should be a sequence of keywords and
  72 values:
  73
  74 :type specifies the connection type, one of the following:
  75   nil or `network'
  76              -- Begin with an ordinary network connection, and if
  77                 the parameters :success and :capability-command
  78                 are also supplied, try to upgrade to an encrypted
  79                 connection via STARTTLS.  Even if that
  80                 fails (e.g. if HOST does not support TLS), retain
  81                 an unencrypted connection.
  82   `plain'    -- An ordinary, unencrypted network connection.
  83   `starttls' -- Begin with an ordinary connection, and try
  84                 upgrading via STARTTLS.  If that fails for any
  85                 reason, drop the connection; in that case the
  86                 returned object is a killed process.
  87   `tls'      -- A TLS connection.
  88   `ssl'      -- Equivalent to `tls'.
  89   `shell'    -- A shell connection.
  90
  91 :return-list specifies this function's return value.
  92   If omitted or nil, return a process object.  A non-nil means to
  93   return (PROC . PROPS), where PROC is a process object and PROPS
  94   is a plist of connection properties, with these keywords:
  95    :greeting -- the greeting returned by HOST (a string), or nil.
  96    :capabilities -- a string representing HOST's capabilities,
  97                     or nil if none could be found.
  98    :type -- the resulting connection type; `plain' (unencrypted)
  99             or `tls' (TLS-encrypted).
 100
 101 :end-of-command specifies a regexp matching the end of a command.
 102
 103 :end-of-capability specifies a regexp matching the end of the
 104   response to the command specified for :capability-command.
 105   It defaults to the regexp specified for :end-of-command.
 106
 107 :success specifies a regexp matching a message indicating a
 108   successful STARTTLS negotiation.  For instance, the default
 109   should be \"^3\" for an NNTP connection.
 110
 111 :capability-command specifies a command used to query the HOST
 112   for its capabilities.  For instance, for IMAP this should be
 113   \"1 CAPABILITY\\r\\n\".
 114
 115 :starttls-function specifies a function for handling STARTTLS.
 116   This function should take one parameter, the response to the
 117   capability command, and should return the command to switch on
 118   STARTTLS if the server supports STARTTLS, and nil otherwise.
 119
 120 :always-query-capabilities says whether to query the server for
 121   capabilities, even if we're doing a `plain' network connection.
 122
 123 :client-certificate should either be a list where the first
 124   element is the certificate key file name, and the second
 125   element is the certificate file name itself, or t, which
 126   means that `auth-source' will be queried for the key and the
 127   certificate.  This parameter will only be used when doing TLS
 128   or STARTTLS connections.
 129
 130 :use-starttls-if-possible is a boolean that says to do opportunistic
 131 STARTTLS upgrades even if Emacs doesn't have built-in TLS functionality.
 132
 133 :warn-unless-encrypted is a boolean which, if :return-list is
 134 non-nil, is used warn the user if the connection isn't encrypted.
 135
 136 :nogreeting is a boolean that can be used to inhibit waiting for
 137 a greeting from the server.
 138
 139 :nowait, if non-nil, says the connection should be made
 140 asynchronously, if possible.
 141
 142 :shell-command is a format-spec string that can be used if :type
 143 is `shell'.  It has two specs, %s for host and %p for port
 144 number.  Example: \"ssh gateway nc %s %p\".
 145
 146 :tls-parameters is a list that should be supplied if you're
 147 opening a TLS connection.  The first element is the TLS
 148 type (either `gnutls-x509pki' or `gnutls-anon'), and the
 149 remaining elements should be a keyword list accepted by
 150 gnutls-boot (as returned by `gnutls-boot-parameters')."
 151   (unless (featurep 'make-network-process)
 152     (error "Emacs was compiled without networking support"))
 153   (let ((type (plist-get parameters :type))
 154         (return-list (plist-get parameters :return-list)))
 155     (if (and (not return-list)
 156              (or (eq type 'plain)
 157                  (and (memq type '(nil network))
 158                       (not (and (plist-get parameters :success)
 159                                 (plist-get parameters :capability-command))))))
 160         ;; The simplest case: wrapper around `make-network-process'.
 161         (make-network-process :name name :buffer buffer
 162                               :host (puny-encode-domain host) :service service
 163                               :nowait (plist-get parameters :nowait)
 164                               :tls-parameters
 165                               (plist-get parameters :tls-parameters))
 166       (let ((work-buffer (or buffer
 167                              (generate-new-buffer " *stream buffer*")))
 168             (fun (cond ((and (eq type 'plain)
 169                              (not (plist-get parameters
 170                                              :always-query-capabilities)))
 171                         'network-stream-open-plain)
 172                        ((memq type '(nil network starttls plain))
 173                         'network-stream-open-starttls)
 174                        ((memq type '(tls ssl)) 'network-stream-open-tls)
 175                        ((eq type 'shell) 'network-stream-open-shell)
 176                        (t (error "Invalid connection type %s" type))))
 177             result)
 178         (unwind-protect
 179             (setq result (funcall fun name work-buffer host service parameters))
 180           (unless buffer
 181             (and (processp (car result))
 182                  (set-process-buffer (car result) nil))
 183             (kill-buffer work-buffer)))
 184         (if return-list
 185             (list (car result)
 186                   :greeting     (nth 1 result)
 187                   :capabilities (nth 2 result)
 188                   :type         (nth 3 result)
 189                   :error        (nth 4 result))
 190           (car result))))))
 191
 192 (defun network-stream-certificate (host service parameters)
 193   (let ((spec (plist-get :client-certificate parameters)))
 194     (cond
 195      ((listp spec)
 196       ;; Either nil or a list with a key/certificate pair.
 197       spec)
 198      ((eq spec t)
 199       (let* ((auth-info
 200               (car (auth-source-search :max 1
 201                                        :host host
 202                                        :port service)))
 203              (key (plist-get auth-info :key))
 204              (cert (plist-get auth-info :cert)))
 205         (and key cert
 206              (list key cert)))))))
 207
 208 ;;;###autoload
 209 (defalias 'open-protocol-stream 'open-network-stream)
 210 (define-obsolete-function-alias 'open-protocol-stream 'open-network-stream
 211   "26.1")
 212
 213 (defun network-stream-open-plain (name buffer host service parameters)
 214   (let ((start (with-current-buffer buffer (point)))
 215         (stream (make-network-process :name name :buffer buffer
 216                                       :host (puny-encode-domain host)
 217                                       :service service
 218                                       :nowait (plist-get parameters :nowait))))
 219     (when (plist-get parameters :warn-unless-encrypted)
 220       (setq stream (nsm-verify-connection stream host service nil t)))
 221     (list stream
 222           (network-stream-get-response stream start
 223                                        (plist-get parameters :end-of-command))
 224           nil
 225           'plain)))
 226
 227 (defun network-stream-open-starttls (name buffer host service parameters)
 228   (let* ((start (with-current-buffer buffer (point)))
 229          (require-tls    (eq (plist-get parameters :type) 'starttls))
 230          (starttls-function  (plist-get parameters :starttls-function))
 231          (success-string     (plist-get parameters :success))
 232          (capability-command (plist-get parameters :capability-command))
 233          (eoc                (plist-get parameters :end-of-command))
 234          (eo-capa            (or (plist-get parameters :end-of-capability)
 235                                  eoc))
 236          ;; Return (STREAM GREETING CAPABILITIES RESULTING-TYPE)
 237          (stream (make-network-process :name name :buffer buffer
 238                                        :host (puny-encode-domain host)
 239                                        :service service))
 240          (greeting (and (not (plist-get parameters :nogreeting))
 241                         (network-stream-get-response stream start eoc)))
 242          (capabilities (network-stream-command stream capability-command
 243                                                eo-capa))
 244          (resulting-type 'plain)
 245          starttls-available starttls-command error)
 246
 247     ;; First check whether the server supports STARTTLS at all.
 248     (when (and capabilities success-string starttls-function)
 249       (setq starttls-command
 250             (funcall starttls-function capabilities)))
 251     ;; If we have built-in STARTTLS support, try to upgrade the
 252     ;; connection.
 253     (when (and starttls-command
 254                (setq starttls-available
 255                      (or (gnutls-available-p)
 256                          (and (or require-tls
 257                                   (plist-get parameters :use-starttls-if-possible))
 258                               (starttls-available-p))))
 259                (not (eq (plist-get parameters :type) 'plain)))
 260       ;; If using external STARTTLS, drop this connection and start
 261       ;; anew with `starttls-open-stream'.
 262       (unless (gnutls-available-p)
 263         (delete-process stream)
 264         (setq start (with-current-buffer buffer (point-max)))
 265         (let* ((starttls-extra-arguments
 266                 (if (or require-tls
 267                         (member "--insecure" starttls-extra-arguments))
 268                     starttls-extra-arguments
 269                   ;; For opportunistic TLS upgrades, we don't really
 270                   ;; care about the identity of the peer.
 271                   (cons "--insecure" starttls-extra-arguments)))
 272                (starttls-extra-args starttls-extra-args)
 273                (cert (network-stream-certificate host service parameters)))
 274           ;; There are client certificates requested, so add them to
 275           ;; the command line.
 276           (when cert
 277             (setq starttls-extra-arguments
 278                   (nconc (list "--x509keyfile" (expand-file-name (nth 0 cert))
 279                                "--x509certfile" (expand-file-name (nth 1 cert)))
 280                          starttls-extra-arguments)
 281                   starttls-extra-args
 282                   (nconc (list "--key-file" (expand-file-name (nth 0 cert))
 283                                "--cert-file" (expand-file-name (nth 1 cert)))
 284                          starttls-extra-args)))
 285           (setq stream (starttls-open-stream name buffer host service)))
 286         (network-stream-get-response stream start eoc)
 287         ;; Requery capabilities for protocols that require it; i.e.,
 288         ;; EHLO for SMTP.
 289         (when (plist-get parameters :always-query-capabilities)
 290           (network-stream-command stream capability-command eo-capa)))
 291       (when (let ((response
 292                    (network-stream-command stream starttls-command eoc)))
 293               (and response (string-match success-string response)))
 294         ;; The server said it was OK to begin STARTTLS negotiations.
 295         (if (gnutls-available-p)
 296             (let ((cert (network-stream-certificate host service parameters)))
 297               (condition-case nil
 298                   (gnutls-negotiate :process stream :hostname host
 299                                     :keylist (and cert (list cert)))
 300                 ;; If we get a gnutls-specific error (for instance if
 301                 ;; the certificate the server gives us is completely
 302                 ;; syntactically invalid), then close the connection
 303                 ;; and possibly (further down) try to create a
 304                 ;; non-encrypted connection.
 305                 (gnutls-error
 306                  (delete-process stream))))
 307           (unless (starttls-negotiate stream)
 308             (delete-process stream)))
 309         (if (memq (process-status stream) '(open run))
 310             (setq resulting-type 'tls)
 311           ;; We didn't successfully negotiate STARTTLS; if TLS
 312           ;; isn't demanded, reopen an unencrypted connection.
 313           (unless require-tls
 314             (setq stream
 315                   (make-network-process :name name :buffer buffer
 316                                         :host (puny-encode-domain host)
 317                                         :service service))
 318             (network-stream-get-response stream start eoc)))
 319         (unless (process-live-p stream)
 320           (error "Unable to negotiate a TLS connection with %s/%s"
 321                  host service))
 322         ;; Re-get the capabilities, which may have now changed.
 323         (setq capabilities
 324               (network-stream-command stream capability-command eo-capa))))
 325
 326     ;; If TLS is mandatory, close the connection if it's unencrypted.
 327     (when (and require-tls
 328                ;; ... but Emacs wasn't able to -- either no built-in
 329                ;; support, or no gnutls-cli installed.
 330                (eq resulting-type 'plain))
 331       (setq error
 332             (if (or (null starttls-command)
 333                     starttls-available)
 334                 "Server does not support TLS"
 335               ;; See `starttls-available-p'.  If this predicate
 336               ;; changes to allow running under Windows, the error
 337               ;; message below should be amended.
 338               (if (memq system-type '(windows-nt ms-dos))
 339                   (concat "Emacs does not support TLS")
 340                 (concat "Emacs does not support TLS, and no external `"
 341                         (if starttls-use-gnutls
 342                             starttls-gnutls-program
 343                           starttls-program)
 344                         "' program was found"))))
 345       (delete-process stream)
 346       (setq stream nil))
 347     ;; Check certificate validity etc.
 348     (when (gnutls-available-p)
 349       (setq stream (nsm-verify-connection
 350                     stream host service
 351                     (eq resulting-type 'tls)
 352                     (plist-get parameters :warn-unless-encrypted))))
 353     ;; Return value:
 354     (list stream greeting capabilities resulting-type error)))
 355
 356 (defun network-stream-command (stream command eoc)
 357   (when command
 358     (let ((start (with-current-buffer (process-buffer stream) (point-max))))
 359       (process-send-string stream command)
 360       (network-stream-get-response stream start eoc))))
 361
 362 (defun network-stream-get-response (stream start end-of-command)
 363   (when end-of-command
 364     (with-current-buffer (process-buffer stream)
 365       (save-excursion
 366         (goto-char start)
 367         (while (and (memq (process-status stream) '(open run))
 368                     (not (re-search-forward end-of-command nil t)))
 369           (accept-process-output stream 0 50)
 370           (goto-char start))
 371         ;; Return the data we got back, or nil if the process died.
 372         (unless (= start (point))
 373           (buffer-substring start (point)))))))
 374
 375 (defun network-stream-open-tls (name buffer host service parameters)
 376   (with-current-buffer buffer
 377     (let* ((start (point-max))
 378            (stream
 379             (if (gnutls-available-p)
 380                 (open-gnutls-stream name buffer host service
 381                                     (plist-get parameters :nowait))
 382               (open-tls-stream name buffer host service)))
 383            (eoc (plist-get parameters :end-of-command)))
 384       (if (plist-get parameters :nowait)
 385           (list stream nil nil 'tls)
 386         ;; Check certificate validity etc.
 387         (when (and (gnutls-available-p) stream)
 388           (setq stream (nsm-verify-connection stream host service)))
 389         (if (null stream)
 390             (list nil nil nil 'plain)
 391           ;; If we're using tls.el, we have to delete the output from
 392           ;; openssl/gnutls-cli.
 393           (when (and (not (gnutls-available-p))
 394                      eoc)
 395             (network-stream-get-response stream start eoc)
 396             (goto-char (point-min))
 397             (when (re-search-forward eoc nil t)
 398               (goto-char (match-beginning 0))
 399               (delete-region (point-min) (line-beginning-position))))
 400           (let ((capability-command (plist-get parameters :capability-command))
 401                 (eo-capa (or (plist-get parameters :end-of-capability)
 402                              eoc)))
 403             (list stream
 404                   (network-stream-get-response stream start eoc)
 405                   (network-stream-command stream capability-command eo-capa)
 406                   'tls)))))))
 407
 408 (defun network-stream-open-shell (name buffer host service parameters)
 409   (require 'format-spec)
 410   (let* ((capability-command (plist-get parameters :capability-command))
 411          (eoc                (plist-get parameters :end-of-command))
 412          (start (with-current-buffer buffer (point)))
 413          (stream (let ((process-connection-type nil))
 414                    (start-process name buffer shell-file-name
 415                                   shell-command-switch
 416                                   (format-spec
 417                                    (plist-get parameters :shell-command)
 418                                    (format-spec-make
 419                                     ?s host
 420                                     ?p service))))))
 421     (list stream
 422           (network-stream-get-response stream start eoc)
 423           (network-stream-command stream capability-command
 424                                   (or (plist-get parameters :end-of-capability)
 425                                       eoc))
 426           'plain)))
 427
 428 (provide 'network-stream)
 429
 430 ;;; network-stream.el ends here