From 4150b563e7877a0c65233c9e7bd3fa64a7a14342 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fabi=C3=A1n=20Ezequiel=20Gallina?= Date: Fri, 21 Aug 2015 19:06:57 -0300 Subject: [PATCH] python.el: Enhancements to process environment setup. * lisp/progmodes/python.el (python-shell-process-environment) (python-shell-extra-pythonpaths, python-shell-exec-path) (python-shell-virtualenv-root): Update docstring. Remove :safe. (python-shell-setup-codes): Remove :safe. (python-shell-remote-exec-path): New defcustom. (python-shell--add-to-path-with-priority): New macro. (python-shell-calculate-pythonpath): Give priority to python-shell-extra-pythonpaths. Update docstring. (python-shell-calculate-process-environment): Give priority to python-shell-process-environment. Update docstring. (python-shell-calculate-exec-path): Give priority to python-shell-exec-path and calculated virtualenv bin directory. Update docstring. (python-shell-tramp-refresh-remote-path): New function. (python-shell-with-environment): Use it when working remotely and do not modify tramp-remote-path. Allow nesting. (python-shell-calculate-command): Remove useless python-shell-with-environment call. * test/automated/python-tests.el (python-shell-calculate-pythonpath-1) (python-shell-calculate-pythonpath-2) (python-shell-calculate-process-environment-6) (python-shell-calculate-process-environment-7) (python-shell-calculate-process-environment-8) (python-shell-calculate-exec-path-3) (python-shell-calculate-exec-path-4) (python-shell-calculate-exec-path-5) (python-shell-calculate-exec-path-6) (python-shell-with-environment-3): New tests. (python-shell-calculate-process-environment-2) (python-shell-calculate-process-environment-3) (python-shell-calculate-process-environment-4) (python-shell-calculate-process-environment-5) (python-shell-calculate-exec-path-1) (python-shell-calculate-exec-path-2) (python-shell-with-environment-1) (python-shell-with-environment-2): Update and simplify. --- lisp/progmodes/python.el | 207 +++++++++++++++++++++++------------------ test/automated/python-tests.el | 207 +++++++++++++++++++++++++++++------------ 2 files changed, 264 insertions(+), 150 deletions(-) diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el index 189cd3759f5..e6592fbd2c9 100644 --- a/lisp/progmodes/python.el +++ b/lisp/progmodes/python.el @@ -1957,42 +1957,52 @@ Python shell. See commentary for details." :safe 'booleanp) (defcustom python-shell-process-environment nil - "List of environment variables for Python shell. -This variable follows the same rules as `process-environment' -since it merges with it before the process creation routines are -called. When this variable is nil, the Python shell is run with -the default `process-environment'." + "List of overridden environment variables for subprocesses to inherit. +Each element should be a string of the form ENVVARNAME=VALUE. +When this variable is non-nil, values are exported into the +process environment before starting it. Any variables already +present in the current environment are superseded by variables +set here." :type '(repeat string) - :group 'python - :safe 'listp) + :group 'python) (defcustom python-shell-extra-pythonpaths nil "List of extra pythonpaths for Python shell. -The values of this variable are added to the existing value of -PYTHONPATH in the `process-environment' variable." +When this variable is non-nil, values added at the beginning of +the PYTHONPATH before starting processes. Any values present +here that already exists in PYTHONPATH are moved to the beginning +of the list so that they are prioritized when looking for +modules." :type '(repeat string) - :group 'python - :safe 'listp) + :group 'python) (defcustom python-shell-exec-path nil - "List of path to search for binaries. -This variable follows the same rules as `exec-path' since it -merges with it before the process creation routines are called. -When this variable is nil, the Python shell is run with the -default `exec-path'." + "List of paths for searching executables. +When this variable is non-nil, values added at the beginning of +the PATH before starting processes. Any values present here that +already exists in PATH are moved to the beginning of the list so +that they are prioritized when looking for executables." :type '(repeat string) - :group 'python - :safe 'listp) + :group 'python) + +(defcustom python-shell-remote-exec-path nil + "List of paths to be ensured remotely for searching executables. +When this variable is non-nil, values are exported into remote +hosts PATH before starting processes. Values defined in +`python-shell-exec-path' will take precedence to paths defined +here. Normally you wont use this variable directly unless you +plan to ensure a particular set of paths to all Python shell +executed through tramp connections." + :type '(repeat string) + :group 'python) (defcustom python-shell-virtualenv-root nil "Path to virtualenv root. -This variable, when set to a string, makes the values stored in -`python-shell-process-environment' and `python-shell-exec-path' -to be modified properly so shells are started with the specified +This variable, when set to a string, makes the environment to be +modified such that shells are started within the specified virtualenv." :type '(choice (const nil) string) - :group 'python - :safe 'stringp) + :group 'python) (define-obsolete-variable-alias 'python-shell-virtualenv-path 'python-shell-virtualenv-root "25.1") @@ -2002,8 +2012,7 @@ virtualenv." python-eldoc-setup-code) "List of code run by `python-shell-send-setup-codes'." :type '(repeat symbol) - :group 'python - :safe 'listp) + :group 'python) (defcustom python-shell-compilation-regexp-alist `((,(rx line-start (1+ (any " \t")) "File \"" @@ -2020,22 +2029,37 @@ virtualenv." :type '(alist string) :group 'python) +(defmacro python-shell--add-to-path-with-priority (pathvar paths) + "Modify PATHVAR and ensure PATHS are added only once at beginning." + `(dolist (path (reverse ,paths)) + (cl-delete path ,pathvar :test #'string=) + (cl-pushnew path ,pathvar :test #'string=))) + +(defun python-shell-calculate-pythonpath () + "Calculate the PYTHONPATH using `python-shell-extra-pythonpaths'." + (let ((pythonpath + (tramp-compat-split-string + (or (getenv "PYTHONPATH") "") path-separator))) + (python-shell--add-to-path-with-priority + pythonpath python-shell-extra-pythonpaths) + (mapconcat 'identity pythonpath path-separator))) + (defun python-shell-calculate-process-environment () "Calculate `process-environment' or `tramp-remote-process-environment'. -Pre-appends `python-shell-process-environment', sets extra +Prepends `python-shell-process-environment', sets extra pythonpaths from `python-shell-extra-pythonpaths' and sets a few virtualenv related vars. If `default-directory' points to a -remote machine, the returned value is intended for +remote host, the returned value is intended for `tramp-remote-process-environment'." (let* ((remote-p (file-remote-p default-directory)) - (process-environment (append - python-shell-process-environment - (if remote-p - tramp-remote-process-environment - process-environment) nil)) - (virtualenv (if python-shell-virtualenv-root - (directory-file-name python-shell-virtualenv-root) - nil))) + (process-environment (if remote-p + tramp-remote-process-environment + process-environment)) + (virtualenv (when python-shell-virtualenv-root + (directory-file-name python-shell-virtualenv-root)))) + (dolist (env python-shell-process-environment) + (pcase-let ((`(,key ,value) (split-string env "="))) + (setenv key value))) (when python-shell-unbuffered (setenv "PYTHONUNBUFFERED" "1")) (when python-shell-extra-pythonpaths @@ -2047,50 +2071,71 @@ remote machine, the returned value is intended for process-environment)) (defun python-shell-calculate-exec-path () - "Calculate `exec-path' or `tramp-remote-path'. -Pre-appends `python-shell-exec-path' and adds the binary -directory for virtualenv if `python-shell-virtualenv-root' is -set. If `default-directory' points to a remote machine, the -returned value is intended for `tramp-remote-path'." - (let ((path (append - ;; Use nil as the tail so that the list is a full copy, - ;; this is a paranoid safeguard for side-effects. - python-shell-exec-path - (if (file-remote-p default-directory) - tramp-remote-path - exec-path) - nil))) + "Calculate `exec-path'. +Prepends `python-shell-exec-path' and adds the binary directory +for virtualenv if `python-shell-virtualenv-root' is set. If +`default-directory' points to a remote host, the returned value +appends `python-shell-remote-exec-path' instead of `exec-path'." + (let ((new-path (copy-sequence + (if (file-remote-p default-directory) + python-shell-remote-exec-path + exec-path)))) + (python-shell--add-to-path-with-priority + new-path python-shell-exec-path) (if (not python-shell-virtualenv-root) - path - (cons (expand-file-name "bin" python-shell-virtualenv-root) - path)))) + new-path + (python-shell--add-to-path-with-priority + new-path + (list (expand-file-name "bin" python-shell-virtualenv-root))) + new-path))) + +(defun python-shell-tramp-refresh-remote-path (vec paths) + "Update VEC's remote-path giving PATHS priority." + (let ((remote-path (tramp-get-connection-property vec "remote-path" nil))) + (when remote-path + (python-shell--add-to-path-with-priority remote-path paths) + (tramp-set-connection-property vec "remote-path" remote-path) + (tramp-set-remote-path vec)))) + +(defvar python-shell--with-environment-wrapped nil) (defmacro python-shell-with-environment (&rest body) "Modify shell environment during execution of BODY. Temporarily sets `process-environment' and `exec-path' during execution of body. If `default-directory' points to a remote machine then modifies `tramp-remote-process-environment' and -`tramp-remote-path' instead." +`python-shell-remote-exec-path' instead." (declare (indent 0) (debug (body))) - (let ((remote-p (make-symbol "remote-p"))) - `(let* ((,remote-p (file-remote-p default-directory)) - (process-environment - (if ,remote-p - process-environment - (python-shell-calculate-process-environment))) - (tramp-remote-process-environment - (if ,remote-p - (python-shell-calculate-process-environment) - tramp-remote-process-environment)) - (exec-path - (if ,remote-p - exec-path - (python-shell-calculate-exec-path))) - (tramp-remote-path - (if ,remote-p - (python-shell-calculate-exec-path) - tramp-remote-path))) - ,(macroexp-progn body)))) + (let ((vec (make-symbol "vec"))) + `(progn + (if python-shell--with-environment-wrapped + ,(macroexp-progn body) + (let* ((,vec + (when (file-remote-p default-directory) + (ignore-errors + (tramp-dissect-file-name default-directory 'noexpand)))) + (process-environment + (if ,vec + process-environment + (python-shell-calculate-process-environment))) + (exec-path + (if ,vec + exec-path + (python-shell-calculate-exec-path))) + (tramp-remote-process-environment + (if ,vec + (python-shell-calculate-process-environment) + tramp-remote-process-environment)) + (python-shell--with-environment-wrapped t)) + (when (tramp-get-connection-process ,vec) + ;; For already existing connections, modified env vars must + ;; be re-set again. This is a normal thing to happen when + ;; remote dir-locals are read from remote and *then* + ;; processes should be started within the same connection + ;; with env vars calculated from them. + (python-shell-tramp-refresh-remote-path + ,vec (python-shell-calculate-exec-path))) + ,(macroexp-progn body)))))) (defvar python-shell--prompt-calculated-input-regexp nil "Calculated input prompt regexp for inferior python shell. @@ -2271,28 +2316,14 @@ the `buffer-name'." (defun python-shell-calculate-command () "Calculate the string used to execute the inferior Python process." - (python-shell-with-environment - ;; `exec-path' gets tweaked so that virtualenv's specific - ;; `python-shell-interpreter' absolute path can be found by - ;; `executable-find'. - (format "%s %s" - (shell-quote-argument python-shell-interpreter) - python-shell-interpreter-args))) + (format "%s %s" + (shell-quote-argument python-shell-interpreter) + python-shell-interpreter-args)) (define-obsolete-function-alias 'python-shell-parse-command #'python-shell-calculate-command "25.1") -(defun python-shell-calculate-pythonpath () - "Calculate the PYTHONPATH using `python-shell-extra-pythonpaths'." - (let ((pythonpath (getenv "PYTHONPATH")) - (extra (mapconcat 'identity - python-shell-extra-pythonpaths - path-separator))) - (if pythonpath - (concat extra path-separator pythonpath) - extra))) - (defvar python-shell--package-depth 10) (defun python-shell-package-enable (directory package) diff --git a/test/automated/python-tests.el b/test/automated/python-tests.el index d490f7f9df5..1f8533f9b1a 100644 --- a/test/automated/python-tests.el +++ b/test/automated/python-tests.el @@ -2440,107 +2440,190 @@ Using `python-shell-interpreter' and python-shell-interpreter-args) (python-shell-calculate-command))))) +(ert-deftest python-shell-calculate-pythonpath-1 () + "Test PYTHONPATH calculation." + (let ((process-environment '("PYTHONPATH=/path0")) + (python-shell-extra-pythonpaths '("/path1" "/path2"))) + (should (string= (python-shell-calculate-pythonpath) + "/path1:/path2:/path0")))) + +(ert-deftest python-shell-calculate-pythonpath-2 () + "Test existing paths are moved to front." + (let ((process-environment '("PYTHONPATH=/path0:/path1")) + (python-shell-extra-pythonpaths '("/path1" "/path2"))) + (should (string= (python-shell-calculate-pythonpath) + "/path1:/path2:/path0")))) + (ert-deftest python-shell-calculate-process-environment-1 () "Test `python-shell-process-environment' modification." (let* ((python-shell-process-environment '("TESTVAR1=value1" "TESTVAR2=value2")) - (process-environment - (python-shell-calculate-process-environment))) + (process-environment (python-shell-calculate-process-environment))) (should (equal (getenv "TESTVAR1") "value1")) (should (equal (getenv "TESTVAR2") "value2")))) (ert-deftest python-shell-calculate-process-environment-2 () "Test `python-shell-extra-pythonpaths' modification." (let* ((process-environment process-environment) - (original-pythonpath (setenv "PYTHONPATH" "path3")) - (paths '("path1" "path2")) - (python-shell-extra-pythonpaths paths) - (process-environment - (python-shell-calculate-process-environment))) - (should (equal (getenv "PYTHONPATH") - (concat - (mapconcat 'identity paths path-separator) - path-separator original-pythonpath))))) + (original-pythonpath (setenv "PYTHONPATH" "/path0")) + (python-shell-extra-pythonpaths '("/path1" "/path2")) + (process-environment (python-shell-calculate-process-environment))) + (should (equal (getenv "PYTHONPATH") "/path1:/path2:/path0")))) (ert-deftest python-shell-calculate-process-environment-3 () "Test `python-shell-virtualenv-root' modification." - (let* ((python-shell-virtualenv-root - (directory-file-name user-emacs-directory)) + (let* ((python-shell-virtualenv-root "/env") (process-environment - (python-shell-calculate-process-environment))) + (let (process-environment process-environment) + (setenv "PYTHONHOME" "/home") + (setenv "VIRTUAL_ENV") + (python-shell-calculate-process-environment)))) (should (not (getenv "PYTHONHOME"))) - (should (string= (getenv "VIRTUAL_ENV") python-shell-virtualenv-root)))) + (should (string= (getenv "VIRTUAL_ENV") "/env")))) (ert-deftest python-shell-calculate-process-environment-4 () - "Test `python-shell-unbuffered' modification." - (setenv "PYTHONUNBUFFERED") - (let* ((process-environment - (python-shell-calculate-process-environment))) - ;; Defaults to t - (should python-shell-unbuffered) + "Test PYTHONUNBUFFERED when `python-shell-unbuffered' is non-nil." + (let* ((python-shell-unbuffered t) + (process-environment + (let ((process-environment process-environment)) + (setenv "PYTHONUNBUFFERED") + (python-shell-calculate-process-environment)))) (should (string= (getenv "PYTHONUNBUFFERED") "1")))) (ert-deftest python-shell-calculate-process-environment-5 () - (setenv "PYTHONUNBUFFERED") - "Test `python-shell-unbuffered' modification." + "Test PYTHONUNBUFFERED when `python-shell-unbuffered' is nil." (let* ((python-shell-unbuffered nil) (process-environment - (python-shell-calculate-process-environment))) + (let ((process-environment process-environment)) + (setenv "PYTHONUNBUFFERED") + (python-shell-calculate-process-environment)))) (should (not (getenv "PYTHONUNBUFFERED"))))) +(ert-deftest python-shell-calculate-process-environment-6 () + "Test PYTHONUNBUFFERED=1 when `python-shell-unbuffered' is nil." + (let* ((python-shell-unbuffered nil) + (process-environment + (let ((process-environment process-environment)) + (setenv "PYTHONUNBUFFERED" "1") + (python-shell-calculate-process-environment)))) + ;; User default settings must remain untouched: + (should (string= (getenv "PYTHONUNBUFFERED") "1")))) + +(ert-deftest python-shell-calculate-process-environment-7 () + "Test no side-effects on `process-environment'." + (let* ((python-shell-process-environment + '("TESTVAR1=value1" "TESTVAR2=value2")) + (python-shell-virtualenv-root "/env") + (python-shell-unbuffered t) + (python-shell-extra-pythonpaths'("/path1" "/path2")) + (original-process-environment (copy-sequence process-environment))) + (python-shell-calculate-process-environment) + (should (equal process-environment original-process-environment)))) + +(ert-deftest python-shell-calculate-process-environment-8 () + "Test no side-effects on `tramp-remote-process-environment'." + (let* ((default-directory "/ssh::/example/dir/") + (python-shell-process-environment + '("TESTVAR1=value1" "TESTVAR2=value2")) + (python-shell-virtualenv-root "/env") + (python-shell-unbuffered t) + (python-shell-extra-pythonpaths'("/path1" "/path2")) + (original-process-environment + (copy-sequence tramp-remote-process-environment))) + (python-shell-calculate-process-environment) + (should (equal tramp-remote-process-environment original-process-environment)))) + (ert-deftest python-shell-calculate-exec-path-1 () "Test `python-shell-exec-path' modification." - (let* ((original-exec-path exec-path) - (python-shell-exec-path '("path1" "path2")) - (exec-path (python-shell-calculate-exec-path))) - (should (equal - exec-path - (append python-shell-exec-path - original-exec-path))))) + (let* ((exec-path '("/path0")) + (python-shell-exec-path '("/path1" "/path2")) + (new-exec-path (python-shell-calculate-exec-path))) + (should (equal new-exec-path '("/path1" "/path2" "/path0"))))) (ert-deftest python-shell-calculate-exec-path-2 () "Test `python-shell-virtualenv-root' modification." - (let* ((original-exec-path exec-path) - (python-shell-virtualenv-root - (directory-file-name (expand-file-name user-emacs-directory))) - (exec-path (python-shell-calculate-exec-path))) - (should (equal - exec-path - (append (cons - (format "%s/bin" python-shell-virtualenv-root) - original-exec-path)))))) + (let* ((exec-path '("/path0")) + (python-shell-virtualenv-root "/env") + (new-exec-path (python-shell-calculate-exec-path))) + (should (equal new-exec-path '("/env/bin" "/path0"))))) + +(ert-deftest python-shell-calculate-exec-path-3 () + "Test complete `python-shell-virtualenv-root' modification." + (let* ((exec-path '("/path0")) + (python-shell-exec-path '("/path1" "/path2")) + (python-shell-virtualenv-root "/env") + (new-exec-path (python-shell-calculate-exec-path))) + (should (equal new-exec-path '("/env/bin" "/path1" "/path2" "/path0"))))) + +(ert-deftest python-shell-calculate-exec-path-4 () + "Test complete `python-shell-virtualenv-root' with remote." + (let* ((default-directory "/ssh::/example/dir/") + (python-shell-remote-exec-path '("/path0")) + (python-shell-exec-path '("/path1" "/path2")) + (python-shell-virtualenv-root "/env") + (new-exec-path (python-shell-calculate-exec-path))) + (should (equal new-exec-path '("/env/bin" "/path1" "/path2" "/path0"))))) + +(ert-deftest python-shell-calculate-exec-path-5 () + "Test no side-effects on `exec-path'." + (let* ((exec-path '("/path0")) + (python-shell-exec-path '("/path1" "/path2")) + (python-shell-virtualenv-root "/env") + (original-exec-path (copy-sequence exec-path))) + (python-shell-calculate-exec-path) + (should (equal exec-path original-exec-path)))) + +(ert-deftest python-shell-calculate-exec-path-6 () + "Test no side-effects on `python-shell-remote-exec-path'." + (let* ((default-directory "/ssh::/example/dir/") + (python-shell-remote-exec-path '("/path0")) + (python-shell-exec-path '("/path1" "/path2")) + (python-shell-virtualenv-root "/env") + (original-exec-path (copy-sequence python-shell-remote-exec-path))) + (python-shell-calculate-exec-path) + (should (equal python-shell-remote-exec-path original-exec-path)))) (ert-deftest python-shell-with-environment-1 () - "Test with local `default-directory'." - (let* ((original-exec-path exec-path) - (python-shell-virtualenv-root - (directory-file-name (expand-file-name user-emacs-directory)))) + "Test environment with local `default-directory'." + (let* ((exec-path '("/path0")) + (python-shell-exec-path '("/path1" "/path2")) + (original-exec-path exec-path) + (python-shell-virtualenv-root "/env")) (python-shell-with-environment - (should (equal - exec-path - (append (cons - (format "%s/bin" python-shell-virtualenv-root) - original-exec-path)))) + (should (equal exec-path '("/env/bin" "/path1" "/path2" "/path0"))) (should (not (getenv "PYTHONHOME"))) - (should (string= (getenv "VIRTUAL_ENV") python-shell-virtualenv-root))))) + (should (string= (getenv "VIRTUAL_ENV") "/env"))) + (should (equal exec-path original-exec-path)))) (ert-deftest python-shell-with-environment-2 () - "Test with remote `default-directory'." + "Test environment with remote `default-directory'." (let* ((default-directory "/ssh::/example/dir/") - (original-exec-path tramp-remote-path) - (original-process-environment tramp-remote-process-environment) - (python-shell-virtualenv-root - (directory-file-name (expand-file-name user-emacs-directory)))) + (python-shell-remote-exec-path '("/remote1" "/remote2")) + (python-shell-exec-path '("/path1" "/path2")) + (tramp-remote-process-environment '("EMACS=t")) + (original-process-environment (copy-sequence tramp-remote-process-environment)) + (python-shell-virtualenv-root "/env")) (python-shell-with-environment - (should (equal - tramp-remote-path - (append (cons - (format "%s/bin" python-shell-virtualenv-root) - original-exec-path)))) + (should (equal (python-shell-calculate-exec-path) + '("/env/bin" "/path1" "/path2" "/remote1" "/remote2"))) (let ((process-environment tramp-remote-process-environment)) (should (not (getenv "PYTHONHOME"))) - (should (string= (getenv "VIRTUAL_ENV") - python-shell-virtualenv-root)))))) + (should (string= (getenv "VIRTUAL_ENV") "/env")))) + (should (equal tramp-remote-process-environment original-process-environment)))) + +(ert-deftest python-shell-with-environment-3 () + "Test `python-shell-with-environment' is idempotent." + (let* ((python-shell-extra-pythonpaths '("/example/dir/")) + (python-shell-exec-path '("path1" "path2")) + (python-shell-virtualenv-root "/home/user/env") + (single-call + (python-shell-with-environment + (list exec-path process-environment))) + (nested-call + (python-shell-with-environment + (python-shell-with-environment + (list exec-path process-environment))))) + (should (equal single-call nested-call)))) (ert-deftest python-shell-make-comint-1 () "Check comint creation for global shell buffer." -- 2.11.4.GIT