Autodetect Python shell prompts.

* lisp/progmodes/python.el:
(python-shell-interpreter-interactive-arg)
(python-shell-prompt-detect-enabled)
(python-shell-prompt-detect-failure-warning)
(python-shell-prompt-input-regexps)
(python-shell-prompt-output-regexps): New vars.
(python-shell-prompt-calculated-input-regexp)
(python-shell-prompt-calculated-output-regexp): New vars.
(python-shell-get-process-name)
(python-shell-internal-get-process-name)
(python-shell-output-filter)
(python-shell-completion-get-completions): Use them.
(python-shell-prompt-detect)
(python-shell-prompt-validate-regexps): New functions.
(python-shell-prompt-set-calculated-regexps): New function.
(inferior-python-mode): Use it.  Also honor overriden
python-shell-interpreter and python-shell-interpreter-args.
(python-shell-make-comint): Honor overriden
python-shell-interpreter and python-shell-interpreter-args.
(python-shell-get-or-create-process): Make it testable by allowing
to call run-python non-interactively.
(python-util-valid-regexp-p): New function.
(python-shell-prompt-regexp, python-shell-prompt-block-regexp)
(python-shell-prompt-output-regexp)
(python-shell-prompt-pdb-regexp): Use it as defcustom :safe.

* test/automated/python-tests.el (python-shell-make-comint-1):
(python-shell-make-comint-2): Fix indentation.
(python-shell-make-comint-3)
(python-shell-make-comint-4): New tests.
(python-shell-get-or-create-process-1): Fix test.
(python-shell-get-or-create-process-2)
(python-shell-get-or-create-process-3): New tests.
(python-shell-internal-get-or-create-process-1): Fix test.
(python-shell-prompt-detect-1): New test.
(python-shell-prompt-detect-2): New test.  (Bug#17370)
(python-shell-prompt-detect-3)
(python-shell-prompt-detect-4)
(python-shell-prompt-detect-5)
(python-shell-prompt-detect-6)
(python-shell-prompt-validate-regexps-1)
(python-shell-prompt-validate-regexps-2)
(python-shell-prompt-validate-regexps-3)
(python-shell-prompt-validate-regexps-4)
(python-shell-prompt-validate-regexps-5)
(python-shell-prompt-validate-regexps-6)
(python-shell-prompt-validate-regexps-7)
(python-shell-prompt-set-calculated-regexps-1)
(python-shell-prompt-set-calculated-regexps-2)
(python-shell-prompt-set-calculated-regexps-3)
(python-shell-prompt-set-calculated-regexps-4)
(python-shell-prompt-set-calculated-regexps-5)
(python-shell-prompt-set-calculated-regexps-6)
(python-util-valid-regexp-p-1): New tests.

1 files changed, 271 insertions, 45 deletions

diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index 237302f0530..e49d5882a61 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -4,7 +4,7 @@
 
 ;; Author: Fabián E. Gallina <fabian@anue.biz>
 ;; URL: https://github.com/fgallina/python.el
-;; Version: 0.24.2
+;; Version: 0.24.4
 ;; Maintainer: emacs-devel@gnu.org
 ;; Created: Jul 2010
 ;; Keywords: languages
@@ -62,13 +62,28 @@
 ;; (add-hook 'python-mode-hook
 ;;           (lambda () (setq forward-sexp-function nil)))
 
-;; Shell interaction: is provided and allows you to execute easily any
+;; Shell interaction: is provided and allows you to easily execute any
 ;; block of code of your current buffer in an inferior Python process.
+;; This relies upon having prompts for input (e.g. ">>> " and "... "
+;; in standard Python shell) and output (e.g. "Out[1]: " in iPython)
+;; detected properly.  Failing that Emacs may hang but, in the case
+;; that happens, you can recover with \\[keyboard-quit].  To avoid
+;; this issue, a two-step prompt autodetection mechanism is provided:
+;; the first step is manual and consists of a collection of regular
+;; expressions matching common prompts for Python shells stored in
+;; `python-shell-prompt-input-regexps' and
+;; `python-shell-prompt-output-regexps', and dir-local friendly vars
+;; `python-shell-prompt-regexp', `python-shell-prompt-block-regexp',
+;; `python-shell-prompt-output-regexp' which are appended to the
+;; former automatically when a shell spawns; the second step is
+;; automatic and depends on the `python-shell-prompt-detect' helper
+;; function.  See its docstring for details on global variables that
+;; modify its behavior.
 
 ;; Shell completion: hitting tab will try to complete the current
-;; word.  Shell completion is implemented in a manner that if you
-;; change the `python-shell-interpreter' to any other (for example
-;; IPython) it should be easy to integrate another way to calculate
+;; word.  Shell completion is implemented in a way that if you change
+;; the `python-shell-interpreter' to any other (for example IPython)
+;; it should be easy to integrate another way to calculate
 ;; completions.  You just need to specify your custom
 ;; `python-shell-completion-setup-code' and
 ;; `python-shell-completion-string-code'.
@@ -79,8 +94,6 @@
 ;; (setq
 ;;  python-shell-interpreter "ipython"
 ;;  python-shell-interpreter-args ""
-;;  python-shell-prompt-regexp "In \\[[0-9]+\\]: "
-;;  python-shell-prompt-output-regexp "Out\\[[0-9]+\\]: "
 ;;  python-shell-completion-setup-code
 ;;    "from IPython.core.completerlib import module_completion"
 ;;  python-shell-completion-module-string-code
@@ -213,7 +226,9 @@
 ;;; Code:
 
 (require 'ansi-color)
+(require 'cl-lib)
 (require 'comint)
+(require 'json)
 
 ;; Avoid compiler warnings
 (defvar view-return-to-alist)
@@ -1706,33 +1721,56 @@ position, else returns nil."
   :type 'string
   :group 'python)
 
+(defcustom python-shell-interpreter-interactive-arg "-i"
+  "Interpreter argument to force it to run interactively."
+  :type 'string
+  :version "24.4")
+
+(defcustom python-shell-prompt-detect-enabled t
+  "Non-nil enables autodetection of interpreter prompts."
+  :type 'boolean
+  :safe 'booleanp
+  :version "24.4")
+
+(defcustom python-shell-prompt-detect-failure-warning t
+  "Non-nil enables warnings when detection of prompts fail."
+  :type 'boolean
+  :safe 'booleanp
+  :version "24.4")
+
+(defcustom python-shell-prompt-input-regexps
+  '(">>> " "\\.\\.\\. "                 ; Python
+    "In \\[[0-9]+\\]: ")                ; iPython
+  "List of regular expressions matching input prompts."
+  :type '(repeat string)
+  :version "24.4")
+
+(defcustom python-shell-prompt-output-regexps
+  '(""                                  ; Python
+    "Out\\[[0-9]+\\]: ")                ; iPython
+  "List of regular expressions matching output prompts."
+  :type '(repeat string)
+  :version "24.4")
+
 (defcustom python-shell-prompt-regexp ">>> "
-  "Regular expression matching top-level input prompt of Python shell.
+  "Regular expression matching top level input prompt of Python shell.
 It should not contain a caret (^) at the beginning."
-  :type 'string
-  :group 'python
-  :safe 'stringp)
+  :type 'string)
 
-(defcustom python-shell-prompt-block-regexp "[.][.][.] "
+(defcustom python-shell-prompt-block-regexp "\\.\\.\\. "
   "Regular expression matching block input prompt of Python shell.
 It should not contain a caret (^) at the beginning."
-  :type 'string
-  :group 'python
-  :safe 'stringp)
+  :type 'string)
 
 (defcustom python-shell-prompt-output-regexp ""
   "Regular expression matching output prompt of Python shell.
 It should not contain a caret (^) at the beginning."
-  :type 'string
-  :group 'python
-  :safe 'stringp)
+  :type 'string)
 
 (defcustom python-shell-prompt-pdb-regexp "[(<]*[Ii]?[Pp]db[>)]+ "
   "Regular expression matching pdb input prompt of Python shell.
 It should not contain a caret (^) at the beginning."
-  :type 'string
-  :group 'python
-  :safe 'stringp)
+  :type 'string)
 
 (defcustom python-shell-enable-font-lock t
   "Should syntax highlighting be enabled in the Python shell buffer?
@@ -1802,6 +1840,162 @@ virtualenv."
   :type '(alist string)
   :group 'python)
 
+(defvar python-shell--prompt-calculated-input-regexp nil
+  "Calculated input prompt regexp for inferior python shell.
+Do not set this variable directly, instead use
+`python-shell-prompt-set-calculated-regexps'.")
+
+(defvar python-shell--prompt-calculated-output-regexp nil
+  "Calculated output prompt regexp for inferior python shell.
+Do not set this variable directly, instead use
+`python-shell-set-prompt-regexp'.")
+
+(defun python-shell-prompt-detect ()
+  "Detect prompts for the current `python-shell-interpreter'.
+When prompts can be retrieved successfully from the
+`python-shell-interpreter' run with
+`python-shell-interpreter-interactive-arg', returns a list of
+three elements, where the first two are input prompts and the
+last one is an output prompt.  When no prompts can be detected
+and `python-shell-prompt-detect-failure-warning' is non-nil,
+shows a warning with instructions to avoid hangs and returns nil.
+When `python-shell-prompt-detect-enabled' is nil avoids any
+detection and just returns nil."
+  (when python-shell-prompt-detect-enabled
+    (let* ((process-environment (python-shell-calculate-process-environment))
+           (exec-path (python-shell-calculate-exec-path))
+           (python-code-file
+            (python-shell--save-temp-file
+             (concat
+              "import sys\n"
+              "ps = [getattr(sys, 'ps%s' % i, '') for i in range(1,4)]\n"
+              ;; JSON is built manually for compatibility
+              "ps_json = '\\n[\"%s\", \"%s\", \"%s\"]\\n' % tuple(ps)\n"
+              "print (ps_json)\n"
+              "sys.exit(0)\n")))
+           (output
+            (with-temp-buffer
+              (call-process
+               (executable-find python-shell-interpreter)
+               python-code-file
+               '(t nil)
+               nil
+               python-shell-interpreter-interactive-arg)
+              (ignore-errors (delete-file python-code-file))
+              (buffer-string)))
+           (prompts
+            (catch 'prompts
+              (dolist (line (split-string output "\n" t))
+                (let ((res
+                       ;; Check if current line is a valid JSON array
+                       (and (string= (substring line 0 2) "[\"")
+                            (ignore-errors
+                              ;; Return prompts as a list, not vector
+                              (append (json-read-from-string line) nil)))))
+                  ;; The list must contain 3 strings, where the first
+                  ;; is the input prompt, the second is the block
+                  ;; prompt and the last one is the output prompt.  The
+                  ;; input prompt is the only one that can't be empty.
+                  (when (and (= (length res) 3)
+                             (cl-every #'stringp res)
+                             (not (string= (car res) "")))
+                    (throw 'prompts res))))
+              nil)))
+      (when (and (not prompts)
+                 python-shell-prompt-detect-failure-warning)
+        (warn
+         (concat
+          "Python shell prompts cannot be detected.\n"
+          "If your emacs session hangs when starting python shells\n"
+          "recover with `keyboard-quit' and then try fixing the\n"
+          "interactive flag for your interpreter by adjusting the\n"
+          "`python-shell-interpreter-interactive-arg' or add regexps\n"
+          "matching shell prompts in the directory-local friendly vars:\n"
+          "  + `python-shell-prompt-regexp'\n"
+          "  + `python-shell-prompt-block-regexp'\n"
+          "  + `python-shell-prompt-output-regexp'\n"
+          "Or alternatively in:\n"
+          "  + `python-shell-prompt-input-regexps'\n"
+          "  + `python-shell-prompt-output-regexps'")))
+      prompts)))
+
+(defun python-shell-prompt-validate-regexps ()
+  "Validate all user provided regexps for prompts.
+Signals `user-error' if any of these vars contain invalid
+regexps: `python-shell-prompt-regexp',
+`python-shell-prompt-block-regexp',
+`python-shell-prompt-pdb-regexp',
+`python-shell-prompt-output-regexp',
+`python-shell-prompt-input-regexps',
+`python-shell-prompt-output-regexps'."
+  (dolist (symbol (list 'python-shell-prompt-input-regexps
+                        'python-shell-prompt-output-regexps
+                        'python-shell-prompt-regexp
+                        'python-shell-prompt-block-regexp
+                        'python-shell-prompt-pdb-regexp
+                        'python-shell-prompt-output-regexp))
+    (dolist (regexp (let ((regexps (symbol-value symbol)))
+                      (if (listp regexps)
+                          regexps
+                        (list regexps))))
+      (when (not (python-util-valid-regexp-p regexp))
+        (user-error "Invalid regexp %s in `%s'"
+                    regexp symbol)))))
+
+(defun python-shell-prompt-set-calculated-regexps ()
+  "Detect and set input and output prompt regexps.
+Build and set the values for `python-shell-input-prompt-regexp'
+and `python-shell-output-prompt-regexp' using the values from
+`python-shell-prompt-regexp', `python-shell-prompt-block-regexp',
+`python-shell-prompt-pdb-regexp',
+`python-shell-prompt-output-regexp',
+`python-shell-prompt-input-regexps',
+`python-shell-prompt-output-regexps' and detected prompts from
+`python-shell-prompt-detect'."
+  (when (not (and python-shell--prompt-calculated-input-regexp
+                  python-shell--prompt-calculated-output-regexp))
+    (let* ((detected-prompts (python-shell-prompt-detect))
+           (input-prompts nil)
+           (output-prompts nil)
+           (build-regexp
+            (lambda (prompts)
+              (concat "^\\("
+                      (mapconcat #'identity
+                                 (sort prompts
+                                       (lambda (a b)
+                                         (let ((length-a (length a))
+                                               (length-b (length b)))
+                                           (if (= length-a length-b)
+                                               (string< a b)
+                                             (> (length a) (length b))))))
+                                 "\\|")
+                      "\\)"))))
+      ;; Validate ALL regexps
+      (python-shell-prompt-validate-regexps)
+      ;; Collect all user defined input prompts
+      (dolist (prompt (append python-shell-prompt-input-regexps
+                              (list python-shell-prompt-regexp
+                                    python-shell-prompt-block-regexp
+                                    python-shell-prompt-pdb-regexp)))
+        (cl-pushnew prompt input-prompts :test #'string=))
+      ;; Collect all user defined output prompts
+      (dolist (prompt (cons python-shell-prompt-output-regexp
+                            python-shell-prompt-output-regexps))
+        (cl-pushnew prompt output-prompts :test #'string=))
+      ;; Collect detected prompts if any
+      (when detected-prompts
+        (dolist (prompt (butlast detected-prompts))
+          (setq prompt (regexp-quote prompt))
+          (cl-pushnew prompt input-prompts :test #'string=))
+        (cl-pushnew (regexp-quote
+                     (car (last detected-prompts)))
+                    output-prompts :test #'string=))
+      ;; Set input and output prompt regexps from collected prompts
+      (setq python-shell--prompt-calculated-input-regexp
+            (funcall build-regexp input-prompts)
+            python-shell--prompt-calculated-output-regexp
+            (funcall build-regexp output-prompts)))))
+
 (defun python-shell-get-process-name (dedicated)
   "Calculate the appropriate process name for inferior Python process.
 If DEDICATED is t and the variable `buffer-file-name' is non-nil
@@ -1824,10 +2018,10 @@ uniqueness for different types of configurations."
           python-shell-internal-buffer-name
           (md5
            (concat
-            (python-shell-parse-command)
-            python-shell-prompt-regexp
-            python-shell-prompt-block-regexp
-            python-shell-prompt-output-regexp
+            python-shell-interpreter
+            python-shell-interpreter-args
+            python-shell--prompt-calculated-input-regexp
+            python-shell--prompt-calculated-output-regexp
             (mapconcat #'symbol-value python-shell-setup-codes "")
             (mapconcat #'identity python-shell-process-environment "")
             (mapconcat #'identity python-shell-extra-pythonpaths "")
@@ -1921,12 +2115,19 @@ initialization of the interpreter via `python-shell-setup-codes'
 variable.
 
 \(Type \\[describe-mode] in the process buffer for a list of commands.)"
-  (and python-shell--parent-buffer
-       (python-util-clone-local-variables python-shell--parent-buffer))
-  (setq comint-prompt-regexp (format "^\\(?:%s\\|%s\\|%s\\)"
-                                     python-shell-prompt-regexp
-                                     python-shell-prompt-block-regexp
-                                     python-shell-prompt-pdb-regexp))
+  (let ((interpreter python-shell-interpreter)
+        (args python-shell-interpreter-args))
+    (when python-shell--parent-buffer
+      (python-util-clone-local-variables python-shell--parent-buffer))
+    ;; Users can override default values for these vars when calling
+    ;; `run-python'.  This ensures new values let-bound in
+    ;; `python-shell-make-comint' are locally set.
+    (set (make-local-variable 'python-shell-interpreter) interpreter)
+    (set (make-local-variable 'python-shell-interpreter-args) args))
+  (set (make-local-variable 'python-shell--prompt-calculated-input-regexp) nil)
+  (set (make-local-variable 'python-shell--prompt-calculated-output-regexp) nil)
+  (python-shell-prompt-set-calculated-regexps)
+  (setq comint-prompt-regexp python-shell--prompt-calculated-input-regexp)
   (setq mode-line-process '(":%s"))
   (make-local-variable 'comint-output-filter-functions)
   (add-hook 'comint-output-filter-functions
@@ -1989,10 +2190,20 @@ killed."
            (exec-path (python-shell-calculate-exec-path)))
       (when (not (comint-check-proc proc-buffer-name))
         (let* ((cmdlist (split-string-and-unquote cmd))
+               (interpreter (car cmdlist))
+               (args (cdr cmdlist))
                (buffer (apply #'make-comint-in-buffer proc-name proc-buffer-name
-                              (car cmdlist) nil (cdr cmdlist)))
+                              interpreter nil args))
                (python-shell--parent-buffer (current-buffer))
-               (process (get-buffer-process buffer)))
+               (process (get-buffer-process buffer))
+               ;; As the user may have overriden default values for
+               ;; these vars on `run-python', let-binding them allows
+               ;; to have the new right values in all setup code
+               ;; that's is done in `inferior-python-mode', which is
+               ;; important, especially for prompt detection.
+               (python-shell-interpreter interpreter)
+               (python-shell-interpreter-args
+                (mapconcat #'identity args " ")))
           (with-current-buffer buffer
             (inferior-python-mode))
           (accept-process-output process)
@@ -2064,8 +2275,12 @@ startup."
   "Return inferior Python process for current buffer."
   (get-buffer-process (python-shell-get-buffer)))
 
-(defun python-shell-get-or-create-process ()
-  "Get or create an inferior Python process for current buffer and return it."
+(defun python-shell-get-or-create-process (&optional cmd dedicated show)
+  "Get or create an inferior Python process for current buffer and return it.
+Arguments CMD, DEDICATED and SHOW are those of `run-python' and
+are used to start the shell.  If those arguments are not
+provided, `run-python' is called interactively and the user will
+be asked for their values."
   (let* ((dedicated-proc-name (python-shell-get-process-name t))
          (dedicated-proc-buffer-name (format "*%s*" dedicated-proc-name))
          (global-proc-name  (python-shell-get-process-name nil))
@@ -2074,7 +2289,11 @@ startup."
          (global-running (comint-check-proc global-proc-buffer-name))
          (current-prefix-arg 16))
     (when (and (not dedicated-running) (not global-running))
-      (if (call-interactively 'run-python)
+      (if (if (not cmd)
+              ;; XXX: Refactor code such that calling `run-python'
+              ;; interactively is not needed anymore.
+              (call-interactively 'run-python)
+            (run-python cmd dedicated show))
           (setq dedicated-running t)
         (setq global-running t)))
     ;; Always prefer dedicated
@@ -2157,10 +2376,13 @@ detecting a prompt at the end of the buffer."
   (when (string-match
          ;; XXX: It seems on OSX an extra carriage return is attached
          ;; at the end of output, this handles that too.
-         (format "\r?\n\\(?:%s\\|%s\\|%s\\)$"
-                 python-shell-prompt-regexp
-                 python-shell-prompt-block-regexp
-                 python-shell-prompt-pdb-regexp)
+         (concat
+          "\r?\n"
+          ;; Remove initial caret from calculated regexp
+          (replace-regexp-in-string
+           (rx string-start ?^) ""
+           python-shell--prompt-calculated-input-regexp)
+          "$")
          python-shell-output-filter-buffer)
     ;; Output ends when `python-shell-output-filter-buffer' contains
     ;; the prompt attached at the end of it.
@@ -2168,9 +2390,9 @@ detecting a prompt at the end of the buffer."
           python-shell-output-filter-buffer
           (substring python-shell-output-filter-buffer
                      0 (match-beginning 0)))
-    (when (and (> (length python-shell-prompt-output-regexp) 0)
-               (string-match (concat "^" python-shell-prompt-output-regexp)
-                             python-shell-output-filter-buffer))
+    (when (string-match
+           python-shell--prompt-calculated-output-regexp
+           python-shell-output-filter-buffer)
       ;; Some shells, like iPython might append a prompt before the
       ;; output, clean that.
       (setq python-shell-output-filter-buffer
@@ -2456,11 +2678,11 @@ LINE is used to detect the context on how to complete given INPUT."
                 ((and (>
                        (length python-shell-completion-module-string-code) 0)
                       (string-match
-                       (concat "^" python-shell-prompt-regexp) prompt)
+                       python-shell--prompt-calculated-input-regexp prompt)
                       (string-match "^[ \t]*\\(from\\|import\\)[ \t]" line))
                  'import)
                 ((string-match
-                  (concat "^" python-shell-prompt-regexp) prompt)
+                  python-shell--prompt-calculated-input-regexp prompt)
                  'default)
                 (t nil)))
          (completion-code
@@ -3706,6 +3928,10 @@ returned as is."
    ""
    string))
 
+(defun python-util-valid-regexp-p (regexp)
+  "Return non-nil if REGEXP is valid."
+  (ignore-errors (string-match regexp "") t))
+
 
 (defun python-electric-pair-string-delimiter ()
   (when (and electric-pair-mode