python.el: Enhance docstring detection following PEP-257.

* lisp/progmodes/python.el (python-docstring-at-p): Remove function.
(python-info-assignment-statement-p): New function.
(python-info-assignment-continuation-line-p): Use it.
(python-info-docstring-p): New function.
(python-font-lock-syntactic-face-function)
(python-fill-string): Use it.

* test/automated/python-tests.el (python-info-assignment-statement-p-1)
(python-info-assignment-statement-p-2)
(python-info-assignment-statement-p-3, python-info-docstring-p-1)
(python-info-docstring-p-2, python-info-docstring-p-3)
(python-info-docstring-p-4, python-info-docstring-p-5)
(python-info-docstring-p-6): New tests.

4 files changed, 262 insertions, 27 deletions

diff --git a/lisp/ChangeLog b/lisp/ChangeLog
index 8ea7f4a3cc8..c7d4dd5bac0 100644
--- a/lisp/ChangeLog
+++ b/lisp/ChangeLog
@@ -1,3 +1,14 @@
+2015-04-06  Fabián Ezequiel Gallina  <fgallina@gnu.org>
+
+        python.el: Enhance docstring detection following PEP-257.
+
+        * progmodes/python.el (python-docstring-at-p): Remove function.
+        (python-info-assignment-statement-p): New function.
+        (python-info-assignment-continuation-line-p): Use it.
+        (python-info-docstring-p): New function.
+        (python-font-lock-syntactic-face-function)
+        (python-fill-string): Use it.
+
 2015-04-05  Eli Zaretskii  <eliz@gnu.org>
 
         * ses.el (ses-sym-rowcol): Move up, before the first use, to avoid
diff --git a/lisp/progmodes/python.el b/lisp/progmodes/python.el
index 67b44aa1bbe..f402ad83cca 100644
--- a/lisp/progmodes/python.el
+++ b/lisp/progmodes/python.el
@@ -482,19 +482,10 @@ The type returned can be `comment', `string' or `paren'."
   'python-info-ppss-comment-or-string-p
   #'python-syntax-comment-or-string-p "24.3")
 
-(defun python-docstring-at-p (pos)
-  "Check to see if there is a docstring at POS."
-  (save-excursion
-    (goto-char pos)
-    (if (looking-at-p "'''\\|\"\"\"")
-        (progn
-          (python-nav-backward-statement)
-          (looking-at "\\`\\|class \\|def "))
-      nil)))
-
 (defun python-font-lock-syntactic-face-function (state)
+  "Return syntactic face given STATE."
   (if (nth 3 state)
-      (if (python-docstring-at-p (nth 8 state))
+      (if (python-info-docstring-p state)
           font-lock-doc-face
         font-lock-string-face)
     font-lock-comment-face))
@@ -3587,17 +3578,12 @@ JUSTIFY should be used (if applicable) as in `fill-paragraph'."
             (`pep-257 (and multi-line-p (cons nil 2)))
             (`pep-257-nn (and multi-line-p (cons nil 1)))
             (`symmetric (and multi-line-p (cons 1 1)))))
-         (docstring-p (save-excursion
-                        ;; Consider docstrings those strings which
-                        ;; start on a line by themselves.
-                        (python-nav-beginning-of-statement)
-                        (and (= (point) str-start-pos))))
          (fill-paragraph-function))
     (save-restriction
       (narrow-to-region str-start-pos str-end-pos)
       (fill-paragraph justify))
     (save-excursion
-      (when (and docstring-p python-fill-docstring-style)
+      (when (and (python-info-docstring-p) python-fill-docstring-style)
         ;; Add the number of newlines indicated by the selected style
         ;; at the start of the docstring.
         (goto-char (+ str-start-pos num-quotes))
@@ -4423,23 +4409,40 @@ where the continued line ends."
       (when (looking-at (python-rx block-start))
         (point-marker)))))
 
+(defun python-info-assignment-statement-p (&optional current-line-only)
+  "Check if current line is an assignment.
+With argument CURRENT-LINE-ONLY is non-nil, don't follow any
+continuations, just check the if current line is an assignment."
+  (save-excursion
+    (let ((found nil))
+      (if current-line-only
+          (back-to-indentation)
+        (python-nav-beginning-of-statement))
+      (while (and
+              (re-search-forward (python-rx not-simple-operator
+                                            assignment-operator
+                                            (group not-simple-operator))
+                                 (line-end-position) t)
+              (not found))
+        (save-excursion
+          ;; The assignment operator should not be inside a string.
+          (backward-char (length (match-string-no-properties 1)))
+          (setq found (not (python-syntax-context-type)))))
+      (when found
+        (skip-syntax-forward " ")
+        (point-marker)))))
+
+;; TODO: rename to clarify this is only for the first continuation
+;; line or remove it and move its body to `python-indent-context'.
 (defun python-info-assignment-continuation-line-p ()
-  "Check if current line is a continuation of an assignment.
+  "Check if current line is the first continuation of an assignment.
 When current line is continuation of another with an assignment
 return the point of the first non-blank character after the
 operator."
   (save-excursion
     (when (python-info-continuation-line-p)
       (forward-line -1)
-      (back-to-indentation)
-      (when (and (not (looking-at (python-rx block-start)))
-                 (and (re-search-forward (python-rx not-simple-operator
-                                                    assignment-operator
-                                                    not-simple-operator)
-                                         (line-end-position) t)
-                      (not (python-syntax-context-type))))
-        (skip-syntax-forward "\s")
-        (point-marker)))))
+      (python-info-assignment-statement-p t))))
 
 (defun python-info-looking-at-beginning-of-defun (&optional syntax-ppss)
   "Check if point is at `beginning-of-defun' using SYNTAX-PPSS."
@@ -4464,6 +4467,45 @@ operator."
                 (* whitespace) line-end))
     (string-equal "" (match-string-no-properties 1))))
 
+(defun python-info-docstring-p (&optional syntax-ppss)
+  "Return non-nil if point is in a docstring.
+When optional argument SYNTAX-PPSS is given, use that instead of
+point's current `syntax-ppss'."
+  ;;; https://www.python.org/dev/peps/pep-0257/#what-is-a-docstring
+  (save-excursion
+    (when (and syntax-ppss (python-syntax-context 'string syntax-ppss))
+      (goto-char (nth 8 syntax-ppss)))
+    (python-nav-beginning-of-statement)
+    (let ((counter 1)
+          (indentation (current-indentation))
+          (backward-sexp-point)
+          (re (concat "[uU]?[rR]?"
+                      (python-rx string-delimiter))))
+      (when (and
+             (not (python-info-assignment-statement-p))
+             (looking-at-p re)
+             ;; Allow up to two consecutive docstrings only.
+             (>=
+              2
+              (progn
+                (while (save-excursion
+                         (python-nav-backward-sexp)
+                         (setq backward-sexp-point (point))
+                         (and (= indentation (current-indentation))
+                              (looking-at-p
+                               (concat "[uU]?[rR]?"
+                                       (python-rx string-delimiter)))))
+                  ;; Previous sexp was a string, restore point.
+                  (goto-char backward-sexp-point)
+                  (cl-incf counter))
+                counter)))
+        (python-util-forward-comment -1)
+        (python-nav-beginning-of-statement)
+        (cond ((bobp))
+              ((python-info-assignment-statement-p) t)
+              ((python-info-looking-at-beginning-of-defun))
+              (t nil))))))
+
 (defun python-info-encoding-from-cookie ()
   "Detect current buffer's encoding from its coding cookie.
 Returns the encoding as a symbol."
diff --git a/test/ChangeLog b/test/ChangeLog
index f7bec2ee119..813f5dd42b7 100644
--- a/test/ChangeLog
+++ b/test/ChangeLog
@@ -1,3 +1,12 @@
+2015-04-06  Fabián Ezequiel Gallina  <fgallina@gnu.org>
+
+        * automated/python-tests.el (python-info-assignment-statement-p-1)
+        (python-info-assignment-statement-p-2)
+        (python-info-assignment-statement-p-3, python-info-docstring-p-1)
+        (python-info-docstring-p-2, python-info-docstring-p-3)
+        (python-info-docstring-p-4, python-info-docstring-p-5)
+        (python-info-docstring-p-6): New tests.
+
 2015-04-01  Artur Malabarba  <bruce.connor.am@gmail.com>
 
         * automated/package-test.el: Avoid async while testing.
diff --git a/test/automated/python-tests.el b/test/automated/python-tests.el
index b377a26f77a..22c111fc04a 100644
--- a/test/automated/python-tests.el
+++ b/test/automated/python-tests.el
@@ -4273,6 +4273,49 @@ def foo(a,
    (python-tests-look-at "c):")
    (should (not (python-info-block-continuation-line-p)))))
 
+(ert-deftest python-info-assignment-statement-p-1 ()
+  (python-tests-with-temp-buffer
+   "
+data = foo(), bar() \\\\
+       baz(), 4 \\\\
+       5, 6
+"
+   (python-tests-look-at "data = foo(), bar()")
+   (should (python-info-assignment-statement-p))
+   (should (python-info-assignment-statement-p t))
+   (python-tests-look-at "baz(), 4")
+   (should (python-info-assignment-statement-p))
+   (should (not (python-info-assignment-statement-p t)))
+   (python-tests-look-at "5, 6")
+   (should (python-info-assignment-statement-p))
+   (should (not (python-info-assignment-statement-p t)))))
+
+(ert-deftest python-info-assignment-statement-p-2 ()
+  (python-tests-with-temp-buffer
+   "
+data = (foo(), bar()
+        baz(), 4
+        5, 6)
+"
+   (python-tests-look-at "data = (foo(), bar()")
+   (should (python-info-assignment-statement-p))
+   (should (python-info-assignment-statement-p t))
+   (python-tests-look-at "baz(), 4")
+   (should (python-info-assignment-statement-p))
+   (should (not (python-info-assignment-statement-p t)))
+   (python-tests-look-at "5, 6)")
+   (should (python-info-assignment-statement-p))
+   (should (not (python-info-assignment-statement-p t)))))
+
+(ert-deftest python-info-assignment-statement-p-3 ()
+  (python-tests-with-temp-buffer
+   "
+data '=' 42
+"
+   (python-tests-look-at "data '=' 42")
+   (should (not (python-info-assignment-statement-p)))
+   (should (not (python-info-assignment-statement-p t)))))
+
 (ert-deftest python-info-assignment-continuation-line-p-1 ()
   (python-tests-with-temp-buffer
    "
@@ -4360,6 +4403,136 @@ foo = True  # another comment
    (forward-line 1)
    (should (python-info-current-line-empty-p))))
 
+(ert-deftest python-info-docstring-p-1 ()
+  "Test module docstring detection."
+  (python-tests-with-temp-buffer
+   "# -*- coding: utf-8 -*-
+#!/usr/bin/python
+
+'''
+Module Docstring Django style.
+'''
+u'''Additional module docstring.'''
+'''Not a module docstring.'''
+"
+   (python-tests-look-at "Module Docstring Django style.")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "u'''Additional module docstring.'''")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "'''Not a module docstring.'''")
+   (should (not (python-info-docstring-p)))))
+
+(ert-deftest python-info-docstring-p-2 ()
+  "Test variable docstring detection."
+  (python-tests-with-temp-buffer
+   "
+variable = 42
+U'''Variable docstring.'''
+'''Additional variable docstring.'''
+'''Not a variable docstring.'''
+"
+   (python-tests-look-at "Variable docstring.")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "u'''Additional variable docstring.'''")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "'''Not a variable docstring.'''")
+   (should (not (python-info-docstring-p)))))
+
+(ert-deftest python-info-docstring-p-3 ()
+  "Test function docstring detection."
+  (python-tests-with-temp-buffer
+   "
+def func(a, b):
+    r'''
+    Function docstring.
+
+    onetwo style.
+    '''
+    R'''Additional function docstring.'''
+    '''Not a function docstring.'''
+    return a + b
+"
+   (python-tests-look-at "Function docstring.")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "R'''Additional function docstring.'''")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "'''Not a function docstring.'''")
+   (should (not (python-info-docstring-p)))))
+
+(ert-deftest python-info-docstring-p-4 ()
+  "Test class docstring detection."
+  (python-tests-with-temp-buffer
+   "
+class Class:
+    ur'''
+    Class docstring.
+
+    symmetric style.
+    '''
+    uR'''
+    Additional class docstring.
+    '''
+    '''Not a class docstring.'''
+    pass
+"
+   (python-tests-look-at "Class docstring.")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "uR'''")  ;; Additional class docstring
+   (should (python-info-docstring-p))
+   (python-tests-look-at "'''Not a class docstring.'''")
+   (should (not (python-info-docstring-p)))))
+
+(ert-deftest python-info-docstring-p-5 ()
+  "Test class attribute docstring detection."
+  (python-tests-with-temp-buffer
+   "
+class Class:
+    attribute = 42
+    Ur'''
+    Class attribute docstring.
+
+    pep-257 style.
+
+    '''
+    UR'''
+    Additional class attribute docstring.
+    '''
+    '''Not a class attribute docstring.'''
+    pass
+"
+   (python-tests-look-at "Class attribute docstring.")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "UR'''")  ;; Additional class attr docstring
+   (should (python-info-docstring-p))
+   (python-tests-look-at "'''Not a class attribute docstring.'''")
+   (should (not (python-info-docstring-p)))))
+
+(ert-deftest python-info-docstring-p-6 ()
+  "Test class method docstring detection."
+  (python-tests-with-temp-buffer
+   "
+class Class:
+
+    def __init__(self, a, b):
+        self.a = a
+        self.b = b
+
+    def __call__(self):
+        '''Method docstring.
+
+        pep-257-nn style.
+        '''
+        '''Additional method docstring.'''
+        '''Not a method docstring.'''
+        return self.a + self.b
+"
+   (python-tests-look-at "Method docstring.")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "'''Additional method docstring.'''")
+   (should (python-info-docstring-p))
+   (python-tests-look-at "'''Not a method docstring.'''")
+   (should (not (python-info-docstring-p)))))
+
 (ert-deftest python-info-encoding-from-cookie-1 ()
   "Should detect it on first line."
   (python-tests-with-temp-buffer