* lisp/progmodes/ruby-mode.el (ruby-expression-expansion-re): Allow to

start at point, so that expansion starting right after opening
slash in a regexp is recognized.
(ruby-syntax-before-regexp-re): New defvar, extracted from
ruby-syntax-propertize-function.  Since the value of this regexp
is looked up at runtime now, we should be able to turn
`ruby-syntax-methods-before-regexp' into a defcustom later.
(ruby-syntax-propertize-function): Split regexp matching into two
parts, for opening and closing slashes.  That allows us to skip
over string interpolations and support multiline regexps.
Don't call `ruby-syntax-propertize-expansions', instead use another rule
for them, which calls `ruby-syntax-propertize-expansion'.
(ruby-syntax-propertize-expansions): Move `remove-text-properties'
call to `ruby-syntax-propertize-function'.
(ruby-syntax-propertize-expansion): Extracted from
`ruby-syntax-propertize-expansions'.  Handles one expansion.
(ruby-syntax-propertize-heredoc): Explicitly call
`ruby-syntax-propertize-expansions'.
(ruby-syntax-propertize-percent-literal): Leave point right after
the percent symbol, so that the expression expansion rule can
propertize the contents.

* test/automated/ruby-mode-tests.el (ruby-heredoc-highlights-interpolations)
(ruby-regexp-skips-over-interpolation)
(ruby-regexp-continues-till-end-when-unclosed)
(ruby-regexp-can-be-multiline)
(ruby-interpolation-inside-percent-literal): New tests.

* test/indent/ruby.rb: Add multiline regexp example.

5 files changed, 140 insertions, 60 deletions

diff --git a/lisp/ChangeLog b/lisp/ChangeLog
index 52711c47163..903dc74fc7d 100644
--- a/lisp/ChangeLog
+++ b/lisp/ChangeLog
@@ -1,3 +1,27 @@
+2013-05-19  Dmitry Gutov  <dgutov@yandex.ru>
+
+        * progmodes/ruby-mode.el (ruby-expression-expansion-re): Allow to
+        start at point, so that expansion starting right after opening
+        slash in a regexp is recognized.
+        (ruby-syntax-before-regexp-re): New defvar, extracted from
+        ruby-syntax-propertize-function.  Since the value of this regexp
+        is looked up at runtime now, we should be able to turn
+        `ruby-syntax-methods-before-regexp' into a defcustom later.
+        (ruby-syntax-propertize-function): Split regexp matching into two
+        parts, for opening and closing slashes.  That allows us to skip
+        over string interpolations and support multiline regexps.
+        Don't call `ruby-syntax-propertize-expansions', instead use another rule
+        for them, which calls `ruby-syntax-propertize-expansion'.
+        (ruby-syntax-propertize-expansions): Move `remove-text-properties'
+        call to `ruby-syntax-propertize-function'.
+        (ruby-syntax-propertize-expansion): Extracted from
+        `ruby-syntax-propertize-expansions'.  Handles one expansion.
+        (ruby-syntax-propertize-heredoc): Explicitly call
+        `ruby-syntax-propertize-expansions'.
+        (ruby-syntax-propertize-percent-literal): Leave point right after
+        the percent symbol, so that the expression expansion rule can
+        propertize the contents.
+
 2013-05-18  Juri Linkov  <juri@jurta.org>
 
         * man.el (Man-default-man-entry): Remove `-' from the end
diff --git a/lisp/progmodes/ruby-mode.el b/lisp/progmodes/ruby-mode.el
index 3ea55c4eabf..1732e0ba918 100644
--- a/lisp/progmodes/ruby-mode.el
+++ b/lisp/progmodes/ruby-mode.el
@@ -113,7 +113,7 @@
   "Regexp to match the beginning of a heredoc.")
 
   (defconst ruby-expression-expansion-re
-    "[^\\]\\(\\\\\\\\\\)*\\(#\\({[^}\n\\\\]*\\(\\\\.[^}\n\\\\]*\\)*}\\|\\(\\$\\|@\\|@@\\)\\(\\w\\|_\\)+\\)\\)"))
+    "\\(?:[^\\]\\|\\=\\)\\(\\\\\\\\\\)*\\(#\\({[^}\n\\\\]*\\(\\\\.[^}\n\\\\]*\\)*}\\|\\(\\$\\|@\\|@@\\)\\(\\w\\|_\\)+\\)\\)"))
 
 (defun ruby-here-doc-end-match ()
   "Return a regexp to find the end of a heredoc.
@@ -1360,11 +1360,26 @@ If the result is do-end block, it will always be multiline."
           '("gsub" "gsub!" "sub" "sub!" "scan" "split" "split!" "index" "match"
             "assert_match" "Given" "Then" "When")
           "Methods that can take regexp as the first argument.
-It will be properly highlighted even when the call omits parens."))
+It will be properly highlighted even when the call omits parens.")
+
+        (defvar ruby-syntax-before-regexp-re
+          (concat
+           ;; Special tokens that can't be followed by a division operator.
+           "\\(^\\|[[=(,~?:;<>]"
+           ;; Control flow keywords and operators following bol or whitespace.
+           "\\|\\(?:^\\|\\s \\)"
+           (regexp-opt '("if" "elsif" "unless" "while" "until" "when" "and"
+                         "or" "not" "&&" "||"))
+           ;; Method name from the list.
+           "\\|\\_<"
+           (regexp-opt ruby-syntax-methods-before-regexp)
+           "\\)\\s *")
+          "Regexp to match text that can be followed by a regular expression."))
 
       (defun ruby-syntax-propertize-function (start end)
         "Syntactic keywords for Ruby mode.  See `syntax-propertize-function'."
         (goto-char start)
+        (remove-text-properties start end '(ruby-expansion-match-data))
         (ruby-syntax-propertize-heredoc end)
         (ruby-syntax-enclosing-percent-literal end)
         (funcall
@@ -1376,25 +1391,26 @@ It will be properly highlighted even when the call omits parens."))
                         ;; Not within a string.
                         (nth 3 (syntax-ppss (match-beginning 0))))
                 (string-to-syntax "\\"))))
-          ;; Regexps: regexps are distinguished from division because
-          ;; of the keyword, symbol, or method name before them.
-          ((concat
-            ;; Special tokens that can't be followed by a division operator.
-            "\\(^\\|[[=(,~?:;<>]"
-            ;; Control flow keywords and operators following bol or whitespace.
-            "\\|\\(?:^\\|\\s \\)"
-            (regexp-opt '("if" "elsif" "unless" "while" "until" "when" "and"
-                          "or" "not" "&&" "||"))
-            ;; Method name from the list.
-            "\\|\\_<"
-            (regexp-opt ruby-syntax-methods-before-regexp)
-            "\\)\\s *"
-            ;; The regular expression itself.
-            "\\(/\\)[^/\n\\\\]*\\(?:\\\\.[^/\n\\\\]*\\)*\\(/\\)")
-           (3 (unless (nth 3 (syntax-ppss (match-beginning 2)))
-                (put-text-property (match-beginning 2) (match-end 2)
-                                   'syntax-table (string-to-syntax "\"/"))
-                (string-to-syntax "\"/"))))
+          ;; Regular expressions.  Start with matching unescaped slash.
+          ("\\(?:\\=\\|[^\\]\\)\\(?:\\\\\\\\\\)*\\(/\\)"
+           (1 (let ((state (save-excursion (syntax-ppss (match-beginning 1)))))
+                (when (or
+                       ;; Beginning of a regexp.
+                       (and (null (nth 8 state))
+                            (save-excursion
+                              (forward-char -1)
+                              (looking-back ruby-syntax-before-regexp-re
+                                            (point-at-bol))))
+                       ;; End of regexp.  We don't match the whole
+                       ;; regexp at once because it can have
+                       ;; string interpolation inside, or span
+                       ;; several lines.
+                       (eq ?/ (nth 3 state)))
+                  (string-to-syntax "\"/")))))
+          ;; Expression expansions in strings.  We're handling them
+          ;; here, so that the regexp rule never matches inside them.
+          (ruby-expression-expansion-re
+           (0 (ignore (ruby-syntax-propertize-expansion))))
           ("^=en\\(d\\)\\_>" (1 "!"))
           ("^\\(=\\)begin\\_>" (1 "!"))
           ;; Handle here documents.
@@ -1406,8 +1422,7 @@ It will be properly highlighted even when the call omits parens."))
           ;; Handle percent literals: %w(), %q{}, etc.
           ((concat "\\(?:^\\|[[ \t\n<+(,=]\\)" ruby-percent-literal-beg-re)
            (1 (prog1 "|" (ruby-syntax-propertize-percent-literal end)))))
-         (point) end)
-        (ruby-syntax-propertize-expansions start end))
+         (point) end))
 
       (defun ruby-syntax-propertize-heredoc (limit)
         (let ((ppss (syntax-ppss))
@@ -1432,7 +1447,9 @@ It will be properly highlighted even when the call omits parens."))
                                        'syntax-table (string-to-syntax "\""))))
               ;; Make extra sure we don't move back, lest we could fall into an
               ;; inf-loop.
-              (if (< (point) start) (goto-char start))))))
+              (if (< (point) start)
+                  (goto-char start)
+                (ruby-syntax-propertize-expansions start (point)))))))
 
       (defun ruby-syntax-enclosing-percent-literal (limit)
         (let ((state (syntax-ppss))
@@ -1453,44 +1470,47 @@ It will be properly highlighted even when the call omits parens."))
                  (cl (or (cdr (aref (syntax-table) op))
                          (cdr (assoc op '((?< . ?>))))))
                  parse-sexp-lookup-properties)
-            (condition-case nil
-                (progn
-                  (if cl ; Paired delimiters.
-                      ;; Delimiter pairs of the same kind can be nested
-                      ;; inside the literal, as long as they are balanced.
-                      ;; Create syntax table that ignores other characters.
-                      (with-syntax-table (make-char-table 'syntax-table nil)
-                        (modify-syntax-entry op (concat "(" (char-to-string cl)))
-                        (modify-syntax-entry cl (concat ")" ops))
-                        (modify-syntax-entry ?\\ "\\")
-                        (save-restriction
-                          (narrow-to-region (point) limit)
-                          (forward-list))) ; skip to the paired character
-                    ;; Single character delimiter.
-                    (re-search-forward (concat "[^\\]\\(?:\\\\\\\\\\)*"
-                                               (regexp-quote ops)) limit nil))
-                  ;; Found the closing delimiter.
-                  (put-text-property (1- (point)) (point) 'syntax-table
-                                     (string-to-syntax "|")))
-              ;; Unclosed literal, leave the following text unpropertized.
-              ((scan-error search-failed) (goto-char limit))))))
+            (save-excursion
+              (condition-case nil
+                  (progn
+                    (if cl              ; Paired delimiters.
+                        ;; Delimiter pairs of the same kind can be nested
+                        ;; inside the literal, as long as they are balanced.
+                        ;; Create syntax table that ignores other characters.
+                        (with-syntax-table (make-char-table 'syntax-table nil)
+                          (modify-syntax-entry op (concat "(" (char-to-string cl)))
+                          (modify-syntax-entry cl (concat ")" ops))
+                          (modify-syntax-entry ?\\ "\\")
+                          (save-restriction
+                            (narrow-to-region (point) limit)
+                            (forward-list))) ; skip to the paired character
+                      ;; Single character delimiter.
+                      (re-search-forward (concat "[^\\]\\(?:\\\\\\\\\\)*"
+                                                 (regexp-quote ops)) limit nil))
+                    ;; Found the closing delimiter.
+                    (put-text-property (1- (point)) (point) 'syntax-table
+                                       (string-to-syntax "|")))
+                ;; Unclosed literal, do nothing.
+                ((scan-error search-failed)))))))
+
+      (defun ruby-syntax-propertize-expansion ()
+        ;; Save the match data to a text property, for font-locking later.
+        ;; Set the syntax of all double quotes and backticks to punctuation.
+        (let ((beg (match-beginning 2))
+              (end (match-end 2)))
+          (when (and beg (save-excursion (nth 3 (syntax-ppss beg))))
+            (put-text-property beg (1+ beg) 'ruby-expansion-match-data
+                               (match-data))
+            (goto-char beg)
+            (while (re-search-forward "[\"`]" end 'move)
+              (put-text-property (match-beginning 0) (match-end 0)
+                                 'syntax-table (string-to-syntax "."))))))
 
       (defun ruby-syntax-propertize-expansions (start end)
-        (remove-text-properties start end '(ruby-expansion-match-data))
-        (goto-char start)
-        ;; Find all expression expansions and
-        ;; - save the match data to a text property, for font-locking later,
-        ;; - set the syntax of all double quotes and backticks to punctuation.
-        (while (re-search-forward ruby-expression-expansion-re end 'move)
-          (let ((beg (match-beginning 2))
-                (end (match-end 2)))
-            (when (and beg (save-excursion (nth 3 (syntax-ppss beg))))
-              (put-text-property beg (1+ beg) 'ruby-expansion-match-data
-                                 (match-data))
-              (goto-char beg)
-              (while (re-search-forward "[\"`]" end 'move)
-                (put-text-property (match-beginning 0) (match-end 0)
-                                   'syntax-table (string-to-syntax ".")))))))
+        (save-excursion
+          (goto-char start)
+          (while (re-search-forward ruby-expression-expansion-re end 'move)
+            (ruby-syntax-propertize-expansion))))
       )
 
   ;; For Emacsen where syntax-propertize-rules is not (yet) available,
diff --git a/test/ChangeLog b/test/ChangeLog
index c11d5d26c13..5c17e09e962 100644
--- a/test/ChangeLog
+++ b/test/ChangeLog
@@ -1,3 +1,13 @@
+2013-05-19  Dmitry Gutov  <dgutov@yandex.ru>
+
+        * indent/ruby.rb: Add multiline regexp example.
+
+        * automated/ruby-mode-tests.el (ruby-heredoc-highlights-interpolations)
+        (ruby-regexp-skips-over-interpolation)
+        (ruby-regexp-continues-till-end-when-unclosed)
+        (ruby-regexp-can-be-multiline)
+        (ruby-interpolation-inside-percent-literal): New tests.
+
 2013-05-08  Stefan Monnier  <monnier@iro.umontreal.ca>
 
         * indent/ruby.rb: Fix indentation after =; add more cases.
diff --git a/test/automated/ruby-mode-tests.el b/test/automated/ruby-mode-tests.el
index 23dc45ad509..e52927a2968 100644
--- a/test/automated/ruby-mode-tests.el
+++ b/test/automated/ruby-mode-tests.el
@@ -84,6 +84,9 @@ VALUES-PLIST is a list with alternating index and value elements."
 (ert-deftest ruby-singleton-class-no-heredoc-font-lock ()
   (ruby-assert-face "class<<a" 8 nil))
 
+(ert-deftest ruby-heredoc-highlights-interpolations ()
+  (ruby-assert-face "s = <<EOS\n  #{foo}\nEOS" 15 font-lock-variable-name-face))
+
 (ert-deftest ruby-deep-indent ()
   (let ((ruby-deep-arglist nil)
         (ruby-deep-indent-paren '(?\( ?\{ ?\[ ?\] t)))
@@ -109,6 +112,15 @@ VALUES-PLIST is a list with alternating index and value elements."
 (ert-deftest ruby-regexp-starts-after-string ()
   (ruby-assert-state "'(/', /\d+/" 3 ?/ 8))
 
+(ert-deftest ruby-regexp-skips-over-interpolation ()
+  (ruby-assert-state "/#{foobs.join('/')}/" 3 nil))
+
+(ert-deftest ruby-regexp-continues-till-end-when-unclosed ()
+  (ruby-assert-state "/bars" 3 ?/))
+
+(ert-deftest ruby-regexp-can-be-multiline ()
+  (ruby-assert-state "/bars\ntees # toots \nfoos/" 3 nil))
+
 (ert-deftest ruby-indent-simple ()
   (ruby-should-indent-buffer
    "if foo
@@ -325,6 +337,13 @@ VALUES-PLIST is a list with alternating index and value elements."
       (search-forward "tee")
       (should (string= (thing-at-point 'symbol) "tee")))))
 
+(ert-deftest ruby-interpolation-inside-percent-literal ()
+  (let ((s "%( #{boo} )"))
+    (ruby-assert-face s 1 font-lock-string-face)
+    (ruby-assert-face s 4 font-lock-variable-name-face)
+    (ruby-assert-face s 10 font-lock-string-face)
+    (ruby-assert-state s 8 nil)))
+
 (ert-deftest ruby-interpolation-inside-percent-literal-with-paren ()
   :expected-result :failed
   (let ((s "%(^#{\")\"}^)"))
diff --git a/test/indent/ruby.rb b/test/indent/ruby.rb
index 90c6dcdc65c..853f4dbf992 100644
--- a/test/indent/ruby.rb
+++ b/test/indent/ruby.rb
@@ -21,6 +21,11 @@ a = asub / aslb + bsub / bslb;
 # Highlight the regexp after "if".
 x = toto / foo if /do bar/ =~ "dobar"
 
+# Multiline regexp.
+/bars
+ tees # toots
+ nfoos/
+
 def test1(arg)
   puts "hello"
 end
@@ -47,6 +52,8 @@ def test2 (arg)
   case a
   when "a"
     6
+  # Support for this syntax was removed in Ruby 1.9, so we
+  # probably don't need to handle it either.
   # when "b" :
   #   7
   # when "c" : 2