merge trunk

11 files changed, 196 insertions, 89 deletions

diff --git a/admin/ChangeLog b/admin/ChangeLog
index b5b1b75c55c..2da65523116 100644
--- a/admin/ChangeLog
+++ b/admin/ChangeLog
@@ -1,3 +1,18 @@
+2012-10-01  David Engster  <deng@randomsample.de>
+
+        * grammars/bovine-grammar.el (bovine--grammar-newstyle-unquote):
+        Remove.
+        (bovine-grammar-expand-form): Test for emacs-major-version.
+
+        * grammars/c.by: Add EXPLICIT to keyword tokens.
+
+        * grammars/f90.by: Add %provide token.
+
+        * grammar/grammar.wy (semantic-grammar-lexer): Remove, since it
+        was copied to grammar.el.  New %provide token to generate prefix
+        which conforms with Emacs conventions.  Remove lexer definition,
+        which is now in grammar.el.
+
 2012-09-27  Glenn Morris  <rgm@gnu.org>
 
         * admin.el (set-version): Set msdos.c's Vwindow_system_version.
diff --git a/admin/grammars/bovine-grammar.el b/admin/grammars/bovine-grammar.el
index efe4db039c4..a7289f6bafe 100644
--- a/admin/grammars/bovine-grammar.el
+++ b/admin/grammars/bovine-grammar.el
@@ -109,14 +109,6 @@ NAME, ALIASCLASS, DEFINITION and ATTRIBUTES."
 ;; Cache of macro definitions currently in use.
 (defvar bovine--grammar-macros nil)
 
-;; Detect if we have an Emacs with newstyle unquotes allowed outside
-;; of backquote.
-;; This should probably be changed to a test to (= emacs-major-version 24)
-;; when it is released, but at the moment it might be possible that people
-;; are using an older snapshot.
-(defvar bovine--grammar-newstyle-unquote
-  (equal '(\, test) (read ",test")))
-
 (defun bovine-grammar-expand-form (form quotemode &optional inplace)
   "Expand FORM into a new one suitable to the bovine parser.
 FORM is a list in which we are substituting.
@@ -152,7 +144,7 @@ expanded from elsewhere."
               form  (cdr form))
         ;; Hack for dealing with new reading of unquotes outside of
         ;; backquote (introduced in rev. 102591 in emacs-bzr).
-        (when (and bovine--grammar-newstyle-unquote
+        (when (and (>= emacs-major-version 24)
                    (listp first)
                    (or (equal (car first) '\,)
                        (equal (car first) '\,@)))
@@ -456,6 +448,7 @@ Menu items are appended to the common grammar menu.")
 
 (defun bovine-make-parsers ()
   "Generate Emacs' built-in Bovine-based parser files."
+  (interactive)
   (semantic-mode 1)
   ;; Loop through each .by file in current directory, and run
   ;; `semantic-grammar-batch-build-one-package' to build the grammar.
@@ -465,13 +458,14 @@ Menu items are appended to the common grammar menu.")
                (with-current-buffer (find-file-noselect f)
                  (semantic-grammar-create-package))
              (error (message "%s" (error-message-string err)) nil)))
-          lang)
+          lang filename)
       (when (and packagename
-                 (string-match "^semantic-\\(.*\\)-by\\.el\\'" packagename))
+                 (string-match "^.*/\\(.*\\)-by\\.el\\'" packagename))
         (setq lang (match-string 1 packagename))
+        (setq filename (concat lang "-by.el"))
         (with-temp-buffer
-          (insert-file-contents packagename)
-          (setq buffer-file-name (expand-file-name packagename))
+          (insert-file-contents filename)
+          (setq buffer-file-name (expand-file-name filename))
           ;; Fix copyright header:
           (goto-char (point-min))
           (re-search-forward "^;; Author:")
@@ -500,20 +494,14 @@ Menu items are appended to the common grammar menu.")
                   lang ".by.
 
 ;;; Code:
-
-\(require 'semantic/lex)
-\(eval-when-compile (require 'semantic/bovine))\n")
+")
           (goto-char (point-min))
           (delete-region (point-min) (line-end-position))
-          (insert ";;; semantic/bovine/" lang
-                  "-by.el --- Generated parser support file")
+          (insert ";;; " packagename
+                  " --- Generated parser support file")
           (delete-trailing-whitespace)
-          ;; Fix footer:
-          (goto-char (point-max))
-          (re-search-backward ".\n;;; Analyzers")
-          (delete-region (point) (point-max))
-          (insert "(provide 'semantic/bovine/" lang "-by)\n\n")
-          (insert ";;; semantic/bovine/" lang "-by.el ends here\n")
+          (re-search-forward ";;; \\(.*\\) ends here")
+          (replace-match packagename nil nil nil 1)
           (save-buffer))))))
 
 ;;; bovine-grammar.el ends here
diff --git a/admin/grammars/c.by b/admin/grammars/c.by
index 1bdaf8f447b..dfced9813d1 100644
--- a/admin/grammars/c.by
+++ b/admin/grammars/c.by
@@ -1,5 +1,4 @@
 ;;; c.by -- LL grammar for C/C++ language specification
-
 ;; Copyright (C) 1999-2012 Free Software Foundation, Inc.
 ;;
 ;; Author: Eric M. Ludlam <zappo@gnu.org>
@@ -39,6 +38,13 @@
 ;; >  * Can't parse signature element: "RmcBucStatus* rftBucStatus"
 
 %package semantic-c-by
+%provide semantic/bovine/c-by
+
+%{
+(declare-function semantic-c-reconstitute-token "semantic/bovine/c")
+(declare-function semantic-c-reconstitute-template "semantic/bovine/c")
+(declare-function semantic-expand-c-tag "semantic/bovine/c")
+}
 
 %languagemode  c-mode c++-mode
 %start         declaration
@@ -98,6 +104,8 @@
 %put VIRTUAL summary "Method Modifier: virtual <type> <name>(...) ..."
 %token MUTABLE "mutable"
 %put MUTABLE summary "Member Declaration Modifier: mutable <type> <name> ..."
+%token EXPLICIT "explicit"
+%put EXPLICIT summary "Forbids implicit type conversion: explicit <constructor>"
 
 %token STRUCT "struct"
 %put STRUCT summary "Structure Type Declaration: struct [name] { ... };"
@@ -370,6 +378,9 @@ namespacesubparts
  ;; PUBLIC or PRIVATE bits.  Ignore them for now.
   | template
   | using
+ ;; Includes inside namespaces
+  | spp-include
+    (TAG $1 'include :inside-ns t)
   | ;;EMPTY
   ;
 
@@ -1098,6 +1109,8 @@ functionname
 function-pointer
   : LPAREN STAR symbol RPAREN
     ( (concat "*" $3) )
+  | LPAREN symbol RPAREN
+    ( $2 )
   ;
 
 fun-or-proto-end
diff --git a/admin/grammars/grammar.wy b/admin/grammars/grammar.wy
index 1189d6b0886..f89fe6220ff 100644
--- a/admin/grammars/grammar.wy
+++ b/admin/grammars/grammar.wy
@@ -23,6 +23,9 @@
 ;; You should have received a copy of the GNU General Public License
 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
 
+%package semantic-grammar-wy
+%provide semantic/grammar-wy
+
 %{
 (defvar semantic-grammar-lex-c-char-re)
 
@@ -32,8 +35,6 @@
 (defvar semantic-grammar-wy--rindx nil)
 }
 
-%package semantic-grammar-wy
-
 %languagemode wy-mode
 
 ;; Main
@@ -52,6 +53,7 @@
 %keyword LEFT            "%left"
 %keyword NONASSOC        "%nonassoc"
 %keyword PACKAGE         "%package"
+%keyword PROVIDE         "%provide"
 %keyword PREC            "%prec"
 %keyword PUT             "%put"
 %keyword QUOTEMODE       "%quotemode"
@@ -134,6 +136,7 @@ decl:
   | no_default_prec_decl
   | languagemode_decl
   | package_decl
+  | provide_decl
   | precedence_decl
   | put_decl
   | quotemode_decl
@@ -165,6 +168,11 @@ package_decl:
     `(PACKAGE-TAG ',$2 nil)
   ;
 
+provide_decl:
+    PROVIDE SYMBOL
+    `(TAG ',$2 'provide)
+  ;
+
 precedence_decl:
     associativity token_type_opt items
     `(TAG ',$1 'assoc :type ',$2 :value ',$3)
diff --git a/admin/grammars/java-tags.wy b/admin/grammars/java-tags.wy
index b58b96c6bfb..708715533ff 100644
--- a/admin/grammars/java-tags.wy
+++ b/admin/grammars/java-tags.wy
@@ -23,6 +23,7 @@
 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
 
 %package wisent-java-tags-wy
+%provide semantic/wisent/javat-wy
 
 %languagemode  java-mode
 
@@ -444,7 +445,7 @@ class_member_declaration
 ;;; Type Declaration token
 ;; ("NAME" type "TYPE" ( PART-LIST ) ( PARENTS ) EXTRA-SPEC "DOCSTRING")
 interface_declaration
-  : modifiers_opt INTERFACE IDENTIFIER extends_interfaces_opt interface_body
+  : modifiers_opt INTERFACE qualified_name extends_interfaces_opt interface_body
     (TYPE-TAG $3 $2 $5 (if $4 (cons nil $4)) :typemodifiers $1)
   ;
 
@@ -547,7 +548,7 @@ formal_parameters
 ;;; Variable token
 ;; ("NAME" variable "TYPE" DEFAULT-VALUE EXTRA-SPEC "DOCSTRING")
 formal_parameter
-  : formal_parameter_modifier_opt type variable_declarator_id
+  : formal_parameter_modifier_opt type opt_variable_declarator_id
     (VARIABLE-TAG $3 $2 nil :typemodifiers $1)
   ;
 
@@ -582,6 +583,13 @@ variable_declarator
     (cons $1 $region)
   ;
 
+opt_variable_declarator_id
+  : ;; EMPTY
+    (identity "")
+  | variable_declarator_id
+    (identity $1)
+  ;
+
 variable_declarator_id
   : IDENTIFIER dims_opt
     (concat $1 $2)
diff --git a/admin/grammars/js.wy b/admin/grammars/js.wy
index c6b3a58aac3..7b55f5c3834 100644
--- a/admin/grammars/js.wy
+++ b/admin/grammars/js.wy
@@ -58,6 +58,7 @@
 ;; DAMAGE.
 
 %package wisent-javascript-jv-wy
+%provide semantic/wisent/js-wy
 ;; JAVE I prefere ecmascript-mode
 %languagemode ecmascript-mode javascript-mode
 
diff --git a/admin/grammars/make.by b/admin/grammars/make.by
index ee933805cf6..6cff4716f82 100644
--- a/admin/grammars/make.by
+++ b/admin/grammars/make.by
@@ -22,6 +22,7 @@
 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
 
 %package semantic-make-by
+%provide semantic/bovine/make-by
 
 %languagemode  makefile-mode
 %start         Makefile
diff --git a/admin/grammars/python.wy b/admin/grammars/python.wy
index f7808fd20b8..f17f41c9b1b 100644
--- a/admin/grammars/python.wy
+++ b/admin/grammars/python.wy
@@ -88,6 +88,12 @@
 ;; --------
 
 %package wisent-python-wy
+%provide semantic/wisent/python-wy
+
+%{
+(declare-function wisent-python-reconstitute-function-tag "semantic/wisent/python")
+(declare-function wisent-python-reconstitute-class-tag "semantic/wisent/python")
+}
 
 %languagemode python-mode
 
@@ -173,6 +179,7 @@
 %token <punctuation> COMMA      ","
 %token <punctuation> ASSIGN     "="
 %token <punctuation> BACKQUOTE  "`"
+%token <punctuation> AT         "@"
 
 
 ;; -----------------
@@ -307,6 +314,10 @@
 %put     WHILE summary
 "Start a 'while' loop"
 
+%keyword WITH        "with"
+%put     WITH summary
+"Start statement with an associated context object"
+
 %keyword YIELD       "yield"
 %put     YIELD summary
 "Create a generator function"
@@ -545,8 +556,10 @@ import_stmt
 
 ;; dotted_as_name (',' dotted_as_name)*
 dotted_as_name_list
-  : dotted_as_name
-  | dotted_as_name_list COMMA dotted_as_name
+  : dotted_as_name_list COMMA dotted_as_name
+    (cons $3 $1)
+  | dotted_as_name
+    (list $1)
   ;
 
 ;; ('*' | import_as_name (',' import_as_name)*)
@@ -649,6 +662,7 @@ compound_stmt
   | while_stmt
   | for_stmt
   | try_stmt
+  | with_stmt
   | funcdef
   | class_declaration
   ;
@@ -756,13 +770,46 @@ zero_one_or_two_test
   ;
 
 ;;;============================================================================
+;;@@ with_stmt
+;;;============================================================================
+
+;; with_stmt: 'with' test [ with_var ] ':' suite
+with_stmt
+  : WITH test COLON suite
+    (CODE-TAG $1 nil)
+  | WITH test with_var COLON suite
+    (CODE-TAG $1 nil) ;; TODO capture variable
+  ;
+
+with_var
+  : AS expr
+    () ;; TODO capture
+  ;
+
+;;;============================================================================
 ;;;@@ funcdef
 ;;;============================================================================
 
-;; funcdef: 'def' NAME parameters ':' suite
+decorator
+  : AT dotted_name varargslist_opt NEWLINE
+    (FUNCTION-TAG $2 "decorator" $3)
+  ;
+
+decorators
+  : decorator
+    (list $1)
+  | decorator decorators
+    (cons $1 $2)
+  ;
+
+;; funcdef: [decorators] 'def' NAME parameters ':' suite
 funcdef
   : DEF NAME function_parameter_list COLON suite
-    (FUNCTION-TAG $2 nil $3)
+    (wisent-python-reconstitute-function-tag
+     (FUNCTION-TAG $2 nil $3) $5)
+  | decorators DEF NAME function_parameter_list COLON suite
+    (wisent-python-reconstitute-function-tag
+     (FUNCTION-TAG $3 nil $4 :decorators $1) $6)
   ;
 
 function_parameter_list
@@ -798,10 +845,11 @@ function_parameter
 ;; classdef: 'class' NAME ['(' testlist ')'] ':' suite
 class_declaration
   : CLASS NAME paren_class_list_opt COLON suite
-    (TYPE-TAG $2 $1             ;; Name "class"
-              $5                ;; Members
-              (cons $3 nil)     ;; (SUPERCLASSES . INTERFACES)
-              )
+    (wisent-python-reconstitute-class-tag
+     (TYPE-TAG $2 $1             ;; Name "class"
+               $5                ;; Members
+               (cons $3 nil)     ;; (SUPERCLASSES . INTERFACES)
+               ))
   ;
 
 ;; ['(' testlist ')']
diff --git a/admin/grammars/scheme.by b/admin/grammars/scheme.by
index 3925f03c28b..98e75901a71 100644
--- a/admin/grammars/scheme.by
+++ b/admin/grammars/scheme.by
@@ -18,6 +18,7 @@
 ;; along with GNU Emacs.  If not, see <http://www.gnu.org/licenses/>.
 
 %package semantic-scm-by
+%provide semantic/bovine/scm-by
 
 %languagemode  scheme-mode
 %start         scheme
diff --git a/admin/grammars/srecode-template.wy b/admin/grammars/srecode-template.wy
index f38d7eaa2a1..f84a414b46e 100644
--- a/admin/grammars/srecode-template.wy
+++ b/admin/grammars/srecode-template.wy
@@ -28,6 +28,9 @@
 ;; Semantic Recoder templates are based on Google Templates
 ;; and are at the bottom of the Semantic Recoder API.
 
+%package srecode-template-wy
+%provide srecode/srt-wy
+
 %languagemode  srecode-mode
 
 %start template_file
@@ -46,6 +49,15 @@
 %put     TEMPLATE  summary "template <name>\\n <template definition>"
 %keyword SECTIONDICTIONARY "sectiondictionary"
 %put     SECTIONDICTIONARY summary "sectiondictionary <name>\\n <dictionary entries>"
+
+%keyword SECTION  "section"
+%put     SECTION  summary
+         "section <name>\\n <dictionary entries>\\n end"
+
+%keyword END      "end"
+%put     END      summary
+         "section ... end"
+
 %keyword PROMPT "prompt"
 %keyword DEFAULT "default"
 %keyword DEFAULTMACRO "defaultmacro"
@@ -62,7 +74,7 @@
 %token <separator> TEMPLATE_BLOCK "^----"
 
 ;;; Bland default types
-%type <property> ":\\(\\w\\|\\s_\\)*"
+%type <property> syntax ":\\(\\w\\|\\s_\\)*"
 %token <property> property
 
 %type  <symbol>
@@ -134,7 +146,7 @@ insertable-string
 template
   : TEMPLATE templatename opt-dynamic-arguments newline
     opt-string
-    opt-section-dictionaries
+    section-dictionary-list
     TEMPLATE_BLOCK newline
     opt-bind
     (FUNCTION-TAG $2 nil $3 :documentation $5 :code $7
@@ -165,29 +177,52 @@ opt-string
   | ()
   ;
 
-opt-section-dictionaries
-  : () ;; EMPTY
-  | section-dictionary-list
-  ;
-
 section-dictionary-list
-  : one-section-dictionary
-    (list $1)
-  | section-dictionary-list one-section-dictionary
+  : ;; empty
+    ()
+  | section-dictionary-list flat-section-dictionary
+    (append $1 (list $2))
+  | section-dictionary-list section-dictionary
     (append $1 (list $2))
   ;
 
-one-section-dictionary
+flat-section-dictionary
   : SECTIONDICTIONARY string newline
-    variable-list
+    flat-dictionary-entry-list
+    (cons (read $2) $4)
+  ;
+
+flat-dictionary-entry-list
+  : ;; empty
+    ()
+  | flat-dictionary-entry-list flat-dictionary-entry
+    (append $1 $2)
+  ;
+
+flat-dictionary-entry
+  : variable
+    (EXPANDTAG $1)
+  ;
+
+section-dictionary
+  : SECTION string newline
+    dictionary-entry-list
+    END newline
     (cons (read $2) $4)
   ;
 
-variable-list
+dictionary-entry-list
+  : ;; empty
+    ()
+  | dictionary-entry-list dictionary-entry
+    (append $1 $2)
+  ;
+
+dictionary-entry
   : variable
     (EXPANDTAG $1)
-  | variable-list variable
-    (append $1 (EXPANDTAG $2))
+  | section-dictionary
+    (list $1)
   ;
 
 opt-bind
@@ -232,4 +267,4 @@ It ignores whitespace, newlines and comments."
   semantic-lex-default-action
   )
 
-;;; wisent-dot.wy ends here
+;;; srecode-template.wy ends here
diff --git a/admin/grammars/wisent-grammar.el b/admin/grammars/wisent-grammar.el
index 714b5211127..25dba5be2d8 100644
--- a/admin/grammars/wisent-grammar.el
+++ b/admin/grammars/wisent-grammar.el
@@ -209,15 +209,15 @@ See also the function `wisent-skip-token'."
   "Return the list of terminal symbols.
 Keep order of declaration in the WY file without duplicates."
   (let (terms)
-    (mapcar
+    (mapc
      #'(lambda (tag)
-         (mapcar #'(lambda (name)
-                     (add-to-list 'terms (intern name)))
-                 (cons (semantic-tag-name tag)
-                       (semantic-tag-get-attribute tag :rest))))
+         (mapcar #'(lambda (name)
+                     (add-to-list 'terms (intern name)))
+                 (cons (semantic-tag-name tag)
+                       (semantic-tag-get-attribute tag :rest))))
      (semantic--find-tags-by-function
       #'(lambda (tag)
-          (memq (semantic-tag-class tag) '(token keyword)))
+          (memq (semantic-tag-class tag) '(token keyword)))
       (current-buffer)))
     (nreverse terms)))
 
@@ -464,23 +464,20 @@ Menu items are appended to the common grammar menu.")
 ;; DAMAGE.")
 
 (defvar wisent-make-parsers--parser-file-name
-  `(("semantic-grammar-wy.el"
-     "semantic/grammar-wy")
-    ("srecode-template-wy.el"
-     "srecode/srt-wy")
-    ("wisent-javascript-jv-wy.el"
-     "semantic/wisent/js-wy"
+  `(("semantic/grammar-wy.el")
+    ("srecode/srt-wy.el")
+    ("semantic/wisent/js-wy.el"
      "Copyright (C) 1998-2011 Ecma International."
      ,wisent-make-parsers--ecmascript-license)
-    ("wisent-java-tags-wy.el"
-     "semantic/wisent/javat-wy")
-    ("wisent-python-wy.el"
-     "semantic/wisent/python-wy"
-     "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Python Software Foundation; All Rights Reserved."
+    ("semantic/wisent/javat-wy.el")
+    ("semantic/wisent/python-wy.el"
+     "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008,
+\;; 2009, 2010 Python Software Foundation; All Rights Reserved"
      ,wisent-make-parsers--python-license)))
 
 (defun wisent-make-parsers ()
   "Generate Emacs' built-in Wisent-based parser files."
+  (interactive)
   (semantic-mode 1)
   ;; Loop through each .wy file in current directory, and run
   ;; `semantic-grammar-batch-build-one-package' to build the grammar.
@@ -492,16 +489,16 @@ Menu items are appended to the common grammar menu.")
              (error (message "%s" (error-message-string err)) nil)))
           output-data)
       (when (setq output-data (assoc packagename wisent-make-parsers--parser-file-name))
-        (let ((require-name         (nth 1 output-data))
-              (additional-copyright (nth 2 output-data))
-              (additional-license   (nth 3 output-data))
+        (let ((additional-copyright (nth 1 output-data))
+              (additional-license   (nth 2 output-data))
+              (filename (progn (string-match ".*/\\(.*\\)" packagename) (match-string 1 packagename)))
               copyright-end)
           ;; Touch up the generated parsers for Emacs integration.
           (with-temp-buffer
-            (insert-file-contents packagename)
+            (insert-file-contents filename)
             ;; Fix copyright header:
             (goto-char (point-min))
-            (when additional-copyright
+            (when additional-copyright  
               (re-search-forward "Copyright (C).*$")
               (insert "\n;; " additional-copyright))
             (re-search-forward "^;; Author:")
@@ -516,22 +513,14 @@ Menu items are appended to the common grammar menu.")
                         f ".")
             (when additional-license
               (insert "\n" additional-license))
-            (insert "\n\n;;; Code:\n
-\(require 'semantic/lex)\n")
+            (insert "\n\n;;; Code:\n")
             (goto-char (point-min))
             (delete-region (point-min) (line-end-position))
-            (insert ";;; " require-name
-                    ".el --- Generated parser support file")
+            (insert ";;; " packagename
+                    " --- Generated parser support file")
+            (re-search-forward ";;; \\(.*\\) ends here")
+            (replace-match packagename nil nil nil 1)
             (delete-trailing-whitespace)
-            (re-search-forward ";;\n(require 'semantic/lex)\n")
-            (delete-region (match-beginning 0) (match-end 0))
-            ;; Fix footer:
-            (goto-char (point-max))
-            (re-search-backward "^(provide")
-            (delete-region (match-beginning 0) (point-max))
-            (goto-char (point-max))
-            (insert "(provide '" require-name ")\n\n")
-            (insert ";;; " require-name ".el ends here\n")
-            (write-region nil nil (expand-file-name packagename))))))))
+            (write-region nil nil (expand-file-name filename))))))))
 
 ;;; wisent-grammar.el ends here