2005-01-14 Carsten Dominik <dominik@science.uva.nl>

        * reftex-cite.el (reftex-bib-sort-year): Catch the case if the
        year is not given.

        * reftex-ref.el (reftex-replace-prefix-escapes): Added new escapes
        %m and %M, fixed bug with %F by adding save-match-data.
        (reftex-reference): Removed ?. from list of spaces.
        (reftex-label-info): Added automatic label prefix recognition

        * reftex-index.el (reftex-index-next-phrase): Added slave
        parameter to call of `reftex-index-this-phrase'
        (reftex-index-this-phrase): New optional argument
        (reftex-index-region-phrases): Added slave parameter to call of
        `reftex-index-this-phrase'
        (reftex-display-index): New argument redo
        (reftex-index-rescan): Added 'redo to arguments of
        `reftex-display-index'
        (reftex-index-Rescan, reftex-index-revert)
        (reftex-index-switch-index-tag): Added 'redo to arguments of
        `reftex-display-index'
        (reftex-index-make-phrase-regexp): Fixed bug with case-sensitive
        indexing.  Fixed bug with matching is there is a quote before or
        after the word.

        * reftex-cite.el (reftex-all-used-citation-keys): Fix bug when
        collecting citation keys in lines with comments.
        (reftex-citation): Prefix argument no longer rescans the document,
        but forces prompting for optional arguments of cite macros.
        (reftex-do-citation): Prompting for optional arguments
        implemented.

        * reftex-vars.el (reftex-cite-format-builtin): Added optional
        arguments to most cite commands.
        (reftex-cite-cleanup-optional-args): New option
        (reftex-cite-prompt-optional-args): New option.
        (reftex-trust-label-prefix): New option

        * reftex-toc.el (reftex-toc-find-section): Added push-mark before
        changing the position in the buffer.

        * reftex.el (reftex-prefix-to-typekey-alist): New variable
        (reftex-compile-variables): Compute reftex-prefix-to-typekey-alist

11 files changed, 351 insertions, 110 deletions

diff --git a/lisp/textmodes/reftex-auc.el b/lisp/textmodes/reftex-auc.el
index 534e4e7b27b..93cd0757b4e 100644
--- a/lisp/textmodes/reftex-auc.el
+++ b/lisp/textmodes/reftex-auc.el
@@ -1,8 +1,8 @@
 ;;; reftex-auc.el --- RefTeX's interface to AUCTeX
-;; Copyright (c) 1997, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 
 ;; This file is part of GNU Emacs.
 
diff --git a/lisp/textmodes/reftex-cite.el b/lisp/textmodes/reftex-cite.el
index cfebdbdced1..54dc46eac4f 100644
--- a/lisp/textmodes/reftex-cite.el
+++ b/lisp/textmodes/reftex-cite.el
@@ -1,8 +1,8 @@
 ;;; reftex-cite.el --- creating citations with RefTeX
-;; Copyright (c) 1997, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 
 ;; This file is part of GNU Emacs.
 
@@ -49,11 +49,11 @@
  TAB        Enter citation key with completion.
  RET        Accept current entry (also on mouse-2) and create \\cite macro.
  m / u      Mark/Unmark the entry.
+ e / E      Create BibTeX file with all (marked/unmarked) entries
  a / A      Put all (marked) entries into one/many \\cite commands.")
 
 ;; Find bibtex files
 
-
 (defmacro reftex-with-special-syntax-for-bib (&rest body)
   `(let ((saved-syntax (syntax-table)))
      (unwind-protect
@@ -311,8 +311,8 @@
       (not (stringp (car al1))))))
 
 (defun reftex-bib-sort-year (e1 e2)
-  (< (string-to-int (cdr (assoc "year" e1)))
-     (string-to-int (cdr (assoc "year" e2)))))
+  (< (string-to-int (or (cdr (assoc "year" e1)) "0"))
+     (string-to-int (or (cdr (assoc "year" e2)) "0"))))
 
 (defun reftex-bib-sort-year-reverse (e1 e2)
   (> (string-to-int (or (cdr (assoc "year" e1)) "0"))
@@ -597,12 +597,13 @@ to `reftex-cite-format' and inserted into the buffer.
 
 If NO-INSERT is non-nil, nothing is inserted, only the selected key returned.
 
-FORMAT-KEY can be used to pre-select a citation format.
+FORAT-KEY can be used to pre-select a citation format.
 
-When called with one or two `C-u' prefixes, first rescans the document.
-When called with a numeric prefix, make that many citations.  When
-called with point inside the braces of a `\\cite' command, it will
-add another key, ignoring the value of `reftex-cite-format'.
+When called with a `C-u' prefix, prompt for optional arguments in
+cite macros.  When called with a numeric prefix, make that many
+citations.  When called with point inside the braces of a `\\cite'
+command, it will add another key, ignoring the value of
+`reftex-cite-format'.
 
 The regular expression uses an expanded syntax: && is interpreted as `and'.
 Thus, `aaaa&&bbb' matches entries which contain both `aaaa' and `bbb'.
@@ -618,7 +619,7 @@ While entering the regexp, completion on knows citation keys is possible.
   ;; Thus look for the scanning info only if in reftex-mode.
 
   (when reftex-mode
-    (reftex-access-scan-info current-prefix-arg))
+    (reftex-access-scan-info nil))
 
   ;; Call reftex-do-citation, but protected
   (unwind-protect
@@ -629,11 +630,14 @@ While entering the regexp, completion on knows citation keys is possible.
   ;; This really does the work of reftex-citation.
 
   (let* ((format (reftex-figure-out-cite-format arg no-insert format-key))
+         (start 0)
          (docstruct-symbol reftex-docstruct-symbol)
          (selected-entries (reftex-offer-bib-menu))
          (insert-entries selected-entries)
          entry string cite-view)
 
+    (when (stringp selected-entries)
+      (error selected-entries))
     (unless selected-entries (error "Quit"))
 
     (if (stringp selected-entries)
@@ -646,6 +650,7 @@ While entering the regexp, completion on knows citation keys is possible.
 
     (when (eq (car selected-entries) 'concat)
       ;; All keys go into a single command - we need to trick a little
+      ;; FIXME: Unfortunately, this meens that commenting does not work right.
       (pop selected-entries)
       (let ((concat-keys (mapconcat 'car selected-entries ",")))
         (setq insert-entries 
@@ -655,7 +660,7 @@ While entering the regexp, completion on knows citation keys is possible.
 
       ;; We shall insert this into the buffer...
       (message "Formatting...")
-
+      
       (while (setq entry (pop insert-entries))
         ;; Format the citation and insert it
         (setq string (if reftex-format-cite-function
@@ -663,6 +668,24 @@ While entering the regexp, completion on knows citation keys is possible.
                                   (reftex-get-bib-field "&key" entry)
                                   format)
                        (reftex-format-citation entry format)))
+        (when (or (eq reftex-cite-prompt-optional-args t)
+                  (and reftex-cite-prompt-optional-args
+                       (equal arg '(4))))
+          (let ((start 0) (nth 0) value)
+            (while (setq start (string-match "\\[\\]" string start))
+              (setq value (read-string (format "Optional argument %d: "
+                                               (setq nth (1+ nth)))))
+              (setq string (replace-match (concat "[" value "]") t t string))
+              (setq start (1+ start)))))
+        ;; Should we cleanup empty optional arguments?
+        ;; if the first is empty, it can be removed.  If the second is empty,
+        ;; it has to go.
+        (when reftex-cite-cleanup-optional-args
+          (cond 
+           ((string-match "\\[\\]\\(\\[[a-zA-Z0-9., ]+\\]\\)" string)
+            (setq string (replace-match "\\1" nil nil string)))
+           ((string-match "\\[\\]\\[\\]" string)
+            (setq string (replace-match "" t t string)))))
         (insert string))
 
       ;; Reposition cursor?
@@ -842,6 +865,17 @@ While entering the regexp, completion on knows citation keys is possible.
                                 (mapcar 'car (nreverse reftex-select-marked))
                               found-list)))
                 (throw 'done t))
+               ((eq key ?e)
+                ;; Take all (marked), and push the symbol 'concat
+                (reftex-extract-bib-file found-list reftex-select-marked)
+                (setq selected-entries "BibTeX database file created")
+                (throw 'done t))
+               ((eq key ?E)
+                ;; Take all (marked), and push the symbol 'concat
+                (reftex-extract-bib-file found-list reftex-select-marked
+                                         'complement)
+                (setq selected-entries "BibTeX database file created")
+                (throw 'done t))
                ((or (eq key ?\C-m)
                     (eq key 'return))
                 ;; Take selected
@@ -882,6 +916,29 @@ While entering the regexp, completion on knows citation keys is possible.
       (ding)
       found-list)))
 
+(defun reftex-extract-bib-file (all &optional marked complement)
+  ;; Limit FOUND-LIST with more regular expressions
+  (let ((file (read-file-name "File to create: ")))
+    (find-file-other-window file)
+    (if (> (buffer-size) 0)
+        (unless (yes-or-no-p 
+                 (format "Overwrite non-empty file %s? " file))
+          (error "Abort")))
+    (erase-buffer)
+    (setq all (delq nil
+                    (mapcar
+                     (lambda (x)
+                       (if marked
+                           (if (or (and (assoc x marked) (not complement))
+                                   (and (not (assoc x marked)) complement))
+                               (cdr (assoc "&entry" x))
+                             nil)
+                         (cdr (assoc "&entry" x))))
+                     all)))
+    (insert (mapconcat 'identity all "\n\n"))
+    (save-buffer)
+    (goto-char (point-min))))
+
 (defun reftex-insert-bib-matches (list)
   ;; Insert the bib matches and number them correctly
   (let ((mouse-face
@@ -1043,5 +1100,73 @@ While entering the regexp, completion on knows citation keys is possible.
       
     (select-window win)))
 
+;;; Global BibTeX file
+(defun reftex-all-used-citation-keys ()
+  (reftex-access-scan-info)
+  (let ((files (reftex-all-document-files)) file keys kkk kk k)
+    (save-excursion
+      (while (setq file (pop files))
+        (set-buffer (reftex-get-file-buffer-force file 'mark))
+        (save-excursion
+          (save-restriction
+            (widen)
+            (goto-char (point-min))
+            (while (re-search-forward "^[^%\n\r]*\\\\\\(bibentry\\|[a-zA-Z]*cite[a-zA-Z]*\\)\\(\\[[^\\]]*\\]\\)?{\\([^}]+\\)}" nil t)
+              (setq kk (match-string-no-properties 3))
+              (while (string-match "%.*\n?" kk)
+                (setq kk (replace-match "" t t kk)))
+              (setq kk (split-string kk "[, \t\r\n]+"))
+              (while (setq k (pop kk))
+                (or (member k keys)
+                    (setq keys (cons k keys)))))))))
+    (reftex-kill-temporary-buffers)
+    keys))
+
+(defun reftex-create-bibtex-file (bibfile)
+  "Create a new BibTeX database file with all entries referenced in document.
+The command prompts for a filename and writes the collected entries to
+that file.  Only entries referenced in the current document with
+any \\cite-like macros are used. 
+The sequence in the new file is the same as it was in the old database."
+  (interactive "FNew BibTeX file: ")
+  (let ((keys (reftex-all-used-citation-keys))
+        (files (reftex-get-bibfile-list))
+        file key entries beg end entry)
+    (save-excursion
+      (while (setq file (pop files))
+        (set-buffer (reftex-get-file-buffer-force file 'mark))
+        (reftex-with-special-syntax-for-bib
+         (save-excursion
+           (save-restriction
+             (widen)
+             (goto-char (point-min))
+             (while (re-search-forward 
+                     "^[ \t]*@[a-zA-Z]+[ \t]*{\\([^ \t\r\n]+\\),"
+                     nil t)
+               (setq key (match-string 1)
+                     beg (match-beginning 0)
+                     end (progn
+                           (goto-char (match-beginning 1))
+                           (condition-case nil
+                               (up-list 1)
+                             (error (goto-char (match-end 0))))
+                           (point)))
+               (when (member key keys)
+                 (setq entry (buffer-substring beg end)
+                       entries (cons entry entries)
+                       keys (delete key keys)))))))))
+    (find-file-other-window bibfile)
+    (if (> (buffer-size) 0)
+        (unless (yes-or-no-p 
+                 (format "Overwrite non-empty file %s? " bibfile))
+          (error "Abort")))
+    (erase-buffer)
+    (insert (mapconcat 'identity (reverse entries) "\n\n"))
+    (goto-char (point-min))
+    (save-buffer)
+    (message "%d entries extracted and copied to new database"
+             (length entries))))
+
+
 ;;; arch-tag: d53d0a5a-ab32-4b52-a846-2a7c3527cd89
 ;;; reftex-cite.el ends here
diff --git a/lisp/textmodes/reftex-dcr.el b/lisp/textmodes/reftex-dcr.el
index dc47a5e7564..e832465cf17 100644
--- a/lisp/textmodes/reftex-dcr.el
+++ b/lisp/textmodes/reftex-dcr.el
@@ -1,8 +1,8 @@
 ;;; reftex-dcr.el --- viewing cross references and citations with RefTeX
-;; Copyright (c) 1997, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 ;;
 
 ;; This file is part of GNU Emacs.
diff --git a/lisp/textmodes/reftex-global.el b/lisp/textmodes/reftex-global.el
index 84b4c7c67f5..d4d91b476e9 100644
--- a/lisp/textmodes/reftex-global.el
+++ b/lisp/textmodes/reftex-global.el
@@ -1,8 +1,8 @@
 ;;; reftex-global.el --- operations on entire documents with RefTeX
-;; Copyright (c) 1997, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 
 ;; This file is part of GNU Emacs.
 
diff --git a/lisp/textmodes/reftex-index.el b/lisp/textmodes/reftex-index.el
index 71b3b624770..cb02f2e056f 100644
--- a/lisp/textmodes/reftex-index.el
+++ b/lisp/textmodes/reftex-index.el
@@ -1,8 +1,8 @@
 ;;; reftex-index.el --- index support with RefTeX
-;; Copyright (c) 1997, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 
 ;; This file is part of GNU Emacs.
 
@@ -360,7 +360,7 @@ _ ^        Add/Remove parent key (to make this item a subitem).
       (reftex-highlight 0 (match-beginning 0) (match-end 0) (current-buffer)))
     match))
 
-(defun reftex-display-index (&optional tag overriding-restriction
+(defun reftex-display-index (&optional tag overriding-restriction redo
                                        &rest locations)
   "Display a buffer with an index compiled from the current document.
 When the document has multiple indices, first prompts for the correct one.
@@ -387,7 +387,7 @@ With prefix 3, restrict index to region."
          (calling-file (buffer-file-name))
          (restriction
           (or overriding-restriction
-              (and (interactive-p) 
+              (and (not redo) 
                    (reftex-get-restriction current-prefix-arg docstruct))))
          (locations
           ;; See if we are on an index macro as initial position
@@ -427,7 +427,7 @@ With prefix 3, restrict index to region."
     (if restriction
         (setq reftex-index-restriction-indicator (car restriction)
               reftex-index-restriction-data (cdr restriction))
-      (if (interactive-p)
+      (if (not redo)
           (setq reftex-index-restriction-indicator nil
                 reftex-index-restriction-data nil)))
     (when (= (buffer-size) 0)
@@ -703,7 +703,7 @@ The function will go to the section where the entry at point was defined."
               (error "Don't know which file to rescan.  Try `C-u r'")
             (switch-to-buffer (reftex-get-file-buffer-force file))
             (setq current-prefix-arg '(4))
-            (reftex-display-index index-tag nil line)))
+            (reftex-display-index index-tag nil 'redo line)))
       (reftex-index-Rescan))
     (reftex-kill-temporary-buffers)))
 (defun reftex-index-Rescan (&rest ignore)
@@ -714,7 +714,7 @@ The function will go to the section where the entry at point was defined."
     (switch-to-buffer
      (reftex-get-file-buffer-force reftex-last-index-file))
     (setq current-prefix-arg '(16))
-    (reftex-display-index index-tag nil line)))
+    (reftex-display-index index-tag nil 'redo line)))
 (defun reftex-index-revert (&rest ignore)
   "Regenerate the *Index* from the internal lists.  No reparsing os done."
   (interactive)
@@ -727,14 +727,14 @@ The function will go to the section where the entry at point was defined."
     (reftex-erase-buffer buf)
     (setq current-prefix-arg nil
           reftex-last-follow-point 1)
-    (reftex-display-index index-tag nil data line)))
+    (reftex-display-index index-tag nil 'redo data line)))
 (defun reftex-index-switch-index-tag (&rest ignore)
   "Switch to a different index of the same document."
   (interactive)
   (switch-to-buffer
    (reftex-get-file-buffer-force reftex-last-index-file))
   (setq current-prefix-arg nil)
-  (reftex-display-index))
+  (reftex-display-index nil nil 'redo))
 
 (defun reftex-index-restrict-to-section (&optional force)
   "Restrict index to entries defined in same document sect. as entry at point."
@@ -1352,23 +1352,23 @@ Here are all local bindings.
     (if (re-search-forward reftex-index-phrases-phrase-regexp12 nil t)
         (progn
           (goto-char (match-beginning 0))
-          (reftex-index-this-phrase))
+          (reftex-index-this-phrase 'slave))
       (error "No more phrase lines after point"))))
 
-(defun reftex-index-this-phrase ()
+(defun reftex-index-this-phrase (&optional slave)
   "Index the phrase in the current line.
 Does a global search and replace in the entire document.  At each
 match, the user will be asked to confirm the replacement."
   (interactive)
-  (if (interactive-p) (reftex-index-phrases-parse-header t))
+  (if (not slave) (reftex-index-phrases-parse-header t))
   (save-excursion
     (beginning-of-line)
     (cond ((looking-at reftex-index-phrases-comment-regexp)
-           (if (interactive-p) (error "Comment line")))
+           (if (not slave) (error "Comment line")))
           ((looking-at "^[ \t]*$")
-           (if (interactive-p) (error "Empty line")))
+           (if (not slave) (error "Empty line")))
           ((looking-at reftex-index-phrases-macrodef-regexp)
-           (if (interactive-p) (error "Macro definition line")))
+           (if (not slave) (error "Macro definition line")))
           ((looking-at reftex-index-phrases-phrase-regexp12)
            ;; This is a phrase
            (let* ((char (if (not (equal (match-string 1) ""))
@@ -1429,7 +1429,7 @@ Calls `reftex-index-this-phrase' on each line in the region."
   (goto-char beg)
   (while (not (or (eobp)
                   (>= (point) end)))
-    (save-excursion (reftex-index-this-phrase))
+    (save-excursion (reftex-index-this-phrase 'slave))
     (beginning-of-line 2)))
 
 (defun reftex-index-phrases-parse-header (&optional get-files)
@@ -1736,12 +1736,15 @@ With optional arg ALLOW-NEWLINE, allow single newline between words."
                        "\\([ \t]*\\(\n[ \t]*\\)?\\|[ \t]\\)"
                      "\\([ \t]+\\)")))
     (concat (if (and as-words (string-match "\\`\\w" (car words)))
-                "\\<" "")
-            (mapconcat (lambda (w) (regexp-quote (downcase w)))
+                "\\(\\<\\|[`']\\)" "")
+            (mapconcat (lambda (w) (regexp-quote 
+                                    (if reftex-index-phrases-case-fold-search
+                                        (downcase w)
+                                      w)))
                        words space-re)
             (if (and as-words 
                      (string-match "\\w\\'" (nth (1- (length words)) words)))
-                "\\>" ""))))
+                "\\(\\>\\|'\\)" ""))))
 
 (defun reftex-index-simplify-phrase (phrase)
   "Make phrase single spaces and single line."
@@ -1825,6 +1828,8 @@ both ends."
     (unwind-protect
         (while (re-search-forward re nil t)
           (catch 'next-match
+            (if (reftex-in-comment)
+                (throw 'next-match nil))
             (if (and (fboundp reftex-index-verify-function)
                      (not (funcall reftex-index-verify-function)))
                 (throw 'next-match nil))
@@ -1925,7 +1930,7 @@ both ends."
       (reftex-unhighlight 0))))
 
 (defun reftex-index-phrase-match-is-indexed (beg end)
-  ;; CHeck if match is in an argument of an index macro, or if an
+  ;; Check if match is in an argument of an index macro, or if an
   ;; index macro is directly attached to the match.
   (save-excursion
     (goto-char end)
diff --git a/lisp/textmodes/reftex-parse.el b/lisp/textmodes/reftex-parse.el
index 534775bf5a5..630c7101725 100644
--- a/lisp/textmodes/reftex-parse.el
+++ b/lisp/textmodes/reftex-parse.el
@@ -1,8 +1,8 @@
 ;;; reftex-parse.el --- parser functions for RefTeX
-;; Copyright (c) 1997, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 ;;
 
 ;; This file is part of GNU Emacs.
diff --git a/lisp/textmodes/reftex-ref.el b/lisp/textmodes/reftex-ref.el
index 2d4935aedde..43095e2d684 100644
--- a/lisp/textmodes/reftex-ref.el
+++ b/lisp/textmodes/reftex-ref.el
@@ -1,8 +1,8 @@
 ;;; reftex-ref.el --- code to create labels and references with RefTeX
-;; Copyright (c) 1997, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 
 ;; This file is part of GNU Emacs.
 
@@ -96,14 +96,27 @@ If optional BOUND is an integer, limit backward searches to that point."
 
 (defun reftex-label-info (label &optional file bound derive env-or-mac)
   ;; Return info list on LABEL at point.
-  (let* ((env-or-mac (or env-or-mac (reftex-label-location bound)))
-         (typekey (nth 1 (assoc env-or-mac reftex-env-or-mac-alist)))
+  (let* ((prefix (if (string-match "^[a-zA-Z0-9]+:" label)
+                     (match-string 0 label)))
+         (typekey (cdr (assoc prefix reftex-prefix-to-typekey-alist)))
          (file (or file (buffer-file-name)))
-         (parse (nth 2 (assoc env-or-mac reftex-env-or-mac-alist)))
-         (text (reftex-short-context env-or-mac parse reftex-location-start
-                                     derive))
+         (trust reftex-trust-label-prefix)
          (in-comment (reftex-in-comment)))
-    (list label typekey text file in-comment)))
+    (if (and typekey
+             (cond ((eq trust t) t)
+                   ((null trust) nil)
+                   ((stringp trust) (string-match trust typekey))
+                   ((listp trust) (member typekey trust))
+                   (t nil)))
+        (list label typekey
+              (reftex-nicify-text (reftex-context-substring))
+              file in-comment)
+      (let* ((env-or-mac (or env-or-mac (reftex-label-location bound)))
+             (typekey (nth 1 (assoc env-or-mac reftex-env-or-mac-alist)))
+             (parse (nth 2 (assoc env-or-mac reftex-env-or-mac-alist)))
+             (text (reftex-short-context env-or-mac parse reftex-location-start
+                                         derive)))
+        (list label typekey text file in-comment)))))
 
 ;;; Creating labels ---------------------------------------------------------
 
@@ -296,35 +309,43 @@ also applies `reftex-translate-to-ascii-function' to the string."
       (while (string-match "\\%\\([a-zA-Z]\\)" prefix num)
         (setq letter (match-string 1 prefix))
         (setq replace
-              (cond
-               ((equal letter "f")
-                (file-name-sans-extension
-                 (file-name-nondirectory (buffer-file-name))))
-               ((equal letter "F")
-                (let ((masterdir (file-name-directory (reftex-TeX-master-file)))
-                      (file (file-name-sans-extension (buffer-file-name))))
-                  (if (string-match (concat "\\`" (regexp-quote masterdir))
-                                    file)
-                      (substring file (length masterdir))
-                    file)))
-               ((equal letter "u")
-                (or (user-login-name) ""))
-               ((equal letter "S")
-                (let* (macro level-exp level)
-                  (save-excursion
-                    (save-match-data
-                      (when (re-search-backward reftex-section-regexp nil t)
-                        (setq macro (reftex-match-string 2)
-                              level-exp (cdr (assoc macro reftex-section-levels-all))
-                              level (if (symbolp level-exp)
-                                        (abs (save-match-data
-                                               (funcall level-exp)))
-                                      (abs level-exp))))
-                      (cdr (or (assoc macro reftex-section-prefixes)
-                               (assoc level reftex-section-prefixes)
-                               (assq t reftex-section-prefixes)
-                               (list t "sec:")))))))
-               (t "")))
+              (save-match-data
+                (cond
+                 ((equal letter "f")
+                  (file-name-sans-extension
+                   (file-name-nondirectory (buffer-file-name))))
+                 ((equal letter "F")
+                  (let ((masterdir (file-name-directory (reftex-TeX-master-file)))
+                        (file (file-name-sans-extension (buffer-file-name))))
+                    (if (string-match (concat "\\`" (regexp-quote masterdir))
+                                      file)
+                        (substring file (length masterdir))
+                      file)))
+                 ((equal letter "m")
+                  (file-name-sans-extension
+                   (file-name-nondirectory (reftex-TeX-master-file))))
+                 ((equal letter "M")
+                  (file-name-nondirectory
+                   (substring (file-name-directory (reftex-TeX-master-file))
+                              0 -1)))
+                 ((equal letter "u")
+                  (or (user-login-name) ""))
+                 ((equal letter "S")
+                  (let* (macro level-exp level)
+                    (save-excursion
+                      (save-match-data
+                        (when (re-search-backward reftex-section-regexp nil t)
+                          (setq macro (reftex-match-string 2)
+                                level-exp (cdr (assoc macro reftex-section-levels-all))
+                                level (if (symbolp level-exp)
+                                          (abs (save-match-data
+                                                 (funcall level-exp)))
+                                        (abs level-exp))))
+                        (cdr (or (assoc macro reftex-section-prefixes)
+                                 (assoc level reftex-section-prefixes)
+                                 (assq t reftex-section-prefixes)
+                                 (list t "sec:")))))))
+                 (t ""))))
         (setq num (1- (+ (match-beginning 1) (length replace)))
               prefix (replace-match replace nil nil prefix)))
       prefix)))
@@ -449,7 +470,7 @@ When called with 2 C-u prefix args, disable magic word recognition."
 
         ;; remove ~ if we do already have a space
         (when (and (= ?~ (string-to-char form))
-                   (member (preceding-char) '(?\ ?\t ?\n ?. ?~)))
+                   (member (preceding-char) '(?\ ?\t ?\n ?~)))
           (setq form (substring form 1)))
         ;; do we have a special format?
         (setq reftex-format-ref-function
diff --git a/lisp/textmodes/reftex-sel.el b/lisp/textmodes/reftex-sel.el
index 3074e83b681..8b0ebb2ca2b 100644
--- a/lisp/textmodes/reftex-sel.el
+++ b/lisp/textmodes/reftex-sel.el
@@ -1,8 +1,8 @@
 ;;; reftex-sel.el --- the selection modes for RefTeX
-;; Copyright (c) 1997, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 
 ;; This file is part of GNU Emacs.
 
@@ -607,13 +607,11 @@ Useful for large TOC's."
     (setq ovl (make-overlay boe eoe))
     (push (list data ovl separator) reftex-select-marked)
     (overlay-put ovl 'face reftex-select-mark-face)
-    (if (featurep 'xemacs)
-        ;; before-string property is broken in Emacs
-        (overlay-put ovl 'before-string
-                     (if separator
-                         (format "*%c%d* " separator
-                                 (length reftex-select-marked))
-                       (format "*%d*  " (length reftex-select-marked)))))
+    (overlay-put ovl 'before-string
+                 (if separator
+                     (format "*%c%d* " separator
+                             (length reftex-select-marked))
+                   (format "*%d*  " (length reftex-select-marked))))
     (message "Entry has mark no. %d" (length reftex-select-marked))))
 
 (defun reftex-select-mark-comma ()
@@ -725,7 +723,7 @@ Useful for large TOC's."
       (define-key reftex-select-label-map (car x) (cdr x)))
 
 ;; Specific bindings in reftex-select-bib-map
-(loop for key across "grRaA" do
+(loop for key across "grRaAeE" do
       (define-key reftex-select-bib-map (vector (list key))
         (list 'lambda '() 
               "Press `?' during selection to find out about this key."
diff --git a/lisp/textmodes/reftex-toc.el b/lisp/textmodes/reftex-toc.el
index 8d217f08dfb..0f0b72332b8 100644
--- a/lisp/textmodes/reftex-toc.el
+++ b/lisp/textmodes/reftex-toc.el
@@ -2,7 +2,7 @@
 ;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 
 ;; This file is part of GNU Emacs.
 
@@ -754,13 +754,13 @@ if these sets are sorted blocks in the alist."
   "Make sure all files of the document are being visited by buffers,
 and that the scanning info is absolutely up to date.
 We do this by rescanning with reftex-keep-temporary-buffers bound to t.
-The variable PRO-OR-DE is assumed to be dynamically scoped into this function.
+The variable PRO-OR-DE is assumed to be dynamically scoped into thes function.
 When finished, we exit with an error message."
   (let ((reftex-keep-temporary-buffers t))
     (reftex-toc-Rescan)
     (reftex-toc-restore-region start-line mark-line)
     (throw 'exit
-           "TOC had to be updated first.  Please check selection and repeat the command.")))
+           (format "TOC had to be updated first.  Please check selection and repeat the command." pro-or-de))))
 
 (defun reftex-toc-rename-label ()
   "Rename the currently selected label in the *TOC* buffer.
@@ -868,6 +868,7 @@ label prefix determines the wording of a reference."
            ((and (markerp marker) (marker-buffer marker))
             ;; Buffer is still live and we have the marker.  Should be easy.
             (switch-to-buffer-other-window (marker-buffer marker))
+            (push-mark nil)
             (goto-char (marker-position marker))
             (or (looking-at (regexp-quote literal))
                 (looking-at (reftex-make-regexp-allow-for-ctrl-m literal))
@@ -1088,4 +1089,4 @@ always show the current section in connection with the option
 
 
 ;;; arch-tag: 92400ce2-0b86-4c89-a606-4ed71acea17e
-;;; reftex-toc.el ends here
\ No newline at end of file
+;;; reftex-toc.el ends here
diff --git a/lisp/textmodes/reftex-vars.el b/lisp/textmodes/reftex-vars.el
index 53357342c51..a7c0cb1c1ad 100644
--- a/lisp/textmodes/reftex-vars.el
+++ b/lisp/textmodes/reftex-vars.el
@@ -1,8 +1,8 @@
 ;;; reftex-vars.el --- configuration variables for RefTeX
-;; Copyright (c) 1997, 1998, 1999, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 
 ;; This file is part of GNU Emacs.
 
@@ -128,13 +128,13 @@ distribution.  Mixed-case symbols are convenience aliases.")
 
 (defconst reftex-cite-format-builtin
   '((default "Default macro \\cite{%l}"
-      "\\cite{%l}")
+      "\\cite[]{%l}")
     (natbib "The Natbib package"
-     ((?\C-m . "\\cite{%l}")
-      (?t    . "\\citet{%l}")
-      (?T    . "\\citet*{%l}")
-      (?p    . "\\citep{%l}")
-      (?P    . "\\citep*{%l}")
+     ((?\C-m . "\\cite[][]{%l}")
+      (?t    . "\\citet[][]{%l}")
+      (?T    . "\\citet*[][]{%l}")
+      (?p    . "\\citep[][]{%l}")
+      (?P    . "\\citep*[][]{%l}")
       (?e    . "\\citep[e.g.][]{%l}")
       (?s    . "\\citep[see][]{%l}")
       (?a    . "\\citeauthor{%l}")
@@ -157,8 +157,8 @@ distribution.  Mixed-case symbols are convenience aliases.")
     (bibentry "The Bibentry package"
       "\\bibentry{%l}")
     (harvard "The Harvard package"
-     ((?\C-m . "\\cite{%l}")
-      (?p    . "\\cite{%l}")
+     ((?\C-m . "\\cite[]{%l}")
+      (?p    . "\\cite[]{%l}")
       (?t    . "\\citeasnoun{%l}")
       (?n    . "\\citeasnoun{%l}")
       (?s    . "\\possessivecite{%l}")
@@ -166,17 +166,17 @@ distribution.  Mixed-case symbols are convenience aliases.")
       (?y    . "\\citeyear{%l}")
       (?a    . "\\citename{%l}")))
     (chicago "The Chicago package"
-     ((?\C-m . "\\cite{%l}")
-      (?t    . "\\citeN{%l}")
+     ((?\C-m . "\\cite[]{%l}")
+      (?t    . "\\citeN[]{%l}")
       (?T    . "\\shortciteN{%l}")
-      (?p    . "\\cite{%l}")
+      (?p    . "\\cite[]{%l}")
       (?P    . "\\shortcite{%l}")
       (?a    . "\\citeA{%l}")
       (?A    . "\\shortciteA{%l}")
       (?y    . "\\citeyear{%l}")))
     (astron "The Astron package"
-     ((?\C-m . "\\cite{%l}")
-      (?p    . "\\cite{%l}" )
+     ((?\C-m . "\\cite[]{%l}")
+      (?p    . "\\cite[]{%l}" )
       (?t    . "%2a (\\cite{%l})")))
     (author-year "Do-it-yourself Author-year"
      ((?\C-m . "\\cite{%l}")
@@ -484,6 +484,8 @@ LABEL-PREFIX
     empty string.  The prefix may contain the following `%' escapes:
        %f   Current file name with directory and extension stripped.
        %F   Current file name relative to directory of master file.
+       %m   Master file name, directory and extension stripped.
+       %M   Directory name (without path) where master file is located.
        %u   User login name, on systems which support this.
        %S   A section prefix derived with variable `reftex-section-prefixes'.
 
@@ -631,6 +633,43 @@ the final regular expression - so %s will be replaced with the environment
 or macro."
   :group 'reftex-defining-label-environments
   :type '(repeat (cons (symbol) (regexp))))
+
+(defcustom reftex-trust-label-prefix nil
+  "Non-nil means, trust the label prefix when determining label type.
+It is customary to use special label prefixes to distinguish different label
+types.  The label prefixes have no syntactic meaning in LaTeX (unless
+special packages like fancyref) are being used.  RefTeX can and by
+default does parse around each label to detect the correct label type,
+but this process can be slow when a document contains thousands of
+labels.  If you use label prefixes consistently, you may speed up
+document parsing by setting this variable to a non-nil value.  RefTeX
+will then compare the label prefix with the prefixes found in
+`reftex-label-alist' and derive the correct label type in this way.
+Possible values for this option are:
+
+t          This means to trust any label prefixes found.
+regexp     If a regexp, only prefixes matched by the regexp are trusted.
+list       List of accepted prefixes, as strings.  The colon is part of
+           the prefix, e.g. (\"fn:\" \"eqn:\" \"item:\").
+nil        Never trust a label prefix.
+
+The only disadvantage of using this feature is that the label context
+displayed in the label selection buffer along with each label is
+simply some text after the label definition.  This is no problem if you
+place labels keeping this in mind (e.g. *before* the equation, *at
+the beginning* of a fig/tab caption ...).  Anyway, it is probably best
+to use the regexp or the list value types to fine-tune this feature.
+For example, if your document contains thousands of footnotes with
+labels fn:xxx, you may want to set this variable to the value \"^fn:$\" or
+\(\"fn:\").  Then RefTeX will still do extensive parsing for any
+non-footnote labels."
+  :group 'reftex-defining-label-environments
+  :type '(choice
+          (const :tag "Always" t)
+          (const :tag "Never" nil)
+          (regexp)
+          (repeat :tag "List"
+                  (string :tag "prefix (with colon)"))))
   
 (defcustom reftex-special-environment-functions nil
   "List of functions to be called when trying to figure out current environment.
@@ -1010,6 +1049,9 @@ display, and for (setq reftex-comment-citations t).
 %< as a special operator kills punctuation and space around it after the 
 string has been formatted.
 
+A pair of square brackets indicates an optional argument, and RefTeX
+will prompt for the values of these arguments.
+
 Beware that all this only works with BibTeX database files.  When
 citations are made from the \\bibitems in an explicit thebibliography
 environment, only %l is available.
@@ -1042,6 +1084,42 @@ E.g.: (setq reftex-cite-format 'natbib)"
             (cons (character :tag "Key character" ?\r)
                   (string    :tag "Format string" "")))))
 
+(defcustom reftex-cite-prompt-optional-args 'maybe
+  "*Non-nil means, prompt for empty optional arguments in cite macros.
+When an entry in `reftex-cite-format' ist given with square brackets to
+indicate optional arguments (for example \\cite[][]{%l}), RefTeX can
+prompt for values.  Possible values are:
+
+nil     Never prompt for optional arguments
+t       Always prompt
+maybe   Prompt only if `reftex-citation' was called with C-u prefix arg
+
+Unnecessary empty optional arguments are removed before insertion into
+the buffer.  See `reftex-cite-cleanup-optional-args'."
+  :group 'reftex-citation-support
+  :type '(choice
+          (const :tag "Always" t)
+          (const :tag "When called with prefix arg" maybe)
+          (const :tag "Never" nil)))
+
+(defcustom reftex-cite-cleanup-optional-args t
+  "*Non-nil means, remove unnecessary empty optional arguments in cite macros.
+The cite macros provided by some packages (for example
+natbib) allow specifying two optional arguments, one for a prefix to
+the citation, and a second for a postfix.  When only one optional
+argument is given, it is interpreted as postfix.  When this option is
+t, RefTeX removes unnecessary empty optional arguments from the cite
+macro before insertion.  For example, it will change
+    \\cite[][]{Jones}              -> \\cite{Jones}
+    \\cite[][Chapter 1]{Jones}     -> \\cite[Chapter 1]{Jones}
+    \\cite[see][]{Jones}           -> \\cite[see][]{Jones}
+    \\cite[see][Chapter 1]{Jones}  -> \\cite{Jones}
+Is is possible that other packages have other conventions about which
+optional argument is interpreted how - that is why this cleaning up
+can be turned off."
+  :group 'reftex-citation-support
+  :type 'boolean)
+
 (defcustom reftex-comment-citations nil
   "*Non-nil means add a comment for each citation describing the full entry.
 The comment is formatted according to `reftex-cite-comment-format'."
diff --git a/lisp/textmodes/reftex.el b/lisp/textmodes/reftex.el
index c00400a7b96..e177805e9c4 100644
--- a/lisp/textmodes/reftex.el
+++ b/lisp/textmodes/reftex.el
@@ -1,8 +1,8 @@
 ;;; reftex.el --- minor mode for doing \label, \ref, \cite, \index in LaTeX
-;; Copyright (c) 1997, 1998, 1999, 2000, 2003 Free Software Foundation, Inc.
+;; Copyright (c) 1997, 1998, 1999, 2000, 2003, 2004 Free Software Foundation, Inc.
 
 ;; Author: Carsten Dominik <dominik@science.uva.nl>
-;; Version: 4.21
+;; Version: 4.26
 ;; Keywords: tex
 
 ;; This file is part of GNU Emacs.
@@ -300,7 +300,7 @@
 ;;; Define the formal stuff for a minor mode named RefTeX.
 ;;;
 
-(defconst reftex-version "RefTeX version 4.21"
+(defconst reftex-version "RefTeX version 4.26"
   "Version string for RefTeX.")
 
 (defvar reftex-mode nil
@@ -669,6 +669,8 @@ the label information is recompiled on next use."
 
 ;; Alist relating magic words to a label type.
 (defvar reftex-words-to-typekey-alist nil)
+;; Alist relating label prefixes to a label type.
+(defvar reftex-prefix-to-typekey-alist nil)
 
 ;; The last list-of-labels entry used in a reference.
 (defvar reftex-last-used-reference (list nil nil nil nil))
@@ -750,6 +752,7 @@ the label information is recompiled on next use."
     reftex-typekey-to-format-alist
     reftex-typekey-to-prefix-alist
     reftex-words-to-typekey-alist
+    reftex-prefix-to-typekey-alist
     reftex-type-query-prompt
     reftex-type-query-help
 
@@ -904,6 +907,8 @@ This enforces rescanning the buffer on next use."
         macro verify repeat nindex tag key toc-level toc-levels)
 
     (setq reftex-words-to-typekey-alist nil
+          reftex-prefix-to-typekey-alist 
+          '(("sec:" . "s") ("cha:" . "s") ("chap:" . "s"))
           reftex-typekey-list nil
           reftex-typekey-to-format-alist nil
           reftex-typekey-to-prefix-alist nil
@@ -946,6 +951,10 @@ This enforces rescanning the buffer on next use."
         (if typekey
             (add-to-list 'reftex-typekey-list typekey))
         (if (and typekey prefix
+                 (not (assoc prefix reftex-prefix-to-typekey-alist)))
+            (add-to-list 'reftex-prefix-to-typekey-alist
+                         (cons prefix typekey)))
+        (if (and typekey prefix
                  (not (assoc typekey reftex-typekey-to-prefix-alist)))
             (add-to-list 'reftex-typekey-to-prefix-alist
                          (cons typekey prefix)))
@@ -1676,6 +1685,7 @@ When DIE is non-nil, throw an error if file not found."
  "Make a citation using BibTeX database files." t)
 (autoload 'reftex-default-bibliography "reftex-cite")
 (autoload 'reftex-bib-or-thebib "reftex-cite")
+(autoload 'reftex-create-bibtex-file "reftex-cite")
 
 ;;; =========================================================================
 ;;;
@@ -2439,6 +2449,7 @@ IGNORE-WORDS List of words which should be removed from the string."
     ["Restore from File"      (reftex-access-parse-file 'restore) t])
    ("Global Actions"
     ["Search Whole Document"  reftex-search-document t]
+    ["Search Again"           tags-loop-continue t]
     ["Replace in Document"    reftex-query-replace-document t]
     ["Grep on Document"       reftex-grep-document t]
     "--"
@@ -2447,6 +2458,8 @@ IGNORE-WORDS List of words which should be removed from the string."
     ["Change Label and Refs"  reftex-change-label t]
     ["Renumber Simple Labels" reftex-renumber-simple-labels t]
     "--"
+    ["Create BibTeX File"     reftex-create-bibtex-file t]
+    "--"
     ["Create TAGS File"       reftex-create-tags-file t]
     "--"
     ["Save Document"          reftex-save-all-document-buffers t])