Merge from origin/emacs-29

267fc6d00c4 ruby-smie-rules: Fix misindentation of a method call afte...
0fde314f6f6 * lib-src/etags.c (process_file_name): Free malloc'ed var...
dde9d149af3 ; Improve documentation of loading *.eln files
7c552be89da ; Another doc fix in eglot.el
75c65fcc98e ; Fix last change
a3d15c1f749 ; Fix last change
ca79b138d42 Eglot: rename and redocument encoding-related functions (...
3e3e6d71be7 Eglot: support positionEncoding LSP capability (bug#61726)
b0e87e930e8 Eglot: use faster strategy for moving to LSP positions (b...
5b174b96834 Fix mule-tests in UTF-8 locales
5256392a7ec Fix 'vertical-motion' when display strings are around
0db88d625a7 ; * src/treesit.c (treesit_predicate_match): Fix typo.

10 files changed, 153 insertions, 86 deletions

diff --git a/doc/emacs/building.texi b/doc/emacs/building.texi
index 3f6a418de1a..f82b605598e 100644
--- a/doc/emacs/building.texi
+++ b/doc/emacs/building.texi
@@ -1556,18 +1556,26 @@ command prompts for a @dfn{library name} rather than a file name; it
 searches through each directory in the Emacs Lisp load path, trying to
 find a file matching that library name.  If the library name is
 @samp{@var{foo}}, it tries looking for files named
-@file{@var{foo}.elc}, @file{@var{foo}.el}, and @file{@var{foo}}.  The
-default behavior is to load the first file found.  This command
-prefers @file{.elc} files over @file{.el} files because compiled files
-load and run faster.  If it finds that @file{@var{lib}.el} is newer
-than @file{@var{lib}.elc}, it issues a warning, in case someone made
-changes to the @file{.el} file and forgot to recompile it, but loads
-the @file{.elc} file anyway.  (Due to this behavior, you can save
-unfinished edits to Emacs Lisp source files, and not recompile until
-your changes are ready for use.)  If you set the option
-@code{load-prefer-newer} to a non-@code{nil} value, however, then
-rather than the procedure described above, Emacs loads whichever
-version of the file is newest.
+@file{@var{foo}.elc}, @file{@var{foo}.el}, and @file{@var{foo}}.  (If
+Emacs was built with native compilation enabled, @code{load-library}
+looks for a @samp{.eln} file that corresponds to @file{@var{foo}.el}
+and loads it instead of @file{@var{foo}.elc}.)  The default behavior
+is to load the first file found.  This command prefers @file{.eln}
+files over @file{.elc} files, and prefers @file{.elc} files over
+@file{.el} files, because compiled files load and run faster.  If it
+finds that @file{@var{lib}.el} is newer than @file{@var{lib}.elc}, it
+issues a warning, in case someone made changes to the @file{.el} file
+and forgot to recompile it, but loads the @file{.elc} file anyway.
+(Due to this behavior, you can save unfinished edits to Emacs Lisp
+source files, and not recompile until your changes are ready for use.)
+If you set the option @code{load-prefer-newer} to a non-@code{nil}
+value, however, then rather than the procedure described above, Emacs
+loads whichever version of the file is newest.  If Emacs was built
+with native compilation, and it cannot find the @samp{.eln} file
+corresponding to @file{@var{lib}.el}, it will load a
+@file{@var{lib}.elc} and start native compilation of
+@file{@var{lib}.el} in the background, then load the @samp{.eln} file
+when it finishes compilation.
 
   Emacs Lisp programs usually load Emacs Lisp files using the
 @code{load} function.  This is similar to @code{load-library}, but is
@@ -1604,6 +1612,11 @@ It is customary to put locally installed libraries in the
 @code{load-path}, or in some subdirectory of @file{site-lisp}.  This
 way, you don't need to modify the default value of @code{load-path}.
 
+@vindex native-comp-eln-load-path
+  Similarly to @code{load-path}, the list of directories where Emacs
+looks for @file{*.eln} files with natively-compiled Lisp code is
+specified by the variable @code{native-comp-eln-load-path}.
+
 @cindex autoload
   Some commands are @dfn{autoloaded}; when you run them, Emacs
 automatically loads the associated library first.  For instance, the
diff --git a/doc/lispref/loading.texi b/doc/lispref/loading.texi
index dbbdc767738..5c84ba4b1eb 100644
--- a/doc/lispref/loading.texi
+++ b/doc/lispref/loading.texi
@@ -75,17 +75,20 @@ file exists, and Emacs was compiled with native-compilation support
 (@pxref{Native Compilation}), @code{load} attempts to find a
 corresponding @samp{.eln} file, and if found, loads it instead of
 @file{@var{filename}.elc}.  Otherwise, it loads
-@file{@var{filename}.elc}.  If there is no file by that name, then
-@code{load} looks for a file named @file{@var{filename}.el}.  If that
-file exists, it is loaded.  If Emacs was compiled with support for
-dynamic modules (@pxref{Dynamic Modules}), @code{load} next looks for
-a file named @file{@var{filename}.@var{ext}}, where @var{ext} is a
-system-dependent file-name extension of shared libraries.  Finally, if
-neither of those names is found, @code{load} looks for a file named
-@var{filename} with nothing appended, and loads it if it exists.  (The
-@code{load} function is not clever about looking at @var{filename}.
-In the perverse case of a file named @file{foo.el.el}, evaluation of
-@code{(load "foo.el")} will indeed find it.)
+@file{@var{filename}.elc} (and starts a background native compilation
+to produce the missing @samp{.eln} file, followed by loading that
+file).  If there is no @file{@var{filename}.elc}, then @code{load}
+looks for a file named @file{@var{filename}.el}.  If that file exists,
+it is loaded.  If Emacs was compiled with support for dynamic modules
+(@pxref{Dynamic Modules}), @code{load} next looks for a file named
+@file{@var{filename}.@var{ext}}, where @var{ext} is a system-dependent
+file-name extension of shared libraries (@samp{.so} on GNU and Unix
+systems).  Finally, if neither of those names is found, @code{load}
+looks for a file named @var{filename} with nothing appended, and loads
+it if it exists.  (The @code{load} function is not clever about
+looking at @var{filename}.  In the perverse case of a file named
+@file{foo.el.el}, evaluation of @code{(load "foo.el")} will indeed
+find it.)
 
 If Auto Compression mode is enabled, as it is by default, then if
 @code{load} can not find a file, it searches for a compressed version
diff --git a/lib-src/etags.c b/lib-src/etags.c
index 2628849d78e..cb842dbf669 100644
--- a/lib-src/etags.c
+++ b/lib-src/etags.c
@@ -1732,6 +1732,8 @@ process_file_name (char *file, language *lang)
           char *cmd = xmalloc (buf_len);
           snprintf (cmd, buf_len, "%s %s > %s",
                     compr->command, new_real_name, new_tmp_name);
+          free (new_real_name);
+          free (new_tmp_name);
 #endif
           inf = (system (cmd) == -1
                  ? NULL
diff --git a/lisp/progmodes/eglot.el b/lisp/progmodes/eglot.el
index d0e899cbed5..4203e4a336e 100644
--- a/lisp/progmodes/eglot.el
+++ b/lisp/progmodes/eglot.el
@@ -816,6 +816,7 @@ treated as in `eglot--dbind'."
                                        `(:valueSet
                                          [,@(mapcar
                                              #'car eglot--tag-faces)])))
+            :general (list :positionEncodings ["utf-32" "utf-8" "utf-16"])
             :experimental eglot--{})))
 
 (cl-defgeneric eglot-workspace-folders (server)
@@ -1439,20 +1440,32 @@ CONNECT-ARGS are passed as additional arguments to
   (let ((warning-minimum-level :error))
     (display-warning 'eglot (apply #'format format args) :warning)))
 
-(defun eglot-current-column () (- (point) (line-beginning-position)))
-
-(defvar eglot-current-column-function #'eglot-lsp-abiding-column
-  "Function to calculate the current column.
-
-This is the inverse operation of
-`eglot-move-to-column-function' (which see).  It is a function of
-no arguments returning a column number.  For buffers managed by
-fully LSP-compliant servers, this should be set to
-`eglot-lsp-abiding-column' (the default), and
-`eglot-current-column' for all others.")
-
-(defun eglot-lsp-abiding-column (&optional lbp)
-  "Calculate current COLUMN as defined by the LSP spec.
+
+;;; Encoding fever
+;;;
+(define-obsolete-function-alias
+  'eglot-lsp-abiding-column 'eglot-utf-16-linepos "29.1")
+(define-obsolete-function-alias
+  'eglot-current-column 'eglot-utf-32-linepos "29.1")
+(define-obsolete-variable-alias
+  'eglot-current-column-function 'eglot-current-linepos-function "29.1")
+
+(defvar eglot-current-linepos-function #'eglot-utf-16-linepos
+  "Function calculating position relative to line beginning.
+
+This is the inverse of `eglot-move-to-linepos-function' (which see).
+It is a function of no arguments returning the number of code units
+or bytes or codepoints corresponding to the current position of point,
+relative to line beginning, as expected by the function that is the
+value of `eglot-move-to-linepos-function'.")
+
+(defun eglot-utf-8-linepos ()
+  "Calculate number of UTF-8 bytes from line beginning."
+  (length (encode-coding-region (line-beginning-position) (point)
+                                'utf-8-unix t)))
+
+(defun eglot-utf-16-linepos (&optional lbp)
+  "Calculate number of UTF-16 code units from position given by LBP.
 LBP defaults to `line-beginning-position'."
   (/ (- (length (encode-coding-region (or lbp (line-beginning-position))
                                       ;; Fix github#860
@@ -1460,51 +1473,71 @@ LBP defaults to `line-beginning-position'."
         2)
      2))
 
+(defun eglot-utf-32-linepos ()
+  "Calculate number of Unicode codepoints from line beginning."
+  (- (point) (line-beginning-position)))
+
 (defun eglot--pos-to-lsp-position (&optional pos)
   "Convert point POS to LSP position."
   (eglot--widening
    ;; LSP line is zero-origin; emacs is one-origin.
    (list :line (1- (line-number-at-pos pos t))
          :character (progn (when pos (goto-char pos))
-                           (funcall eglot-current-column-function)))))
-
-(defvar eglot-move-to-column-function #'eglot-move-to-lsp-abiding-column
-  "Function to move to a column reported by the LSP server.
-
-According to the standard, LSP column/character offsets are based
-on a count of UTF-16 code units, not actual visual columns.  So
-when LSP says position 3 of a line containing just \"aXbc\",
-where X is a multi-byte character, it actually means `b', not
-`c'. However, many servers don't follow the spec this closely.
-
-For buffers managed by fully LSP-compliant servers, this should
-be set to `eglot-move-to-lsp-abiding-column' (the default), and
-`eglot-move-to-column' for all others.")
-
-(defun eglot-move-to-column (column)
-  "Move to COLUMN without closely following the LSP spec."
+                           (funcall eglot-current-linepos-function)))))
+
+(define-obsolete-function-alias
+  'eglot-move-to-current-column 'eglot-move-to-utf-32-linepos "29.1")
+(define-obsolete-function-alias
+  'eglot-move-to-lsp-abiding-column 'eglot-move-to-utf-16-linepos "29.1")
+(define-obsolete-variable-alias
+'eglot-move-to-column-function 'eglot-move-to-linepos-function "29.1")
+
+(defvar eglot-move-to-linepos-function #'eglot-move-to-utf-16-linepos
+  "Function to move to a position within a line reported by the LSP server.
+
+Per the LSP spec, character offsets in LSP Position objects count
+UTF-16 code units, not actual code points.  So when LSP says
+position 3 of a line containing just \"aXbc\", where X is a funny
+looking character in the UTF-16 \"supplementary plane\", it
+actually means `b', not `c'.  The default value
+`eglot-move-to-utf-16-linepos' accounts for this.
+
+This variable can also be set to `eglot-move-to-utf-8-linepos' or
+`eglot-move-to-utf-32-linepos' for servers not closely following
+the spec.  Also, since LSP 3.17 server and client may agree on an
+encoding and Eglot will set this variable automatically.")
+
+(defun eglot-move-to-utf-8-linepos (n)
+  "Move to line's Nth byte as computed by LSP's UTF-8 criterion."
+  (let* ((bol (line-beginning-position))
+         (goal-byte (+ (position-bytes bol) n))
+         (eol (line-end-position)))
+    (goto-char bol)
+    (while (and (< (position-bytes (point)) goal-byte) (< (point) eol))
+      ;; raw bytes take 2 bytes in the buffer
+      (when (>= (char-after) #x3fff80) (setq goal-byte (1+ goal-byte)))
+      (forward-char 1))))
+
+(defun eglot-move-to-utf-16-linepos (n)
+  "Move to line's Nth code unit as computed by LSP's UTF-16 criterion."
+  (let* ((bol (line-beginning-position))
+         (goal-char (+ bol n))
+         (eol (line-end-position)))
+    (goto-char bol)
+    (while (and (< (point) goal-char) (< (point) eol))
+      ;; code points in the "supplementary place" use two code units
+      (when (<= #x010000 (char-after) #x10ffff) (setq goal-char (1- goal-char)))
+      (forward-char 1))))
+
+(defun eglot-move-to-utf-32-linepos (n)
+  "Move to line's Nth codepoint as computed by LSP's UTF-32 criterion."
   ;; We cannot use `move-to-column' here, because it moves to *visual*
-  ;; columns, which can be different from LSP columns in case of
+  ;; columns, which can be different from LSP characters in case of
   ;; `whitespace-mode', `prettify-symbols-mode', etc.  (github#296,
   ;; github#297)
-  (goto-char (min (+ (line-beginning-position) column)
+  (goto-char (min (+ (line-beginning-position) n)
                   (line-end-position))))
 
-(defun eglot-move-to-lsp-abiding-column (column)
-  "Move to COLUMN abiding by the LSP spec."
-  (save-restriction
-    (cl-loop
-     with lbp = (line-beginning-position)
-     initially
-     (narrow-to-region lbp (line-end-position))
-     (move-to-column column)
-     for diff = (- column
-                   (eglot-lsp-abiding-column lbp))
-     until (zerop diff)
-     do (condition-case eob-err
-            (forward-char (/ (if (> diff 0) (1+ diff) (1- diff)) 2))
-          (end-of-buffer (cl-return eob-err))))))
-
 (defun eglot--lsp-position-to-point (pos-plist &optional marker)
   "Convert LSP position POS-PLIST to Emacs point.
 If optional MARKER, return a marker instead"
@@ -1515,16 +1548,17 @@ If optional MARKER, return a marker instead"
       (forward-line (min most-positive-fixnum
                          (plist-get pos-plist :line)))
       (unless (eobp) ;; if line was excessive leave point at eob
-        (let ((tab-width 1)
-              (col (plist-get pos-plist :character)))
+        (let ((col (plist-get pos-plist :character)))
           (unless (wholenump col)
             (eglot--warn
              "Caution: LSP server sent invalid character position %s. Using 0 instead."
              col)
             (setq col 0))
-          (funcall eglot-move-to-column-function col)))
+          (funcall eglot-move-to-linepos-function col)))
       (if marker (copy-marker (point-marker)) (point)))))
 
+
+;;; More helpers
 (defconst eglot--uri-path-allowed-chars
   (let ((vec (copy-sequence url-path-allowed-chars)))
     (aset vec ?: nil) ;; see github#639
@@ -1758,6 +1792,14 @@ Use `eglot-managed-p' to determine if current buffer is managed.")
   :init-value nil :lighter nil :keymap eglot-mode-map
   (cond
    (eglot--managed-mode
+    (pcase (plist-get (eglot--capabilities (eglot-current-server))
+                      :positionEncoding)
+      ("utf-32"
+       (eglot--setq-saving eglot-current-linepos-function #'eglot-utf-32-linepos)
+       (eglot--setq-saving eglot-move-to-linepos-function #'eglot-move-to-utf-32-linepos))
+      ("utf-8"
+       (eglot--setq-saving eglot-current-linepos-function #'eglot-utf-8-linepos)
+       (eglot--setq-saving eglot-move-to-linepos-function #'eglot-move-to-utf-8-linepos)))
     (add-hook 'after-change-functions 'eglot--after-change nil t)
     (add-hook 'before-change-functions 'eglot--before-change nil t)
     (add-hook 'kill-buffer-hook #'eglot--managed-mode-off nil t)
@@ -2591,7 +2633,7 @@ Try to visit the target file for a richer summary line."
                      (add-face-text-property hi-beg hi-end 'xref-match
                                              t substring)
                      (list substring (line-number-at-pos (point) t)
-                           (eglot-current-column) (- end beg))))))
+                           (eglot-utf-32-linepos) (- end beg))))))
        (`(,summary ,line ,column ,length)
         (cond
          (visiting (with-current-buffer visiting (funcall collect)))
diff --git a/lisp/progmodes/ruby-mode.el b/lisp/progmodes/ruby-mode.el
index dba9ff0a846..559b62fef54 100644
--- a/lisp/progmodes/ruby-mode.el
+++ b/lisp/progmodes/ruby-mode.el
@@ -909,7 +909,9 @@ This only affects the output of the command `ruby-toggle-block'."
                      "<<=" ">>=" "&&=" "||=" "and" "or"))
      (cond
       ((not ruby-after-operator-indent)
-       (ruby-smie--indent-to-stmt ruby-indent-level))
+       (ruby-smie--indent-to-stmt (if (smie-indent--hanging-p)
+                                      ruby-indent-level
+                                    0)))
       ((and (smie-rule-parent-p ";" nil)
             (smie-indent--hanging-p))
        ruby-indent-level)))
diff --git a/src/indent.c b/src/indent.c
index 6de18d749ca..08d2bf5ea28 100644
--- a/src/indent.c
+++ b/src/indent.c
@@ -2401,7 +2401,7 @@ whether or not it is currently displayed in some window.  */)
              last line that it occupies.  */
           if (it_start < ZV)
             {
-              if ((it.bidi_it.scan_dir > 0)
+              if ((it.bidi_it.scan_dir >= 0 || it.vpos == vpos_init)
                   ? IT_CHARPOS (it) < it_start
                   : IT_CHARPOS (it) > it_start)
                 {
diff --git a/src/treesit.c b/src/treesit.c
index ef0f2407840..5a4fe3e8803 100644
--- a/src/treesit.c
+++ b/src/treesit.c
@@ -2484,7 +2484,7 @@ treesit_predicate_match (Lisp_Object args, struct capture_range captures)
 {
   if (XFIXNUM (Flength (args)) != 2)
     xsignal2 (Qtreesit_query_error,
-              build_string ("Predicate `equal' requires two "
+              build_string ("Predicate `match' requires two "
                             "arguments but only given"),
               Flength (args));
 
diff --git a/test/lisp/international/mule-tests.el b/test/lisp/international/mule-tests.el
index 6b76e35ae22..3e0c5bf9f4b 100644
--- a/test/lisp/international/mule-tests.el
+++ b/test/lisp/international/mule-tests.el
@@ -119,9 +119,10 @@ provide HTML fragments.  Some tests override those variables."
 (ert-deftest sgml-html-meta-no-post-less-than-10lines ()
   "No '</head>', detect charset in the first 10 lines."
   (let ((sgml-html-meta-post ""))
-    (should (eq 'utf-8 (sgml-html-meta-run
-                        (concat "\n\n\n\n\n\n\n\n\n"
-                                "<meta charset='utf-8'>"))))))
+    (should (eq 'utf-8 (coding-system-base
+                        (sgml-html-meta-run
+                         (concat "\n\n\n\n\n\n\n\n\n"
+                                 "<meta charset='utf-8'>")))))))
 
 (ert-deftest sgml-html-meta-no-post-10lines ()
   "No '</head>', do not detect charset after the first 10 lines."
diff --git a/test/lisp/progmodes/eglot-tests.el b/test/lisp/progmodes/eglot-tests.el
index 4b6528351b2..5d5de59a19a 100644
--- a/test/lisp/progmodes/eglot-tests.el
+++ b/test/lisp/progmodes/eglot-tests.el
@@ -856,8 +856,8 @@ pylsp prefers autopep over yafp, despite its README stating the contrary."
            '((c-mode . ("clangd")))))
       (with-current-buffer
           (eglot--find-file-noselect "project/foo.c")
-        (setq-local eglot-move-to-column-function #'eglot-move-to-lsp-abiding-column)
-        (setq-local eglot-current-column-function #'eglot-lsp-abiding-column)
+        (setq-local eglot-move-to-linepos-function #'eglot-move-to-utf-16-linepos)
+        (setq-local eglot-current-linepos-function #'eglot-utf-16-linepos)
         (eglot--sniffing (:client-notifications c-notifs)
           (eglot--tests-connect)
           (end-of-line)
@@ -866,12 +866,12 @@ pylsp prefers autopep over yafp, despite its README stating the contrary."
           (eglot--wait-for (c-notifs 2) (&key params &allow-other-keys)
             (should (equal 71 (cadddr (cadadr (aref (cadddr params) 0))))))
           (beginning-of-line)
-          (should (eq eglot-move-to-column-function #'eglot-move-to-lsp-abiding-column))
-          (funcall eglot-move-to-column-function 71)
+          (should (eq eglot-move-to-linepos-function #'eglot-move-to-utf-16-linepos))
+          (funcall eglot-move-to-linepos-function 71)
           (should (looking-at "p")))))))
 
 (ert-deftest eglot-test-lsp-abiding-column ()
-  "Test basic `eglot-lsp-abiding-column' and `eglot-move-to-lsp-abiding-column'."
+  "Test basic LSP character counting logic."
   (skip-unless (executable-find "clangd"))
   (eglot-tests--lsp-abiding-column-1))
 
diff --git a/test/lisp/progmodes/ruby-mode-resources/ruby-after-operator-indent.rb b/test/lisp/progmodes/ruby-mode-resources/ruby-after-operator-indent.rb
index 25cd8736f97..e339d229d3e 100644
--- a/test/lisp/progmodes/ruby-mode-resources/ruby-after-operator-indent.rb
+++ b/test/lisp/progmodes/ruby-mode-resources/ruby-after-operator-indent.rb
@@ -10,6 +10,10 @@ qux = 4 + 5 *
 foo = obj.bar { |m| tee(m) } +
   obj.qux { |m| hum(m) }
 
+some_variable = abc + some_method(
+  some_argument
+)
+
 foo.
   bar
   .baz