[latex3-commits] [git/LaTeX3-latex3-latex3] main: Fix interaction of cs and catcodes in regex replacements (fixes #909) (b067a0512)

[Bruno Le Floch]

Repository : https://github.com/latex3/latex3
On branch  : main
Link       : https://github.com/latex3/latex3/commit/b067a0512726d580bc80734d5a300fd68709358d

>---------------------------------------------------------------

commit b067a0512726d580bc80734d5a300fd68709358d
Author: Bruno Le Floch <blflatex at gmail.com>
Date:   Tue May 11 02:26:28 2021 +0200

    Fix interaction of cs and catcodes in regex replacements (fixes #909)


>---------------------------------------------------------------

b067a0512726d580bc80734d5a300fd68709358d
 l3kernel/CHANGELOG.md             |  1 +
 l3kernel/l3regex.dtx              | 26 +++++++++++++++-----------
 l3kernel/testfiles/m3regex001.lvt | 13 +++++++++++++
 l3kernel/testfiles/m3regex001.tlg | 13 +++++++++++++
 4 files changed, 42 insertions(+), 11 deletions(-)

diff --git a/l3kernel/CHANGELOG.md b/l3kernel/CHANGELOG.md
index 341961b0c..7dabcdb5a 100644
--- a/l3kernel/CHANGELOG.md
+++ b/l3kernel/CHANGELOG.md
@@ -14,6 +14,7 @@ this project uses date-based 'snapshot' version identifiers.
 ### Fixed
 - Loading of backend in generic DVI mode (issue #905)
 - Make `\keyval_parse:nnn` alignment-safe (issue #896)
+- Control sequences and category codes in regex replacements (issue #909)
 
 ### Changed
 - Speed up \group_align_safe_begin: (pull #906)
diff --git a/l3kernel/l3regex.dtx b/l3kernel/l3regex.dtx
index b6ea1f31e..aca9d3989 100644
--- a/l3kernel/l3regex.dtx
+++ b/l3kernel/l3regex.dtx
@@ -5656,17 +5656,21 @@
 %    \begin{macrocode}
 \cs_new_protected:Npn \@@_replacement_normal:n #1
   {
-    \tl_if_empty:NTF \l_@@_replacement_category_tl
-      { \@@_replacement_normal_aux:N #1 }
-      { % (
-        \token_if_eq_charcode:NNTF #1 )
-          {
-            \seq_pop:NN \l_@@_replacement_category_seq
-              \l_@@_replacement_category_tl
-          }
-          {
-            \use:c { @@_replacement_c_ \l_@@_replacement_category_tl :w }
-            ? #1
+    \int_compare:nNnTF { \l_@@_replacement_csnames_int } > 0
+      { \exp_args:No \@@_replacement_put:n { \token_to_str:N #1 } }
+      {
+        \tl_if_empty:NTF \l_@@_replacement_category_tl
+          { \@@_replacement_normal_aux:N #1 }
+          { % (
+            \token_if_eq_charcode:NNTF #1 )
+              {
+                \seq_pop:NN \l_@@_replacement_category_seq
+                  \l_@@_replacement_category_tl
+              }
+              {
+                \use:c { @@_replacement_c_ \l_@@_replacement_category_tl :w }
+                ? #1
+              }
           }
       }
   }
diff --git a/l3kernel/testfiles/m3regex001.lvt b/l3kernel/testfiles/m3regex001.lvt
index c98dab9ec..c49e37d98 100644
--- a/l3kernel/testfiles/m3regex001.lvt
+++ b/l3kernel/testfiles/m3regex001.lvt
@@ -237,5 +237,18 @@
     \test:nn { (?:) } { abc }
   }
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\TEST { catcodes~and~cs~in~replacement }
+  {
+    \cs_set_nopar:cpn { x ? y } { y ? z }
+    \cs_set_nopar:cpn { y ? z ^ } { z . a }
+    \char_set_catcode_active:n { `? }
+    \char_set_catcode_parameter:n { `^ }
+    \regex_replace_all:nnN { .* }
+      { \c{x\?y} \u{x\?y} \c{\u{x\?y}\^} \? }
+      \l_tmpa_tl
+    \tl_analysis_show:N \l_tmpa_tl
+  }
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \END
diff --git a/l3kernel/testfiles/m3regex001.tlg b/l3kernel/testfiles/m3regex001.tlg
index f1003919e..fb1ba2410 100644
--- a/l3kernel/testfiles/m3regex001.tlg
+++ b/l3kernel/testfiles/m3regex001.tlg
@@ -161,3 +161,16 @@ TRUE
 TRUE
 {} {} {} {} 
 ============================================================
+============================================================
+TEST 12: catcodes and cs in replacement
+============================================================
+The token list \l_tmpa_tl contains the tokens:
+>  \x?y (control sequence=macro:->y?z)
+>  y (the letter y)
+>  ? (the character ?)
+>  z (the letter z)
+>  \y?z^ (control sequence=macro:->z.a)
+>  ? (active character=undefined).
+<recently read> }
+l. ...  }
+============================================================