[latex3-commits] [git/LaTeX3-latex3-latex3] main: Fix catcode for NFD creation of non-ASCII chars (46686d702)

[Joseph Wright]

Repository : https://github.com/latex3/latex3
On branch  : main
Link       : https://github.com/latex3/latex3/commit/46686d702b60e13ee197ff02b19e64e9e9dd0f01

>---------------------------------------------------------------

commit 46686d702b60e13ee197ff02b19e64e9e9dd0f01
Author: Joseph Wright <joseph.wright at morningstar2.co.uk>
Date:   Tue Feb 14 22:40:50 2023 +0000

    Fix catcode for NFD creation of non-ASCII chars


>---------------------------------------------------------------

46686d702b60e13ee197ff02b19e64e9e9dd0f01
 l3kernel/CHANGELOG.md                      |  3 +++
 l3kernel/l3unicode.dtx                     | 15 ++++++++++++++-
 l3kernel/testfiles/m3unicode001.luatex.tlg |  5 +++++
 l3kernel/testfiles/m3unicode001.lvt        |  7 ++++++-
 l3kernel/testfiles/m3unicode001.tlg        |  5 +++++
 l3kernel/testfiles/m3unicode001.xetex.tlg  |  5 +++++
 6 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/l3kernel/CHANGELOG.md b/l3kernel/CHANGELOG.md
index 651c19d35..0b57d9d1f 100644
--- a/l3kernel/CHANGELOG.md
+++ b/l3kernel/CHANGELOG.md
@@ -7,6 +7,9 @@ this project uses date-based 'snapshot' version identifiers.
 
 ## [Unreleased]
 
+### Changed
+- More defensive approach in `\codepoint_to_nfd:n`
+
 ### Fixed
 - CJK character handling for (u)pTeX (issue
   [\#1171](https://github.com/latex3/latex3/issues/1171))
diff --git a/l3kernel/l3unicode.dtx b/l3kernel/l3unicode.dtx
index 36d572237..911aab327 100644
--- a/l3kernel/l3unicode.dtx
+++ b/l3kernel/l3unicode.dtx
@@ -347,7 +347,7 @@
 % \end{macro}
 % \end{macro}
 %
-% \begin{macro}[EXP]{\codepoint_to_nfd:n}
+% \begin{macro}[EXP]{\codepoint_to_nfd:n, \@@_to_nfd:n}
 % \begin{macro}[EXP]{\@@_to_nfd:nn}
 % \begin{macro}[EXP]{\@@_to_nfd:nnn}
 % \begin{macro}[EXP]{\@@_to_nfd:nnnn}
@@ -356,7 +356,20 @@
 %   we use the same path for all engines.
 %    \begin{macrocode}
 \cs_new:Npn \codepoint_to_nfd:n #1
+  { \exp_args:Ne \@@_to_nfd:n { \int_eval:n {#1} } }
+\cs_new:Npn \@@_to_nfd:n #1
   { \@@_to_nfd:nn {#1} { \char_value_catcode:n {#1} } }
+\bool_lazy_or:nnF
+  { \sys_if_engine_luatex_p: }
+  { \sys_if_engine_xetex_p: }
+  {
+    \cs_gset:Npn \@@_to_nfd:n #1
+      {
+        \int_compare:nNnTF {#1} > { "80 }
+          { \@@_to_nfd:nn {#1} { 12 } }
+          { \@@_to_nfd:nn {#1} { \char_value_catcode:n {#1} } }
+      }
+  }
 \cs_new:Npn \@@_to_nfd:nn #1#2
   {
     \exp_args:Ne \@@_to_nfd:nnn
diff --git a/l3kernel/testfiles/m3unicode001.luatex.tlg b/l3kernel/testfiles/m3unicode001.luatex.tlg
index 539023027..355dd0eb0 100644
--- a/l3kernel/testfiles/m3unicode001.luatex.tlg
+++ b/l3kernel/testfiles/m3unicode001.luatex.tlg
@@ -28,3 +28,8 @@ A
 Î
 Ή
 ============================================================
+============================================================
+TEST 4: Non-ASCII to ASCII mapping
+============================================================
+;
+============================================================
diff --git a/l3kernel/testfiles/m3unicode001.lvt b/l3kernel/testfiles/m3unicode001.lvt
index 7a5ff1a6b..b90aca5c1 100644
--- a/l3kernel/testfiles/m3unicode001.lvt
+++ b/l3kernel/testfiles/m3unicode001.lvt
@@ -1,5 +1,5 @@
 %
-% Copyright (C) 2022 The LaTeX Project
+% Copyright (C) 2022,2023 The LaTeX Project
 %
 \documentclass{minimal}
 \input{regression-test}
@@ -50,4 +50,9 @@
     \codepoint_to_nfd:n { "0389 }
   }
 
+\TESTEXP { Non-ASCII~to~ASCII~mapping }
+  {
+    \codepoint_to_nfd:n { "037E } %  ; -> ; (Greek question mark to semicolon)
+  }
+
 \END
\ No newline at end of file
diff --git a/l3kernel/testfiles/m3unicode001.tlg b/l3kernel/testfiles/m3unicode001.tlg
index cb4249d1f..78c5a1d72 100644
--- a/l3kernel/testfiles/m3unicode001.tlg
+++ b/l3kernel/testfiles/m3unicode001.tlg
@@ -28,3 +28,8 @@ A
 I^^cc^^82
 ^^ce^^97^^cc^^81
 ============================================================
+============================================================
+TEST 4: Non-ASCII to ASCII mapping
+============================================================
+;
+============================================================
diff --git a/l3kernel/testfiles/m3unicode001.xetex.tlg b/l3kernel/testfiles/m3unicode001.xetex.tlg
index 539023027..355dd0eb0 100644
--- a/l3kernel/testfiles/m3unicode001.xetex.tlg
+++ b/l3kernel/testfiles/m3unicode001.xetex.tlg
@@ -28,3 +28,8 @@ A
 Î
 Ή
 ============================================================
+============================================================
+TEST 4: Non-ASCII to ASCII mapping
+============================================================
+;
+============================================================