[latex3-commits] [git/LaTeX3-latex3-latex3] master: Allow for \active chars when setting up UTF-8 data (fixes #627) (09d7bc400)

[Joseph Wright]

Repository : https://github.com/latex3/latex3
On branch  : master
Link       : https://github.com/latex3/latex3/commit/09d7bc400953a3d6ae5a7f7f5bc761690c3c1c68

>---------------------------------------------------------------

commit 09d7bc400953a3d6ae5a7f7f5bc761690c3c1c68
Author: Joseph Wright <joseph.wright at morningstar2.co.uk>
Date:   Tue Sep 10 09:19:37 2019 +0100

    Allow for \active chars when setting up UTF-8 data (fixes #627)


>---------------------------------------------------------------

09d7bc400953a3d6ae5a7f7f5bc761690c3c1c68
 l3kernel/CHANGELOG.md     |  4 ++++
 l3kernel/l3candidates.dtx |  5 ++++-
 l3kernel/l3unicode.dtx    | 44 +++++++++++++++++++++++++-------------------
 3 files changed, 33 insertions(+), 20 deletions(-)

diff --git a/l3kernel/CHANGELOG.md b/l3kernel/CHANGELOG.md
index 505bb1f9c..61cbd7f84 100644
--- a/l3kernel/CHANGELOG.md
+++ b/l3kernel/CHANGELOG.md
@@ -7,6 +7,10 @@ this project uses date-based 'snapshot' version identifiers.
 
 ## [Unreleased]
 
+### Fixed
+
+- Loading Unicode data when some chars may be active (see #627)
+
 ## [2019-09-08]
 
 ### Fixed
diff --git a/l3kernel/l3candidates.dtx b/l3kernel/l3candidates.dtx
index 88c10ab72..a45e5cdd6 100644
--- a/l3kernel/l3candidates.dtx
+++ b/l3kernel/l3candidates.dtx
@@ -3673,7 +3673,10 @@
   {
     \group_begin:
       \cs_set:Npn \@@_tmp:n #1
-        { \char_generate:nn {#1} { \char_value_catcode:n {#1} } }
+        {
+          \exp_after:wN \exp_after:wN \exp_after:wN \exp_not:N
+            \char_generate:nn {#1} { \char_value_catcode:n {#1} }
+        }
       \tl_const:Nx \c_@@_std_sigma_tl    { \@@_tmp:n { "03C3 } }
       \tl_const:Nx \c_@@_final_sigma_tl  { \@@_tmp:n { "03C2 } }
       \tl_const:Nx \c_@@_accents_lt_tl
diff --git a/l3kernel/l3unicode.dtx b/l3kernel/l3unicode.dtx
index 74a115670..6f794156a 100644
--- a/l3kernel/l3unicode.dtx
+++ b/l3kernel/l3unicode.dtx
@@ -84,29 +84,35 @@
   {
     \group_begin:
 %    \end{macrocode}
-%   Set up a private copy of the char-generation primitive.
+%   Access the primitive but suppress further expansion: active chars are
+%   otherwise an issue.
 %    \begin{macrocode}
-      \cs_set_eq:NN \@@_generate:w \tex_Uchar:D
+      \cs_set:Npn \@@_generate_char:n #1
+        { \tex_detokenize:D \tex_expandafter:D { \tex_Uchar:D " #1 } }
 %    \end{macrocode}
-%   A fast local implementation for generating characters.
+%   A fast local implementation for generating characters; the chars may
+%   be active, so we prevent further expansion.
 %    \begin{macrocode}
       \cs_set:Npx \@@_generate:n #1
         {
-          \sys_if_engine_luatex:TF
+          \exp_not:N \tex_unexpanded:D \exp_not:N \exp_after:wN
             {
-              \exp_not:N \lua_now:e
+              \sys_if_engine_luatex:TF
                 {
-                  l3kernel.charcat
-                    (
-                      \exp_not:N \tex_number:D #1 ,
-                      \exp_not:N \tex_the:D \tex_catcode:D #1
-                    )
+                  \exp_not:N \tex_directlua:D
+                    {
+                      l3kernel.charcat
+                        (
+                          \exp_not:N \tex_number:D #1 ,
+                          \exp_not:N \tex_the:D \tex_catcode:D #1
+                        )
+                    }
+                }
+                {
+                  \exp_not:N \tex_Ucharcat:D
+                    #1 ~
+                    \tex_catcode:D #1 ~
                 }
-            }
-            {
-              \exp_not:N \tex_Ucharcat:D
-                #1 ~
-                \tex_catcode:D #1 ~
             }
         } 
 %    \end{macrocode}
@@ -127,7 +133,7 @@
             \cs_set_nopar:Npn \l_@@_tmpb_tl {#5}
             \reverse_if:N \if_meaning:w \l_@@_tmpa_tl \l_@@_tmpb_tl
               \tl_const:cx
-                { c_@@_mixed_case_ \@@_generate:w "#1 _tl }
+                { c_@@_mixed_case_ \@@_generate_char:n {#1} _tl }
                 { \@@_generate:n { "#7 } }
             \fi:
           \fi:
@@ -155,7 +161,7 @@
             \reverse_if:N \if_int_compare:w
               \char_value_lccode:n {"#1} = "#3 ~
               \tl_const:cx
-                { c_@@_fold_case_ \@@_generate:w "#1 _tl }
+                { c_@@_fold_case_ \@@_generate_char:n {#1} _tl }
                 { \@@_generate:n { "#3 } }
             \fi:
           \else:
@@ -166,7 +172,7 @@
         }
       \cs_set_protected:Npn \@@_data_auxii:w #1 ~ #2 ~ #3 ~ #4 \q_stop
         {
-          \tl_const:cx { c_@@_fold_case_ \@@_generate:w "#1 _tl }
+          \tl_const:cx { c_@@_fold_case_ \@@_generate_char:n {#1} _tl }
             {
               \@@_generate:n { "#2 }
               \@@_generate:n { "#3 }
@@ -200,7 +206,7 @@
         {
           \tl_if_empty:nF {#4}
             {
-              \tl_const:cx { c_@@_ #2 _case_ \@@_generate:w "#1 _tl }
+              \tl_const:cx { c_@@_ #2 _case_ \@@_generate_char:n {#1} _tl }
                 {
                   \@@_generate:n { "#3 }
                   \@@_generate:n { "#4 }