[latex3-commits] [git/LaTeX3-latex3-latex3] peek-regex: Remove further use of toks while matching in l3regex (a52542707)
Bruno Le Floch
blflatex at gmail.com
Mon Jul 20 02:19:19 CEST 2020
Repository : https://github.com/latex3/latex3
On branch : peek-regex
Link : https://github.com/latex3/latex3/commit/a525427072ce490936eaea9a374524980f8bdd27
>---------------------------------------------------------------
commit a525427072ce490936eaea9a374524980f8bdd27
Author: Bruno Le Floch <blflatex at gmail.com>
Date: Sat Jul 18 19:18:41 2020 +0200
Remove further use of toks while matching in l3regex
The eventual aim is to let users run code while the regex is being
matched. For this we need to remove all abuses of TeX registers.
The only one left is toks used for storing states of the nfa.
>---------------------------------------------------------------
a525427072ce490936eaea9a374524980f8bdd27
l3kernel/l3regex.dtx | 195 +++++++++++++++++++--------------------------------
1 file changed, 73 insertions(+), 122 deletions(-)
diff --git a/l3kernel/l3regex.dtx b/l3kernel/l3regex.dtx
index 38bff2520..cac170975 100644
--- a/l3kernel/l3regex.dtx
+++ b/l3kernel/l3regex.dtx
@@ -885,14 +885,6 @@
% of all submatches, and then the ends of all submatches. The
% \meta{threads} are ordered starting from the best to the least
% preferred.
-% \item \cs{g_@@_charcode_intarray} and \cs{g_@@_catcode_intarray} hold the
-% character codes and category codes of tokens at each
-% \meta{position} in the query.
-% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and
-% end-group character tokens which appear before that point in the
-% token list.
-% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}-
-% and \texttt{x}-expand to the \meta{position}-th token in the query.
% \item \cs{g_@@_submatch_prev_intarray}, \cs{g_@@_submatch_begin_intarray}
% and \cs{g_@@_submatch_end_intarray} hold, for each submatch (as would
% be extracted by \cs{regex_extract_all:nnN}), the place where the
@@ -903,6 +895,14 @@
% block corresponding to one match with all its submatches stored in
% consecutive entries.
% \end{itemize}
+% When actually building the result,
+% \begin{itemize}
+% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}-
+% and \texttt{x}-expand to the \meta{position}-th token in the query.
+% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and
+% end-group character tokens which appear before that point in the
+% token list.
+% \end{itemize}
%
% The code is structured as follows. Variables are introduced in the
% relevant section. First we present some generic helper functions. Then
@@ -1085,19 +1085,6 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\g_@@_charcode_intarray, \g_@@_catcode_intarray, \g_@@_balance_intarray}
-% The first thing we do when matching is to go once through the query
-% token list and store the information for each token into
-% \cs{g_@@_charcode_intarray}, \cs{g_@@_catcode_intarray} and \tn{toks}
-% registers. We also store the balance of begin-group/end-group
-% characters into \cs{g_@@_balance_intarray}.
-% \begin{macrocode}
-\intarray_new:Nn \g_@@_charcode_intarray { 65536 }
-\intarray_new:Nn \g_@@_catcode_intarray { 65536 }
-\intarray_new:Nn \g_@@_balance_intarray { 65536 }
-% \end{macrocode}
-% \end{variable}
-%
% \begin{variable}{\l_@@_balance_int}
% During this phase, \cs{l_@@_balance_int} counts the balance of
% begin-group and end-group character tokens which appear before a
@@ -1108,15 +1095,6 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_@@_cs_name_tl}
-% This variable is used in \cs{@@_item_cs:n} to store the csname of
-% the currently-tested token when the regex contains a sub-regex for
-% testing csnames.
-% \begin{macrocode}
-\tl_new:N \l_@@_cs_name_tl
-% \end{macrocode}
-% \end{variable}
-%
% \subsubsection{Testing characters}
%
% \begin{macro}{\c_@@_ascii_min_int, \c_@@_ascii_max_control_int, \c_@@_ascii_max_int}
@@ -1368,21 +1346,18 @@
% First test the catcode of the current token to be zero.
% Then perform the matching test, and break if the csname
% indeed matches.
-% We store the cs name before building states for the cs, as those
-% states may overlap with toks registers storing the user's input.
% \begin{macrocode}
\cs_new_protected:Npn \@@_item_cs:n #1
{
\int_compare:nNnT \l_@@_curr_catcode_int = 0
{
\group_begin:
- \tl_set:Nx \l_@@_cs_name_tl { \@@_curr_cs_to_str: }
\@@_single_match:
\@@_disable_submatches:
\@@_build_for_cs:n {#1}
\bool_set_eq:NN \l_@@_saved_success_bool
\g_@@_success_bool
- \exp_args:NV \@@_match_cs:n \l_@@_cs_name_tl
+ \exp_args:Nx \@@_match_cs:n { \@@_curr_cs_to_str: }
\if_meaning:w \c_true_bool \g_@@_success_bool
\group_insert_after:N \@@_break_true:w
\fi:
@@ -4491,9 +4466,8 @@
% }
% The tokens in the query are indexed from \texttt{min_pos} for the
% first to $\texttt{max_pos}-1$ for the last, and their information is
-% stored in several arrays and \tn{toks} registers with those numbers. We
-% don't start from $0$ because the \tn{toks} registers with low
-% numbers are used to hold the states of the \textsc{nfa}. We match
+% stored in several arrays and \tn{toks} registers with those numbers.
+% We match
% without backtracking, keeping all threads in lockstep at the
% \texttt{curr_pos} in the query. The starting point of the current
% match attempt is \texttt{start_pos}, and \texttt{success_pos},
@@ -4514,11 +4488,13 @@
% \l_@@_curr_catcode_int,
% \l_@@_curr_token_tl,
% \l_@@_last_char_int,
+% \l_@@_last_char_success_int,
% \l_@@_case_changed_char_int
% }
% The character and category codes of the token at the current
% position and a token list expanding to that token; the character
% code of the token at the previous position;
+% the character code of the token just before a successful match;
% and the character code of the result of changing the case of the
% current token (|A-Z|$\leftrightarrow$|a-z|). This last integer is
% only computed when necessary, and is otherwise \cs{c_max_int}. The
@@ -4529,6 +4505,7 @@
\int_new:N \l_@@_curr_catcode_int
\tl_new:N \l_@@_curr_token_tl
\int_new:N \l_@@_last_char_int
+\int_new:N \l_@@_last_char_success_int
\int_new:N \l_@@_case_changed_char_int
% \end{macrocode}
% \end{variable}
@@ -4676,8 +4653,7 @@
%
% \begin{macro}{\@@_match:n, \@@_match_cs:n}
% \begin{macro}{\@@_match_init:}
-% First store the query into \tn{toks} registers and arrays (see
-% \cs{@@_query_set:nnn}). Then initialize the variables that should
+% Initialize the variables that should
% be set once for each user function (even for multiple
% matches). Namely, the overall matching is not yet successful; none of
% the states should be marked as visited (\cs{g_@@_state_active_intarray}), and
@@ -4688,36 +4664,19 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_match:n #1
{
- \int_zero:N \l_@@_balance_int
- \int_set:Nn \l_@@_curr_pos_int { 1 + \l_@@_max_state_int }
- \@@_query_set:nnn { } { -1 } { -2 }
- \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int
- \tl_analysis_map_inline:nn {#1}
- { \@@_query_set:nnn {##1} {"##3} {##2} }
- \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
- \@@_query_set:nnn { } { -1 } { -2 }
+ \int_set:Nn \l_@@_min_pos_int { 2 }
+ \int_set:Nn \l_@@_max_pos_int { 2 + \tl_count_tokens:n {#1} }
\@@_match_init:
\@@_match_once_init:
\tl_analysis_map_inline:nn {#1}
{ \@@_match_one_token:nnN {##1} {##2} ##3 }
- \@@_match_one_token:nnN { } { -2 } 9
+ \@@_match_one_token:nnN { } { -2 } F
\prg_break_point:Nn \@@_maplike_break: { }
}
\cs_new_protected:Npn \@@_match_cs:n #1
{
- \int_zero:N \l_@@_balance_int
- \int_set:Nn \l_@@_curr_pos_int
- { 1 + \int_max:nn \l_@@_max_state_int \l_@@_max_pos_int }
- \@@_query_set:nnn { } { -1 } { -2 }
- \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int
- \str_map_inline:nn {#1}
- {
- \@@_query_set:nnn { \exp_not:n {##1} }
- { \tl_if_blank:nTF {##1} { 10 } { 12 } }
- { `##1 }
- }
- \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
- \@@_query_set:nnn { } { -1 } { -2 }
+ \int_set:Nn \l_@@_min_pos_int { 2 + \l_@@_max_pos_int }
+ \int_set:Nn \l_@@_max_pos_int { \l_@@_min_pos_int + \str_count:n {#1} }
\int_set_eq:NN \l_@@_min_thread_int \l_@@_max_thread_int
\@@_match_init:
\@@_match_once_init:
@@ -4727,7 +4686,7 @@
{ \@@_match_one_token:nnN {##1} {`##1} A }
{ \@@_match_one_token:nnN {##1} {`##1} C }
}
- \@@_match_one_token:nnN { } { -2 } 9
+ \@@_match_one_token:nnN { } { -2 } F
\prg_break_point:Nn \@@_maplike_break: { }
}
\cs_new_protected:Npn \@@_match_init:
@@ -4763,8 +4722,10 @@
% successful, and has no submatches yet; clear the array of active
% threads, and put the starting state $0$ in it. We are then almost
% ready to read our first token in the query, but we actually start
-% one position earlier than the start, and \texttt{get} that token, to
-% set \texttt{last_char} properly for word boundaries.
+% one position earlier than the start because
+% \cs{@@_match_one_token:nnN} increments \cs{l_@@_curr_pos_int} and
+% saves \cs{l_@@_curr_char_int} as the \texttt{last_char} so that word
+% boundaries can be correctly identified.
% \begin{macrocode}
\cs_new_protected:Npn \@@_match_once_init:
{
@@ -4785,7 +4746,7 @@
\@@_store_state:n { \l_@@_min_state_int }
\int_set:Nn \l_@@_curr_pos_int
{ \l_@@_start_pos_int - 1 }
- \@@_query_get:
+ \int_set_eq:NN \l_@@_curr_char_int \l_@@_last_char_success_int
\exp_args:NNf \@@_match_once_init_aux:
\tl_map_inline:nn { \exp_after:wN \l_@@_matched_analysis_tl \l_@@_curr_analysis_tl }
{ \@@_match_one_token:nnN ##1 }
@@ -4853,9 +4814,6 @@
\tl_set:Nn \l_@@_curr_token_tl {#1}
\int_set:Nn \l_@@_curr_char_int {#2}
\int_set:Nn \l_@@_curr_catcode_int { "#3 }
- \if_int_compare:w 9 = \l_@@_curr_catcode_int
- \int_set:Nn \l_@@_curr_catcode_int { -1 } % ^^A todo
- \fi:
\tl_put_right:No \l_@@_matched_analysis_tl { \l_@@_curr_analysis_tl }
\tl_set:Nn \l_@@_curr_analysis_tl { { {#1} {#2} #3 } }
\use:x
@@ -4888,53 +4846,6 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}{\@@_query_set:nnn}
-% The arguments are: tokens that \texttt{o} and \texttt{x} expand to
-% one token of the query, the catcode, and the character code. Store
-% those, and the current brace balance (used later to check for
-% overall brace balance) in a \tn{toks} register and some arrays,
-% then update the \texttt{balance}.
-% \begin{macrocode}
-\cs_new_protected:Npn \@@_query_set:nnn #1#2#3
- {
- \__kernel_intarray_gset:Nnn \g_@@_charcode_intarray
- { \l_@@_curr_pos_int } {#3}
- \__kernel_intarray_gset:Nnn \g_@@_catcode_intarray
- { \l_@@_curr_pos_int } {#2}
- \__kernel_intarray_gset:Nnn \g_@@_balance_intarray
- { \l_@@_curr_pos_int } { \l_@@_balance_int }
- \@@_toks_set:Nn \l_@@_curr_pos_int {#1}
- \int_incr:N \l_@@_curr_pos_int
- \if_case:w #2 \exp_stop_f:
- \or: \int_incr:N \l_@@_balance_int
- \or: \int_decr:N \l_@@_balance_int
- \fi:
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}{\@@_query_get:}
-% Extract the current character and category codes at the current
-% position from the appropriate arrays.
-% \begin{macrocode}
-\cs_new_protected:Npn \@@_query_get:
- {
- \int_set:Nn \l_@@_curr_char_int
- {
- \__kernel_intarray_item:Nn \g_@@_charcode_intarray
- { \l_@@_curr_pos_int }
- }
- \int_set:Nn \l_@@_curr_catcode_int
- {
- \__kernel_intarray_item:Nn \g_@@_catcode_intarray
- { \l_@@_curr_pos_int }
- }
- \tl_set:Nx \l_@@_curr_token_tl
- { \tex_the:D \tex_toks:D \l_@@_curr_pos_int }
- }
-% \end{macrocode}
-% \end{macro}
-%
% \subsubsection{Using states of the \textsc{nfa}}
%
% \begin{macro}{\@@_use_state:}
@@ -5141,6 +5052,7 @@
\bool_set_eq:NN \l_@@_empty_success_bool
\l_@@_fresh_thread_bool
\int_set_eq:NN \l_@@_success_pos_int \l_@@_curr_pos_int
+ \int_set_eq:NN \l_@@_last_char_success_int \l_@@_last_char_int
\tl_clear:N \l_@@_matched_analysis_tl
\tl_set_eq:NN \l_@@_success_submatches_tl
\l_@@_curr_submatches_tl
@@ -6023,6 +5935,14 @@
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{\g_@@_balance_intarray}
+% The first thing we do when matching is to store the balance of
+% begin-group/end-group characters into \cs{g_@@_balance_intarray}.
+% \begin{macrocode}
+\intarray_new:Nn \g_@@_balance_intarray { 65536 }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{macro}{\@@_return:}
% This function triggers either \cs{prg_return_false:} or
% \cs{prg_return_true:} as appropriate to whether a match was found or
@@ -6039,6 +5959,35 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\@@_query_set:n, \@@_query_set_aux:nN}
+% To easily extract subsets of the input once we found the positions
+% at which to cut, store the input tokens one by one into successive
+% \tn{toks} registers. Also store the brace balance (used to check
+% for overall brace balance) in an array.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_query_set:n #1
+ {
+ \int_zero:N \l_@@_balance_int
+ \int_set:Nn \l_@@_curr_pos_int { 1 }
+ \@@_query_set_aux:nN { } F
+ \tl_analysis_map_inline:nn {#1}
+ { \@@_query_set_aux:nN {##1} ##3 }
+ \@@_query_set_aux:nN { } F
+ }
+\cs_new_protected:Npn \@@_query_set_aux:nN #1#2
+ {
+ \@@_toks_set:Nn \l_@@_curr_pos_int {#1}
+ \__kernel_intarray_gset:Nnn \g_@@_balance_intarray
+ { \l_@@_curr_pos_int } { \l_@@_balance_int }
+ \int_incr:N \l_@@_curr_pos_int
+ \if_case:w "#2 \exp_stop_f:
+ \or: \int_incr:N \l_@@_balance_int
+ \or: \int_decr:N \l_@@_balance_int
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Matching}
%
% \begin{macro}{\@@_if_match:nn}
@@ -6094,7 +6043,7 @@
#1
\@@_match:n {#2}
\@@_extract:
- \@@_group_end_extract_seq:N #3
+ \@@_group_end_extract_seq:Nn #3 {#2}
}
\cs_new_protected:Npn \@@_extract_all:nnN #1#2#3
{
@@ -6102,7 +6051,7 @@
\@@_multi_match:n { \@@_extract: }
#1
\@@_match:n {#2}
- \@@_group_end_extract_seq:N #3
+ \@@_group_end_extract_seq:Nn #3 {#2}
}
% \end{macrocode}
% \end{macro}
@@ -6141,7 +6090,6 @@
}
#1
\@@_match:n {#2}
-%<assert>\assert_int:n { \l_@@_curr_pos_int = \l_@@_max_pos_int }
\__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray
{ \l_@@_submatch_int } { 0 }
\__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray
@@ -6156,12 +6104,12 @@
\int_decr:N \l_@@_submatch_int
\fi:
\fi:
- \@@_group_end_extract_seq:N #3
+ \@@_group_end_extract_seq:Nn #3 {#2}
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\@@_group_end_extract_seq:N}
+% \begin{macro}{\@@_group_end_extract_seq:Nn}
% The end-points of submatches are stored as entries of two arrays
% from \cs{l_@@_min_submatch_int} to
% \cs{l_@@_submatch_int} (exclusive). Extract the relevant ranges
@@ -6170,8 +6118,9 @@
% whenever we see too many begin-group or end-group tokens in a
% submatch.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_group_end_extract_seq:N #1
+\cs_new_protected:Npn \@@_group_end_extract_seq:Nn #1#2
{
+ \@@_query_set:n {#2}
\flag_clear:n { @@_begin }
\flag_clear:n { @@_end }
\seq_set_from_function:NnN \l_@@_internal_seq
@@ -6287,11 +6236,12 @@
\@@_single_match:
#1
\@@_replacement:n {#2}
- \exp_args:No \@@_match:n { #3 }
+ \exp_args:No \@@_match:n {#3}
\if_meaning:w \c_false_bool \g_@@_success_bool
\group_end:
\else:
\@@_extract:
+ \exp_args:No \@@_query_set:n {#3}
\int_set:Nn \l_@@_balance_int
{
\@@_replacement_balance_one_match:n
@@ -6333,6 +6283,7 @@
#1
\@@_replacement:n {#2}
\exp_args:No \@@_match:n {#3}
+ \exp_args:No \@@_query_set:n {#3}
\int_set:Nn \l_@@_balance_int
{
0
More information about the latex3-commits
mailing list.