texlive[59990] Master/texmf-dist: babel (19jul21)

commits+karl at tug.org commits+karl at tug.org
Mon Jul 19 22:10:36 CEST 2021


Revision: 59990
          http://tug.org/svn/texlive?view=revision&revision=59990
Author:   karl
Date:     2021-07-19 22:10:36 +0200 (Mon, 19 Jul 2021)
Log Message:
-----------
babel (19jul21)

Modified Paths:
--------------
    trunk/Master/texmf-dist/doc/latex/babel/README.md
    trunk/Master/texmf-dist/doc/latex/babel/babel.pdf
    trunk/Master/texmf-dist/source/latex/babel/babel.dtx
    trunk/Master/texmf-dist/source/latex/babel/babel.ins
    trunk/Master/texmf-dist/source/latex/babel/bbcompat.dtx
    trunk/Master/texmf-dist/source/latex/babel/locale.zip
    trunk/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua
    trunk/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua
    trunk/Master/texmf-dist/tex/generic/babel/babel.def
    trunk/Master/texmf-dist/tex/generic/babel/babel.sty
    trunk/Master/texmf-dist/tex/generic/babel/hyphen.cfg
    trunk/Master/texmf-dist/tex/generic/babel/luababel.def
    trunk/Master/texmf-dist/tex/generic/babel/nil.ldf
    trunk/Master/texmf-dist/tex/generic/babel/xebabel.def

Added Paths:
-----------
    trunk/Master/texmf-dist/tex/generic/babel/babel-transforms.lua

Modified: trunk/Master/texmf-dist/doc/latex/babel/README.md
===================================================================
--- trunk/Master/texmf-dist/doc/latex/babel/README.md	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/doc/latex/babel/README.md	2021-07-19 20:10:36 UTC (rev 59990)
@@ -1,4 +1,4 @@
-## Babel 3.61
+## Babel 3.62
 
 This package manages culturally-determined typographical (and other)
 rules, and hyphenation patterns for a wide range of languages. Many
@@ -8,9 +8,9 @@
 
 The latest stable version is available on <https://ctan.org/pkg/babel>.
 
-Changes in version 3.61 are described in:
+Changes in version 3.62 are described in:
 
-https://latex3.github.io/babel/news/whats-new-in-babel-3.61.html
+https://latex3.github.io/babel/news/whats-new-in-babel-3.62.html
 
 Apart from the manual, you can find information on some aspects of babel at:
 
@@ -46,16 +46,13 @@
 
 ### Summary of Latest changes
 ```
-3.61   2021-06-28
-       * Improved justification=kashida/elongated: hboxes (lua).
-       * Transform danda.nobreak for several Indic scripts (lua):
-         Assamese, Bengali, Gujarati, Hindi, Kannada, Malayalam,
-         Marathi, Oriya, Tamil, Telugu.
-       * Improved \babelprovide when used to reconfigure a language.
+3.62   2021-07-19
+       * No more errors with unknown languages in aux file.
+       * Preliminary support for \AddToHook.
+       * Tentative extension for provide=.
        * Fixes:
-         - Partial fix for #114 (bad breaks and spacing with
-           \selectlanguage).
-         - \shorthandoff*{^} caused error for \section command (#129).
+         - Locale dependent quotes with CJK (#131).
+         - Babel and hyperref prevent changes to the \LaTeX logo (#138).
 ```
 
 ### Previous changes

Modified: trunk/Master/texmf-dist/doc/latex/babel/babel.pdf
===================================================================
(Binary files differ)

Modified: trunk/Master/texmf-dist/source/latex/babel/babel.dtx
===================================================================
--- trunk/Master/texmf-dist/source/latex/babel/babel.dtx	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/source/latex/babel/babel.dtx	2021-07-19 20:10:36 UTC (rev 59990)
@@ -32,7 +32,7 @@
 %
 % \iffalse
 %<*filedriver>
-\ProvidesFile{babel.dtx}[2021/06/28 v3.61 The Babel package]
+\ProvidesFile{babel.dtx}[2021/07/19 v3.62 The Babel package]
 \documentclass{ltxdoc}
 \GetFileInfo{babel.dtx}
 \usepackage{fontspec}
@@ -697,6 +697,14 @@
   code with an additional grouping level.
 \end{warning}
 
+\begin{warning}
+  |\selectlanguage| should not be used inside some boxed environments
+  (like floats or |minipage|) to switch the language if you need the
+  information written to the |aux| be correctly synchronized. This
+  rarely happens, but if it were the case, you must use |otherlanguage|
+  instead.
+\end{warning}
+
 \Describe{\foreignlanguage}{\oarg{option-list}\marg{language}\marg{text}}
 
 The command |\foreignlanguage| takes two arguments; the second argument
@@ -936,6 +944,12 @@
 your own, you may want to switch them off with the package option
 |shorthands=off|, as described below.
 
+\begin{warning}
+  It is worth emphasizing these macros are meant for temporary changes.
+  Whenever possible and if there are not conflicts with other packages,
+  shorthands must be always enabled (or disabled).
+\end{warning}
+
 \Describe{\useshorthands}{%
 \colorbox{thegrey}{\ttfamily\hskip-.2em*\hskip-.2em}%
 \marg{char}}
@@ -1264,7 +1278,6 @@
 classes and packages, and as a last resort in case there are, for some
 reason, incompatible languages. It can be used if you just want to
 select the hyphenation patterns of a single language, too.
-% TODO: example
 
 \Describe\AfterBabelLanguage{\marg{option-name}\marg{code}}
 
@@ -1292,6 +1305,13 @@
 \end{verbatim}
 \end{example}
 
+\begin{note}
+  With a recent version of \LaTeX, an alternative method to execute
+  some code just after an |ldf| file is loaded is with |\AddToHook| and
+  the hook |file/after/<language>.ldf|. \Babel{} does not predeclare
+  it, and you have to do it yourself with |\NewHook| or |\ProvideHook|.
+\end{note}
+
 \begin{warning}
   Currently this option is not compatible with languages loaded on the
   fly.
@@ -3120,12 +3140,12 @@
 \Describe{\babelposthyphenation}{\marg{hyphenrules-name}%
           \marg{lua-pattern}\marg{replacement}}
 
-\New{3.37-3.39} \textit{With \luatex{}} it is now possible to define
+\New{3.37-3.39} \textit{With \luatex{}} it is possible to define
 non-standard hyphenation rules, like |f-f| $\to$ |ff-f|, repeated
 hyphens, ranked ruled (or more precisely, ‘penalized’ hyphenation
-points), and so on. Only a few rules are currently provided (see
-below), but they can be defined as shown in the following example,
-where |{1}| is the first captured char (between |()| in the pattern):
+points), and so on. A few rules are currently provided (see above), but
+they can be defined as shown in the following example, where |{1}| is
+the first captured char (between |()| in the pattern):
 \begin{verbatim}
 \babelposthyphenation{german}{([fmtrp]) | {1}}
 {
@@ -3690,15 +3710,24 @@
 events. Some hooks are predefined when \luatex{} and \xetex{} are
 used.
 
+\New{3.62} This is not the only way to inject code at those points. The
+events listed below can be used as a hook name in |\AddToHook| in the
+form |babel/|\marg{name}, but there is a limitation, because the
+parameters passed with the \babel{} mechanism are not allowed. The
+|\AddToHook| mechanism does \textit{not} replace the current one in
+`babel`. Its main advantage is you can reconfigure `babel` even before
+loading it. See the example below.
+
 \Describe{\AddBabelHook}{\oarg{lang}\marg{name}\marg{event}\marg{code}}
 
-The same name can be applied to several events. Hooks may be enabled
-and disabled for all defined events with |\EnableBabelHook|\marg{name},
-|\DisableBabelHook|\marg{name}. Names containing the string |babel| are
-reserved (they are used, for example, by |\useshortands*| to add a hook
-for the event |afterextras|). \New{3.33} They may be also applied to a
-specific language with the optional argument; language-specific
-settings are executed after global ones.
+The same name can be applied to several events. Hooks with a certain
+\marg{name} may be enabled and disabled for all defined events with
+|\EnableBabelHook|\marg{name}, |\DisableBabelHook|\marg{name}. Names
+containing the string |babel| are reserved (they are used, for example,
+by |\useshortands*| to add a hook for the event |afterextras|).
+\New{3.33} They may be also applied to a specific language with the
+optional argument; language-specific settings are executed after global
+ones.
 
 Current events are the following; in some of them you can use one to
 three \TeX{} parameters (|#1|, |#2|, |#3|), with the meaning given:
@@ -3771,6 +3800,27 @@
   file. Used by \file{luababel.def}.
 \end{description}
 
+\begin{example}
+The generic unlocalized \LaTeX{} hooks are predefined, so that you can
+write:
+\begin{verbatim}
+\AddToHook{babel/afterextras}{\frenchspacing} 
+\end{verbatim}
+which is executed always after the extras for the language being
+selected (and just before the non-localized hooks defined with
+|\AddBabelHook|).
+
+In addition, locale-specific hooks in the form
+|babel/|\marg{hook-name}|/|\marg{language-name} are \textit{recognized}
+(executed just before the localized \babel{} hooks), but they are \textit{not
+predefined}. You have to do it yourself. For example, to set
+|\frenchspacing| only in |bengali|:
+\begin{verbatim}
+\ProvideHook{babel/afterextras/bengali}
+\AddToHook{babel/afterextras/bengali}{\frenchspacing}
+\end{verbatim}
+\end{example}
+
 \Describe{\BabelContentsFiles}{}
 \New{3.9a} This macro contains a list of ``toc'' types
 requiring a command to switch the language. Its default value is
@@ -3951,7 +4001,8 @@
 \item \verb|\bibitem| is out of sync with \verb|\selectlanguage| in
   the \file{.aux} file. The reason is \verb|\bibitem| uses
   \verb|\immediate| (and others, in fact), while
-  \verb|\selectlanguage| doesn't. There is no known workaround.
+  \verb|\selectlanguage| doesn't. There is a similar issue with floats,
+  too. There is no known workaround.
 \item Babel does not take into account |\normalsfcodes| and
   (non-)French spacing is not always properly (un)set by
   languages. However, problems are unlikely to happen and therefore
@@ -4973,8 +5024,8 @@
 % \section{Tools}
 %
 %    \begin{macrocode}
-%<<version=3.61>>
-%<<date=2021/06/28>>
+%<<version=3.62>>
+%<<date=2021/07/19>>
 %    \end{macrocode}
 %
 % \textbf{Do not use the following macros in \texttt{ldf} files. They
@@ -5174,6 +5225,8 @@
 %
 % \begin{macro}{\bbl at replace}
 %
+% Returns implicitly |\toks@| with the modified string. 
+%
 %    \begin{macrocode}
 \def\bbl at replace#1#2#3{% in #1 -> repl #2 by #3
   \toks@{}%
@@ -5293,6 +5346,16 @@
   \fi}
 %    \end{macrocode}
 %
+% An alternative to |\IfFormatAtLeastTF| for old versions. Temporary.
+%
+%    \begin{macrocode}  
+\ifx\IfFormatAtLeastTF\@undefined
+  \def\bbl at ifformatlater{\@ifl at t@r\fmtversion}
+\else
+  \let\bbl at ifformatlater\IfFormatAtLeastTF
+\fi
+%    \end{macrocode}
+%
 % The following adds some code to |\extras...| both before and after,
 % while avoiding doing it twice. It's somewhat convoluted, to deal with
 % |#|'s.
@@ -5348,8 +5411,8 @@
 %
 % \begin{macro}{\last at language}
 %
-% Another counter is used to store the last language defined.  For
-% pre-3.0 formats an extra counter has to be allocated.
+% Another counter is used to keep track of the allocated languages.
+% \TeX{} and \LaTeX{} reserves for this purpose the count 19.
 %
 % \begin{macro}{\addlanguage}
 %
@@ -5358,8 +5421,7 @@
 %
 %    \begin{macrocode}
 %<<*Define core switching macros>>
-%<<*Define core switching macros>>
-\countdef\last at language=19  % TODO. why? remove?
+\countdef\last at language=19
 \def\addlanguage{\csname newlanguage\endcsname}
 %<</Define core switching macros>>
 %    \end{macrocode}
@@ -5369,7 +5431,7 @@
 %
 %    Now we make sure all required files are loaded.  When the command
 %    |\AtBeginDocument| doesn't exist we assume that we are dealing
-%    with a plain-based format or \LaTeX2.09. In that case the file
+%    with a plain-based format. In that case the file
 %    \file{plain.def} is needed (which also defines
 %    |\AtBeginDocument|, and therefore it is not loaded twice). We
 %    need the first part when the format is created, and |\orig at dump|
@@ -5405,6 +5467,7 @@
    \ifx\directlua\@undefined\else
      \directlua{ Babel = Babel or {}
        Babel.debug = true }%
+     \input{babel-debug.tex}%
    \fi}
   {\providecommand\bbl at trace[1]{}%
    \let\bbl at debug\@gobble
@@ -5665,9 +5728,6 @@
 %    \begin{macrocode}
 \ifx\bbl at opt@provide\@nnil\else % Tests. Ignore.
   \chardef\bbl at iniflag\@ne
-  \bbl at replace\bbl at opt@provide{;}{,}
-  \bbl at add\bbl at opt@provide{,import}
-  \show\bbl at opt@provide
 \fi
 % 
 %    \end{macrocode}
@@ -6295,10 +6355,10 @@
 %
 % \begin{macro}{\substitutefontfamily}
 %
+% This command is deprecated. Use the tools provides by \LaTeX.
 % The command |\substitutefontfamily| creates an
 % \file{.fd} file on the fly. The first argument is an encoding
 % mnemonic, the second and third arguments are font family names.
-% This command is deprecated. Use the tools provides by \LaTeX.
 %
 %    \begin{macrocode}
 \def\substitutefontfamily#1#2#3{%
@@ -6328,52 +6388,48 @@
 %
 %  Because documents may use non-ASCII font encodings, we make sure
 %  that the logos of \TeX\ and \LaTeX\ always come out in the right
-%  encoding. There is a list of non-ASCII encodings. Unfortunately,
-%  \textsf{fontenc} deletes its package options, so we must guess which
-%  encodings has been loaded by traversing |\@filelist| to search for
-%  \m{enc}|enc.def|. If a non-ASCII has been loaded, we define versions
-%  of |\TeX| and |\LaTeX| for them using |\ensureascii|. The default
-%  ASCII encoding is set, too (in reverse order): the ``main'' encoding
-%  (when the document begins), the last loaded, or |OT1|.
+%  encoding. There is a list of non-ASCII encodings. Requested
+%  encodings are currently stored in |\@fontenc at load@list|. If a
+%  non-ASCII has been loaded, we define versions of |\TeX| and |\LaTeX|
+%  for them using |\ensureascii|. The default ASCII encoding is set,
+%  too (in reverse order): the ``main'' encoding (when the document
+%  begins), the last loaded, or |OT1|.
 %
 %  \begin{macro}{\ensureascii}
 %
 %    \begin{macrocode}
 \bbl at trace{Encoding and fonts}
-\newcommand\BabelNonASCII{LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU,PU,PD1}
+\newcommand\BabelNonASCII{LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU}
 \newcommand\BabelNonText{TS1,T3,TS3}
 \let\org at TeX\TeX
 \let\org at LaTeX\LaTeX
 \let\ensureascii\@firstofone
 \AtBeginDocument{%
-  \in at false
-  \bbl at foreach\BabelNonASCII{% is there a text non-ascii enc?
-    \ifin@\else
-      \lowercase{\bbl at xin@{,#1enc.def,}{,\@filelist,}}%
+  \def\@elt#1{,#1,}%
+  \edef\bbl at tempa{\expandafter\@gobbletwo\@fontenc at load@list}%
+  \let\@elt\relax
+  \let\bbl at tempb\@empty
+  \def\bbl at tempc{OT1}%
+  \bbl at foreach\BabelNonASCII{% LGR loaded in a non-standard way
+    \bbl at ifunset{T@#1}{}{\def\bbl at tempb{#1}}}%
+  \bbl at foreach\bbl at tempa{%
+    \bbl at xin@{#1}{\BabelNonASCII}%
+    \ifin@
+      \def\bbl at tempb{#1}% Store last non-ascii
+    \else\bbl at xin@{#1}{\BabelNonText}% Pass
+      \ifin@\else
+        \def\bbl at tempc{#1}% Store last ascii
+      \fi
     \fi}%
-  \ifin@ % if a text non-ascii has been loaded
-    \def\ensureascii#1{{\fontencoding{OT1}\selectfont#1}}%
-    \DeclareTextCommandDefault{\TeX}{\org at TeX}%
-    \DeclareTextCommandDefault{\LaTeX}{\org at LaTeX}%
-    \def\bbl at tempb#1\@@{\uppercase{\bbl at tempc#1}ENC.DEF\@empty\@@}%
-    \def\bbl at tempc#1ENC.DEF#2\@@{%
-      \ifx\@empty#2\else
-        \bbl at ifunset{T@#1}%
-          {}%
-          {\bbl at xin@{,#1,}{,\BabelNonASCII,\BabelNonText,}%
-           \ifin@
-             \DeclareTextCommand{\TeX}{#1}{\ensureascii{\org at TeX}}%
-             \DeclareTextCommand{\LaTeX}{#1}{\ensureascii{\org at LaTeX}}%
-           \else
-             \def\ensureascii##1{{\fontencoding{#1}\selectfont##1}}%
-           \fi}%
-      \fi}%
-    \bbl at foreach\@filelist{\bbl at tempb#1\@@}%  TODO - \@@ de mas??
+  \ifx\bbl at tempb\@empty\else
     \bbl at xin@{,\cf at encoding,}{,\BabelNonASCII,\BabelNonText,}%
     \ifin@\else
-      \edef\ensureascii#1{{%
-        \noexpand\fontencoding{\cf at encoding}\noexpand\selectfont#1}}%
+      \edef\bbl at tempc{\cf at encoding}% The default if ascii wins
     \fi
+    \edef\ensureascii#1{%
+      {\noexpand\fontencoding{\bbl at tempc}\noexpand\selectfont#1}}%
+    \DeclareTextCommandDefault{\TeX}{\ensureascii{\org at TeX}}%
+    \DeclareTextCommandDefault{\LaTeX}{\ensureascii{\org at LaTeX}}%
   \fi}
 %    \end{macrocode}
 %
@@ -6415,16 +6471,12 @@
      \ifx\cf at encoding\bbl at t@one
        \xdef\latinencoding{\bbl at t@one}%
      \else
-       \ifx\@fontenc at load@list\@undefined
-         \@ifl at aded{def}{t1enc}{\xdef\latinencoding{\bbl at t@one}}{}%
-       \else
-         \def\@elt#1{,#1,}%
-         \edef\bbl at tempa{\expandafter\@gobbletwo\@fontenc at load@list}%
-         \let\@elt\relax
-         \bbl at xin@{,T1,}\bbl at tempa
-         \ifin@
-           \xdef\latinencoding{\bbl at t@one}%
-         \fi
+       \def\@elt#1{,#1,}%
+       \edef\bbl at tempa{\expandafter\@gobbletwo\@fontenc at load@list}%
+       \let\@elt\relax
+       \bbl at xin@{,T1,}\bbl at tempa
+       \ifin@
+         \xdef\latinencoding{\bbl at t@one}%
        \fi
      \fi}}
 %    \end{macrocode}
@@ -6457,6 +6509,19 @@
 \fi
 %    \end{macrocode}
 %
+% For several functions, we need to execute some code with
+% |\selectfont|. With \LaTeX\ 2021-06-01, there is a hook for this
+% purpose, but in older versions the \LaTeX{} command is patched (the
+% latter solution will be eventually removed).
+%
+%    \begin{macrocode}
+\bbl at ifformatlater{2021-06-01}%
+  {\def\bbl at patchfont#1{\AddToHook{selectfont}{#1}}}
+  {\def\bbl at patchfont#1{%
+     \expandafter\bbl at add\csname selectfont \endcsname{#1}%
+     \expandafter\bbl at toglobal\csname selectfont \endcsname}}
+%    \end{macrocode}
+%
 % \end{macro}
 %
 % \subsection{Basic bidi support}
@@ -6494,79 +6559,11 @@
 %   shows, vertical typesetting is possible, too.
 % \end{itemize}
 %
-% As a frist step, add a handler for bidi and digits (and potentially
-% other processes) just before \textsf{luaoftload} is applied, which is
-% loaded by default by \LaTeX. Just in case, consider the possibility
-% it has not been loaded.
-%
 %    \begin{macrocode}
-\ifodd\bbl at engine
-  \def\bbl at activate@preotf{%
-    \let\bbl at activate@preotf\relax  % only once
-    \directlua{
-      Babel = Babel or {}
-      %
-      function Babel.pre_otfload_v(head)
-        if Babel.numbers and Babel.digits_mapped then
-          head = Babel.numbers(head)
-        end
-        if Babel.bidi_enabled then
-          head = Babel.bidi(head, false, dir)
-        end
-        return head
-      end
-      %
-      function Babel.pre_otfload_h(head, gc, sz, pt, dir)
-        if Babel.numbers and Babel.digits_mapped then
-          head = Babel.numbers(head)
-        end
-        if Babel.bidi_enabled then
-          head = Babel.bidi(head, false, dir)
-        end
-        return head
-      end
-      %
-      luatexbase.add_to_callback('pre_linebreak_filter',
-        Babel.pre_otfload_v,
-        'Babel.pre_otfload_v',
-        luatexbase.priority_in_callback('pre_linebreak_filter',
-          'luaotfload.node_processor') or nil)
-      %
-      luatexbase.add_to_callback('hpack_filter',
-        Babel.pre_otfload_h,
-        'Babel.pre_otfload_h',
-        luatexbase.priority_in_callback('hpack_filter',
-          'luaotfload.node_processor') or nil)
-    }}
-\fi
-%    \end{macrocode}
-%
-% The basic setup. In luatex, the output is modified at a very low
-% level to set the |\bodydir| to the |\pagedir|.
-%
-%    \begin{macrocode}
 \bbl at trace{Loading basic (internal) bidi support}
 \ifodd\bbl at engine
+\else % TODO. Move to txtbabel
   \ifnum\bbl at bidimode>100 \ifnum\bbl at bidimode<200
-    \let\bbl at beforeforeign\leavevmode
-    \AtEndOfPackage{\EnableBabelHook{babel-bidi}}
-    \RequirePackage{luatexbase}
-    \bbl at activate@preotf
-    \directlua{
-      require('babel-data-bidi.lua')
-      \ifcase\expandafter\@gobbletwo\the\bbl at bidimode\or
-        require('babel-bidi-basic.lua')
-      \or
-        require('babel-bidi-basic-r.lua')
-      \fi}
-    % TODO - to locale_props, not as separate attribute
-    \newattribute\bbl at attr@dir
-    % TODO. I don't like it, hackish:
-    \bbl at exp{\output{\bodydir\pagedir\the\output}}
-    \AtEndOfPackage{\EnableBabelHook{babel-bidi}}
-  \fi\fi
-\else
-  \ifnum\bbl at bidimode>100 \ifnum\bbl at bidimode<200
     \bbl at error
       {The bidi method 'basic' is available only in\\%
        luatex. I'll continue with 'bidi=default', so\\%
@@ -6597,11 +6594,13 @@
     \fi
   \fi
 \fi
+% TODO? Separate:
 \ifnum\bbl at bidimode=\@ne
   \let\bbl at beforeforeign\leavevmode
   \ifodd\bbl at engine
     \newattribute\bbl at attr@dir
-    \bbl at exp{\output{\bodydir\pagedir\the\output}}%
+    \directlua{ Babel.attr_dir = luatexbase.registernumber'bbl at attr@dir' }
+    \bbl at exp{\output{\bodydir\pagedir\the\output}}
   \fi
   \AtEndOfPackage{%
     \EnableBabelHook{babel-bidi}%
@@ -6660,55 +6659,10 @@
 %    \end{macrocode}
 %
 % Now the engine-dependent macros. TODO. Must be moved to the engine
-% files?
+% files.
 %
 %    \begin{macrocode} 
 \ifodd\bbl at engine  % luatex=1
-  \chardef\bbl at thetextdir\z@
-  \chardef\bbl at thepardir\z@
-  \def\bbl at getluadir#1{%
-    \directlua{
-      if tex.#1dir == 'TLT' then
-        tex.sprint('0')
-      elseif tex.#1dir == 'TRT' then
-        tex.sprint('1')
-      end}}
-  \def\bbl at setluadir#1#2#3{% 1=text/par.. 2=\textdir.. 3=0 lr/1 rl
-    \ifcase#3\relax
-      \ifcase\bbl at getluadir{#1}\relax\else
-        #2 TLT\relax
-      \fi
-    \else
-      \ifcase\bbl at getluadir{#1}\relax
-        #2 TRT\relax
-      \fi
-    \fi}
-  \def\bbl at textdir#1{%
-    \bbl at setluadir{text}\textdir{#1}%
-    \chardef\bbl at thetextdir#1\relax
-    \setattribute\bbl at attr@dir{\numexpr\bbl at thepardir*3+#1}}
-  \def\bbl at pardir#1{%
-    \bbl at setluadir{par}\pardir{#1}%
-    \chardef\bbl at thepardir#1\relax}
-  \def\bbl at bodydir{\bbl at setluadir{body}\bodydir}
-  \def\bbl at pagedir{\bbl at setluadir{page}\pagedir}
-  \def\bbl at dirparastext{\pardir\the\textdir\relax}%   %%%%
-  % Sadly, we have to deal with boxes in math with basic.
-  % Activated every math with the package option bidi=:
-  \ifnum\bbl at bidimode>\z@
-    \def\bbl at mathboxdir{%
-      \ifcase\bbl at thetextdir\relax
-        \everyhbox{\bbl at mathboxdir@aux L}%
-      \else
-        \everyhbox{\bbl at mathboxdir@aux R}%
-       \fi}
-    \def\bbl at mathboxdir@aux#1{%
-      \@ifnextchar\egroup{}{\textdir T#1T\relax}}
-    \frozen at everymath\expandafter{%
-      \expandafter\bbl at mathboxdir\the\frozen at everymath}
-    \frozen at everydisplay\expandafter{%
-      \expandafter\bbl at mathboxdir\the\frozen at everydisplay}
-  \fi
 \else % pdftex=0, xetex=2
   \newcount\bbl at dirlevel
   \chardef\bbl at thetextdir\z@
@@ -7053,9 +7007,16 @@
     \bbl at ldfinit
     \let\CurrentOption\bbl at opt@main
     \ifx\bbl at opt@provide\@nnil
-      \bbl at exp{\\\babelprovide[import,main]{\bbl at opt@main}}
+      \bbl at exp{\\\babelprovide[import,main]{\bbl at opt@main}}%
     \else
-      \bbl at exp{\\\babelprovide[\bbl at opt@provide,main]{\bbl at opt@main}}%
+      \bbl at exp{\\\bbl at forkv{\@nameuse{@raw at opt@babel.sty}}}{%
+        \bbl at xin@{,provide,}{,#1,}%
+        \ifin@
+          \def\bbl at opt@provide{#2}%
+          \bbl at replace\bbl at opt@provide{;}{,}%
+        \fi}%
+      \bbl at exp{%
+        \\\babelprovide[\bbl at opt@provide,import,main]{\bbl at opt@main}}%
     \fi
     \bbl at afterldf{}%
   \else % case 0,2
@@ -7118,7 +7079,7 @@
 %    \end{macrocode}
 %
 % The file |babel.def| expects some definitions made in the \LaTeXe{}
-% style file. So, In \LaTeX2.09 and Plain{} we must provide at least
+% style file. So, in Plain{} we must provide at least
 % some predefined values as well some tools to set them (even if not
 % all options are available). There are no package options, and
 % therefore and alternative mechanism is provided. For the moment,
@@ -7154,16 +7115,6 @@
 \fi
 %    \end{macrocode}
 %
-% Exit immediately with 2.09. An error is raised by the sty file, but
-% also try to minimize the number of errors.
-%
-%    \begin{macrocode}
-\ifx\bbl at trace\@undefined
-  \let\LdfInit\endinput
-  \def\ProvidesLanguage#1{\endinput}
-\endinput\fi % Same line!
-%    \end{macrocode}
-%
 % And continue.
 %
 % \section{Multiple languages}
@@ -7210,7 +7161,7 @@
 %    exists. Otherwise raises and error.
 %
 %    The argument of |\bbl at fixname| has to be a macro name, as it may get
-%    ``fixed'' if casing (lc/uc) is wrong. It's intented to fix a
+%    ``fixed'' if casing (lc/uc) is wrong. It's an attempt to fix a
 %    long-standing bug when |\foreignlanguage| and the like appear in
 %    a |\MakeXXXcase|. However, a lowercase form is not imposed to
 %    improve backward compatibility (perhaps you defined a language
@@ -7371,8 +7322,9 @@
 \ifx\@undefined\protect\let\protect\relax\fi
 %    \end{macrocode}
 %
-% The following definition is preserved for backwards compatibility. It
-% is related to a trick for 2.09.
+% The following definition is preserved for backwards compatibility
+% (eg, \textsf{arabi}, \textsf{koma}). It is related to a trick for
+% 2.09, now discarded.
 %
 %    \begin{macrocode}
 \let\xstring\string
@@ -7417,7 +7369,15 @@
 %    \begin{macrocode}
 \def\bbl at push@language{%
   \ifx\languagename\@undefined\else
-    \xdef\bbl at language@stack{\languagename+\bbl at language@stack}%
+    \ifx\currentgrouplevel\@undefined
+      \xdef\bbl at language@stack{\languagename+\bbl at language@stack}%
+    \else
+      \ifnum\currentgrouplevel=\z@
+        \xdef\bbl at language@stack{\languagename+}%
+      \else
+        \xdef\bbl at language@stack{\languagename+\bbl at language@stack}%
+      \fi    
+    \fi
   \fi}
 %    \end{macrocode}
 %
@@ -7603,10 +7563,10 @@
       \let\bbl at select@type\z@
       \expandafter\bbl at switch\expandafter{\languagename}%
     \fi}}
-\def\babel at aux#1#2{% TODO. See how to avoid undefined nil's
+\def\babel at aux#1#2{%
   \select at language{#1}%
-  \bbl at foreach\BabelContentsFiles{%
-    \@writefile{##1}{\babel at toc{#1}{#2}}}}% %% TODO - ok in plain?
+  \bbl at foreach\BabelContentsFiles{% \relax -> don't assume vertical mode
+    \@writefile{##1}{\babel at toc{#1}{#2}\relax}}}% TODO - plain?
 \def\babel at toc#1#2{%
   \select at language{#1}}
 %    \end{macrocode}
@@ -8288,10 +8248,12 @@
 \newcommand\EnableBabelHook[1]{\bbl at csarg\let{hk@#1}\@firstofone}
 \newcommand\DisableBabelHook[1]{\bbl at csarg\let{hk@#1}\@gobble}
 \def\bbl at usehooks#1#2{%
+  \ifx\UseHook\@undefined\else\UseHook{babel/#1}\fi
   \def\bbl at elth##1{%
     \bbl at cs{hk@##1}{\bbl at cs{ev@##1@#1@}#2}}%
   \bbl at cs{ev@#1@}%
   \ifx\languagename\@undefined\else % Test required for Plain (?)
+    \ifx\UseHook\@undefined\else\UseHook{babel/#1/\languagename}\fi
     \def\bbl at elth##1{%
       \bbl at cs{hk@##1}{\bbl at cl{ev@##1@#1}#2}}%
     \bbl at cl{ev@#1}%
@@ -8311,6 +8273,10 @@
   beforeextras=0,afterextras=0,stopcommands=0,stringprocess=0,%
   hyphenation=2,initiateactive=3,afterreset=0,foreign=0,foreign*=0,%
   beforestart=0,languagename=2}
+\ifx\NewHook\@undefined\else
+  \def\bbl at tempa#1=#2\@@{\NewHook{babel/#1}}
+  \bbl at foreach\bbl at evargs{\bbl at tempa#1\@@}
+\fi
 %    \end{macrocode}
 %
 % \begin{macro}{\babelensure}
@@ -8499,9 +8465,7 @@
   \let\BabelModifiers\relax
   \let\bbl at screset\relax}%
 \def\ldf at finish#1{%
-  \ifx\loadlocalcfg\@undefined\else % For LaTeX 209
-    \loadlocalcfg{#1}%
-  \fi
+  \loadlocalcfg{#1}%
   \bbl at afterldf{#1}%
   \expandafter\main at language\expandafter{#1}%
   \catcode`\@=\atcatcode \let\atcatcode\relax
@@ -8543,10 +8507,12 @@
 %
 %    \begin{macrocode}
 \def\bbl at beforestart{%
+  \def\@nolanerr##1{%
+    \bbl at warning{Undefined language '##1' in aux.\\Reported}}%
   \bbl at usehooks{beforestart}{}%
   \global\let\bbl at beforestart\relax}
 \AtBeginDocument{%
-  \@nameuse{bbl at beforestart}%
+  {\@nameuse{bbl at beforestart}}%  Group!
   \if at filesw
     \providecommand\babel at aux[2]{}%
     \immediate\write\@mainaux{%
@@ -10920,11 +10886,11 @@
   \ifx\bbl at KVP@captions\@nil
     \let\bbl at KVP@captions\bbl at KVP@import
   \fi
-  % ==
+  % == 
   \ifx\bbl at KVP@transforms\@nil\else
     \bbl at replace\bbl at KVP@transforms{ }{,}%
   \fi
-  % Load ini
+  % == Load ini ==
   \ifcase\bbl at howloaded
     \bbl at provide@new{#2}%
   \else
@@ -11007,7 +10973,7 @@
         end}%
       \ifx\bbl at mapselect\@undefined  % TODO. almost the same as mapfont
         \AtBeginDocument{%
-          \expandafter\bbl at add\csname selectfont \endcsname{{\bbl at mapselect}}%
+          \bbl at patchfont{{\bbl at mapselect}}%
           {\selectfont}}%
         \def\bbl at mapselect{%
           \let\bbl at mapselect\relax
@@ -11033,9 +10999,9 @@
                  {See the manual for details.}}}%
     \bbl at ifunset{bbl at lsys@\languagename}{\bbl at provide@lsys{\languagename}}{}%
     \bbl at ifunset{bbl at wdir@\languagename}{\bbl at provide@dirs{\languagename}}{}%
-    \ifx\bbl at mapselect\@undefined % TODO. See onchar
+    \ifx\bbl at mapselect\@undefined % TODO. See onchar.
       \AtBeginDocument{%
-        \expandafter\bbl at add\csname selectfont \endcsname{{\bbl at mapselect}}%
+        \bbl at patchfont{{\bbl at mapselect}}%
         {\selectfont}}%
       \def\bbl at mapselect{%
         \let\bbl at mapselect\relax
@@ -11056,6 +11022,24 @@
     \bbl at csarg\edef{intsp@#2}{\bbl at KVP@intraspace}%
   \fi
   \bbl at provide@intraspace
+  % == Line breaking: CJK quotes ==
+  \ifcase\bbl at engine\or
+    \bbl at xin@{/c}{/\bbl at cl{lnbrk}}%
+    \ifin@
+      \bbl at ifunset{bbl at quote@\languagename}{}%
+        {\directlua{
+           Babel.locale_props[\the\localeid].cjk_quotes = {}
+           local cs = 'op'
+           for c in string.utfvalues(%
+               [[\csname bbl at quote@\languagename\endcsname]]) do
+             if Babel.cjk_characters[c].c == 'qu' then
+               Babel.locale_props[\the\localeid].cjk_quotes[c] = cs
+             end
+             cs = ( cs == 'op') and 'cl' or 'op'
+           end
+        }}%
+    \fi
+  \fi
   % == Line breaking: justification ==
   \ifx\bbl at KVP@justification\@nil\else
      \let\bbl at KVP@linebreaking\bbl at KVP@justification
@@ -11128,7 +11112,7 @@
              table.pack(string.utfvalue('\bbl at cl{dgnat}'))
            if not Babel.numbers then
              function Babel.numbers(head)
-               local LOCALE = luatexbase.registernumber'bbl at attr@locale'
+               local LOCALE = Babel.attr_locale
                local GLYPH = node.id'glyph'
                local inmath = false
                for item in node.traverse(head) do
@@ -11255,13 +11239,13 @@
     \StartBabelCommands*{#1}{captions}%
       \bbl at read@ini{\bbl at KVP@captions}2%   % Here all letters cat = 11
     \EndBabelCommands
- \fi
- \ifx\bbl at KVP@import\@nil\else
-   \StartBabelCommands*{#1}{date}%
-     \bbl at savetoday
-     \bbl at savedate
-   \EndBabelCommands
   \fi
+  \ifx\bbl at KVP@import\@nil\else
+    \StartBabelCommands*{#1}{date}%
+      \bbl at savetoday
+      \bbl at savedate
+    \EndBabelCommands
+  \fi
   % == hyphenrules (also in new) ==
   \ifx\bbl at lbkflag\@empty
     \bbl at provide@hyphens{#1}%
@@ -11598,6 +11582,7 @@
     \bbl at exportkey{intsp}{typography.intraspace}{}%
     \bbl at exportkey{frspc}{typography.frenchspacing}{u}%
     \bbl at exportkey{chrng}{characters.ranges}{}%
+    \bbl at exportkey{quote}{characters.delimiters.quotes}{}%
     \bbl at exportkey{dgnat}{numbers.digits.native}{}%
     \ifnum#1=\tw@           % only (re)new
       \bbl at exportkey{rqtex}{identification.require.babel}{}%
@@ -11895,7 +11880,10 @@
 % \textbf{Dates} will require some macros for the basic formatting.
 % They may be redefined by language, so ``semi-public'' names (camel
 % case) are used. Oddly enough, the CLDR places particles like “de”
-% inconsistently in either in the date or in the month name.
+% inconsistently in either in the date or in the month name. Note after
+% |\bbl at replace| |\toks@| contains the resulting string, which is used
+% by |\bbl at replace@finish at iii| (this implicit behavior doesn't seem a
+% good idea, but it’s efficient).
 %
 %    \begin{macrocode}
 \let\bbl at calendar\@empty
@@ -11919,7 +11907,7 @@
        range 0-9999.}%
       {There is little you can do. Sorry.}%
   \fi\fi\fi\fi}}
-\newcommand\BabelDateyyyy[1]{{\number#1}} % FIXME - add leading 0
+\newcommand\BabelDateyyyy[1]{{\number#1}} % TODO - add leading 0
 \def\bbl at replace@finish at iii#1{%
   \bbl at exp{\def\\#1####1####2####3{\the\toks@}}}
 \def\bbl at TG@@date{%
@@ -11936,8 +11924,6 @@
   \bbl at replace\bbl at toreplace{[y|}{\bbl at datecntr[####1|}%
   \bbl at replace\bbl at toreplace{[m|}{\bbl at datecntr[####2|}%
   \bbl at replace\bbl at toreplace{[d|}{\bbl at datecntr[####3|}%
-% Note after \bbl at replace \toks@ contains the resulting string.
-% TODO - Using this implicit behavior doesn't seem a good idea.
   \bbl at replace@finish at iii\bbl at toreplace}
 \def\bbl at datecntr{\expandafter\bbl at xdatecntr\expandafter}
 \def\bbl at xdatecntr[#1|#2]{\localenumeral{#2}{#1}}
@@ -12000,10 +11986,8 @@
              \expandafter\@secondoftwo  % to execute right now
            \fi
            \AtBeginDocument{%
-             \expandafter\bbl at add
-             \csname selectfont \endcsname{\bbl at xenohyph}%
-             \expandafter\selectlanguage\expandafter{\languagename}%
-             \expandafter\bbl at toglobal\csname selectfont \endcsname}%
+             \bbl at patchfont{\bbl at xenohyph}%
+             \expandafter\selectlanguage\expandafter{\languagename}}%
         \fi}}%
   \fi
   \bbl at csarg\bbl at toglobal{lsys@#1}}
@@ -12297,10 +12281,11 @@
   \directlua{ Babel.ignore_pre_char = function(node)
       return false
     end }}
-% TODO: use babel name, override 
+%    \end{macrocode}
 %
-% As the final task, load the code for lua.
+% As the final task, load the code for lua. TODO: use babel name, override 
 %
+%    \begin{macrocode}
 \ifx\directlua\@undefined\else
   \ifx\bbl at luapatterns\@undefined
     \input luababel.def
@@ -12328,23 +12313,10 @@
 %
 % The following code is meant to be read by ini\TeX\ because it
 % should instruct \TeX\ to read hyphenation patterns. To this end the
-% \texttt{docstrip} option \texttt{patterns} can be used to include
+% \texttt{docstrip} option \texttt{patterns} is used to include
 % this code in the file \file{hyphen.cfg}. Code is written with lower
 % level macros.
 %
-% To make sure that \LaTeX$\:$2.09 executes the |\@begindocumenthook|
-% we would want to alter |\begin{document}|, but as this done too often
-% already, we add the new code at the front of |\@preamblecmds|. But we
-% can only do that after it has been defined, so we add this piece of
-% code to |\dump|.
-%
-% This new definition starts by adding an instruction to write a
-% message on the terminal and in the transcript file to inform the
-% user of the preloaded hyphenation patterns.
-%
-% Then everything is restored to the old situation and the format
-% is dumped.
-%
 %    \begin{macrocode}
 <@Make sure ProvidesFile is defined@>
 \ProvidesFile{hyphen.cfg}[<@date@> <@version@> Babel hyphens]
@@ -12353,15 +12325,6 @@
 \def\bbl at date{<@date@>}
 \ifx\AtBeginDocument\@undefined
   \def\@empty{}
-  \let\orig at dump\dump
-  \def\dump{%
-    \ifx\@ztryfc\@undefined
-    \else
-      \toks0=\expandafter{\@preamblecmds}%
-      \edef\@preamblecmds{\noexpand\@begindocumenthook\the\toks0}%
-      \def\@begindocumenthook{}%
-    \fi
-    \let\dump\orig at dump\let\orig at dump\@undefined\dump}
 \fi
 <@Define core switching macros@>
 %    \end{macrocode}
@@ -12744,7 +12707,7 @@
   \ExplSyntaxOn
   \catcode`\ =10
   \def\bbl at loadfontspec{%
-    \usepackage{fontspec}%
+    \usepackage{fontspec}%  TODO. Apply patch always
     \expandafter
     \def\csname msg~text~>~fontspec/language-not-exist\endcsname##1##2##3##4{%
       Font '\l_fontspec_fontname_tl' is using the\\%
@@ -12778,9 +12741,7 @@
 \newcommand\bbl at bblfont[2][]{% 1=features 2=fontname, @font=rm|sf|tt
   \bbl at ifunset{\bbl at tempb family}%
     {\bbl at providefam{\bbl at tempb}}%
-    {\bbl at exp{%
-      \\\bbl at sreplace\<\bbl at tempb family >%
-        {\@nameuse{\bbl at tempb default}}{\<\bbl at tempb default>}}}%
+    {}%
   % For the default font, just in case:
   \bbl at ifunset{bbl at lsys@\languagename}{\bbl at provide@lsys{\languagename}}{}%
   \expandafter\bbl at ifblank\expandafter{\bbl at tempa}%
@@ -12794,7 +12755,7 @@
 %    \end{macrocode}
 %
 % If the family in the previous command does not exist, it must be
-% defined. Here is how:
+% defined. Here is how: 
 %
 %    \begin{macrocode}
 \def\bbl at providefam#1{%
@@ -12803,12 +12764,15 @@
     \\\bbl at add@list\\\bbl at font@fams{#1}%
     \\\DeclareRobustCommand\<#1family>{%
       \\\not at math@alphabet\<#1family>\relax
-      \\\fontfamily\<#1default>\\\selectfont}%
+      % \\\prepare at family@series at update{#1}\<#1default>% TODO. Fails
+      \\\fontfamily\<#1default>%
+      \<ifx>\\\UseHooks\\\@undefined\<else>\\\UseHook{#1family}\<fi>%
+      \\\selectfont}%
     \\\DeclareTextFontCommand{\<text#1>}{\<#1family>}}}
 %    \end{macrocode}
 %
 % The following macro is activated when the hook |babel-fontspec| is
-% enabled. But before we define a macro for a warning, which sets a
+% enabled. But before, we define a macro for a warning, which sets a
 % flag to avoid duplicate them.
 %
 %    \begin{macrocode}
@@ -13117,10 +13081,7 @@
         \ifx\AtBeginDocument\@notprerr
           \expandafter\@secondoftwo  % to execute right now
         \fi
-        \AtBeginDocument{%
-          \expandafter\bbl at add
-          \csname selectfont \endcsname{\bbl at ispacesize}%
-          \expandafter\bbl at toglobal\csname selectfont \endcsname}%
+        \AtBeginDocument{\bbl at patchfont{\bbl at xenohyph}}%
       \fi}%
   \fi}
 \ifx\DisableBabelHook\@undefined\endinput\fi
@@ -13511,7 +13472,7 @@
 \endgroup
 \ifx\newattribute\@undefined\else
   \newattribute\bbl at attr@locale
-  \directlua{ Babel.attr_locale = luatexbase.registernumber'bbl at attr@locale'}
+  \directlua{ Babel.attr_locale = luatexbase.registernumber'bbl at attr@locale' }
   \AddBabelHook{luatex}{beforeextras}{%
     \setattribute\bbl at attr@locale\localeid}
 \fi
@@ -13733,11 +13694,15 @@
           local lang = item.lang
 
           local LOCALE = node.get_attribute(item,
-                luatexbase.registernumber'bbl at attr@locale')
+                Babel.attr_locale)
           local props = Babel.locale_props[LOCALE]
 
           local class = Babel.cjk_class[item.char].c
 
+          if props.cjk_quotes and props.cjk_quotes[item.char] then
+            class = props.cjk_quotes[item.char]
+          end
+
           if class == 'cp' then class = 'cl' end % )] as CL
           if class == 'id' then class = 'I' end
 
@@ -13856,8 +13821,9 @@
 \gdef\bbl at arabicjust{%
   \let\bbl at arabicjust\relax
   \newattribute\bblar at kashida
+  \directlua{ Babel.attr_kashida = luatexbase.registernumber'bblar at kashida' }%
   \bblar at kashida=\z@
-  \expandafter\bbl at add\csname selectfont \endcsname{{\bbl at parsejalt}}%
+  \bbl at patchfont{{\bbl at parsejalt}}%
   \directlua{
     Babel.arabic.elong_map   = Babel.arabic.elong_map or {}
     Babel.arabic.elong_map[\the\localeid]   = {}
@@ -13962,9 +13928,9 @@
   local elong_map = Babel.arabic.elong_map
   local last_line
   local GLYPH = node.id'glyph'
-  local KASHIDA = luatexbase.registernumber'bblar at kashida'
-  local LOCALE = luatexbase.registernumber'bbl at attr@locale'
-  
+  local KASHIDA = Babel.attr_kashida
+  local LOCALE = Babel.attr_locale
+
   if line == nil then
     line = {}
     line.glue_sign = 1
@@ -14157,7 +14123,7 @@
 function Babel.locale_map(head)
   if not Babel.locale_mapped then return head end
 
-  local LOCALE = luatexbase.registernumber'bbl at attr@locale'
+  local LOCALE = Babel.attr_locale
   local GLYPH = node.id('glyph')
   local inmath = false
   local toloc_save
@@ -14263,534 +14229,11 @@
 %
 % Post-handling hyphenation patterns for non-standard rules, like |ff|
 % to |ff-f|. There are still some issues with speed (not very slow, but
-% still slow).
-%
-% After declaring the table containing the patterns with their
-% replacements, we define some auxiliary functions: |str_to_nodes|
-% converts the string returned by a function to a node list, taking the
-% node at |base| as a model (font, language, etc.); |fetch_word|
-% fetches a series of glyphs and discretionaries, which |pattern| is
-% matched against (if there is a match, it is called again before
-% trying other patterns, and this is very likely the main bottleneck).
-%
-% |post_hyphenate_replace| is the callback applied after
-% |lang.hyphenate|. This means the automatic hyphenation points are
-% known. As empty captures return a byte position (as explained in the
-% \luatex{} manual), we must convert it to a utf8 position. With
-% |first|, the last byte can be the leading byte in a utf8 sequence, so
-% we just remove it and add 1 to the resulting length. With |last| we
-% must take into account the capture position points to the next
-% character. Here |word_head| points to the starting node of the text
-% to be matched.
+% still slow). The Lua code is below.
 %  
 %    \begin{macrocode}
-\begingroup % TODO - to a lua file
-\catcode`\~=12
-\catcode`\#=12
-\catcode`\%=12
-\catcode`\&=14
 \directlua{
-  Babel.linebreaking.replacements = {}
-  Babel.linebreaking.replacements[0] = {}  &% pre
-  Babel.linebreaking.replacements[1] = {}  &% post
-
-  &% Discretionaries contain strings as nodes
-  function Babel.str_to_nodes(fn, matches, base)
-    local n, head, last    
-    if fn == nil then return nil end
-    for s in string.utfvalues(fn(matches)) do
-      if base.id == 7 then 
-        base = base.replace
-      end
-      n = node.copy(base)
-      n.char    = s
-      if not head then
-        head = n
-      else
-        last.next = n
-      end
-      last = n
-    end 
-    return head
-  end
-
-  Babel.fetch_subtext = {}
-
-  Babel.ignore_pre_char = function(node)
-    return (node.lang == \the\l at nohyphenation)
-  end
-
-  &% Merging both functions doesn't seen feasible, because there are too
-  &% many differences.
-  Babel.fetch_subtext[0] = function(head)
-    local word_string = ''
-    local word_nodes = {}
-    local lang
-    local item = head
-    local inmath = false
-
-    while item do
-
-      if item.id == 11 then
-        inmath = (item.subtype == 0)
-      end
-
-      if inmath then
-        &% pass
-
-      elseif item.id == 29 then
-        local locale = node.get_attribute(item, Babel.attr_locale)
-
-        if lang == locale or lang == nil then
-          lang = lang or locale
-          if Babel.ignore_pre_char(item) then  
-            word_string = word_string .. Babel.us_char
-          else
-            word_string = word_string .. unicode.utf8.char(item.char)
-          end
-          word_nodes[#word_nodes+1] = item
-        else
-          break
-        end
-
-      elseif item.id == 12 and item.subtype == 13 then
-        word_string = word_string .. ' '
-        word_nodes[#word_nodes+1] = item
-
-      &% Ignore leading unrecognized nodes, too.
-      elseif word_string ~= '' then
-        word_string = word_string .. Babel.us_char   
-        word_nodes[#word_nodes+1] = item  &% Will be ignored
-      end
-
-      item = item.next
-    end
-
-    &% Here and above we remove some trailing chars but not the
-    &% corresponding nodes. But they aren't accessed. 
-    if word_string:sub(-1) == ' ' then
-      word_string = word_string:sub(1,-2)
-    end
-    word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
-    return word_string, word_nodes, item, lang
-  end
-
-  Babel.fetch_subtext[1] = function(head)
-    local word_string = ''
-    local word_nodes = {}
-    local lang
-    local item = head
-    local inmath = false
-
-    while item do
-
-      if item.id == 11 then
-        inmath = (item.subtype == 0)
-      end
-
-      if inmath then
-        &% pass
-
-      elseif item.id == 29 then
-        if item.lang == lang or lang == nil then
-          if (item.char ~= 124) and (item.char ~= 61) then &% not =, not |
-            lang = lang or item.lang
-            word_string = word_string .. unicode.utf8.char(item.char)
-            word_nodes[#word_nodes+1] = item
-          end
-        else
-          break
-        end
-
-      elseif item.id == 7 and item.subtype == 2 then
-        word_string = word_string .. '='
-        word_nodes[#word_nodes+1] = item
-
-      elseif item.id == 7 and item.subtype == 3 then
-        word_string = word_string .. '|'       
-        word_nodes[#word_nodes+1] = item
-
-      &% (1) Go to next word if nothing was found, and (2) implictly
-      &% remove leading USs.
-      elseif word_string == '' then
-        &% pass
-
-      &% This is the responsible for splitting by words.
-      elseif (item.id == 12 and item.subtype == 13) then
-        break
-
-      else 
-        word_string = word_string .. Babel.us_char   
-        word_nodes[#word_nodes+1] = item  &% Will be ignored
-      end
-
-      item = item.next
-    end
-
-    word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
-    return word_string, word_nodes, item, lang
-  end
-
-  function Babel.pre_hyphenate_replace(head)
-    Babel.hyphenate_replace(head, 0)
-  end
-
-  function Babel.post_hyphenate_replace(head)
-    Babel.hyphenate_replace(head, 1)
-  end
-
-  function Babel.debug_hyph(w, wn, sc, first, last, last_match)
-    local ss = ''
-    for pp = 1, 40 do
-      if wn[pp] then
-        if wn[pp].id == 29 then
-          ss = ss .. unicode.utf8.char(wn[pp].char)
-        else
-          ss = ss .. '{' .. wn[pp].id .. '}'
-        end
-      end
-    end
-    print('nod', ss)
-    print('lst_m',
-      string.rep(' ', unicode.utf8.len(
-         string.sub(w, 1, last_match))-1) .. '>')
-    print('str', w)
-    print('sc', string.rep(' ', sc-1) .. '^')
-    if first == last then
-      print('f=l', string.rep(' ', first-1) .. '!')
-    else
-      print('f/l', string.rep(' ', first-1) .. '[' ..
-        string.rep(' ', last-first-1) .. ']')
-    end
-  end
-
-  Babel.us_char = string.char(31)
-
-  function Babel.hyphenate_replace(head, mode)
-    local u = unicode.utf8
-    local lbkr = Babel.linebreaking.replacements[mode]
-
-    local word_head = head
-
-    while true do  &% for each subtext block
-
-      local w, w_nodes, nw, lang = Babel.fetch_subtext[mode](word_head)
-
-      if Babel.debug then
-        print()
-        print((mode == 0) and '@@@@<' or '@@@@>', w)
-      end
-
-      if nw == nil and w == '' then break end
-
-      if not lang then goto next end
-      if not lbkr[lang] then goto next end
-
-      &% For each saved (pre|post)hyphenation. TODO. Reconsider how
-      &% loops are nested.
-      for k=1, #lbkr[lang] do
-        local p = lbkr[lang][k].pattern
-        local r = lbkr[lang][k].replace
-
-        if Babel.debug then
-          print('*****', p, mode)
-        end
-
-        &% This variable is set in some cases below to the first *byte*
-        &% after the match, either as found by u.match (faster) or the
-        &% computed position based on sc if w has changed.
-        local last_match = 0
-        local step = 0
-
-        &% For every match.
-        while true do
-          if Babel.debug then
-            print('=====')
-          end
-          local new  &% used when inserting and removing nodes
-
-          local matches = { u.match(w, p, last_match) }
-          
-          if #matches < 2 then break end
-
-          &% Get and remove empty captures (with ()'s, which return a
-          &% number with the position), and keep actual captures
-          &% (from (...)), if any, in matches.
-          local first = table.remove(matches, 1)
-          local last  = table.remove(matches, #matches)
-          &% Non re-fetched substrings may contain \31, which separates
-          &% subsubstrings.
-          if string.find(w:sub(first, last-1), Babel.us_char) then break end
-
-          local save_last = last &% with A()BC()D, points to D
-
-          &% Fix offsets, from bytes to unicode. Explained above.
-          first = u.len(w:sub(1, first-1)) + 1
-          last  = u.len(w:sub(1, last-1)) &% now last points to C
-
-          &% This loop stores in n small table the nodes
-          &% corresponding to the pattern. Used by 'data' to provide a
-          &% predictable behavior with 'insert' (now w_nodes is modified on
-          &% the fly), and also access to 'remove'd nodes.
-          local sc = first-1           &% Used below, too
-          local data_nodes = {}
-
-          for q = 1, last-first+1 do
-            data_nodes[q] = w_nodes[sc+q]
-          end
-
-          &% This loop traverses the matched substring and takes the
-          &% corresponding action stored in the replacement list.
-          &% sc = the position in substr nodes / string
-          &% rc = the replacement table index
-          local rc = 0
-
-          while rc < last-first+1 do &% for each replacement
-            if Babel.debug then
-              print('.....', rc + 1)
-            end
-            sc = sc + 1
-            rc = rc + 1
-
-            if Babel.debug then
-              Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
-              local ss = ''
-              for itt in node.traverse(head) do
-               if itt.id == 29 then
-                 ss = ss .. unicode.utf8.char(itt.char)
-               else
-                 ss = ss .. '{' .. itt.id .. '}'
-               end
-              end
-              print('*****************', ss)
-
-            end
-
-            local crep = r[rc]
-            local item = w_nodes[sc]
-            local item_base = item
-            local placeholder = Babel.us_char
-            local d
-
-            if crep and crep.data then
-              item_base = data_nodes[crep.data]
-            end
-            
-            if crep then
-              step = crep.step or 0
-            end
-
-            if crep and next(crep) == nil then &% = {}
-              last_match = save_last    &% Optimization
-              goto next
-
-            elseif crep == nil or crep.remove then
-              node.remove(head, item)
-              table.remove(w_nodes, sc)
-              w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1)
-              sc = sc - 1  &% Nothing has been inserted.
-              last_match = utf8.offset(w, sc+1+step)
-              goto next
-
-            elseif crep and crep.kashida then &% Experimental
-              node.set_attribute(item,
-                 luatexbase.registernumber'bblar at kashida', 
-                 crep.kashida)
-              last_match = utf8.offset(w, sc+1+step)
-              goto next
-
-            elseif crep and crep.string then
-              local str = crep.string(matches)
-              if str == '' then  &% Gather with nil
-                node.remove(head, item)
-                table.remove(w_nodes, sc)
-                w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1)
-                sc = sc - 1  &% Nothing has been inserted.
-              else
-                local loop_first = true
-                for s in string.utfvalues(str) do
-                  d = node.copy(item_base)
-                  d.char = s
-                  if loop_first then
-                    loop_first = false
-                    head, new = node.insert_before(head, item, d)
-                    if sc == 1 then
-                      word_head = head
-                    end
-                    w_nodes[sc] = d
-                    w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc+1)
-                  else
-                    sc = sc + 1
-                    head, new = node.insert_before(head, item, d)
-                    table.insert(w_nodes, sc, new)
-                    w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc)
-                  end
-                  if Babel.debug then
-                    print('.....', 'str')
-                    Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
-                  end
-                end  &% for
-                node.remove(head, item)
-              end  &% if ''
-              last_match = utf8.offset(w, sc+1+step)
-              goto next
-
-            elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then
-              d = node.new(7, 0)   &% (disc, discretionary)
-              d.pre     = Babel.str_to_nodes(crep.pre, matches, item_base)
-              d.post    = Babel.str_to_nodes(crep.post, matches, item_base)
-              d.replace = Babel.str_to_nodes(crep.no, matches, item_base)
-              d.attr = item_base.attr
-              if crep.pre == nil then  &% TeXbook p96
-                d.penalty = crep.penalty or tex.hyphenpenalty
-              else
-                d.penalty = crep.penalty or tex.exhyphenpenalty
-              end
-              placeholder = '|'
-              head, new = node.insert_before(head, item, d)
-
-            elseif mode == 0 and crep and (crep.pre or crep.no or crep.post) then
-              &% ERROR
-
-            elseif crep and crep.penalty then
-              d = node.new(14, 0)   &% (penalty, userpenalty)
-              d.attr = item_base.attr
-              d.penalty = crep.penalty
-              head, new = node.insert_before(head, item, d)
-
-            elseif crep and crep.space then
-              &% 655360 = 10 pt = 10 * 65536 sp
-              d = node.new(12, 13)      &% (glue, spaceskip)
-              local quad = font.getfont(item_base.font).size or 655360
-              node.setglue(d, crep.space[1] * quad,
-                              crep.space[2] * quad,
-                              crep.space[3] * quad)
-              if mode == 0 then
-                placeholder = ' '
-              end
-              head, new = node.insert_before(head, item, d)
-
-            elseif crep and crep.spacefactor then
-              d = node.new(12, 13)      &% (glue, spaceskip)
-              local base_font = font.getfont(item_base.font)
-              node.setglue(d,
-                crep.spacefactor[1] * base_font.parameters['space'],
-                crep.spacefactor[2] * base_font.parameters['space_stretch'],
-                crep.spacefactor[3] * base_font.parameters['space_shrink'])
-              if mode == 0 then
-                placeholder = ' '
-              end
-              head, new = node.insert_before(head, item, d)
-
-            elseif mode == 0 and crep and crep.space then
-              &% ERROR
-
-            end  &% ie replacement cases
-
-            &% Shared by disc, space and penalty.
-            if sc == 1 then
-              word_head = head
-            end
-            if crep.insert then
-              w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc)
-              table.insert(w_nodes, sc, new)
-              last = last + 1
-            else
-              w_nodes[sc] = d
-              node.remove(head, item)
-              w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc+1)
-            end
-
-            last_match = utf8.offset(w, sc+1+step)
-
-            ::next::
-
-          end  &% for each replacement
-
-          if Babel.debug then
-              print('.....', '/')
-              Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
-          end
-
-        end  &% for match
-
-      end  &% for patterns
-
-      ::next::
-      word_head = nw
-    end  &% for substring
-    return head
-  end
-
-  &% This table stores capture maps, numbered consecutively
-  Babel.capture_maps = {}
-
-  &% The following functions belong to the next macro
-  function Babel.capture_func(key, cap)
-    local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]"
-    local cnt
-    local u = unicode.utf8
-    ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map)
-    if cnt == 0 then
-      ret = u.gsub(ret, '{(%x%x%x%x+)}', 
-            function (n)
-              return u.char(tonumber(n, 16))
-            end)
-    end
-    ret = ret:gsub("%[%[%]%]%.%.", '')
-    ret = ret:gsub("%.%.%[%[%]%]", '')
-    return key .. [[=function(m) return ]] .. ret .. [[ end]]
-  end
-
-  function Babel.capt_map(from, mapno)
-    return Babel.capture_maps[mapno][from] or from
-  end
-
-  &% Handle the {n|abc|ABC} syntax in captures
-  function Babel.capture_func_map(capno, from, to)
-    local u = unicode.utf8
-    from = u.gsub(from, '{(%x%x%x%x+)}', 
-         function (n)
-           return u.char(tonumber(n, 16))
-         end)
-    to = u.gsub(to, '{(%x%x%x%x+)}', 
-         function (n)
-           return u.char(tonumber(n, 16))
-         end)
-    local froms = {}
-    for s in string.utfcharacters(from) do
-      table.insert(froms, s)
-    end
-    local cnt = 1
-    table.insert(Babel.capture_maps, {})
-    local mlen = table.getn(Babel.capture_maps)
-    for s in string.utfcharacters(to) do
-      Babel.capture_maps[mlen][froms[cnt]] = s
-      cnt = cnt + 1
-    end
-    return "]]..Babel.capt_map(m[" .. capno .. "]," ..
-           (mlen) .. ").." .. "[["
-  end
-  
-  &% Create/Extend reversed sorted list of kashida weights:
-  function Babel.capture_kashida(key, wt)
-    wt = tonumber(wt)
-    if Babel.kashida_wts then
-      for p, q in ipairs(Babel.kashida_wts) do
-        if wt  == q then
-          break
-        elseif wt > q then
-          table.insert(Babel.kashida_wts, p, wt)
-          break
-        elseif table.getn(Babel.kashida_wts) == p then
-          table.insert(Babel.kashida_wts, wt)
-        end
-      end
-    else
-      Babel.kashida_wts = { wt }
-    end
-    return 'kashida = ' .. wt
-  end
+  Babel.nohyphenation = \the\l at nohyphenation
 }
 %    \end{macrocode}
 %
@@ -14809,7 +14252,10 @@
 % in macro names (which explains the internal group, too).
 % 
 %    \begin{macrocode}
-\catcode`\#=6
+\begingroup
+\catcode`\~=12
+\catcode`\%=12
+\catcode`\&=14
 \gdef\babelposthyphenation#1#2#3{&%
   \bbl at activateposthyphen
   \begingroup
@@ -14904,15 +14350,135 @@
 \def\bbl at activateposthyphen{%
   \let\bbl at activateposthyphen\relax
   \directlua{
+    require('babel-transforms.lua')
     Babel.linebreaking.add_after(Babel.post_hyphenate_replace)
   }}
 \def\bbl at activateprehyphen{%
   \let\bbl at activateprehyphen\relax
   \directlua{
+    require('babel-transforms.lua')
     Babel.linebreaking.add_before(Babel.pre_hyphenate_replace)
   }}
 %    \end{macrocode}
 %
+% \subsection{Bidi}
+%
+% As a first step, add a handler for bidi and digits (and potentially
+% other processes) just before \textsf{luaoftload} is applied, which is
+% loaded by default by \LaTeX. Just in case, consider the possibility
+% it has not been loaded.
+%
+%    \begin{macrocode}
+\def\bbl at activate@preotf{%
+  \let\bbl at activate@preotf\relax  % only once
+  \directlua{
+    Babel = Babel or {}
+    %
+    function Babel.pre_otfload_v(head)
+      if Babel.numbers and Babel.digits_mapped then
+        head = Babel.numbers(head)
+      end
+      if Babel.bidi_enabled then
+        head = Babel.bidi(head, false, dir)
+      end
+      return head
+    end
+    %
+    function Babel.pre_otfload_h(head, gc, sz, pt, dir)
+      if Babel.numbers and Babel.digits_mapped then
+        head = Babel.numbers(head)
+      end
+      if Babel.bidi_enabled then
+        head = Babel.bidi(head, false, dir)
+      end
+      return head
+    end
+    %
+    luatexbase.add_to_callback('pre_linebreak_filter',
+      Babel.pre_otfload_v,
+      'Babel.pre_otfload_v',
+      luatexbase.priority_in_callback('pre_linebreak_filter',
+        'luaotfload.node_processor') or nil)
+    %
+    luatexbase.add_to_callback('hpack_filter',
+      Babel.pre_otfload_h,
+      'Babel.pre_otfload_h',
+      luatexbase.priority_in_callback('hpack_filter',
+        'luaotfload.node_processor') or nil)
+  }}
+%    \end{macrocode}
+%
+% The basic setup. The output is modified at a very low level to set
+% the |\bodydir| to the |\pagedir|. Sadly, we have to deal with boxes
+% in math with basic, so the |\bbl at mathboxdir| hack is activated every
+% math with the package option bidi=.
+%
+%    \begin{macrocode}
+\ifnum\bbl at bidimode>100 \ifnum\bbl at bidimode<200
+  \let\bbl at beforeforeign\leavevmode
+  \AtEndOfPackage{\EnableBabelHook{babel-bidi}}
+  \RequirePackage{luatexbase}
+  \bbl at activate@preotf
+  \directlua{
+    require('babel-data-bidi.lua')
+    \ifcase\expandafter\@gobbletwo\the\bbl at bidimode\or
+      require('babel-bidi-basic.lua')
+    \or
+      require('babel-bidi-basic-r.lua')
+    \fi}
+  % TODO - to locale_props, not as separate attribute
+  \newattribute\bbl at attr@dir
+  \directlua{ Babel.attr_dir = luatexbase.registernumber'bbl at attr@dir' }
+  % TODO. I don't like it, hackish:
+  \bbl at exp{\output{\bodydir\pagedir\the\output}}
+  \AtEndOfPackage{\EnableBabelHook{babel-bidi}}
+\fi\fi
+\chardef\bbl at thetextdir\z@
+\chardef\bbl at thepardir\z@
+\def\bbl at getluadir#1{%
+  \directlua{
+    if tex.#1dir == 'TLT' then
+      tex.sprint('0')
+    elseif tex.#1dir == 'TRT' then
+      tex.sprint('1')
+    end}}
+\def\bbl at setluadir#1#2#3{% 1=text/par.. 2=\textdir.. 3=0 lr/1 rl
+  \ifcase#3\relax
+    \ifcase\bbl at getluadir{#1}\relax\else
+      #2 TLT\relax
+    \fi
+  \else
+    \ifcase\bbl at getluadir{#1}\relax
+      #2 TRT\relax
+    \fi
+  \fi}
+\def\bbl at textdir#1{%
+  \bbl at setluadir{text}\textdir{#1}%
+  \chardef\bbl at thetextdir#1\relax
+  \setattribute\bbl at attr@dir{\numexpr\bbl at thepardir*3+#1}}
+\def\bbl at pardir#1{%
+  \bbl at setluadir{par}\pardir{#1}%
+  \chardef\bbl at thepardir#1\relax}
+\def\bbl at bodydir{\bbl at setluadir{body}\bodydir}
+\def\bbl at pagedir{\bbl at setluadir{page}\pagedir}
+\def\bbl at dirparastext{\pardir\the\textdir\relax}%   %%%%
+%
+\ifnum\bbl at bidimode>\z@
+  \def\bbl at mathboxdir{%
+    \ifcase\bbl at thetextdir\relax
+      \everyhbox{\bbl at mathboxdir@aux L}%
+    \else
+      \everyhbox{\bbl at mathboxdir@aux R}%
+     \fi}
+  \def\bbl at mathboxdir@aux#1{%
+    \@ifnextchar\egroup{}{\textdir T#1T\relax}}
+  \frozen at everymath\expandafter{%
+    \expandafter\bbl at mathboxdir\the\frozen at everymath}
+  \frozen at everydisplay\expandafter{%
+    \expandafter\bbl at mathboxdir\the\frozen at everydisplay}
+\fi
+%    \end{macrocode}
+%
 % \subsection{Layout}
 %
 % Unlike \xetex{}, \luatex{} requires only minimal changes for
@@ -15135,8 +14701,509 @@
 %</luatex>
 %    \end{macrocode}
 %
-% \subsection{Auto bidi with \texttt{basic} and \texttt{basic-r}}
+% \subsection{Lua: transforms}
 %
+% After declaring the table containing the patterns with their
+% replacements, we define some auxiliary functions: |str_to_nodes|
+% converts the string returned by a function to a node list, taking the
+% node at |base| as a model (font, language, etc.); |fetch_word|
+% fetches a series of glyphs and discretionaries, which |pattern| is
+% matched against (if there is a match, it is called again before
+% trying other patterns, and this is very likely the main bottleneck).
+%
+% |post_hyphenate_replace| is the callback applied after
+% |lang.hyphenate|. This means the automatic hyphenation points are
+% known. As empty captures return a byte position (as explained in the
+% \luatex{} manual), we must convert it to a utf8 position. With
+% |first|, the last byte can be the leading byte in a utf8 sequence, so
+% we just remove it and add 1 to the resulting length. With |last| we
+% must take into account the capture position points to the next
+% character. Here |word_head| points to the starting node of the text
+% to be matched.
+%
+%    \begin{macrocode}
+%<*transforms>
+Babel.linebreaking.replacements = {}
+Babel.linebreaking.replacements[0] = {}  -- pre
+Babel.linebreaking.replacements[1] = {}  -- post
+
+-- Discretionaries contain strings as nodes
+function Babel.str_to_nodes(fn, matches, base)
+  local n, head, last    
+  if fn == nil then return nil end
+  for s in string.utfvalues(fn(matches)) do
+    if base.id == 7 then 
+      base = base.replace
+    end
+    n = node.copy(base)
+    n.char    = s
+    if not head then
+      head = n
+    else
+      last.next = n
+    end
+    last = n
+  end 
+  return head
+end
+
+Babel.fetch_subtext = {}
+
+Babel.ignore_pre_char = function(node)
+  return (node.lang == Babel.nohyphenation)
+end
+
+-- Merging both functions doesn't seen feasible, because there are too
+-- many differences.
+Babel.fetch_subtext[0] = function(head)
+  local word_string = ''
+  local word_nodes = {}
+  local lang
+  local item = head
+  local inmath = false
+
+  while item do
+
+    if item.id == 11 then
+      inmath = (item.subtype == 0)
+    end
+
+    if inmath then
+      -- pass
+
+    elseif item.id == 29 then
+      local locale = node.get_attribute(item, Babel.attr_locale)
+
+      if lang == locale or lang == nil then
+        lang = lang or locale
+        if Babel.ignore_pre_char(item) then  
+          word_string = word_string .. Babel.us_char
+        else
+          word_string = word_string .. unicode.utf8.char(item.char)
+        end
+        word_nodes[#word_nodes+1] = item
+      else
+        break
+      end
+
+    elseif item.id == 12 and item.subtype == 13 then
+      word_string = word_string .. ' '
+      word_nodes[#word_nodes+1] = item
+
+    -- Ignore leading unrecognized nodes, too.
+    elseif word_string ~= '' then
+      word_string = word_string .. Babel.us_char   
+      word_nodes[#word_nodes+1] = item  -- Will be ignored
+    end
+
+    item = item.next
+  end
+
+  -- Here and above we remove some trailing chars but not the
+  -- corresponding nodes. But they aren't accessed. 
+  if word_string:sub(-1) == ' ' then
+    word_string = word_string:sub(1,-2)
+  end
+  word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
+  return word_string, word_nodes, item, lang
+end
+
+Babel.fetch_subtext[1] = function(head)
+  local word_string = ''
+  local word_nodes = {}
+  local lang
+  local item = head
+  local inmath = false
+
+  while item do
+
+    if item.id == 11 then
+      inmath = (item.subtype == 0)
+    end
+
+    if inmath then
+      -- pass
+
+    elseif item.id == 29 then
+      if item.lang == lang or lang == nil then
+        if (item.char ~= 124) and (item.char ~= 61) then -- not =, not |
+          lang = lang or item.lang
+          word_string = word_string .. unicode.utf8.char(item.char)
+          word_nodes[#word_nodes+1] = item
+        end
+      else
+        break
+      end
+
+    elseif item.id == 7 and item.subtype == 2 then
+      word_string = word_string .. '='
+      word_nodes[#word_nodes+1] = item
+
+    elseif item.id == 7 and item.subtype == 3 then
+      word_string = word_string .. '|'       
+      word_nodes[#word_nodes+1] = item
+
+    -- (1) Go to next word if nothing was found, and (2) implicitly
+    -- remove leading USs.
+    elseif word_string == '' then
+      -- pass
+
+    -- This is the responsible for splitting by words.
+    elseif (item.id == 12 and item.subtype == 13) then
+      break
+
+    else 
+      word_string = word_string .. Babel.us_char   
+      word_nodes[#word_nodes+1] = item  -- Will be ignored
+    end
+
+    item = item.next
+  end
+
+  word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
+  return word_string, word_nodes, item, lang
+end
+
+function Babel.pre_hyphenate_replace(head)
+  Babel.hyphenate_replace(head, 0)
+end
+
+function Babel.post_hyphenate_replace(head)
+  Babel.hyphenate_replace(head, 1)
+end
+
+Babel.us_char = string.char(31)
+
+function Babel.hyphenate_replace(head, mode)
+  local u = unicode.utf8
+  local lbkr = Babel.linebreaking.replacements[mode]
+
+  local word_head = head
+
+  while true do  -- for each subtext block
+
+    local w, w_nodes, nw, lang = Babel.fetch_subtext[mode](word_head)
+
+    if Babel.debug then
+      print()
+      print((mode == 0) and '@@@@<' or '@@@@>', w)
+    end
+
+    if nw == nil and w == '' then break end
+
+    if not lang then goto next end
+    if not lbkr[lang] then goto next end
+
+    -- For each saved (pre|post)hyphenation. TODO. Reconsider how
+    -- loops are nested.
+    for k=1, #lbkr[lang] do
+      local p = lbkr[lang][k].pattern
+      local r = lbkr[lang][k].replace
+
+      if Babel.debug then
+        print('*****', p, mode)
+      end
+
+      -- This variable is set in some cases below to the first *byte*
+      -- after the match, either as found by u.match (faster) or the
+      -- computed position based on sc if w has changed.
+      local last_match = 0
+      local step = 0
+
+      -- For every match.
+      while true do
+        if Babel.debug then
+          print('=====')
+        end
+        local new  -- used when inserting and removing nodes
+
+        local matches = { u.match(w, p, last_match) }
+
+        if #matches < 2 then break end
+
+        -- Get and remove empty captures (with ()'s, which return a
+        -- number with the position), and keep actual captures
+        -- (from (...)), if any, in matches.
+        local first = table.remove(matches, 1)
+        local last  = table.remove(matches, #matches)
+        -- Non re-fetched substrings may contain \31, which separates
+        -- subsubstrings.
+        if string.find(w:sub(first, last-1), Babel.us_char) then break end
+
+        local save_last = last -- with A()BC()D, points to D
+
+        -- Fix offsets, from bytes to unicode. Explained above.
+        first = u.len(w:sub(1, first-1)) + 1
+        last  = u.len(w:sub(1, last-1)) -- now last points to C
+
+        -- This loop stores in n small table the nodes
+        -- corresponding to the pattern. Used by 'data' to provide a
+        -- predictable behavior with 'insert' (now w_nodes is modified on
+        -- the fly), and also access to 'remove'd nodes.
+        local sc = first-1           -- Used below, too
+        local data_nodes = {}
+
+        for q = 1, last-first+1 do
+          data_nodes[q] = w_nodes[sc+q]
+        end
+
+        -- This loop traverses the matched substring and takes the
+        -- corresponding action stored in the replacement list.
+        -- sc = the position in substr nodes / string
+        -- rc = the replacement table index
+        local rc = 0
+
+        while rc < last-first+1 do -- for each replacement
+          if Babel.debug then
+            print('.....', rc + 1)
+          end
+          sc = sc + 1
+          rc = rc + 1
+
+          if Babel.debug then
+            Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
+            local ss = ''
+            for itt in node.traverse(head) do
+             if itt.id == 29 then
+               ss = ss .. unicode.utf8.char(itt.char)
+             else
+               ss = ss .. '{' .. itt.id .. '}'
+             end
+            end
+            print('*****************', ss)
+
+          end
+
+          local crep = r[rc]
+          local item = w_nodes[sc]
+          local item_base = item
+          local placeholder = Babel.us_char
+          local d
+
+          if crep and crep.data then
+            item_base = data_nodes[crep.data]
+          end
+
+          if crep then
+            step = crep.step or 0
+          end
+
+          if crep and next(crep) == nil then -- = {}
+            last_match = save_last    -- Optimization
+            goto next
+
+          elseif crep == nil or crep.remove then
+            node.remove(head, item)
+            table.remove(w_nodes, sc)
+            w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1)
+            sc = sc - 1  -- Nothing has been inserted.
+            last_match = utf8.offset(w, sc+1+step)
+            goto next
+
+          elseif crep and crep.kashida then -- Experimental
+            node.set_attribute(item,
+               Babel.attr_kashida, 
+               crep.kashida)
+            last_match = utf8.offset(w, sc+1+step)
+            goto next
+
+          elseif crep and crep.string then
+            local str = crep.string(matches)
+            if str == '' then  -- Gather with nil
+              node.remove(head, item)
+              table.remove(w_nodes, sc)
+              w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1)
+              sc = sc - 1  -- Nothing has been inserted.
+            else
+              local loop_first = true
+              for s in string.utfvalues(str) do
+                d = node.copy(item_base)
+                d.char = s
+                if loop_first then
+                  loop_first = false
+                  head, new = node.insert_before(head, item, d)
+                  if sc == 1 then
+                    word_head = head
+                  end
+                  w_nodes[sc] = d
+                  w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc+1)
+                else
+                  sc = sc + 1
+                  head, new = node.insert_before(head, item, d)
+                  table.insert(w_nodes, sc, new)
+                  w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc)
+                end
+                if Babel.debug then
+                  print('.....', 'str')
+                  Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
+                end
+              end  -- for
+              node.remove(head, item)
+            end  -- if ''
+            last_match = utf8.offset(w, sc+1+step)
+            goto next
+
+          elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then
+            d = node.new(7, 0)   -- (disc, discretionary)
+            d.pre     = Babel.str_to_nodes(crep.pre, matches, item_base)
+            d.post    = Babel.str_to_nodes(crep.post, matches, item_base)
+            d.replace = Babel.str_to_nodes(crep.no, matches, item_base)
+            d.attr = item_base.attr
+            if crep.pre == nil then  -- TeXbook p96
+              d.penalty = crep.penalty or tex.hyphenpenalty
+            else
+              d.penalty = crep.penalty or tex.exhyphenpenalty
+            end
+            placeholder = '|'
+            head, new = node.insert_before(head, item, d)
+
+          elseif mode == 0 and crep and (crep.pre or crep.no or crep.post) then
+            -- ERROR
+
+          elseif crep and crep.penalty then
+            d = node.new(14, 0)   -- (penalty, userpenalty)
+            d.attr = item_base.attr
+            d.penalty = crep.penalty
+            head, new = node.insert_before(head, item, d)
+
+          elseif crep and crep.space then
+            -- 655360 = 10 pt = 10 * 65536 sp
+            d = node.new(12, 13)      -- (glue, spaceskip)
+            local quad = font.getfont(item_base.font).size or 655360
+            node.setglue(d, crep.space[1] * quad,
+                            crep.space[2] * quad,
+                            crep.space[3] * quad)
+            if mode == 0 then
+              placeholder = ' '
+            end
+            head, new = node.insert_before(head, item, d)
+
+          elseif crep and crep.spacefactor then
+            d = node.new(12, 13)      -- (glue, spaceskip)
+            local base_font = font.getfont(item_base.font)
+            node.setglue(d,
+              crep.spacefactor[1] * base_font.parameters['space'],
+              crep.spacefactor[2] * base_font.parameters['space_stretch'],
+              crep.spacefactor[3] * base_font.parameters['space_shrink'])
+            if mode == 0 then
+              placeholder = ' '
+            end
+            head, new = node.insert_before(head, item, d)
+
+          elseif mode == 0 and crep and crep.space then
+            -- ERROR
+
+          end  -- ie replacement cases
+
+          -- Shared by disc, space and penalty.
+          if sc == 1 then
+            word_head = head
+          end
+          if crep.insert then
+            w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc)
+            table.insert(w_nodes, sc, new)
+            last = last + 1
+          else
+            w_nodes[sc] = d
+            node.remove(head, item)
+            w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc+1)
+          end
+
+          last_match = utf8.offset(w, sc+1+step)
+
+          ::next::
+
+        end  -- for each replacement
+
+        if Babel.debug then
+            print('.....', '/')
+            Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
+        end
+
+      end  -- for match
+
+    end  -- for patterns
+
+    ::next::
+    word_head = nw
+  end  -- for substring
+  return head
+end
+
+-- This table stores capture maps, numbered consecutively
+Babel.capture_maps = {}
+
+-- The following functions belong to the next macro
+function Babel.capture_func(key, cap)
+  local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]"
+  local cnt
+  local u = unicode.utf8
+  ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map)
+  if cnt == 0 then
+    ret = u.gsub(ret, '{(%x%x%x%x+)}', 
+          function (n)
+            return u.char(tonumber(n, 16))
+          end)
+  end
+  ret = ret:gsub("%[%[%]%]%.%.", '')
+  ret = ret:gsub("%.%.%[%[%]%]", '')
+  return key .. [[=function(m) return ]] .. ret .. [[ end]]
+end
+
+function Babel.capt_map(from, mapno)
+  return Babel.capture_maps[mapno][from] or from
+end
+
+-- Handle the {n|abc|ABC} syntax in captures
+function Babel.capture_func_map(capno, from, to)
+  local u = unicode.utf8
+  from = u.gsub(from, '{(%x%x%x%x+)}', 
+       function (n)
+         return u.char(tonumber(n, 16))
+       end)
+  to = u.gsub(to, '{(%x%x%x%x+)}', 
+       function (n)
+         return u.char(tonumber(n, 16))
+       end)
+  local froms = {}
+  for s in string.utfcharacters(from) do
+    table.insert(froms, s)
+  end
+  local cnt = 1
+  table.insert(Babel.capture_maps, {})
+  local mlen = table.getn(Babel.capture_maps)
+  for s in string.utfcharacters(to) do
+    Babel.capture_maps[mlen][froms[cnt]] = s
+    cnt = cnt + 1
+  end
+  return "]]..Babel.capt_map(m[" .. capno .. "]," ..
+         (mlen) .. ").." .. "[["
+end
+
+-- Create/Extend reversed sorted list of kashida weights:
+function Babel.capture_kashida(key, wt)
+  wt = tonumber(wt)
+  if Babel.kashida_wts then
+    for p, q in ipairs(Babel.kashida_wts) do
+      if wt  == q then
+        break
+      elseif wt > q then
+        table.insert(Babel.kashida_wts, p, wt)
+        break
+      elseif table.getn(Babel.kashida_wts) == p then
+        table.insert(Babel.kashida_wts, wt)
+      end
+    end
+  else
+    Babel.kashida_wts = { wt }
+  end
+  return 'kashida = ' .. wt
+end
+%</transforms>
+%    \end{macrocode}
+%
+% \subsection{Lua: Auto bidi with \texttt{basic} and \texttt{basic-r}}
+%
 % The file \textsf{babel-data-bidi.lua} currently only contains data. It is
 % a large and boring file and it is not shown here (see the generated file),
 % but here is a sample:
@@ -21371,7 +21438,7 @@
       if new_dir then
         attr_dir = 0
         for at in node.traverse(item.attr) do
-          if at.number == luatexbase.registernumber'bbl at attr@dir' then
+          if at.number == Babel.attr_dir then
             attr_dir = at.value % 3
           end
         end
@@ -21625,7 +21692,7 @@
   local has_en = false
   local first_et = nil
 
-  local ATDIR = luatexbase.registernumber'bbl at attr@dir'
+  local ATDIR = Babel.attr_dir
 
   local save_outer
   local temp = node.get_attribute(head, ATDIR)
@@ -22645,7 +22712,7 @@
 \fi
 %    \end{macrocode}
 %
-%    To prevent wasting two counters in \LaTeX$\:$2.09 (because
+%    To prevent wasting two counters in \LaTeX\ (because
 %    counters with the same name are allocated later by it) we reset
 %    the counter that holds the next free counter (|\count10|).
 %

Modified: trunk/Master/texmf-dist/source/latex/babel/babel.ins
===================================================================
--- trunk/Master/texmf-dist/source/latex/babel/babel.ins	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/source/latex/babel/babel.ins	2021-07-19 20:10:36 UTC (rev 59990)
@@ -26,7 +26,7 @@
 %% and covered by LPPL is defined by the unpacking scripts (with
 %% extension .ins) which are part of the distribution.
 %%
-\def\filedate{2021/06/28}
+\def\filedate{2021/07/19}
 \def\batchfile{babel.ins}
 \input docstrip.tex
 
@@ -221,6 +221,7 @@
 \def\MetaPrefix{--}
 \usepreamble\luapreamble
 \nopostamble
+\generate{\file{babel-transforms.lua}{\from{babel.dtx}{transforms}}}
 \generate{\file{babel-data-bidi.lua}{\from{babel.dtx}{bididata}}}
 \generate{\file{babel-data-cjk.lua}{\from{babel.dtx}{cjkdata}}}
 \generate{\file{babel-bidi-basic-r.lua}{\from{babel.dtx}{basic-r}}}

Modified: trunk/Master/texmf-dist/source/latex/babel/bbcompat.dtx
===================================================================
--- trunk/Master/texmf-dist/source/latex/babel/bbcompat.dtx	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/source/latex/babel/bbcompat.dtx	2021-07-19 20:10:36 UTC (rev 59990)
@@ -30,7 +30,7 @@
 %
 % \iffalse
 %<*dtx>
-\ProvidesFile{bbcompat.dtx}[2021/06/28 v3.61]
+\ProvidesFile{bbcompat.dtx}[2021/07/19 v3.62]
 %</dtx>
 %
 %% File 'bbcompat.dtx'

Modified: trunk/Master/texmf-dist/source/latex/babel/locale.zip
===================================================================
(Binary files differ)

Modified: trunk/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua
===================================================================
--- trunk/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/tex/generic/babel/babel-bidi-basic-r.lua	2021-07-19 20:10:36 UTC (rev 59990)
@@ -100,7 +100,7 @@
       if new_dir then
         attr_dir = 0
         for at in node.traverse(item.attr) do
-          if at.number == luatexbase.registernumber'bbl at attr@dir' then
+          if at.number == Babel.attr_dir then
             attr_dir = at.value % 3
           end
         end

Modified: trunk/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua
===================================================================
--- trunk/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/tex/generic/babel/babel-bidi-basic.lua	2021-07-19 20:10:36 UTC (rev 59990)
@@ -106,7 +106,7 @@
   local has_en = false
   local first_et = nil
 
-  local ATDIR = luatexbase.registernumber'bbl at attr@dir'
+  local ATDIR = Babel.attr_dir
 
   local save_outer
   local temp = node.get_attribute(head, ATDIR)

Added: trunk/Master/texmf-dist/tex/generic/babel/babel-transforms.lua
===================================================================
--- trunk/Master/texmf-dist/tex/generic/babel/babel-transforms.lua	                        (rev 0)
+++ trunk/Master/texmf-dist/tex/generic/babel/babel-transforms.lua	2021-07-19 20:10:36 UTC (rev 59990)
@@ -0,0 +1,511 @@
+--
+-- This is file `babel-transforms.lua',
+-- generated with the docstrip utility.
+--
+-- The original source files were:
+--
+-- babel.dtx  (with options: `transforms')
+-- 
+--
+-- Copyright (C) 2012-2021 Javier Bezos and Johannes L. Braams.
+-- Copyright (C) 1989-2012 Johannes L. Braams and
+--           any individual authors listed elsewhere in this file.
+-- All rights reserved.
+--
+--
+-- This file is part of the Babel system.
+-- --------------------------------------
+--
+-- It may be distributed and/or modified under the
+-- conditions of the LaTeX Project Public License, either version 1.3
+-- of this license or (at your option) any later version.
+-- The latest version of this license is in
+--   http://www.latex-project.org/lppl.txt
+-- and version 1.3 or later is part of all distributions of LaTeX
+-- version 2003/12/01 or later.
+--
+-- This work has the LPPL maintenance status "maintained".
+--
+-- The Current Maintainer of this work is Javier Bezos.
+--
+-- The list of derived (unpacked) files belonging to the distribution
+-- and covered by LPPL is defined by the unpacking scripts (with
+-- extension |.ins|) which are part of the distribution.
+--
+
+Babel.linebreaking.replacements = {}
+Babel.linebreaking.replacements[0] = {}  -- pre
+Babel.linebreaking.replacements[1] = {}  -- post
+
+-- Discretionaries contain strings as nodes
+function Babel.str_to_nodes(fn, matches, base)
+  local n, head, last
+  if fn == nil then return nil end
+  for s in string.utfvalues(fn(matches)) do
+    if base.id == 7 then
+      base = base.replace
+    end
+    n = node.copy(base)
+    n.char    = s
+    if not head then
+      head = n
+    else
+      last.next = n
+    end
+    last = n
+  end
+  return head
+end
+
+Babel.fetch_subtext = {}
+
+Babel.ignore_pre_char = function(node)
+  return (node.lang == Babel.nohyphenation)
+end
+
+-- Merging both functions doesn't seen feasible, because there are too
+-- many differences.
+Babel.fetch_subtext[0] = function(head)
+  local word_string = ''
+  local word_nodes = {}
+  local lang
+  local item = head
+  local inmath = false
+
+  while item do
+
+    if item.id == 11 then
+      inmath = (item.subtype == 0)
+    end
+
+    if inmath then
+      -- pass
+
+    elseif item.id == 29 then
+      local locale = node.get_attribute(item, Babel.attr_locale)
+
+      if lang == locale or lang == nil then
+        lang = lang or locale
+        if Babel.ignore_pre_char(item) then
+          word_string = word_string .. Babel.us_char
+        else
+          word_string = word_string .. unicode.utf8.char(item.char)
+        end
+        word_nodes[#word_nodes+1] = item
+      else
+        break
+      end
+
+    elseif item.id == 12 and item.subtype == 13 then
+      word_string = word_string .. ' '
+      word_nodes[#word_nodes+1] = item
+
+    -- Ignore leading unrecognized nodes, too.
+    elseif word_string ~= '' then
+      word_string = word_string .. Babel.us_char
+      word_nodes[#word_nodes+1] = item  -- Will be ignored
+    end
+
+    item = item.next
+  end
+
+  -- Here and above we remove some trailing chars but not the
+  -- corresponding nodes. But they aren't accessed.
+  if word_string:sub(-1) == ' ' then
+    word_string = word_string:sub(1,-2)
+  end
+  word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
+  return word_string, word_nodes, item, lang
+end
+
+Babel.fetch_subtext[1] = function(head)
+  local word_string = ''
+  local word_nodes = {}
+  local lang
+  local item = head
+  local inmath = false
+
+  while item do
+
+    if item.id == 11 then
+      inmath = (item.subtype == 0)
+    end
+
+    if inmath then
+      -- pass
+
+    elseif item.id == 29 then
+      if item.lang == lang or lang == nil then
+        if (item.char ~= 124) and (item.char ~= 61) then -- not =, not |
+          lang = lang or item.lang
+          word_string = word_string .. unicode.utf8.char(item.char)
+          word_nodes[#word_nodes+1] = item
+        end
+      else
+        break
+      end
+
+    elseif item.id == 7 and item.subtype == 2 then
+      word_string = word_string .. '='
+      word_nodes[#word_nodes+1] = item
+
+    elseif item.id == 7 and item.subtype == 3 then
+      word_string = word_string .. '|'
+      word_nodes[#word_nodes+1] = item
+
+    -- (1) Go to next word if nothing was found, and (2) implicitly
+    -- remove leading USs.
+    elseif word_string == '' then
+      -- pass
+
+    -- This is the responsible for splitting by words.
+    elseif (item.id == 12 and item.subtype == 13) then
+      break
+
+    else
+      word_string = word_string .. Babel.us_char
+      word_nodes[#word_nodes+1] = item  -- Will be ignored
+    end
+
+    item = item.next
+  end
+
+  word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
+  return word_string, word_nodes, item, lang
+end
+
+function Babel.pre_hyphenate_replace(head)
+  Babel.hyphenate_replace(head, 0)
+end
+
+function Babel.post_hyphenate_replace(head)
+  Babel.hyphenate_replace(head, 1)
+end
+
+Babel.us_char = string.char(31)
+
+function Babel.hyphenate_replace(head, mode)
+  local u = unicode.utf8
+  local lbkr = Babel.linebreaking.replacements[mode]
+
+  local word_head = head
+
+  while true do  -- for each subtext block
+
+    local w, w_nodes, nw, lang = Babel.fetch_subtext[mode](word_head)
+
+    if Babel.debug then
+      print()
+      print((mode == 0) and '@@@@<' or '@@@@>', w)
+    end
+
+    if nw == nil and w == '' then break end
+
+    if not lang then goto next end
+    if not lbkr[lang] then goto next end
+
+    -- For each saved (pre|post)hyphenation. TODO. Reconsider how
+    -- loops are nested.
+    for k=1, #lbkr[lang] do
+      local p = lbkr[lang][k].pattern
+      local r = lbkr[lang][k].replace
+
+      if Babel.debug then
+        print('*****', p, mode)
+      end
+
+      -- This variable is set in some cases below to the first *byte*
+      -- after the match, either as found by u.match (faster) or the
+      -- computed position based on sc if w has changed.
+      local last_match = 0
+      local step = 0
+
+      -- For every match.
+      while true do
+        if Babel.debug then
+          print('=====')
+        end
+        local new  -- used when inserting and removing nodes
+
+        local matches = { u.match(w, p, last_match) }
+
+        if #matches < 2 then break end
+
+        -- Get and remove empty captures (with ()'s, which return a
+        -- number with the position), and keep actual captures
+        -- (from (...)), if any, in matches.
+        local first = table.remove(matches, 1)
+        local last  = table.remove(matches, #matches)
+        -- Non re-fetched substrings may contain \31, which separates
+        -- subsubstrings.
+        if string.find(w:sub(first, last-1), Babel.us_char) then break end
+
+        local save_last = last -- with A()BC()D, points to D
+
+        -- Fix offsets, from bytes to unicode. Explained above.
+        first = u.len(w:sub(1, first-1)) + 1
+        last  = u.len(w:sub(1, last-1)) -- now last points to C
+
+        -- This loop stores in n small table the nodes
+        -- corresponding to the pattern. Used by 'data' to provide a
+        -- predictable behavior with 'insert' (now w_nodes is modified on
+        -- the fly), and also access to 'remove'd nodes.
+        local sc = first-1           -- Used below, too
+        local data_nodes = {}
+
+        for q = 1, last-first+1 do
+          data_nodes[q] = w_nodes[sc+q]
+        end
+
+        -- This loop traverses the matched substring and takes the
+        -- corresponding action stored in the replacement list.
+        -- sc = the position in substr nodes / string
+        -- rc = the replacement table index
+        local rc = 0
+
+        while rc < last-first+1 do -- for each replacement
+          if Babel.debug then
+            print('.....', rc + 1)
+          end
+          sc = sc + 1
+          rc = rc + 1
+
+          if Babel.debug then
+            Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
+            local ss = ''
+            for itt in node.traverse(head) do
+             if itt.id == 29 then
+               ss = ss .. unicode.utf8.char(itt.char)
+             else
+               ss = ss .. '{' .. itt.id .. '}'
+             end
+            end
+            print('*****************', ss)
+
+          end
+
+          local crep = r[rc]
+          local item = w_nodes[sc]
+          local item_base = item
+          local placeholder = Babel.us_char
+          local d
+
+          if crep and crep.data then
+            item_base = data_nodes[crep.data]
+          end
+
+          if crep then
+            step = crep.step or 0
+          end
+
+          if crep and next(crep) == nil then -- = {}
+            last_match = save_last    -- Optimization
+            goto next
+
+          elseif crep == nil or crep.remove then
+            node.remove(head, item)
+            table.remove(w_nodes, sc)
+            w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1)
+            sc = sc - 1  -- Nothing has been inserted.
+            last_match = utf8.offset(w, sc+1+step)
+            goto next
+
+          elseif crep and crep.kashida then -- Experimental
+            node.set_attribute(item,
+               Babel.attr_kashida,
+               crep.kashida)
+            last_match = utf8.offset(w, sc+1+step)
+            goto next
+
+          elseif crep and crep.string then
+            local str = crep.string(matches)
+            if str == '' then  -- Gather with nil
+              node.remove(head, item)
+              table.remove(w_nodes, sc)
+              w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1)
+              sc = sc - 1  -- Nothing has been inserted.
+            else
+              local loop_first = true
+              for s in string.utfvalues(str) do
+                d = node.copy(item_base)
+                d.char = s
+                if loop_first then
+                  loop_first = false
+                  head, new = node.insert_before(head, item, d)
+                  if sc == 1 then
+                    word_head = head
+                  end
+                  w_nodes[sc] = d
+                  w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc+1)
+                else
+                  sc = sc + 1
+                  head, new = node.insert_before(head, item, d)
+                  table.insert(w_nodes, sc, new)
+                  w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc)
+                end
+                if Babel.debug then
+                  print('.....', 'str')
+                  Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
+                end
+              end  -- for
+              node.remove(head, item)
+            end  -- if ''
+            last_match = utf8.offset(w, sc+1+step)
+            goto next
+
+          elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then
+            d = node.new(7, 0)   -- (disc, discretionary)
+            d.pre     = Babel.str_to_nodes(crep.pre, matches, item_base)
+            d.post    = Babel.str_to_nodes(crep.post, matches, item_base)
+            d.replace = Babel.str_to_nodes(crep.no, matches, item_base)
+            d.attr = item_base.attr
+            if crep.pre == nil then  -- TeXbook p96
+              d.penalty = crep.penalty or tex.hyphenpenalty
+            else
+              d.penalty = crep.penalty or tex.exhyphenpenalty
+            end
+            placeholder = '|'
+            head, new = node.insert_before(head, item, d)
+
+          elseif mode == 0 and crep and (crep.pre or crep.no or crep.post) then
+            -- ERROR
+
+          elseif crep and crep.penalty then
+            d = node.new(14, 0)   -- (penalty, userpenalty)
+            d.attr = item_base.attr
+            d.penalty = crep.penalty
+            head, new = node.insert_before(head, item, d)
+
+          elseif crep and crep.space then
+            -- 655360 = 10 pt = 10 * 65536 sp
+            d = node.new(12, 13)      -- (glue, spaceskip)
+            local quad = font.getfont(item_base.font).size or 655360
+            node.setglue(d, crep.space[1] * quad,
+                            crep.space[2] * quad,
+                            crep.space[3] * quad)
+            if mode == 0 then
+              placeholder = ' '
+            end
+            head, new = node.insert_before(head, item, d)
+
+          elseif crep and crep.spacefactor then
+            d = node.new(12, 13)      -- (glue, spaceskip)
+            local base_font = font.getfont(item_base.font)
+            node.setglue(d,
+              crep.spacefactor[1] * base_font.parameters['space'],
+              crep.spacefactor[2] * base_font.parameters['space_stretch'],
+              crep.spacefactor[3] * base_font.parameters['space_shrink'])
+            if mode == 0 then
+              placeholder = ' '
+            end
+            head, new = node.insert_before(head, item, d)
+
+          elseif mode == 0 and crep and crep.space then
+            -- ERROR
+
+          end  -- ie replacement cases
+
+          -- Shared by disc, space and penalty.
+          if sc == 1 then
+            word_head = head
+          end
+          if crep.insert then
+            w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc)
+            table.insert(w_nodes, sc, new)
+            last = last + 1
+          else
+            w_nodes[sc] = d
+            node.remove(head, item)
+            w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc+1)
+          end
+
+          last_match = utf8.offset(w, sc+1+step)
+
+          ::next::
+
+        end  -- for each replacement
+
+        if Babel.debug then
+            print('.....', '/')
+            Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
+        end
+
+      end  -- for match
+
+    end  -- for patterns
+
+    ::next::
+    word_head = nw
+  end  -- for substring
+  return head
+end
+
+-- This table stores capture maps, numbered consecutively
+Babel.capture_maps = {}
+
+-- The following functions belong to the next macro
+function Babel.capture_func(key, cap)
+  local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]"
+  local cnt
+  local u = unicode.utf8
+  ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map)
+  if cnt == 0 then
+    ret = u.gsub(ret, '{(%x%x%x%x+)}',
+          function (n)
+            return u.char(tonumber(n, 16))
+          end)
+  end
+  ret = ret:gsub("%[%[%]%]%.%.", '')
+  ret = ret:gsub("%.%.%[%[%]%]", '')
+  return key .. [[=function(m) return ]] .. ret .. [[ end]]
+end
+
+function Babel.capt_map(from, mapno)
+  return Babel.capture_maps[mapno][from] or from
+end
+
+-- Handle the {n|abc|ABC} syntax in captures
+function Babel.capture_func_map(capno, from, to)
+  local u = unicode.utf8
+  from = u.gsub(from, '{(%x%x%x%x+)}',
+       function (n)
+         return u.char(tonumber(n, 16))
+       end)
+  to = u.gsub(to, '{(%x%x%x%x+)}',
+       function (n)
+         return u.char(tonumber(n, 16))
+       end)
+  local froms = {}
+  for s in string.utfcharacters(from) do
+    table.insert(froms, s)
+  end
+  local cnt = 1
+  table.insert(Babel.capture_maps, {})
+  local mlen = table.getn(Babel.capture_maps)
+  for s in string.utfcharacters(to) do
+    Babel.capture_maps[mlen][froms[cnt]] = s
+    cnt = cnt + 1
+  end
+  return "]]..Babel.capt_map(m[" .. capno .. "]," ..
+         (mlen) .. ").." .. "[["
+end
+
+-- Create/Extend reversed sorted list of kashida weights:
+function Babel.capture_kashida(key, wt)
+  wt = tonumber(wt)
+  if Babel.kashida_wts then
+    for p, q in ipairs(Babel.kashida_wts) do
+      if wt  == q then
+        break
+      elseif wt > q then
+        table.insert(Babel.kashida_wts, p, wt)
+        break
+      elseif table.getn(Babel.kashida_wts) == p then
+        table.insert(Babel.kashida_wts, wt)
+      end
+    end
+  else
+    Babel.kashida_wts = { wt }
+  end
+  return 'kashida = ' .. wt
+end


Property changes on: trunk/Master/texmf-dist/tex/generic/babel/babel-transforms.lua
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Modified: trunk/Master/texmf-dist/tex/generic/babel/babel.def
===================================================================
--- trunk/Master/texmf-dist/tex/generic/babel/babel.def	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/tex/generic/babel/babel.def	2021-07-19 20:10:36 UTC (rev 59990)
@@ -40,7 +40,7 @@
     \wlog{File: #1 #4 #3 <#2>}%
     \let\ProvidesFile\@undefined}
 \fi
-\ProvidesFile{babel.def}[2021/06/28 3.61 Babel common definitions]
+\ProvidesFile{babel.def}[2021/07/19 3.62 Babel common definitions]
 \ifx\AtBeginDocument\@undefined  % TODO. change test.
     % == Code for plain ==
 \def\@empty{}
@@ -389,17 +389,13 @@
   \expandafter\newif\csname ifbbl at single\endcsname
   \chardef\bbl at bidimode\z@
 \fi
-\ifx\bbl at trace\@undefined
-  \let\LdfInit\endinput
-  \def\ProvidesLanguage#1{\endinput}
-\endinput\fi % Same line!
 \ifx\language\@undefined
   \csname newcount\endcsname\language
 \fi
-\countdef\last at language=19  % TODO. why? remove?
+\countdef\last at language=19
 \def\addlanguage{\csname newlanguage\endcsname}
-\def\bbl at version{3.61}
-\def\bbl at date{2021/06/28}
+\def\bbl at version{3.62}
+\def\bbl at date{2021/07/19}
 \def\adddialect#1#2{%
   \global\chardef#1#2\relax
   \bbl at usehooks{adddialect}{{#1}{#2}}%
@@ -522,7 +518,15 @@
 \def\bbl at language@stack{}
 \def\bbl at push@language{%
   \ifx\languagename\@undefined\else
-    \xdef\bbl at language@stack{\languagename+\bbl at language@stack}%
+    \ifx\currentgrouplevel\@undefined
+      \xdef\bbl at language@stack{\languagename+\bbl at language@stack}%
+    \else
+      \ifnum\currentgrouplevel=\z@
+        \xdef\bbl at language@stack{\languagename+}%
+      \else
+        \xdef\bbl at language@stack{\languagename+\bbl at language@stack}%
+      \fi
+    \fi
   \fi}
 \def\bbl at pop@lang#1+#2\@@{%
   \edef\languagename{#1}%
@@ -631,10 +635,10 @@
       \let\bbl at select@type\z@
       \expandafter\bbl at switch\expandafter{\languagename}%
     \fi}}
-\def\babel at aux#1#2{% TODO. See how to avoid undefined nil's
+\def\babel at aux#1#2{%
   \select at language{#1}%
-  \bbl at foreach\BabelContentsFiles{%
-    \@writefile{##1}{\babel at toc{#1}{#2}}}}% %% TODO - ok in plain?
+  \bbl at foreach\BabelContentsFiles{% \relax -> don't assume vertical mode
+    \@writefile{##1}{\babel at toc{#1}{#2}\relax}}}% TODO - plain?
 \def\babel at toc#1#2{%
   \select at language{#1}}
 \newif\ifbbl at usedategroup
@@ -1094,6 +1098,11 @@
   \else
     \expandafter\@firstofone
   \fi}
+\ifx\IfFormatAtLeastTF\@undefined
+  \def\bbl at ifformatlater{\@ifl at t@r\fmtversion}
+\else
+  \let\bbl at ifformatlater\IfFormatAtLeastTF
+\fi
 \def\bbl at extras@wrap#1#2#3{% 1:in-test, 2:before, 3:after
   \toks@\expandafter\expandafter\expandafter{%
     \csname extras\languagename\endcsname}%
@@ -1171,10 +1180,12 @@
 \newcommand\EnableBabelHook[1]{\bbl at csarg\let{hk@#1}\@firstofone}
 \newcommand\DisableBabelHook[1]{\bbl at csarg\let{hk@#1}\@gobble}
 \def\bbl at usehooks#1#2{%
+  \ifx\UseHook\@undefined\else\UseHook{babel/#1}\fi
   \def\bbl at elth##1{%
     \bbl at cs{hk@##1}{\bbl at cs{ev@##1@#1@}#2}}%
   \bbl at cs{ev@#1@}%
   \ifx\languagename\@undefined\else % Test required for Plain (?)
+    \ifx\UseHook\@undefined\else\UseHook{babel/#1/\languagename}\fi
     \def\bbl at elth##1{%
       \bbl at cs{hk@##1}{\bbl at cl{ev@##1@#1}#2}}%
     \bbl at cl{ev@#1}%
@@ -1185,6 +1196,10 @@
   beforeextras=0,afterextras=0,stopcommands=0,stringprocess=0,%
   hyphenation=2,initiateactive=3,afterreset=0,foreign=0,foreign*=0,%
   beforestart=0,languagename=2}
+\ifx\NewHook\@undefined\else
+  \def\bbl at tempa#1=#2\@@{\NewHook{babel/#1}}
+  \bbl at foreach\bbl at evargs{\bbl at tempa#1\@@}
+\fi
 \bbl at trace{Defining babelensure}
 \newcommand\babelensure[2][]{%  TODO - revise test files
   \AddBabelHook{babel-ensure}{afterextras}{%
@@ -1287,9 +1302,7 @@
   \let\BabelModifiers\relax
   \let\bbl at screset\relax}%
 \def\ldf at finish#1{%
-  \ifx\loadlocalcfg\@undefined\else % For LaTeX 209
-    \loadlocalcfg{#1}%
-  \fi
+  \loadlocalcfg{#1}%
   \bbl at afterldf{#1}%
   \expandafter\main at language\expandafter{#1}%
   \catcode`\@=\atcatcode \let\atcatcode\relax
@@ -1303,10 +1316,12 @@
   \bbl at id@assign
   \bbl at patterns{\languagename}}
 \def\bbl at beforestart{%
+  \def\@nolanerr##1{%
+    \bbl at warning{Undefined language '##1' in aux.\\Reported}}%
   \bbl at usehooks{beforestart}{}%
   \global\let\bbl at beforestart\relax}
 \AtBeginDocument{%
-  \@nameuse{bbl at beforestart}%
+  {\@nameuse{bbl at beforestart}}%  Group!
   \if at filesw
     \providecommand\babel at aux[2]{}%
     \immediate\write\@mainaux{%
@@ -2459,7 +2474,7 @@
   \ifx\bbl at KVP@transforms\@nil\else
     \bbl at replace\bbl at KVP@transforms{ }{,}%
   \fi
-  % Load ini
+  % == Load ini ==
   \ifcase\bbl at howloaded
     \bbl at provide@new{#2}%
   \else
@@ -2542,7 +2557,7 @@
         end}%
       \ifx\bbl at mapselect\@undefined  % TODO. almost the same as mapfont
         \AtBeginDocument{%
-          \expandafter\bbl at add\csname selectfont \endcsname{{\bbl at mapselect}}%
+          \bbl at patchfont{{\bbl at mapselect}}%
           {\selectfont}}%
         \def\bbl at mapselect{%
           \let\bbl at mapselect\relax
@@ -2568,9 +2583,9 @@
                  {See the manual for details.}}}%
     \bbl at ifunset{bbl at lsys@\languagename}{\bbl at provide@lsys{\languagename}}{}%
     \bbl at ifunset{bbl at wdir@\languagename}{\bbl at provide@dirs{\languagename}}{}%
-    \ifx\bbl at mapselect\@undefined % TODO. See onchar
+    \ifx\bbl at mapselect\@undefined % TODO. See onchar.
       \AtBeginDocument{%
-        \expandafter\bbl at add\csname selectfont \endcsname{{\bbl at mapselect}}%
+        \bbl at patchfont{{\bbl at mapselect}}%
         {\selectfont}}%
       \def\bbl at mapselect{%
         \let\bbl at mapselect\relax
@@ -2591,6 +2606,24 @@
     \bbl at csarg\edef{intsp@#2}{\bbl at KVP@intraspace}%
   \fi
   \bbl at provide@intraspace
+  % == Line breaking: CJK quotes ==
+  \ifcase\bbl at engine\or
+    \bbl at xin@{/c}{/\bbl at cl{lnbrk}}%
+    \ifin@
+      \bbl at ifunset{bbl at quote@\languagename}{}%
+        {\directlua{
+           Babel.locale_props[\the\localeid].cjk_quotes = {}
+           local cs = 'op'
+           for c in string.utfvalues(%
+               [[\csname bbl at quote@\languagename\endcsname]]) do
+             if Babel.cjk_characters[c].c == 'qu' then
+               Babel.locale_props[\the\localeid].cjk_quotes[c] = cs
+             end
+             cs = ( cs == 'op') and 'cl' or 'op'
+           end
+        }}%
+    \fi
+  \fi
   % == Line breaking: justification ==
   \ifx\bbl at KVP@justification\@nil\else
      \let\bbl at KVP@linebreaking\bbl at KVP@justification
@@ -2663,7 +2696,7 @@
              table.pack(string.utfvalue('\bbl at cl{dgnat}'))
            if not Babel.numbers then
              function Babel.numbers(head)
-               local LOCALE = luatexbase.registernumber'bbl at attr@locale'
+               local LOCALE = Babel.attr_locale
                local GLYPH = node.id'glyph'
                local inmath = false
                for item in node.traverse(head) do
@@ -2782,13 +2815,13 @@
     \StartBabelCommands*{#1}{captions}%
       \bbl at read@ini{\bbl at KVP@captions}2%   % Here all letters cat = 11
     \EndBabelCommands
- \fi
- \ifx\bbl at KVP@import\@nil\else
-   \StartBabelCommands*{#1}{date}%
-     \bbl at savetoday
-     \bbl at savedate
-   \EndBabelCommands
   \fi
+  \ifx\bbl at KVP@import\@nil\else
+    \StartBabelCommands*{#1}{date}%
+      \bbl at savetoday
+      \bbl at savedate
+    \EndBabelCommands
+  \fi
   % == hyphenrules (also in new) ==
   \ifx\bbl at lbkflag\@empty
     \bbl at provide@hyphens{#1}%
@@ -3046,6 +3079,7 @@
     \bbl at exportkey{intsp}{typography.intraspace}{}%
     \bbl at exportkey{frspc}{typography.frenchspacing}{u}%
     \bbl at exportkey{chrng}{characters.ranges}{}%
+    \bbl at exportkey{quote}{characters.delimiters.quotes}{}%
     \bbl at exportkey{dgnat}{numbers.digits.native}{}%
     \ifnum#1=\tw@           % only (re)new
       \bbl at exportkey{rqtex}{identification.require.babel}{}%
@@ -3304,7 +3338,7 @@
        range 0-9999.}%
       {There is little you can do. Sorry.}%
   \fi\fi\fi\fi}}
-\newcommand\BabelDateyyyy[1]{{\number#1}} % FIXME - add leading 0
+\newcommand\BabelDateyyyy[1]{{\number#1}} % TODO - add leading 0
 \def\bbl at replace@finish at iii#1{%
   \bbl at exp{\def\\#1####1####2####3{\the\toks@}}}
 \def\bbl at TG@@date{%
@@ -3372,10 +3406,8 @@
              \expandafter\@secondoftwo  % to execute right now
            \fi
            \AtBeginDocument{%
-             \expandafter\bbl at add
-             \csname selectfont \endcsname{\bbl at xenohyph}%
-             \expandafter\selectlanguage\expandafter{\languagename}%
-             \expandafter\bbl at toglobal\csname selectfont \endcsname}%
+             \bbl at patchfont{\bbl at xenohyph}%
+             \expandafter\selectlanguage\expandafter{\languagename}}%
         \fi}}%
   \fi
   \bbl at csarg\bbl at toglobal{lsys@#1}}

Modified: trunk/Master/texmf-dist/tex/generic/babel/babel.sty
===================================================================
--- trunk/Master/texmf-dist/tex/generic/babel/babel.sty	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/tex/generic/babel/babel.sty	2021-07-19 20:10:36 UTC (rev 59990)
@@ -34,7 +34,7 @@
 %%
 
 \NeedsTeXFormat{LaTeX2e}[2005/12/01]
-\ProvidesPackage{babel}[2021/06/28 3.61 The Babel package]
+\ProvidesPackage{babel}[2021/07/19 3.62 The Babel package]
 \@ifpackagewith{babel}{debug}
   {\providecommand\bbl at trace[1]{\message{^^J[ #1 ]}}%
    \let\bbl at debug\@firstofone
@@ -41,6 +41,7 @@
    \ifx\directlua\@undefined\else
      \directlua{ Babel = Babel or {}
        Babel.debug = true }%
+     \input{babel-debug.tex}%
    \fi}
   {\providecommand\bbl at trace[1]{}%
    \let\bbl at debug\@gobble
@@ -217,6 +218,11 @@
   \else
     \expandafter\@firstofone
   \fi}
+\ifx\IfFormatAtLeastTF\@undefined
+  \def\bbl at ifformatlater{\@ifl at t@r\fmtversion}
+\else
+  \let\bbl at ifformatlater\IfFormatAtLeastTF
+\fi
 \def\bbl at extras@wrap#1#2#3{% 1:in-test, 2:before, 3:after
   \toks@\expandafter\expandafter\expandafter{%
     \csname extras\languagename\endcsname}%
@@ -419,9 +425,6 @@
 \ProcessOptions*
 \ifx\bbl at opt@provide\@nnil\else % Tests. Ignore.
   \chardef\bbl at iniflag\@ne
-  \bbl at replace\bbl at opt@provide{;}{,}
-  \bbl at add\bbl at opt@provide{,import}
-  \show\bbl at opt@provide
 \fi
 \bbl at trace{Conditional loading of shorthands}
 \def\bbl at sh@string#1{%
@@ -665,40 +668,37 @@
   }
 \@onlypreamble\substitutefontfamily
 \bbl at trace{Encoding and fonts}
-\newcommand\BabelNonASCII{LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU,PU,PD1}
+\newcommand\BabelNonASCII{LGR,X2,OT2,OT3,OT6,LHE,LWN,LMA,LMC,LMS,LMU}
 \newcommand\BabelNonText{TS1,T3,TS3}
 \let\org at TeX\TeX
 \let\org at LaTeX\LaTeX
 \let\ensureascii\@firstofone
 \AtBeginDocument{%
-  \in at false
-  \bbl at foreach\BabelNonASCII{% is there a text non-ascii enc?
-    \ifin@\else
-      \lowercase{\bbl at xin@{,#1enc.def,}{,\@filelist,}}%
+  \def\@elt#1{,#1,}%
+  \edef\bbl at tempa{\expandafter\@gobbletwo\@fontenc at load@list}%
+  \let\@elt\relax
+  \let\bbl at tempb\@empty
+  \def\bbl at tempc{OT1}%
+  \bbl at foreach\BabelNonASCII{% LGR loaded in a non-standard way
+    \bbl at ifunset{T@#1}{}{\def\bbl at tempb{#1}}}%
+  \bbl at foreach\bbl at tempa{%
+    \bbl at xin@{#1}{\BabelNonASCII}%
+    \ifin@
+      \def\bbl at tempb{#1}% Store last non-ascii
+    \else\bbl at xin@{#1}{\BabelNonText}% Pass
+      \ifin@\else
+        \def\bbl at tempc{#1}% Store last ascii
+      \fi
     \fi}%
-  \ifin@ % if a text non-ascii has been loaded
-    \def\ensureascii#1{{\fontencoding{OT1}\selectfont#1}}%
-    \DeclareTextCommandDefault{\TeX}{\org at TeX}%
-    \DeclareTextCommandDefault{\LaTeX}{\org at LaTeX}%
-    \def\bbl at tempb#1\@@{\uppercase{\bbl at tempc#1}ENC.DEF\@empty\@@}%
-    \def\bbl at tempc#1ENC.DEF#2\@@{%
-      \ifx\@empty#2\else
-        \bbl at ifunset{T@#1}%
-          {}%
-          {\bbl at xin@{,#1,}{,\BabelNonASCII,\BabelNonText,}%
-           \ifin@
-             \DeclareTextCommand{\TeX}{#1}{\ensureascii{\org at TeX}}%
-             \DeclareTextCommand{\LaTeX}{#1}{\ensureascii{\org at LaTeX}}%
-           \else
-             \def\ensureascii##1{{\fontencoding{#1}\selectfont##1}}%
-           \fi}%
-      \fi}%
-    \bbl at foreach\@filelist{\bbl at tempb#1\@@}%  TODO - \@@ de mas??
+  \ifx\bbl at tempb\@empty\else
     \bbl at xin@{,\cf at encoding,}{,\BabelNonASCII,\BabelNonText,}%
     \ifin@\else
-      \edef\ensureascii#1{{%
-        \noexpand\fontencoding{\cf at encoding}\noexpand\selectfont#1}}%
+      \edef\bbl at tempc{\cf at encoding}% The default if ascii wins
     \fi
+    \edef\ensureascii#1{%
+      {\noexpand\fontencoding{\bbl at tempc}\noexpand\selectfont#1}}%
+    \DeclareTextCommandDefault{\TeX}{\ensureascii{\org at TeX}}%
+    \DeclareTextCommandDefault{\LaTeX}{\ensureascii{\org at LaTeX}}%
   \fi}
 \AtEndOfPackage{\edef\latinencoding{\cf at encoding}}
 \AtBeginDocument{%
@@ -713,16 +713,12 @@
      \ifx\cf at encoding\bbl at t@one
        \xdef\latinencoding{\bbl at t@one}%
      \else
-       \ifx\@fontenc at load@list\@undefined
-         \@ifl at aded{def}{t1enc}{\xdef\latinencoding{\bbl at t@one}}{}%
-       \else
-         \def\@elt#1{,#1,}%
-         \edef\bbl at tempa{\expandafter\@gobbletwo\@fontenc at load@list}%
-         \let\@elt\relax
-         \bbl at xin@{,T1,}\bbl at tempa
-         \ifin@
-           \xdef\latinencoding{\bbl at t@one}%
-         \fi
+       \def\@elt#1{,#1,}%
+       \edef\bbl at tempa{\expandafter\@gobbletwo\@fontenc at load@list}%
+       \let\@elt\relax
+       \bbl at xin@{,T1,}\bbl at tempa
+       \ifin@
+         \xdef\latinencoding{\bbl at t@one}%
        \fi
      \fi}}
 \DeclareRobustCommand{\latintext}{%
@@ -733,67 +729,15 @@
 \else
   \DeclareTextFontCommand{\textlatin}{\latintext}
 \fi
-\ifodd\bbl at engine
-  \def\bbl at activate@preotf{%
-    \let\bbl at activate@preotf\relax  % only once
-    \directlua{
-      Babel = Babel or {}
-      %
-      function Babel.pre_otfload_v(head)
-        if Babel.numbers and Babel.digits_mapped then
-          head = Babel.numbers(head)
-        end
-        if Babel.bidi_enabled then
-          head = Babel.bidi(head, false, dir)
-        end
-        return head
-      end
-      %
-      function Babel.pre_otfload_h(head, gc, sz, pt, dir)
-        if Babel.numbers and Babel.digits_mapped then
-          head = Babel.numbers(head)
-        end
-        if Babel.bidi_enabled then
-          head = Babel.bidi(head, false, dir)
-        end
-        return head
-      end
-      %
-      luatexbase.add_to_callback('pre_linebreak_filter',
-        Babel.pre_otfload_v,
-        'Babel.pre_otfload_v',
-        luatexbase.priority_in_callback('pre_linebreak_filter',
-          'luaotfload.node_processor') or nil)
-      %
-      luatexbase.add_to_callback('hpack_filter',
-        Babel.pre_otfload_h,
-        'Babel.pre_otfload_h',
-        luatexbase.priority_in_callback('hpack_filter',
-          'luaotfload.node_processor') or nil)
-    }}
-\fi
+\bbl at ifformatlater{2021-06-01}%
+  {\def\bbl at patchfont#1{\AddToHook{selectfont}{#1}}}
+  {\def\bbl at patchfont#1{%
+     \expandafter\bbl at add\csname selectfont \endcsname{#1}%
+     \expandafter\bbl at toglobal\csname selectfont \endcsname}}
 \bbl at trace{Loading basic (internal) bidi support}
 \ifodd\bbl at engine
+\else % TODO. Move to txtbabel
   \ifnum\bbl at bidimode>100 \ifnum\bbl at bidimode<200
-    \let\bbl at beforeforeign\leavevmode
-    \AtEndOfPackage{\EnableBabelHook{babel-bidi}}
-    \RequirePackage{luatexbase}
-    \bbl at activate@preotf
-    \directlua{
-      require('babel-data-bidi.lua')
-      \ifcase\expandafter\@gobbletwo\the\bbl at bidimode\or
-        require('babel-bidi-basic.lua')
-      \or
-        require('babel-bidi-basic-r.lua')
-      \fi}
-    % TODO - to locale_props, not as separate attribute
-    \newattribute\bbl at attr@dir
-    % TODO. I don't like it, hackish:
-    \bbl at exp{\output{\bodydir\pagedir\the\output}}
-    \AtEndOfPackage{\EnableBabelHook{babel-bidi}}
-  \fi\fi
-\else
-  \ifnum\bbl at bidimode>100 \ifnum\bbl at bidimode<200
     \bbl at error
       {The bidi method 'basic' is available only in\\%
        luatex. I'll continue with 'bidi=default', so\\%
@@ -828,7 +772,8 @@
   \let\bbl at beforeforeign\leavevmode
   \ifodd\bbl at engine
     \newattribute\bbl at attr@dir
-    \bbl at exp{\output{\bodydir\pagedir\the\output}}%
+    \directlua{ Babel.attr_dir = luatexbase.registernumber'bbl at attr@dir' }
+    \bbl at exp{\output{\bodydir\pagedir\the\output}}
   \fi
   \AtEndOfPackage{%
     \EnableBabelHook{babel-bidi}%
@@ -878,51 +823,6 @@
 \AddBabelHook{babel-bidi}{afterextras}{\bbl at switchdir}
 \DisableBabelHook{babel-bidi}
 \ifodd\bbl at engine  % luatex=1
-  \chardef\bbl at thetextdir\z@
-  \chardef\bbl at thepardir\z@
-  \def\bbl at getluadir#1{%
-    \directlua{
-      if tex.#1dir == 'TLT' then
-        tex.sprint('0')
-      elseif tex.#1dir == 'TRT' then
-        tex.sprint('1')
-      end}}
-  \def\bbl at setluadir#1#2#3{% 1=text/par.. 2=\textdir.. 3=0 lr/1 rl
-    \ifcase#3\relax
-      \ifcase\bbl at getluadir{#1}\relax\else
-        #2 TLT\relax
-      \fi
-    \else
-      \ifcase\bbl at getluadir{#1}\relax
-        #2 TRT\relax
-      \fi
-    \fi}
-  \def\bbl at textdir#1{%
-    \bbl at setluadir{text}\textdir{#1}%
-    \chardef\bbl at thetextdir#1\relax
-    \setattribute\bbl at attr@dir{\numexpr\bbl at thepardir*3+#1}}
-  \def\bbl at pardir#1{%
-    \bbl at setluadir{par}\pardir{#1}%
-    \chardef\bbl at thepardir#1\relax}
-  \def\bbl at bodydir{\bbl at setluadir{body}\bodydir}
-  \def\bbl at pagedir{\bbl at setluadir{page}\pagedir}
-  \def\bbl at dirparastext{\pardir\the\textdir\relax}%   %%%%
-  % Sadly, we have to deal with boxes in math with basic.
-  % Activated every math with the package option bidi=:
-  \ifnum\bbl at bidimode>\z@
-    \def\bbl at mathboxdir{%
-      \ifcase\bbl at thetextdir\relax
-        \everyhbox{\bbl at mathboxdir@aux L}%
-      \else
-        \everyhbox{\bbl at mathboxdir@aux R}%
-       \fi}
-    \def\bbl at mathboxdir@aux#1{%
-      \@ifnextchar\egroup{}{\textdir T#1T\relax}}
-    \frozen at everymath\expandafter{%
-      \expandafter\bbl at mathboxdir\the\frozen at everymath}
-    \frozen at everydisplay\expandafter{%
-      \expandafter\bbl at mathboxdir\the\frozen at everydisplay}
-  \fi
 \else % pdftex=0, xetex=2
   \newcount\bbl at dirlevel
   \chardef\bbl at thetextdir\z@
@@ -1158,9 +1058,16 @@
     \bbl at ldfinit
     \let\CurrentOption\bbl at opt@main
     \ifx\bbl at opt@provide\@nnil
-      \bbl at exp{\\\babelprovide[import,main]{\bbl at opt@main}}
+      \bbl at exp{\\\babelprovide[import,main]{\bbl at opt@main}}%
     \else
-      \bbl at exp{\\\babelprovide[\bbl at opt@provide,main]{\bbl at opt@main}}%
+      \bbl at exp{\\\bbl at forkv{\@nameuse{@raw at opt@babel.sty}}}{%
+        \bbl at xin@{,provide,}{,#1,}%
+        \ifin@
+          \def\bbl at opt@provide{#2}%
+          \bbl at replace\bbl at opt@provide{;}{,}%
+        \fi}%
+      \bbl at exp{%
+        \\\babelprovide[\bbl at opt@provide,import,main]{\bbl at opt@main}}%
     \fi
     \bbl at afterldf{}%
   \else % case 0,2

Modified: trunk/Master/texmf-dist/tex/generic/babel/hyphen.cfg
===================================================================
--- trunk/Master/texmf-dist/tex/generic/babel/hyphen.cfg	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/tex/generic/babel/hyphen.cfg	2021-07-19 20:10:36 UTC (rev 59990)
@@ -38,26 +38,17 @@
     \wlog{File: #1 #4 #3 <#2>}%
     \let\ProvidesFile\@undefined}
 \fi
-\ProvidesFile{hyphen.cfg}[2021/06/28 3.61 Babel hyphens]
+\ProvidesFile{hyphen.cfg}[2021/07/19 3.62 Babel hyphens]
 \xdef\bbl at format{\jobname}
-\def\bbl at version{3.61}
-\def\bbl at date{2021/06/28}
+\def\bbl at version{3.62}
+\def\bbl at date{2021/07/19}
 \ifx\AtBeginDocument\@undefined
   \def\@empty{}
-  \let\orig at dump\dump
-  \def\dump{%
-    \ifx\@ztryfc\@undefined
-    \else
-      \toks0=\expandafter{\@preamblecmds}%
-      \edef\@preamblecmds{\noexpand\@begindocumenthook\the\toks0}%
-      \def\@begindocumenthook{}%
-    \fi
-    \let\dump\orig at dump\let\orig at dump\@undefined\dump}
 \fi
 \ifx\language\@undefined
   \csname newcount\endcsname\language
 \fi
-\countdef\last at language=19  % TODO. why? remove?
+\countdef\last at language=19
 \def\addlanguage{\csname newlanguage\endcsname}
 \def\process at line#1#2 #3 #4 {%
   \ifx=#1%

Modified: trunk/Master/texmf-dist/tex/generic/babel/luababel.def
===================================================================
--- trunk/Master/texmf-dist/tex/generic/babel/luababel.def	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/tex/generic/babel/luababel.def	2021-07-19 20:10:36 UTC (rev 59990)
@@ -252,7 +252,7 @@
 \endgroup
 \ifx\newattribute\@undefined\else
   \newattribute\bbl at attr@locale
-  \directlua{ Babel.attr_locale = luatexbase.registernumber'bbl at attr@locale'}
+  \directlua{ Babel.attr_locale = luatexbase.registernumber'bbl at attr@locale' }
   \AddBabelHook{luatex}{beforeextras}{%
     \setattribute\bbl at attr@locale\localeid}
 \fi
@@ -433,11 +433,15 @@
           local lang = item.lang
 
           local LOCALE = node.get_attribute(item,
-                luatexbase.registernumber'bbl at attr@locale')
+                Babel.attr_locale)
           local props = Babel.locale_props[LOCALE]
 
           local class = Babel.cjk_class[item.char].c
 
+          if props.cjk_quotes and props.cjk_quotes[item.char] then
+            class = props.cjk_quotes[item.char]
+          end
+
           if class == 'cp' then class = 'cl' end % )] as CL
           if class == 'id' then class = 'I' end
 
@@ -551,8 +555,9 @@
 \gdef\bbl at arabicjust{%
   \let\bbl at arabicjust\relax
   \newattribute\bblar at kashida
+  \directlua{ Babel.attr_kashida = luatexbase.registernumber'bblar at kashida' }%
   \bblar at kashida=\z@
-  \expandafter\bbl at add\csname selectfont \endcsname{{\bbl at parsejalt}}%
+  \bbl at patchfont{{\bbl at parsejalt}}%
   \directlua{
     Babel.arabic.elong_map   = Babel.arabic.elong_map or {}
     Babel.arabic.elong_map[\the\localeid]   = {}
@@ -651,8 +656,8 @@
   local elong_map = Babel.arabic.elong_map
   local last_line
   local GLYPH = node.id'glyph'
-  local KASHIDA = luatexbase.registernumber'bblar at kashida'
-  local LOCALE = luatexbase.registernumber'bbl at attr@locale'
+  local KASHIDA = Babel.attr_kashida
+  local LOCALE = Babel.attr_locale
 
   if line == nil then
     line = {}
@@ -770,7 +775,7 @@
   \ExplSyntaxOn
   \catcode`\ =10
   \def\bbl at loadfontspec{%
-    \usepackage{fontspec}%
+    \usepackage{fontspec}%  TODO. Apply patch always
     \expandafter
     \def\csname msg~text~>~fontspec/language-not-exist\endcsname##1##2##3##4{%
       Font '\l_fontspec_fontname_tl' is using the\\%
@@ -804,9 +809,7 @@
 \newcommand\bbl at bblfont[2][]{% 1=features 2=fontname, @font=rm|sf|tt
   \bbl at ifunset{\bbl at tempb family}%
     {\bbl at providefam{\bbl at tempb}}%
-    {\bbl at exp{%
-      \\\bbl at sreplace\<\bbl at tempb family >%
-        {\@nameuse{\bbl at tempb default}}{\<\bbl at tempb default>}}}%
+    {}%
   % For the default font, just in case:
   \bbl at ifunset{bbl at lsys@\languagename}{\bbl at provide@lsys{\languagename}}{}%
   \expandafter\bbl at ifblank\expandafter{\bbl at tempa}%
@@ -823,7 +826,10 @@
     \\\bbl at add@list\\\bbl at font@fams{#1}%
     \\\DeclareRobustCommand\<#1family>{%
       \\\not at math@alphabet\<#1family>\relax
-      \\\fontfamily\<#1default>\\\selectfont}%
+      % \\\prepare at family@series at update{#1}\<#1default>% TODO. Fails
+      \\\fontfamily\<#1default>%
+      \<ifx>\\\UseHooks\\\@undefined\<else>\\\UseHook{#1family}\<fi>%
+      \\\selectfont}%
     \\\DeclareTextFontCommand{\<text#1>}{\<#1family>}}}
 \def\bbl at nostdfont#1{%
   \bbl at ifunset{bbl at WFF@\f at family}%
@@ -1029,7 +1035,7 @@
 function Babel.locale_map(head)
   if not Babel.locale_mapped then return head end
 
-  local LOCALE = luatexbase.registernumber'bbl at attr@locale'
+  local LOCALE = Babel.attr_locale
   local GLYPH = node.id('glyph')
   local inmath = false
   local toloc_save
@@ -1125,515 +1131,13 @@
     Babel.chr_to_loc[\the\count@] =
       \bbl at ifblank{#1}{-1000}{\the\bbl at cs{id@@#1}}\space
   }}
-\begingroup % TODO - to a lua file
+\directlua{
+  Babel.nohyphenation = \the\l at nohyphenation
+}
+\begingroup
 \catcode`\~=12
-\catcode`\#=12
 \catcode`\%=12
 \catcode`\&=14
-\directlua{
-  Babel.linebreaking.replacements = {}
-  Babel.linebreaking.replacements[0] = {}  &% pre
-  Babel.linebreaking.replacements[1] = {}  &% post
-
-  &% Discretionaries contain strings as nodes
-  function Babel.str_to_nodes(fn, matches, base)
-    local n, head, last
-    if fn == nil then return nil end
-    for s in string.utfvalues(fn(matches)) do
-      if base.id == 7 then
-        base = base.replace
-      end
-      n = node.copy(base)
-      n.char    = s
-      if not head then
-        head = n
-      else
-        last.next = n
-      end
-      last = n
-    end
-    return head
-  end
-
-  Babel.fetch_subtext = {}
-
-  Babel.ignore_pre_char = function(node)
-    return (node.lang == \the\l at nohyphenation)
-  end
-
-  &% Merging both functions doesn't seen feasible, because there are too
-  &% many differences.
-  Babel.fetch_subtext[0] = function(head)
-    local word_string = ''
-    local word_nodes = {}
-    local lang
-    local item = head
-    local inmath = false
-
-    while item do
-
-      if item.id == 11 then
-        inmath = (item.subtype == 0)
-      end
-
-      if inmath then
-        &% pass
-
-      elseif item.id == 29 then
-        local locale = node.get_attribute(item, Babel.attr_locale)
-
-        if lang == locale or lang == nil then
-          lang = lang or locale
-          if Babel.ignore_pre_char(item) then
-            word_string = word_string .. Babel.us_char
-          else
-            word_string = word_string .. unicode.utf8.char(item.char)
-          end
-          word_nodes[#word_nodes+1] = item
-        else
-          break
-        end
-
-      elseif item.id == 12 and item.subtype == 13 then
-        word_string = word_string .. ' '
-        word_nodes[#word_nodes+1] = item
-
-      &% Ignore leading unrecognized nodes, too.
-      elseif word_string ~= '' then
-        word_string = word_string .. Babel.us_char
-        word_nodes[#word_nodes+1] = item  &% Will be ignored
-      end
-
-      item = item.next
-    end
-
-    &% Here and above we remove some trailing chars but not the
-    &% corresponding nodes. But they aren't accessed.
-    if word_string:sub(-1) == ' ' then
-      word_string = word_string:sub(1,-2)
-    end
-    word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
-    return word_string, word_nodes, item, lang
-  end
-
-  Babel.fetch_subtext[1] = function(head)
-    local word_string = ''
-    local word_nodes = {}
-    local lang
-    local item = head
-    local inmath = false
-
-    while item do
-
-      if item.id == 11 then
-        inmath = (item.subtype == 0)
-      end
-
-      if inmath then
-        &% pass
-
-      elseif item.id == 29 then
-        if item.lang == lang or lang == nil then
-          if (item.char ~= 124) and (item.char ~= 61) then &% not =, not |
-            lang = lang or item.lang
-            word_string = word_string .. unicode.utf8.char(item.char)
-            word_nodes[#word_nodes+1] = item
-          end
-        else
-          break
-        end
-
-      elseif item.id == 7 and item.subtype == 2 then
-        word_string = word_string .. '='
-        word_nodes[#word_nodes+1] = item
-
-      elseif item.id == 7 and item.subtype == 3 then
-        word_string = word_string .. '|'
-        word_nodes[#word_nodes+1] = item
-
-      &% (1) Go to next word if nothing was found, and (2) implictly
-      &% remove leading USs.
-      elseif word_string == '' then
-        &% pass
-
-      &% This is the responsible for splitting by words.
-      elseif (item.id == 12 and item.subtype == 13) then
-        break
-
-      else
-        word_string = word_string .. Babel.us_char
-        word_nodes[#word_nodes+1] = item  &% Will be ignored
-      end
-
-      item = item.next
-    end
-
-    word_string = unicode.utf8.gsub(word_string, Babel.us_char .. '+$', '')
-    return word_string, word_nodes, item, lang
-  end
-
-  function Babel.pre_hyphenate_replace(head)
-    Babel.hyphenate_replace(head, 0)
-  end
-
-  function Babel.post_hyphenate_replace(head)
-    Babel.hyphenate_replace(head, 1)
-  end
-
-  function Babel.debug_hyph(w, wn, sc, first, last, last_match)
-    local ss = ''
-    for pp = 1, 40 do
-      if wn[pp] then
-        if wn[pp].id == 29 then
-          ss = ss .. unicode.utf8.char(wn[pp].char)
-        else
-          ss = ss .. '{' .. wn[pp].id .. '}'
-        end
-      end
-    end
-    print('nod', ss)
-    print('lst_m',
-      string.rep(' ', unicode.utf8.len(
-         string.sub(w, 1, last_match))-1) .. '>')
-    print('str', w)
-    print('sc', string.rep(' ', sc-1) .. '^')
-    if first == last then
-      print('f=l', string.rep(' ', first-1) .. '!')
-    else
-      print('f/l', string.rep(' ', first-1) .. '[' ..
-        string.rep(' ', last-first-1) .. ']')
-    end
-  end
-
-  Babel.us_char = string.char(31)
-
-  function Babel.hyphenate_replace(head, mode)
-    local u = unicode.utf8
-    local lbkr = Babel.linebreaking.replacements[mode]
-
-    local word_head = head
-
-    while true do  &% for each subtext block
-
-      local w, w_nodes, nw, lang = Babel.fetch_subtext[mode](word_head)
-
-      if Babel.debug then
-        print()
-        print((mode == 0) and '@@@@<' or '@@@@>', w)
-      end
-
-      if nw == nil and w == '' then break end
-
-      if not lang then goto next end
-      if not lbkr[lang] then goto next end
-
-      &% For each saved (pre|post)hyphenation. TODO. Reconsider how
-      &% loops are nested.
-      for k=1, #lbkr[lang] do
-        local p = lbkr[lang][k].pattern
-        local r = lbkr[lang][k].replace
-
-        if Babel.debug then
-          print('*****', p, mode)
-        end
-
-        &% This variable is set in some cases below to the first *byte*
-        &% after the match, either as found by u.match (faster) or the
-        &% computed position based on sc if w has changed.
-        local last_match = 0
-        local step = 0
-
-        &% For every match.
-        while true do
-          if Babel.debug then
-            print('=====')
-          end
-          local new  &% used when inserting and removing nodes
-
-          local matches = { u.match(w, p, last_match) }
-
-          if #matches < 2 then break end
-
-          &% Get and remove empty captures (with ()'s, which return a
-          &% number with the position), and keep actual captures
-          &% (from (...)), if any, in matches.
-          local first = table.remove(matches, 1)
-          local last  = table.remove(matches, #matches)
-          &% Non re-fetched substrings may contain \31, which separates
-          &% subsubstrings.
-          if string.find(w:sub(first, last-1), Babel.us_char) then break end
-
-          local save_last = last &% with A()BC()D, points to D
-
-          &% Fix offsets, from bytes to unicode. Explained above.
-          first = u.len(w:sub(1, first-1)) + 1
-          last  = u.len(w:sub(1, last-1)) &% now last points to C
-
-          &% This loop stores in n small table the nodes
-          &% corresponding to the pattern. Used by 'data' to provide a
-          &% predictable behavior with 'insert' (now w_nodes is modified on
-          &% the fly), and also access to 'remove'd nodes.
-          local sc = first-1           &% Used below, too
-          local data_nodes = {}
-
-          for q = 1, last-first+1 do
-            data_nodes[q] = w_nodes[sc+q]
-          end
-
-          &% This loop traverses the matched substring and takes the
-          &% corresponding action stored in the replacement list.
-          &% sc = the position in substr nodes / string
-          &% rc = the replacement table index
-          local rc = 0
-
-          while rc < last-first+1 do &% for each replacement
-            if Babel.debug then
-              print('.....', rc + 1)
-            end
-            sc = sc + 1
-            rc = rc + 1
-
-            if Babel.debug then
-              Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
-              local ss = ''
-              for itt in node.traverse(head) do
-               if itt.id == 29 then
-                 ss = ss .. unicode.utf8.char(itt.char)
-               else
-                 ss = ss .. '{' .. itt.id .. '}'
-               end
-              end
-              print('*****************', ss)
-
-            end
-
-            local crep = r[rc]
-            local item = w_nodes[sc]
-            local item_base = item
-            local placeholder = Babel.us_char
-            local d
-
-            if crep and crep.data then
-              item_base = data_nodes[crep.data]
-            end
-
-            if crep then
-              step = crep.step or 0
-            end
-
-            if crep and next(crep) == nil then &% = {}
-              last_match = save_last    &% Optimization
-              goto next
-
-            elseif crep == nil or crep.remove then
-              node.remove(head, item)
-              table.remove(w_nodes, sc)
-              w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1)
-              sc = sc - 1  &% Nothing has been inserted.
-              last_match = utf8.offset(w, sc+1+step)
-              goto next
-
-            elseif crep and crep.kashida then &% Experimental
-              node.set_attribute(item,
-                 luatexbase.registernumber'bblar at kashida',
-                 crep.kashida)
-              last_match = utf8.offset(w, sc+1+step)
-              goto next
-
-            elseif crep and crep.string then
-              local str = crep.string(matches)
-              if str == '' then  &% Gather with nil
-                node.remove(head, item)
-                table.remove(w_nodes, sc)
-                w = u.sub(w, 1, sc-1) .. u.sub(w, sc+1)
-                sc = sc - 1  &% Nothing has been inserted.
-              else
-                local loop_first = true
-                for s in string.utfvalues(str) do
-                  d = node.copy(item_base)
-                  d.char = s
-                  if loop_first then
-                    loop_first = false
-                    head, new = node.insert_before(head, item, d)
-                    if sc == 1 then
-                      word_head = head
-                    end
-                    w_nodes[sc] = d
-                    w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc+1)
-                  else
-                    sc = sc + 1
-                    head, new = node.insert_before(head, item, d)
-                    table.insert(w_nodes, sc, new)
-                    w = u.sub(w, 1, sc-1) .. u.char(s) .. u.sub(w, sc)
-                  end
-                  if Babel.debug then
-                    print('.....', 'str')
-                    Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
-                  end
-                end  &% for
-                node.remove(head, item)
-              end  &% if ''
-              last_match = utf8.offset(w, sc+1+step)
-              goto next
-
-            elseif mode == 1 and crep and (crep.pre or crep.no or crep.post) then
-              d = node.new(7, 0)   &% (disc, discretionary)
-              d.pre     = Babel.str_to_nodes(crep.pre, matches, item_base)
-              d.post    = Babel.str_to_nodes(crep.post, matches, item_base)
-              d.replace = Babel.str_to_nodes(crep.no, matches, item_base)
-              d.attr = item_base.attr
-              if crep.pre == nil then  &% TeXbook p96
-                d.penalty = crep.penalty or tex.hyphenpenalty
-              else
-                d.penalty = crep.penalty or tex.exhyphenpenalty
-              end
-              placeholder = '|'
-              head, new = node.insert_before(head, item, d)
-
-            elseif mode == 0 and crep and (crep.pre or crep.no or crep.post) then
-              &% ERROR
-
-            elseif crep and crep.penalty then
-              d = node.new(14, 0)   &% (penalty, userpenalty)
-              d.attr = item_base.attr
-              d.penalty = crep.penalty
-              head, new = node.insert_before(head, item, d)
-
-            elseif crep and crep.space then
-              &% 655360 = 10 pt = 10 * 65536 sp
-              d = node.new(12, 13)      &% (glue, spaceskip)
-              local quad = font.getfont(item_base.font).size or 655360
-              node.setglue(d, crep.space[1] * quad,
-                              crep.space[2] * quad,
-                              crep.space[3] * quad)
-              if mode == 0 then
-                placeholder = ' '
-              end
-              head, new = node.insert_before(head, item, d)
-
-            elseif crep and crep.spacefactor then
-              d = node.new(12, 13)      &% (glue, spaceskip)
-              local base_font = font.getfont(item_base.font)
-              node.setglue(d,
-                crep.spacefactor[1] * base_font.parameters['space'],
-                crep.spacefactor[2] * base_font.parameters['space_stretch'],
-                crep.spacefactor[3] * base_font.parameters['space_shrink'])
-              if mode == 0 then
-                placeholder = ' '
-              end
-              head, new = node.insert_before(head, item, d)
-
-            elseif mode == 0 and crep and crep.space then
-              &% ERROR
-
-            end  &% ie replacement cases
-
-            &% Shared by disc, space and penalty.
-            if sc == 1 then
-              word_head = head
-            end
-            if crep.insert then
-              w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc)
-              table.insert(w_nodes, sc, new)
-              last = last + 1
-            else
-              w_nodes[sc] = d
-              node.remove(head, item)
-              w = u.sub(w, 1, sc-1) .. placeholder .. u.sub(w, sc+1)
-            end
-
-            last_match = utf8.offset(w, sc+1+step)
-
-            ::next::
-
-          end  &% for each replacement
-
-          if Babel.debug then
-              print('.....', '/')
-              Babel.debug_hyph(w, w_nodes, sc, first, last, last_match)
-          end
-
-        end  &% for match
-
-      end  &% for patterns
-
-      ::next::
-      word_head = nw
-    end  &% for substring
-    return head
-  end
-
-  &% This table stores capture maps, numbered consecutively
-  Babel.capture_maps = {}
-
-  &% The following functions belong to the next macro
-  function Babel.capture_func(key, cap)
-    local ret = "[[" .. cap:gsub('{([0-9])}', "]]..m[%1]..[[") .. "]]"
-    local cnt
-    local u = unicode.utf8
-    ret, cnt = ret:gsub('{([0-9])|([^|]+)|(.-)}', Babel.capture_func_map)
-    if cnt == 0 then
-      ret = u.gsub(ret, '{(%x%x%x%x+)}',
-            function (n)
-              return u.char(tonumber(n, 16))
-            end)
-    end
-    ret = ret:gsub("%[%[%]%]%.%.", '')
-    ret = ret:gsub("%.%.%[%[%]%]", '')
-    return key .. [[=function(m) return ]] .. ret .. [[ end]]
-  end
-
-  function Babel.capt_map(from, mapno)
-    return Babel.capture_maps[mapno][from] or from
-  end
-
-  &% Handle the {n|abc|ABC} syntax in captures
-  function Babel.capture_func_map(capno, from, to)
-    local u = unicode.utf8
-    from = u.gsub(from, '{(%x%x%x%x+)}',
-         function (n)
-           return u.char(tonumber(n, 16))
-         end)
-    to = u.gsub(to, '{(%x%x%x%x+)}',
-         function (n)
-           return u.char(tonumber(n, 16))
-         end)
-    local froms = {}
-    for s in string.utfcharacters(from) do
-      table.insert(froms, s)
-    end
-    local cnt = 1
-    table.insert(Babel.capture_maps, {})
-    local mlen = table.getn(Babel.capture_maps)
-    for s in string.utfcharacters(to) do
-      Babel.capture_maps[mlen][froms[cnt]] = s
-      cnt = cnt + 1
-    end
-    return "]]..Babel.capt_map(m[" .. capno .. "]," ..
-           (mlen) .. ").." .. "[["
-  end
-
-  &% Create/Extend reversed sorted list of kashida weights:
-  function Babel.capture_kashida(key, wt)
-    wt = tonumber(wt)
-    if Babel.kashida_wts then
-      for p, q in ipairs(Babel.kashida_wts) do
-        if wt  == q then
-          break
-        elseif wt > q then
-          table.insert(Babel.kashida_wts, p, wt)
-          break
-        elseif table.getn(Babel.kashida_wts) == p then
-          table.insert(Babel.kashida_wts, wt)
-        end
-      end
-    else
-      Babel.kashida_wts = { wt }
-    end
-    return 'kashida = ' .. wt
-  end
-}
-\catcode`\#=6
 \gdef\babelposthyphenation#1#2#3{&%
   \bbl at activateposthyphen
   \begingroup
@@ -1727,13 +1231,114 @@
 \def\bbl at activateposthyphen{%
   \let\bbl at activateposthyphen\relax
   \directlua{
+    require('babel-transforms.lua')
     Babel.linebreaking.add_after(Babel.post_hyphenate_replace)
   }}
 \def\bbl at activateprehyphen{%
   \let\bbl at activateprehyphen\relax
   \directlua{
+    require('babel-transforms.lua')
     Babel.linebreaking.add_before(Babel.pre_hyphenate_replace)
   }}
+\def\bbl at activate@preotf{%
+  \let\bbl at activate@preotf\relax  % only once
+  \directlua{
+    Babel = Babel or {}
+    %
+    function Babel.pre_otfload_v(head)
+      if Babel.numbers and Babel.digits_mapped then
+        head = Babel.numbers(head)
+      end
+      if Babel.bidi_enabled then
+        head = Babel.bidi(head, false, dir)
+      end
+      return head
+    end
+    %
+    function Babel.pre_otfload_h(head, gc, sz, pt, dir)
+      if Babel.numbers and Babel.digits_mapped then
+        head = Babel.numbers(head)
+      end
+      if Babel.bidi_enabled then
+        head = Babel.bidi(head, false, dir)
+      end
+      return head
+    end
+    %
+    luatexbase.add_to_callback('pre_linebreak_filter',
+      Babel.pre_otfload_v,
+      'Babel.pre_otfload_v',
+      luatexbase.priority_in_callback('pre_linebreak_filter',
+        'luaotfload.node_processor') or nil)
+    %
+    luatexbase.add_to_callback('hpack_filter',
+      Babel.pre_otfload_h,
+      'Babel.pre_otfload_h',
+      luatexbase.priority_in_callback('hpack_filter',
+        'luaotfload.node_processor') or nil)
+  }}
+\ifnum\bbl at bidimode>100 \ifnum\bbl at bidimode<200
+  \let\bbl at beforeforeign\leavevmode
+  \AtEndOfPackage{\EnableBabelHook{babel-bidi}}
+  \RequirePackage{luatexbase}
+  \bbl at activate@preotf
+  \directlua{
+    require('babel-data-bidi.lua')
+    \ifcase\expandafter\@gobbletwo\the\bbl at bidimode\or
+      require('babel-bidi-basic.lua')
+    \or
+      require('babel-bidi-basic-r.lua')
+    \fi}
+  % TODO - to locale_props, not as separate attribute
+  \newattribute\bbl at attr@dir
+  \directlua{ Babel.attr_dir = luatexbase.registernumber'bbl at attr@dir' }
+  % TODO. I don't like it, hackish:
+  \bbl at exp{\output{\bodydir\pagedir\the\output}}
+  \AtEndOfPackage{\EnableBabelHook{babel-bidi}}
+\fi\fi
+\chardef\bbl at thetextdir\z@
+\chardef\bbl at thepardir\z@
+\def\bbl at getluadir#1{%
+  \directlua{
+    if tex.#1dir == 'TLT' then
+      tex.sprint('0')
+    elseif tex.#1dir == 'TRT' then
+      tex.sprint('1')
+    end}}
+\def\bbl at setluadir#1#2#3{% 1=text/par.. 2=\textdir.. 3=0 lr/1 rl
+  \ifcase#3\relax
+    \ifcase\bbl at getluadir{#1}\relax\else
+      #2 TLT\relax
+    \fi
+  \else
+    \ifcase\bbl at getluadir{#1}\relax
+      #2 TRT\relax
+    \fi
+  \fi}
+\def\bbl at textdir#1{%
+  \bbl at setluadir{text}\textdir{#1}%
+  \chardef\bbl at thetextdir#1\relax
+  \setattribute\bbl at attr@dir{\numexpr\bbl at thepardir*3+#1}}
+\def\bbl at pardir#1{%
+  \bbl at setluadir{par}\pardir{#1}%
+  \chardef\bbl at thepardir#1\relax}
+\def\bbl at bodydir{\bbl at setluadir{body}\bodydir}
+\def\bbl at pagedir{\bbl at setluadir{page}\pagedir}
+\def\bbl at dirparastext{\pardir\the\textdir\relax}%   %%%%
+\ifnum\bbl at bidimode>\z@
+  \def\bbl at mathboxdir{%
+    \ifcase\bbl at thetextdir\relax
+      \everyhbox{\bbl at mathboxdir@aux L}%
+    \else
+      \everyhbox{\bbl at mathboxdir@aux R}%
+     \fi}
+  \def\bbl at mathboxdir@aux#1{%
+    \@ifnextchar\egroup{}{\textdir T#1T\relax}}
+  \frozen at everymath\expandafter{%
+    \expandafter\bbl at mathboxdir\the\frozen at everymath}
+  \frozen at everydisplay\expandafter{%
+    \expandafter\bbl at mathboxdir\the\frozen at everydisplay}
+\fi
 \bbl at trace{Redefinitions for bidi layout}
 \ifx\@eqnnum\@undefined\else
   \ifx\bbl at attr@dir\@undefined\else

Modified: trunk/Master/texmf-dist/tex/generic/babel/nil.ldf
===================================================================
--- trunk/Master/texmf-dist/tex/generic/babel/nil.ldf	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/tex/generic/babel/nil.ldf	2021-07-19 20:10:36 UTC (rev 59990)
@@ -33,7 +33,7 @@
 %% extension |.ins|) which are part of the distribution.
 %%
 
-\ProvidesLanguage{nil}[2021/06/28 3.61 Nil language]
+\ProvidesLanguage{nil}[2021/07/19 3.62 Nil language]
 \LdfInit{nil}{datenil}
 \ifx\l at nil\@undefined
   \newlanguage\l at nil

Modified: trunk/Master/texmf-dist/tex/generic/babel/xebabel.def
===================================================================
--- trunk/Master/texmf-dist/tex/generic/babel/xebabel.def	2021-07-19 20:09:28 UTC (rev 59989)
+++ trunk/Master/texmf-dist/tex/generic/babel/xebabel.def	2021-07-19 20:10:36 UTC (rev 59990)
@@ -87,10 +87,7 @@
         \ifx\AtBeginDocument\@notprerr
           \expandafter\@secondoftwo  % to execute right now
         \fi
-        \AtBeginDocument{%
-          \expandafter\bbl at add
-          \csname selectfont \endcsname{\bbl at ispacesize}%
-          \expandafter\bbl at toglobal\csname selectfont \endcsname}%
+        \AtBeginDocument{\bbl at patchfont{\bbl at xenohyph}}%
       \fi}%
   \fi}
 \ifx\DisableBabelHook\@undefined\endinput\fi
@@ -102,7 +99,7 @@
   \ExplSyntaxOn
   \catcode`\ =10
   \def\bbl at loadfontspec{%
-    \usepackage{fontspec}%
+    \usepackage{fontspec}%  TODO. Apply patch always
     \expandafter
     \def\csname msg~text~>~fontspec/language-not-exist\endcsname##1##2##3##4{%
       Font '\l_fontspec_fontname_tl' is using the\\%
@@ -136,9 +133,7 @@
 \newcommand\bbl at bblfont[2][]{% 1=features 2=fontname, @font=rm|sf|tt
   \bbl at ifunset{\bbl at tempb family}%
     {\bbl at providefam{\bbl at tempb}}%
-    {\bbl at exp{%
-      \\\bbl at sreplace\<\bbl at tempb family >%
-        {\@nameuse{\bbl at tempb default}}{\<\bbl at tempb default>}}}%
+    {}%
   % For the default font, just in case:
   \bbl at ifunset{bbl at lsys@\languagename}{\bbl at provide@lsys{\languagename}}{}%
   \expandafter\bbl at ifblank\expandafter{\bbl at tempa}%
@@ -155,7 +150,10 @@
     \\\bbl at add@list\\\bbl at font@fams{#1}%
     \\\DeclareRobustCommand\<#1family>{%
       \\\not at math@alphabet\<#1family>\relax
-      \\\fontfamily\<#1default>\\\selectfont}%
+      % \\\prepare at family@series at update{#1}\<#1default>% TODO. Fails
+      \\\fontfamily\<#1default>%
+      \<ifx>\\\UseHooks\\\@undefined\<else>\\\UseHook{#1family}\<fi>%
+      \\\selectfont}%
     \\\DeclareTextFontCommand{\<text#1>}{\<#1family>}}}
 \def\bbl at nostdfont#1{%
   \bbl at ifunset{bbl at WFF@\f at family}%



More information about the tex-live-commits mailing list.