texlive[64883] Master/texmf-dist: piton (31oct22)

commits+karl at tug.org commits+karl at tug.org
Mon Oct 31 20:49:20 CET 2022


Revision: 64883
          http://tug.org/svn/texlive?view=revision&revision=64883
Author:   karl
Date:     2022-10-31 20:49:19 +0100 (Mon, 31 Oct 2022)
Log Message:
-----------
piton (31oct22)

Modified Paths:
--------------
    trunk/Master/texmf-dist/doc/lualatex/piton/piton-french.pdf
    trunk/Master/texmf-dist/doc/lualatex/piton/piton-french.tex
    trunk/Master/texmf-dist/doc/lualatex/piton/piton.pdf
    trunk/Master/texmf-dist/source/lualatex/piton/piton.dtx
    trunk/Master/texmf-dist/tex/lualatex/piton/piton.sty

Modified: trunk/Master/texmf-dist/doc/lualatex/piton/piton-french.pdf
===================================================================
(Binary files differ)

Modified: trunk/Master/texmf-dist/doc/lualatex/piton/piton-french.tex
===================================================================
--- trunk/Master/texmf-dist/doc/lualatex/piton/piton-french.tex	2022-10-31 19:48:53 UTC (rev 64882)
+++ trunk/Master/texmf-dist/doc/lualatex/piton/piton-french.tex	2022-10-31 19:49:19 UTC (rev 64883)
@@ -97,8 +97,10 @@
 from math import pi
 
 def arctan(x,n=10):
-    '''Compute the value of arctan(x)
-       n is the number of terms if the sum'''
+    """Compute the mathematical value of arctan(x)
+
+    n is the number of terms in the sum
+    """
     if x < 0:
         return -arctan(-x) # appel récursif
     elif x > 1: 
@@ -163,40 +165,50 @@
 effectués par cette commande est le groupe TeX courant.\footnote{On rappelle que tout environnement LaTeX est, en particulier, un groupe.}
 
 \begin{itemize}
-\item La clé |gobble| peut comme valeur un entier positif $n$ : les $n$ premiers caractères de chaque ligne sont
-alors retirés (avant formatage du code) dans les environnements |{Piton}|.
+\item La clé |gobble| prend comme valeur un entier positif $n$ : les $n$ premiers caractères de chaque ligne sont 
+alors retirés (avant formatage du code) dans les environnements |{Piton}|. Ces $n$ caractères ne sont pas
+nécessairement des espaces.
 
 \item Quand la clé |auto-gobble| est activée, l'extension \pkg{piton} détermine la valeur minimale $n$ du nombre
 d'espaces successifs débutant chaque ligne (non vide) de l'environnment |{Piton}| et applique |gobble| avec cette
 valeur de~$n$.
 
-\item Quand la clé |env-gobble| est activée, \pkg{piton} applique la clé |gobble| avec une valeur de $n$ égale au
-nombre d'espaces sur la ligne précédant le |\end{Piton}| (si cette ligne ne comporte que des espaces).
+\item Quand la clé |env-gobble| est activée, \pkg{piton} analyse la dernière ligne de l'environnement, c'est-à-dire
+celle qui contient le |\end{Piton}| et détermine si cette ligne ne comporte que des espaces suivis par
+|\end{Piton}|. Si c'est le cas, \pkg{piton} calcule le nombre $n$ de ces espaces et applique |gobble| avec cette
+valeur de~$n$. Le nom de cette clé vient de \emph{environment gobble}: le nombre d'espaces à retirer ne dépend que
+de la position des délimiteurs |\begin{Piton}| et |\end{Piton}| de l'environnement.
 
-\item Avec la clé |line-numbers|, les lignes \emph{non vides} sont numérotées dans les environnements \verb|{Piton}|
-et dans les listings produits par la commande |\PitonInputFile|.
+\item Avec la clé |line-numbers|, les lignes \emph{non vides} sont numérotées (à gauche) dans les environnements
+\verb|{Piton}| et dans les listings produits par la commande |\PitonInputFile|.
 
 \item Avec la clé |all-line-numbers|, \emph{toutes} les lignes sont numérotées, y compris les lignes vides.
 
+\item La clé |left-margin| fixe une marge sur la gauche. Cette clé peut être utile, en particulier, en conjonction
+avec l'une des clés |line-numbers| et |all-line-numbers| si on ne souhaite pas que les numéros de ligne soient dans
+une position en débordement sur la gauche. Voir un exemple à la partie \ref{example-numbering} p.~\pageref{example-numbering}.
+
 \item Avec la clé |resume|, le compteur de lignes n'est pas remis à zéro comme il l'est normalement au début d'un
-environnement |{Piton}| ou bien au début d'un listing produit par |\PitonInputFile|.
+environnement |{Piton}| ou bien au début d'un listing produit par |\PitonInputFile|. Cela permet de poursuivre la
+numérotation d'un environnement à l'autre.
 
 \item La clé |splittable| autorise un saut de page dans les environnements |{Piton}| et les listings produits par
 |\PitonInputFile|.
 
+\smallskip
+\colorbox{yellow!50}{\textbf{Nouveau 0.9}}\enskip On peut donner comme valeur à la clé |splittable| un entier
+naturel~$n$. Avec une telle valeur, les environnements |{Piton}| et les listings produits par |\PitonInputFile|
+sont sécables mais aucune coupure ne pourra avoir lieu entre les $n$~premières lignes, ni entre les $n$~dernières.
+La valeur par défaut de la clé |splittable| vaut en fait $1$, qui autorise les sauts de page partout.
+
 \item La clé |background-color| fixe la couleur de fond des environnements |{Piton}| et des listings produits par
 |\PitonInputFile| (ce fond a une largeur égale à la valeur courante de |\linewidth|). Même avec une couleur de
-fond, les sauts de page sont possibles, à partir du moment où la clé |splittable| est utilisée.\footnote{Un
-  environnement |{Piton}| est sécable même dans un environnement de \pkg{tcolorbox}, à partir du moment où la clé
-  |breakable| de \pkg{tcolorbox} est utilisée. On précise cela parce que, en revanche, un environnement de
-  \pkg{tcolorbox} inclus dans un autre environnement de \pkg{tcolorbox} n'est pas sécable, même quand les deux
-  utilisent la clé |breakable|.}
+fond, les sauts de page sont possibles, à partir du moment où la clé |splittable| est utilisée.\footnote{Avec la
+  clé |splittable|, un environnement |{Piton}| est sécable même dans un environnement de \pkg{tcolorbox} (à partir
+  du moment où la clé |breakable| de \pkg{tcolorbox} est utilisée). On précise cela parce que, en revanche, un
+  environnement de \pkg{tcolorbox} inclus dans un autre environnement de \pkg{tcolorbox} n'est pas sécable, même
+  quand les deux utilisent la clé |breakable|.}\end{itemize}
 
-\item \colorbox{yellow!50}{\textbf{Nouveau 0.8}}\enskip La clé |left-margin| fixe une marge sur la gauche. Cette
-clé peut être utile, en particulier, en conjonction avec l'une des clés |line-numbers| et |all-line-numbers| si on
-ne souhaite pas que les numéros de ligne soient dans une position en débordement sur la gauche.
-\end{itemize}
-
 \bigskip
 
 \begingroup
@@ -207,8 +219,10 @@
     from math import pi
 
     def arctan(x,n=10):
-        '''Compute the value of arctan(x)
-           n is the number of terms if the sum'''
+        """Compute the mathematical value of arctan(x)
+
+        n is the number of terms in the sum
+        """
         if x < 0:
             return -arctan(-x) # appel récursif
         elif x > 1: 
@@ -229,8 +243,10 @@
     from math import pi
 
     def arctan(x,n=10):
-        '''Compute the value of arctan(x)
-           n is the number of terms if the sum'''
+        """Compute the value of arctan(x)
+
+        n is the number of terms in the sum
+        """
         if x < 0:
             return -arctan(-x) # appel récursif
         elif x > 1: 
@@ -343,16 +359,16 @@
 String.Long & les chaînes de caractères longues (entre \texttt{'''} ou \texttt{"""}) sauf les chaînes de
               documentation \\
 String & cette clé fixe à la fois |String.Short| et |String.Long| \\
-String.Doc & les chaînes de documentation \\
-String.Interpol & les éléments syntaxiques des champs des f-strings (c'est-à-dire les caractères \texttt{\{}, \texttt{\}} et \texttt{:}) \\
+String.Doc & les chaînes de documentation (seulement entre |"""| suivant PEP~257) \\
+String.Interpol & les éléments syntaxiques des champs des f-strings (c'est-à-dire les caractères \texttt{\{}et \texttt{\}}) \\
 Operator & les opérateurs suivants : \texttt{!= == << >> - \~{} + / * \% = < > \& .} \verb+|+ \verb|@| \\
 Operator.Word & les opérateurs suivants : |in|, |is|, |and|, |or| et |not| \\
 Name.Builtin & la plupart des fonctions prédéfinies par Python \\
 Name.Function & le nom des fonctions définies par l'utilisateur \emph{au moment de leur définition}, c'est-à-dire 
                 après le mot-clé \verb|def| \\ 
-Name.Decorator & les décorateurs (instructions débutant par \verb|@| dans les classes) \\
+Name.Decorator & les décorateurs (instructions débutant par \verb|@|) \\
 Name.Namespace & le nom des modules (= bibliothèques extérieures) \\
-Name.Class & le nom des classes au moment de leur définition \\
+Name.Class & le nom des classes au moment de leur définition, c'est-à-dire après le mot-clé \verb|class| \\
 Exception & le nom des exceptions prédéfinies (eg: SyntaxError) \\
 Comment & les commentaires commençant par \texttt{\#} \\
 Comment.LaTeX & les commentaires commençant par \texttt{\#\#} qui sont composés en LaTeX par \pkg{piton} (et
@@ -417,8 +433,8 @@
 paire |\footnotemark|--|\footnotetext|. 
 
 \smallskip
-Il est aussi possible d'extraire les notes de pieds de page avec l'extension \pkg{footnote} ou bien l'extension
-\pkg{footnotehyper}.
+Néanmoins, il est également possible d'extraire les notes de pieds de page avec l'extension \pkg{footnote} ou bien
+l'extension pkg{footnotehyper}.
 
 \smallskip
 Si \pkg{piton} est chargée avec l'option |footnote| (avec |\usepackage[footnote]{piton}|) l'extension
@@ -438,9 +454,15 @@
 
 \medskip
 Dans ce document, l'extension \pkg{piton} a été chargée avec l'option |footnotehyper| et c'est pourquoi des notes
-peuvent être mises dans les environnements |{Piton}|: cf. première page de ce document.
+peuvent être mises dans les environnements |{Piton}| : voir un exemple sur la première page de ce document. 
 
+\subsection{Tabulations}
 
+\smallskip
+\colorbox{yellow!50}{\textbf{Nouveau 0.9}}\enskip Même s'il est recommandé d'indenter les listings Python avec des
+espaces (cf. PEP~8), \pkg{piton} accepte les caractères de tabulations (U+0009) en début de ligne. Chaque caractère
+U+0009 est remplacé par $n$ espaces. La valeur initiale de~$n$ est~4 mais on peut la changer avec la clé |tab-size|
+de |\PitonOptions|.
 
 \section{Exemples}
 
@@ -449,6 +471,10 @@
 
 \subsection{Numérotation des lignes}
 
+\label{example-numbering}
+
+On rappelle que l'on peut demander la numérotation des lignes des listings avec la clé |line-numbers| ou la clé |all-line-numbers|.
+
 Par défaut, les numéros de ligne sont composés par \pkg{piton} en débordement à gauche (en utilisant en interne la commande |\llap| de LaTeX).
 
 Si on ne veut pas de débordement, il faut réserver une place à gauche pour les numéros de lignes avec la clé |left-margin|.
@@ -723,7 +749,7 @@
     Name.Builtin = ,
     Name.Function = \bfseries \colorbox{gray!20} ,
     Comment = \color{gray} ,
-    Comment.LaTeX = \color{gray},
+    Comment.LaTeX = \normalfont \color{gray},
     Keyword = \bfseries ,
     Name.Namespace = ,
     Name.Class = , 
@@ -746,7 +772,7 @@
     Name.Builtin = ,
     Name.Function = \bfseries \colorbox{gray!20} ,
     Comment = \color{gray} ,
-    Comment.LaTeX = \color{gray},
+    Comment.LaTeX = \normalfont \color{gray},
     Keyword = \bfseries ,
     Name.Namespace = ,
     Name.Class = , 
@@ -755,14 +781,16 @@
   }
 
 
-\bigskip
+\bigskip 
 
 \begin{Piton}
 from math import pi
 
 def arctan(x,n=10):
-    '''Compute the value of arctan(x)
-       n is the number of terms if the sum'''
+    """Compute the mathematical value of arctan(x)
+
+    n is the number of terms in the sum
+    """
     if x < 0:
         return -arctan(-x) # appel récursif
     elif x > 1: 

Modified: trunk/Master/texmf-dist/doc/lualatex/piton/piton.pdf
===================================================================
(Binary files differ)

Modified: trunk/Master/texmf-dist/source/lualatex/piton/piton.dtx
===================================================================
--- trunk/Master/texmf-dist/source/lualatex/piton/piton.dtx	2022-10-31 19:48:53 UTC (rev 64882)
+++ trunk/Master/texmf-dist/source/lualatex/piton/piton.dtx	2022-10-31 19:49:19 UTC (rev 64883)
@@ -15,8 +15,8 @@
 %
 % \fi
 % \iffalse
-\def\myfileversion{0.8}
-\def\myfiledate{2022/10/23}
+\def\myfileversion{0.9}
+\def\myfiledate{2022/10/31}
 %
 %
 %<*batchfile> 
@@ -58,6 +58,7 @@
 \usepackage{ragged2e}
 \usepackage[footnotehyper,escape-inside=$$]{piton} % $$
 
+
 \def\interitem{\vskip 7mm plus 2 mm minus 3mm}          
 \parindent 0pt
 \skip\footins = 2\bigskipamount
@@ -72,6 +73,9 @@
 %</driver>
 
 % \fi
+%
+% \catcode`\" = 11 
+% 
 % \title{The package \pkg{piton}\thanks{This document corresponds to the
 % version~\myfileversion\space of \pkg{piton}, at the date of~\myfiledate.}} 
 % \author{F. Pantigny \\ \texttt{fpantigny at wanadoo.fr}}
@@ -84,6 +88,7 @@
 % \end{abstract}
 % 
 % 
+%
 % \section{Presentation}
 % 
 %
@@ -105,8 +110,10 @@
 % from math import pi
 %
 % def arctan(x,n=10):
-%     '''Compute the value of arctan(x)
-%        n is the number of terms of the sum'''
+%    """Compute the mathematical value of arctan(x)
+%
+%    n is the number of terms in the sum
+%    """
 %     if x < 0:
 %         return -arctan(-x) # recursive call
 %     elif x > 1: 
@@ -179,47 +186,62 @@
 % \begin{itemize}
 % \item The key |gobble| takes in as value a positive integer~$n$: the first $n$
 % characters are discarded (before the process of highlightning of the code) for
-% each line of the environment |{Piton}|.
+% each line of the environment |{Piton}|. These characters are not necessarily
+% spaces. 
 %
-% \item Then the key |auto-gobble| is in force, the extension \pkg{piton} computes
-% the minimal value $n$ of the number of consecutives space beginning each (non
+% \item When the key |auto-gobble| is in force, the extension \pkg{piton} computes
+% the minimal value $n$ of the number of consecutive spaces beginning each (non
 % empty) line of the environment |{Piton}| and applies |gobble| with that value
 % of~$n$.
 %
-% \item When the key |env-gobble| is in force, \pkg{piton} applies |gobble| with
-% a value of $n$ equal to the number of spaces before |\end{Piton}| on the last
-% line (if that line contains only spaces). The name of that key comes from
-% \emph{environment gobble}: the effect of gobble is set by the position of the
-% commands |\begin{Piton}| and |\end{Piton}| which delimit the current environment.
+% \item When the key |env-gobble| is in force, \pkg{piton} analyzes the last
+% line of the environment |{Piton}|, that is to say the line which contains
+% |\end{Piton}| and determines whether that line contains only spaces followed
+% by the |\end{Piton}|. If we are in that situation, \pkg{piton} computes the
+% number~$n$ of spaces on that line and applies |gobble| with that value of~$n$.
+% The name of that key comes from \emph{environment gobble}: the effect of
+% gobble is set by the position of the commands |\begin{Piton}| and
+% |\end{Piton}| which delimit the current environment.
 %
-% \item With the key |line-numbers|, the \emph{non empty} lines are numbered in
-% the environments |{Piton}| and in the listings resulting from the use of
-% |\PitonInputFile|.
+% \item With the key |line-numbers|, the \emph{non empty} lines are numbered (on
+% the left) in the environments |{Piton}| and in the listings resulting from the
+% use of |\PitonInputFile|.
 % 
 % \item With the key |all-line-numbers|, \emph{all} the lines are numbered,
 % including the empty ones.
 %
-% \item By default, the counter of lines is set to zero at the beginning of each
-% environment |{Piton}| or use of |\PitonInputFile|. With the key |resume|, that
-% reinitialisation is not done.
+% \item With the key |resume| the counter of lines is not set to zero at the
+% beginning of each environment |{Piton}| or use |\PitonInputFile| as it is
+% otherwise. That's allows a numbering of the lines accross several
+% environments. See an example part \ref{example-numbering} on
+% page~\pageref{example-numbering}.
 %
+% \item The key |left-margin| corresponds to a margin on the left. That key may
+% be useful in conjonction with the key |line-numbers| or the key
+% |line-all-numbers| if one does not want the numbers in an overlapping position
+% on the left.
+%
 % \item The key |splittable| allows pages breaks within the environments
 % |{Piton}| and the listings produced by 
 % |\PitonInputFile|. 
 %
+% \smallskip
+% \colorbox{yellow!50}{\textbf{New 0.9}}\enskip It's possible to give as value to
+% the key |splittable| a positive integer~$n$. With that value, the environments
+% |{Piton}| and the listings produced by |\PitonInputFile| are splittable but no
+% page break can occur within the first $n$ lines and within the last $n$ lines.
+% The default value of the key |splittable| is, in fact, $1$, which allows pages
+% breaks everywhere.
+%
 % \item The key |background-color| sets the background color of the environments
 % |{Piton}| and the listings produced by |\PitonInputFile| (that background has
 % a width of |\linewidth|). Even with a background color, the pages breaks are
-% allowed, as soon as the key |splittable| is in force.\footnote{The
-% environments \texttt{\{Piton\}} are breakable, even within a breakable
-% environment of \pkg{tcolorbox}. Remind that an environment of \pkg{tcolorbox}
-% included in another environment of \pkg{tcolorbox} is \emph{not} breakable,
-% even when both environments use the key |breakable| of \pkg{tcolorbox}.}
-%
-% \item \colorbox{yellow!50}{\textbf{New 0.8}}\enskip The key |left-margin|
-% corresponds to a margin on the left. That key may be useful in conjonction
-% with the key |line-numbers| or the key |line-all-numbers| if one does not want
-% the numbers in an overlapping position on the left.
+% allowed, as soon as the key |splittable| is in force.\footnote{With the key
+% |splittable|, the environments \texttt{\{Piton\}} are breakable, even within a
+% (breakable) environment of \pkg{tcolorbox}. Remind that an environment of
+% \pkg{tcolorbox} included in another environment of \pkg{tcolorbox} is
+% \emph{not} breakable, even when both environments use the key |breakable| of
+% \pkg{tcolorbox}.}
 % \end{itemize}
 %
 % \bigskip
@@ -230,10 +252,11 @@
 % ~emphase&\PitonOptions{line-numbers,auto-gobble,background-color = gray!15}@
 % \begin{Piton}
 %     from math import pi
+%     def arctan(x,n=10):
+%         """Compute the mathematical value of arctan(x)
 %
-%     def arctan(x,n=10):
-%         '''Compute the value of arctan(x)
-%            n is the number of terms of the sum'''
+%         n is the number of terms in the sum
+%         """
 %         if x < 0:
 %             return -arctan(-x) # recursive call
 %         elif x > 1: 
@@ -254,8 +277,10 @@
 % from math import pi
 %
 % def arctan(x,n=10):
-%     '''Compute the value of arctan(x)
-%        n is the number of terms of the sum'''
+%     """Compute the mathematical value of arctan(x)
+%                                         
+%     n is the number of terms in the sum
+%     """
 %     if x < 0:
 %         return -arctan(-x) # recursive call
 %     elif x > 1: 
@@ -323,7 +348,7 @@
 % The package \pkg{piton} provides the command |\SetPitonStyle| to customize the
 % different styles used to format the syntactic elements of the Python listings.
 % The customizations done by that command are limited to the current TeX
-% group.\footnote{We remind that an LaTeX environment is, in particuilar, a TeX group.}
+% group.\footnote{We remind that an LaTeX environment is, in particular, a TeX group.}
 %
 % \bigskip
 % The command |\SetPitonStyle| takes in as argument a comma-separated list of
@@ -379,9 +404,10 @@
 % String.Long & the long strings (between \texttt{'''} or \verb|"""|) except the
 % documentation strings \\ 
 % String & that keys sets both |String.Short| and |String.Long| \\
-% String.Doc & the documentation strings \\
+% String.Doc & the documentation strings (only between \texttt{"""} following
+% PEP~257) \\
 % String.Interpol & the syntactic elements of the fields of the f-strings (that
-% is to say the characters \texttt{\{}, \texttt{\}} and \texttt{:}) \\
+% is to say the characters \texttt{\{} and \texttt{\}}) \\
 % Operator & the following operators : \texttt{!= == << >> - \~{} + / * \% = < > \& .} \verb+|+ \verb|@|\\
 % Operator.Word & the following operators : \texttt{in}, \texttt{is},
 % \texttt{and}, \texttt{or} and \texttt{not} \\ 
@@ -388,16 +414,17 @@
 % Name.Builtin & the predefined functions of Python \\
 % Name.Function & the name of the functions defined by the user, at the point of
 % their definition (that is to say after the keyword |def|) \\
-% Name.Decorator & the decorators (instructions beginning by \verb|@| in the classes) \\
+% Name.Decorator & the decorators (instructions beginning by \verb|@|) \\
 % Name.Namespace & the name of the modules (= external libraries) \\
-% Name.Class & the name of the classes at the point of their definition \\
+% Name.Class & the name of the classes at the point of their definition (that is
+% to say after the keyword |class|) \\
 % Exception & the names of the exceptions (eg: \texttt{SyntaxError}) \\
 % Comment & the comments beginning with \verb|#| \\
-% Comment.LaTeX & the comments beginning by \verb|##| which are composed in LaTeX by
-% \pkg{piton} (and called merely ``LaTeX comments'' in this document) \\
+% Comment.LaTeX & the comments beginning by \verb|##|, which are composed in LaTeX by
+% \pkg{piton} (and simply called ``LaTeX comments'' in this document) \\
 % Keyword.Constant & |True|, |False| and |None| \\
 % Keyword & the following keywords :
-%           \ttfamily assert, break, case, continue, del,
+%           \ttfamily as, assert, break, case, continue, def, del,
 %           elif, else, except, exec, finally, for, from, 
 %           global, if, import, lambda, non local,
 %           pass, raise, return, try, while,
@@ -458,8 +485,8 @@
 % pair |\footnotemark|--|\footnotetext|. 
 %
 % \smallskip
-% It's also possible to extract the footnotes with the help of the package
-% \pkg{footnote} or the package \pkg{footnotehyper}.
+% However, it's also possible to extract the footnotes with the help of the
+% package \pkg{footnote} or the package \pkg{footnotehyper}.
 %
 % \smallskip
 % If \pkg{piton} is loaded with the option |footnote| (with
@@ -479,17 +506,30 @@
 % in particular: it must be loaded after the package \pkg{xcolor} and it is not
 % perfectly compatible with \pkg{hyperref}.
 %
-% \medskip
-% In this document, the package \pkg{piton} has been loaded with the option
-% |footnotehyper|: see the first page of this document for an example of a
-% footnote in an environment |{Piton}|.
+% \medskip 
+% In this document, the package \pkg{piton} has been loaded with the
+% option |footnotehyper|. For examples of notes, cf. \ref{notes-examples},
+% p.~\pageref{notes-examples}.
 %
+% \subsection{Tabulations}
 %
+% \smallskip 
+% \colorbox{yellow!50}{\textbf{New 0.9}}\enskip Even though it's recommended to
+% indent the Python listings with spaces (see PEP~8), \pkg{piton} accepts the
+% characters of tabulation (that is to say the characters U+0009) at the
+% beginning of the lines. Each character U+0009 is replaced by $n$~spaces. The
+% initial value of $n$ is $4$ but it's possible to change it with the key
+% |tab-size| of |\PitonOptions|.
+%
 % \section{Examples}
 %
-%
 % \subsection{Line numbering}
 %
+% \label{example-numbering}
+%
+% We remind that it's possible to have an automatic numbering of the lines in
+% the Python listings by using the key |line-numbers| or the key |all-line-numbers|.
+%
 % By default, the numbers of the lines are composed by \pkg{piton} in an
 % overlapping position on the left (by using internally the command |\llap| of LaTeX).
 %
@@ -625,8 +665,10 @@
 %
 % \subsection{Notes in the listings}
 %
+% \label{notes-examples}
+% 
 % In order to be able to extract the notes (which are typeset with the command
-% |\footnote|, the extension \pkg{piton} must be loaded with the key |footnote|
+% |\footnote|), the extension \pkg{piton} must be loaded with the key |footnote|
 % or the key |footenotehyper| as explained in the section \ref{footnote}
 % p.~\pageref{footnote}. In this document, the extension \pkg{piton} has been
 % loaded with the key |footnotehyper|.
@@ -709,7 +751,7 @@
 %
 %
 % \vspace{1cm}
-% If one embed an environment |{Piton}| in an environment |{minipage}|
+% If we embed an environment |{Piton}| in an environment |{minipage}|
 % (typically in order to limit the width of a colored background), it's
 % necessary to embed the whole environment |{minipage}| in an environment
 % |{savenotes}| (of \pkg{footnote} or \pkg{footnotehyper}) in order to have the
@@ -778,7 +820,7 @@
 %     Name.Builtin = ,
 %     Name.Function = \bfseries \colorbox{gray!20} ,
 %     Comment = \color{gray} ,
-%     Comment.LaTeX = \color{gray},
+%     Comment.LaTeX = \normalfont \color{gray},
 %     Keyword = \bfseries ,
 %     Name.Namespace = ,
 %     Name.Class = , 
@@ -792,7 +834,7 @@
 %
 % \setmonofont[Scale=0.85]{DejaVu Sans Mono}
 %
-% \PitonOptions{splittable}
+%  \PitonOptions{splittable}
 %
 % \SetPitonStyle
 %   {
@@ -799,12 +841,12 @@
 %     Number = ,
 %     String = \itshape , 
 %     String.Doc = \color{gray} \slshape ,
+%     Operator.Word = \bfseries ,
 %     Operator = , 
-%     Operator.Word = \bfseries ,
 %     Name.Builtin = ,
 %     Name.Function = \bfseries \colorbox{gray!20} ,
 %     Comment = \color{gray} ,
-%     Comment.LaTeX = \color{gray} ,
+%     Comment.LaTeX = \normalfont \color{gray} ,
 %     Keyword = \bfseries ,
 %     Name.Namespace = ,
 %     Name.Class = , 
@@ -819,8 +861,10 @@
 % from math import pi
 %
 % def arctan(x,n=10):
-%     '''Compute the value of arctan(x)
-%        n is the number of terms if the sum'''
+%     """Compute the mathematical value of arctan(x)
+%                                               
+%     n is the number of terms in the sum
+%     """
 %     if x < 0:
 %         return -arctan(-x) # appel récursif
 %     elif x > 1: 
@@ -837,12 +881,137 @@
 % 
 %
 % \bigskip
-% The previous example has been composed while the key |splittable| (of
-% |\PitonOptions|) is in force. That's why a page break may have occurred.
 % \clearpage
 %
 % \section{Implementation}
 %
+% \subsection{Introduction}
+% 
+% The main job of the package \pkg{piton} is to take in as input a Python
+% listing and to send back to LaTeX as output that code \emph{with interlaced LaTeX
+% instructions of formatting}.
+%
+% In fact, all that job is done by a \textsc{lpeg} called |SyntaxPython|. That
+% \textsc{lpeg}, when matched against the string of a Python listing,
+% returns as capture a Lua table containing data to send to LaTeX. 
+% The only thing to do after will be to apply |tex.tprint| to each element of
+% that table.\footnote{Recall that |tex.tprint| takes in as argument a Lua table whose
+% first component is a ``catcode table'' and the second element a string. The
+% string will be sent to LaTeX with the regime of catcodes specified by the
+% catcode table. If no catcode table is provided, the standard catcodes of LaTeX
+% will be used.}
+% 
+% \bigskip
+% Consider, for example, the following Python code:
+%
+% \begin{Piton}
+% def parity(x):
+%     return x%2
+% \end{Piton}
+%
+% The capture returned by the \pkg{lpeg} |SyntaxPython| against that code is the
+% Lua table containing the following elements :
+%
+% \bigskip
+% \begin{minipage}{\linewidth}
+% \color{gray}
+% 
+% |{ "\\__piton_begin_line:" }|\footnote{Each line of the Python listings will
+% be encapsulated in a pair: \texttt{\textbackslash_@@_begin_line:} --
+% \texttt{\textbackslash@@_end_line:}. The token
+% \texttt{\textbackslash@@_end_line:} must be explicit because it will be used as
+% marker in order to delimit the argument of the command \texttt{\textbackslash
+% @@\_begin\_line:}. Both tokens \texttt{\textbackslash_@@_begin_line:} and
+% \texttt{\textbackslash@@_end_line:} will be nullified in the command
+% \texttt{\textbackslash piton} (since there can't be lines breaks in the
+% argument of a command \texttt{\textbackslash piton}).}  
+% 
+% \texttt{\{ "\{\textbackslash PitonStyle\{Keyword\}\{" \}}\footnote{The
+% lexical elements of Python for which we have a \pkg{piton} style will be
+% formatted via the use of the command \texttt{\textbackslash PitonStyle}.
+% Such an element is typeset in LaTeX via the syntax \texttt{\{\textbackslash
+% PitonStyle\{\textsl{style}\}\{...\}\}} because the instructions inside an \texttt{\textbackslash
+% PitonStyle} may be both semi-global declarations like
+% \texttt{\textbackslash bfseries} and commands with one argument like
+% \texttt{\textbackslash fbox}.}
+%
+% \texttt{\{
+% luatexbase.catcodetables.CatcodeTableOther\footnote{\texttt{luatexbase.catcodetables.CatcodeTableOther} is a mere number which corresponds to the ``catcode table'' whose all characters have the catcode ``other'' (which means that they will be typeset by LaTeX verbatim).}, "def" \} }
+%
+% |{ "}}" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, " " }|
+%
+% |{ "{\PitonStyle{Name.Function}{" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, "parity" }|
+%
+% |{ "}}" }|
+% 
+% |{ luatexbase.catcodetables.CatcodeTableOther, "(" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, "x" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, ")" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, ":" }|
+% 
+% |{ "\\__piton_end_line: \\__piton_newline: \\__piton_begin_line:" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, "    " }|
+%
+% |{ "{\PitonStyle{Keyword}{" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, "return" }|
+%
+% |{ "}}" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, " " }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, "x" }|
+%
+% |{ "{\PitonStyle{Operator}{" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, "&" }|
+%
+% |{ "}}" }|
+%
+% |{ "{\PitonStyle{Number}{" }|
+%
+% |{ luatexbase.catcodetables.CatcodeTableOther, "2" }|
+%
+% |{ "}}" }|
+% 
+% |{ "\\__piton_end_line:" }|
+% 
+% \end{minipage}
+%
+% \bigskip
+% We give now the LaTeX code which is sent back by Lua to TeX (we have written
+% on several lines for legibility but no character |\r| will be sent to LaTeX). The
+% characters which are greyed-out are sent to LaTeX with the catcode ``other''
+% (=12). All the others characters are sent with the regime of catcodes of L3
+% (as set by |\ExplSyntaxOn|)
+%
+% 
+% \begingroup
+% \def\gbox#1{\colorbox{gray!20}{\strut #1}}
+% \setlength{\fboxsep}{1pt}
+% 
+% \begin{Verbatim*}[formatcom = \color{black}]
+% \__piton_begin_line:{\PitonStyle{Keyword}{~gbox#def@}}
+% ~gbox# @{\PitonStyle{Name.Function}{~gbox#parity@}}~gbox#(x):@\__piton_end_line:\__piton_newline:
+% \__piton_begin_line:~gbox#    @{\PitonStyle{Keyword}{~gbox#return@}}
+% ~gbox# x@{\PitonStyle{Operator}{~gbox#%@}}{\PitonStyle{Number}{~gbox#2@}}\__piton_end_line:
+% \end{Verbatim*}
+% \endgroup
+%
+%
+% 
+%
+% \subsection{The L3 part of the implementation}
+%
+% \subsubsection{Declaration of the package}
 %     \begin{macrocode}
 \NeedsTeXFormat{LaTeX2e}
 \RequirePackage{l3keys2e}
@@ -1010,9 +1179,23 @@
 % |\c_@@_footnote_bool| in order to know if we have to insert an environment
 % |{savenotes}|.
 % 
+% \bigskip
+% \subsubsection{Parameters and technical definitions}
+%
+% We will compute (with Lua) the numbers of lines of the Python code and store
+% it in the following counter.
+%    \begin{macrocode}
+\int_new:N \l_@@_nb_lines_int
+%    \end{macrocode}
+% 
+% The following counter will be used to count the lines during the composition.
+% It will count all the lines, empty or not empty. It won't be used to print the
+% numbers of the lines.
+%    \begin{macrocode}
+\int_new:N \g_@@_line_int
+%    \end{macrocode}
+% 
 % \medskip
-% \subsection{Parameters}
-%
 % The following token list will contains the (potential) informations to write
 % on the |aux| (to be used in the next compilation).
 %    \begin{macrocode}
@@ -1020,12 +1203,21 @@
 %    \end{macrocode}
 % 
 % \medskip
-% The following flag corresponds to the key |splittable| of |\PitonOptions|.
+% The following counter corresponds to the key |splittable| of |\PitonOptions|.
+% If the value of |\l_@@_splittable_int| is equal to $n$, then no line break can
+% occur within the first $n$~lines or the last $n$~lines of the listings.
 %    \begin{macrocode}
-\bool_new:N \l_@@_splittable_bool
+\int_new:N \l_@@_splittable_int
 %    \end{macrocode}
 %
 % \medskip
+% An initial value of |splittable| equal to 100 is equivalent to say that the
+% environments |{Piton}| are unbreakable.
+%    \begin{macrocode}
+\int_set:Nn \l_@@_splittable_int { 100 }
+%    \end{macrocode}
+%
+% \medskip
 % The following string corresponds to the key |background-color| of |\PitonOptions|.
 %    \begin{macrocode}
 \str_new:N \l_@@_background_color_str
@@ -1033,9 +1225,12 @@
 %
 % \medskip
 % We will compute the maximal width of the lines of an environment |{Piton}| in
-% |\l_@@_width_dim|.
+% |\g_@@_width_dim|. We need a global variable because when the key |footnote|
+% is in force, each line when be composed in an environment |{savenotes}| and
+% (when |slim| is in force) we need to exit |\g_@@_width_dim| from that
+% environment.
 %    \begin{macrocode}
-\dim_new:N \l_@@_width_dim 
+\dim_new:N \g_@@_width_dim 
 %    \end{macrocode}
 % The value of that dimension as written on the |aux| file will be stored in 
 % |\l_@@_width_on_aux_dim|.
@@ -1061,24 +1256,52 @@
 %    \begin{macrocode}
 \dim_new:N \l_@@_left_margin_dim
 %    \end{macrocode}
-% 
-% \subsection{The gobbling mechanism}
 %
 % \medskip
-% The following integer is the number of characters to gobble on the left side
-% of the Python listings. Of course, the initial value is $0$.
+% The tabulators will be replaced by the content of the following token list.
 %    \begin{macrocode}
-\int_new:N \l_@@_gobble_int 
+\tl_new:N \l_@@_tab_tl
 %    \end{macrocode}
 %
 % \medskip
 %    \begin{macrocode}
-\cs_new_protected:Npn \@@_define_gobble_syntax:n #1
-  { \lua_now:n { define_gobble_syntax(#1) } }
+\cs_new_protected:Npn \@@_set_tab_tl:n #1
+  { 
+    \tl_clear:N \l_@@_tab_tl
+    \prg_replicate:nn { #1 } 
+      { \tl_put_right:Nn \l_@@_tab_tl { ~ } }
+  }
+\@@_set_tab_tl:n { 4 }
 %    \end{macrocode}
+% 
+% \medskip  
+%    \begin{macrocode}
+\cs_new_protected:Npn \@@_newline: 
+  { 
+    \int_gincr:N \g_@@_line_int
+    \int_compare:nNnT \g_@@_line_int > { \l_@@_splittable_int - 1 }
+      {
+        \int_compare:nNnT 
+          { \l_@@_nb_lines_int - \g_@@_line_int } > \l_@@_splittable_int 
+          {
+            \egroup
+            \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } 
+            \newline
+            \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } 
+            \vtop \bgroup
+          }
+     }
+  } 
+%    \end{macrocode}
+% 
+% \medskip
+% The following integer corresponds to the key |gobble|.
+%    \begin{macrocode}
+\int_new:N \l_@@_gobble_int 
+%    \end{macrocode}
 %
-% \medskip
-% \subsection{Treatment of a line of code}
+% \bigskip
+% \subsubsection{Treatment of a line of code}
 % 
 % In the contents provided by Lua, each line of the Python code will be
 % surrounded by |\@@_begin_line:| and |\@@_end_line:|. 
@@ -1086,8 +1309,6 @@
 %    \begin{macrocode}
 \cs_set_protected:Npn \@@_begin_line: #1 \@@_end_line:
   { 
-    \bool_lazy_and:nnT \l_@@_splittable_bool \c_@@_footnote_bool
-      { \begin { savenotes } } 
 %    \end{macrocode}
 %
 % Be careful: there is curryfication in the following lines.
@@ -1115,11 +1336,11 @@
         #1 \hfil 
       } 
 %    \end{macrocode}
-% We compute in |\l_@@_width_dim| the maximal width of the lines of the
+% We compute in |\g_@@_width_dim| the maximal width of the lines of the
 % environments. 
 %    \begin{macrocode}
-    \dim_compare:nNnT { \box_wd:N \l_tmpa_box } > \l_@@_width_dim 
-      { \dim_set:Nn \l_@@_width_dim { \box_wd:N \l_tmpa_box } }
+    \dim_compare:nNnT { \box_wd:N \l_tmpa_box } > \g_@@_width_dim 
+      { \dim_gset:Nn \g_@@_width_dim { \box_wd:N \l_tmpa_box } }
     \box_set_dp:Nn \l_tmpa_box { \box_dp:N \l_tmpa_box + 1.25 pt } 
     \box_set_ht:Nn \l_tmpa_box { \box_ht:N \l_tmpa_box + 1.25 pt } 
     \tl_if_empty:NTF \l_@@_background_color_str
@@ -1139,15 +1360,13 @@
             \box_use_drop:N \l_tmpa_box
           }
       }   
-    \bool_lazy_and:nnT \l_@@_splittable_bool \c_@@_footnote_bool
-      { \end { savenotes } }
     \vspace { - 2.5 pt }
   }
 %    \end{macrocode}
 %
 % 
-% \medskip
-% \subsection{PitonOptions}
+% \bigskip
+% \subsubsection{PitonOptions}
 %
 % \medskip
 % The following parameters correspond to the keys |line-numbers| and
@@ -1184,8 +1403,8 @@
     all-line-numbers .value_forbidden:n = true  ,
     resume           .bool_set:N        = \l_@@_resume_bool ,
     resume           .value_forbidden:n = true ,
-    splittable       .bool_set:N        = \l_@@_splittable_bool ,
-    splittable       .default:n         = true , 
+    splittable       .int_set:N         = \l_@@_splittable_int ,
+    splittable       .default:n         = 1 , 
     background-color .str_set:N         = \l_@@_background_color_str ,
     background-color .value_required:n  = true ,
     slim             .bool_set:N        = \l_@@_slim_bool ,
@@ -1192,11 +1411,14 @@
     slim             .default:n         = true ,
     left-margin      .dim_set:N         = \l_@@_left_margin_dim ,
     left-margin      .value_required:n  = true ,
+    tab-size         .code:n            = \@@_set_tab_tl:n { #1 } ,
+    tab-size         .value_required:n  = true , 
     unknown          .code:n = 
       \msg_error:nn { piton } { Unknown~key~for~PitonOptions }
   }
 %    \end{macrocode}
 %
+% 
 %  \bigskip
 %    \begin{macrocode}
 \msg_new:nnn { piton } { Unknown~key~for~PitonOptions }
@@ -1204,7 +1426,7 @@
     Unknown~key. \\
     The~key~'\l_keys_key_str'~is~unknown~for~\token_to_str:N \PitonOptions.~The~
     available~keys~are:~all-line-numbers,~auto-gobble,~env-gobble,~gobble,~
-    left-margin,~line-numbers,~resume,~slim~and~splittable.\\
+    left-margin,~line-numbers,~resume,~slim~splittable~and~tab-size.\\
     If~you~go~on,~that~key~will~be~ignored.
   }
 %    \end{macrocode}
@@ -1215,14 +1437,15 @@
 \NewDocumentCommand \PitonOptions { } { \keys_set:nn { PitonOptions } }
 %    \end{macrocode}
 %
-% \medskip
-% \subsection{The numbers of the lines}
+% \bigskip
+% \subsubsection{The numbers of the lines}
 %
 % \medskip
 % The following counter will be used to count the lines in the code when the
-% user requires the numbers of the lines to be printed.
+% user requires the numbers of the lines to be printed (with |line-numbers| or
+% |all-line-numbers|). 
 %    \begin{macrocode}
-\int_new:N \g_@@_line_int 
+\int_new:N \g_@@_visual_line_int 
 %    \end{macrocode}
 % 
 % 
@@ -1229,10 +1452,10 @@
 %    \begin{macrocode}
 \cs_new_protected:Npn \@@_print_number:
   {
-    \int_gincr:N \g_@@_line_int 
+    \int_gincr:N \g_@@_visual_line_int 
     \hbox_overlap_left:n
       { 
-        { \color { gray } \footnotesize \int_to_arabic:n \g_@@_line_int }
+        { \color { gray } \footnotesize \int_to_arabic:n \g_@@_visual_line_int }
         \skip_horizontal:n { 0.4 em } 
       }
   }
@@ -1239,8 +1462,8 @@
 %    \end{macrocode}
 % 
 %
-% \medskip
-% \subsection{The command to write on the aux file}
+% \bigskip
+% \subsubsection{The command to write on the aux file}
 %
 %   
 %    \begin{macrocode}
@@ -1270,7 +1493,7 @@
             \tl_gput_right:Nx \g_@@_aux_tl
               {
                 \dim_set:Nn \l_@@_width_on_aux_dim 
-                  { \dim_use:N \l_@@_width_dim + 0.5 em }
+                  { \dim_eval:n { \g_@@_width_dim + 0.5 em } }
               }
           }
       }
@@ -1277,8 +1500,8 @@
   }
 %    \end{macrocode}
 % 
-% \medskip
-% \subsection{The main commands and environments for the final user}
+% \bigskip
+% \subsubsection{The main commands and environments for the final user}
 % 
 %    \begin{macrocode}
 \NewDocumentCommand { \piton } { v }
@@ -1305,29 +1528,48 @@
   }
 %    \end{macrocode}
 % 
-% 
 % \bigskip
+% Despite its name, |\@@_pre_env:| will be used both in |\PitonInputFile| dans
+% in the environments such as |{Piton}|.
 %    \begin{macrocode}
-\NewDocumentCommand { \PitonInputFile } { m }
+\cs_new:Npn \@@_pre_env:
   {
     \int_gincr:N \g_@@_env_int 
     \tl_gclear:N \g_@@_aux_tl
     \tl_if_exist:cT { c_@@ _ \int_use:N \g_@@_env_int _ tl }
       { \use:c { c_@@ _ \int_use:N \g_@@_env_int _ tl } }
-    \bool_if:NF \l_@@_splittable_bool
-      { \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } }
     \dim_compare:nNnT \l_@@_width_on_aux_dim = \c_zero_dim
       { \dim_set_eq:NN \l_@@_width_on_aux_dim \linewidth }
-    \bool_if:NF \l_@@_resume_bool { \int_gzero:N \g_@@_line_int }
+    \bool_if:NF \l_@@_resume_bool { \int_gzero:N \g_@@_visual_line_int }
+    \dim_gzero:N \g_@@_width_dim
+    \int_gzero:N \g_@@_line_int
+    \dim_zero:N \parindent 
+    \dim_zero:N \lineskip 
+  }
+%    \end{macrocode}
+% 
+% \bigskip
+%    \begin{macrocode}
+\NewDocumentCommand { \PitonInputFile } { m }
+  {
     \group_begin:
-      \dim_set_eq:NN \parindent \c_zero_dim 
+      \@@_pre_env:
+      \mode_if_vertical:TF \mode_leave_vertical: \newline 
+%    \end{macrocode}
+% We count with Lua the number of lines of the argument. The result will be
+% stored by Lua in |\l_@@_nb_lines_int|. That information will be used to allow
+% or disallow page breaks.
+%    \begin{macrocode}
+      \lua_now:n { CountLinesFile(token.scan_argument()) } { #1 }  
       \ttfamily
-      \lua_now:e { ParseFile(token.scan_argument()) } { #1 }
-    \@@_width_to_aux:
+      \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } 
+      \vtop \bgroup 
+      \lua_now:n { ParseFile(token.scan_argument()) } { #1 }
+      \egroup 
+      \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } 
+      \@@_width_to_aux:
     \group_end:
     \@@_write_aux:
-    \bool_if:NF \l_@@_splittable_bool
-      { \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } }
   }
 %    \end{macrocode}
 %
@@ -1353,43 +1595,27 @@
       }
           {
             \group_end:
-            \mode_if_vertical:TF 
-              { \mode_leave_vertical: } 
-              \newline 
+            \mode_if_vertical:TF \mode_leave_vertical: \newline 
 %    \end{macrocode}
-% Be careful: there is curryfication in the following code.
+% We count with Lua the number of lines of the argument. The result will be
+% stored by Lua in |\l_@@_nb_lines_int|. That information will be used to allow
+% or disallow page breaks.
 %    \begin{macrocode}
-            \bool_if:NF \l_@@_splittable_bool 
+            \lua_now:n { CountLines(token.scan_argument()) } { ##1 }  
+            \ttfamily
+            \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } 
+            \vtop \bgroup 
+            \lua_now:e 
               { 
-                \bool_if:NT \c_@@_footnote_bool { \begin { savenotes } } 
-                \vbox_top:n 
-              }
-              {
-                \ttfamily
-                \dim_zero:N \lineskip 
-                \int_case:nnF \l_@@_gobble_int 
-                  { 
-                    0 
+                GobbleParse
+                  ( \int_use:N \l_@@_gobble_int , token.scan_argument() ) 
+              } 
+              { ##1 }
+            \vspace { 2.5 pt } 
+            \egroup 
+            \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } 
+            \@@_width_to_aux:
 %    \end{macrocode}
-% Be careful: the last argument is provided by curryfication.
-%    \begin{macrocode}
-                    { \lua_now:n { Parse(token.scan_argument()) } }
-                    { -1 }
-                    { \lua_now:n { AutoGobbleParse(token.scan_argument()) } } 
-                    { -2 }
-                    { \lua_now:n { EnvGobbleParse(token.scan_argument()) } } 
-                  }
-                  {
-                     \exp_args:NV \@@_define_gobble_syntax:n \l_@@_gobble_int 
-                     \lua_now:n { GobbleParse(token.scan_argument()) } 
-                  }
-                { ##1 }
-                \vspace { 2.5 pt } 
-                \@@_width_to_aux:
-              }
-            \bool_if:NF \l_@@_splittable_bool
-              { \bool_if:NT \c_@@_footnote_bool { \end { savenotes } } }
-%    \end{macrocode}
 % The following |\end{#1}| is only for the groups and the stack of
 % environments of LaTeX.
 %    \begin{macrocode}
@@ -1407,17 +1633,10 @@
     \NewDocumentEnvironment { #1 } { #2 } 
       {
         #3 
-        \int_gincr:N \g_@@_env_int 
-        \tl_gclear:N \g_@@_aux_tl
-        \tl_if_exist:cT { c_@@ _ \int_use:N \g_@@_env_int _ tl }
-          { \use:c { c_@@ _ \int_use:N \g_@@_env_int _ tl } }
-        \dim_compare:nNnT \l_@@_width_on_aux_dim = \c_zero_dim
-          { \dim_set_eq:NN \l_@@_width_on_aux_dim \linewidth }
-        \bool_if:NF \l_@@_resume_bool { \int_gzero:N \g_@@_line_int }
+        \@@_pre_env:
         \group_begin:
-        \box_clear:N \l_tmpa_box
         \tl_map_function:nN 
-          { \ \\ \{ \} \$ \& \# \^ \_ \% \~ } 
+          { \ \\ \{ \} \$ \& \# \^ \_ \% \~ \^^I } 
           \char_set_catcode_other:N 
         \use:c { _@@_collect_ #1 :w }
       }
@@ -1443,7 +1662,7 @@
 %    \end{macrocode}
 %
 % \bigskip
-% \subsection{The styles}
+% \subsubsection{The styles}
 % 
 % \medskip
 % The following command is fundamental: it will be used by the Lua code.
@@ -1553,8 +1772,8 @@
   }
 %    \end{macrocode}
 %
-% 
-% \subsection{The initial style}
+% \bigskip
+% \subsubsection{The initial style}
 %
 % The initial style is inspired by the style ``manni'' of Pygments.
 % 
@@ -1581,12 +1800,15 @@
     Name.Type        = \color[HTML]{336666} ,
     InitialValues    = \@@_piton:n ,
     Dict.Value       = \@@_piton:n ,
+    Interpol.Inside  = \color{black}\@@_piton:n ,
     Post.Function    = \@@_piton:n ,
-    Interpol.Inside  = \color{black}\@@_piton:n ,
   }
 %    \end{macrocode}
+% The last style |Post.Function| should be considered as an ``internal style''
+% (not available for the final user).
 %
-% \subsection{Security}
+% \bigskip
+% \subsubsection{Security}
 %
 %     \begin{macrocode}
 \AddToHook { env / piton / begin } 
@@ -1603,371 +1825,739 @@
 % 
 %
 % \bigskip
-% \subsection{The Lua code}
+% \subsection{The Lua part of the implementation}
 %
-% \medskip
+% \bigskip
 %    \begin{macrocode}
 \ExplSyntaxOff
 \RequirePackage{luacode}
 %    \end{macrocode}
 %   
-% \vspace{1cm}
+% \medskip
 %    \begin{macrocode}
 \begin{luacode*}
-local P, S, V , C , Ct, Cc, Cf = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cf
-
-
---[[ By convention, a capture which provides as value a table (and not a string), provides, in fact,
-a string (the first element of the table) which is a formatting LaTeX instruction (it will be
-thrown back to TeX with normal catcodes (ant not ``other'' catcode for everybody).]]
-
-local function L(string)
-  return Cc ( { string } ) 
+%    \end{macrocode}
+%
+% \bigskip
+% \subsubsection{Special functions dealing with LPEG}
+%
+% \medskip
+% We will use the Lua library \pkg{lpeg} which is built in LuaTeX. That's why we
+% define first aliases for several functions of that library.
+%    \begin{macrocode}
+local P, S, V, C, Ct, Cc, Cf = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cf
+%    \end{macrocode}
+% 
+%
+% \bigskip
+% The function |Q| takes in as argument a pattern and returns a \textsc{lpeg}
+% \emph{which does a capture} of the pattern. That capture will be sent to LaTeX
+% with the catcode ``other'' for all the characters: it's suitable for elements
+% of the Python listings that \pkg{piton} will typeset verbatim (thanks to the
+% catcode ``other''). That function will be widely used.
+%    \begin{macrocode}
+local function Q(pattern)
+  return Ct ( Cc ( luatexbase.catcodetables.CatcodeTableOther ) * C ( pattern ) )
 end 
-
+%    \end{macrocode}
+%
+% \bigskip
+% The function |L| takes in as argument a pattern and returns a \textsc{lpeg}
+% \emph{which does a capture} of the pattern. That capture will be sent to LaTeX
+% with standard LaTeX catcodes for all the characters: the elements captured
+% will be formatted as normal LaTeX codes. It's suitable for the ``comment
+% LateX'' in the environments |{Piton}| and the elements beetween
+% ``|escape-inside|''. That function won't be much used.
+%    \begin{macrocode}
+local function L(pattern)
+  return Ct ( C ( pattern ) ) 
+end
+%    \end{macrocode}
+%
+% \bigskip
+% The function |Lc| (the c is for \emph{constant}) takes in as argument a string
+% and returns a \textsc{lpeg} \emph{with does a constant capture} which returns
+% that string. The elements captured will be formatted as L3 code. It will be
+% used to send to LaTeX all the formatting LaTeX instructions we have to insert
+% in order to do the syntactic highlighting (that's the main job of
+% \pkg{piton}). That function will be widely used.
+%    \begin{macrocode}
+local function Lc(string)
+  return Cc ( { luatexbase.catcodetables.expl , string } ) 
+end 
+%    \end{macrocode}
+% 
+% \bigskip
+% The function |K| function creates a \textsc{lpeg} which will return as capture
+% the whole LaTeX code corresponding to a Python chunk (that is to say with the
+% LaTeX formatting instructions corresponding to the syntactic nature of that
+% Python chunk). The first argument is a pattern (that is to say a \textsc{lpeg}
+% without capture) and the second element is a Lua string corresponding to the
+% name of a \pkg{piton} style.
+%    \begin{macrocode}
 local function K(pattern, style)
   return 
-     L ( "{\\PitonStyle{" .. style .. "}{" )
-     * C(pattern)
-     * L ( "}}" )
+     Lc ( "{\\PitonStyle{" .. style .. "}{" )
+     * Q ( pattern )
+     * Lc ( "}}" )
 end
-
-
---[[ The text in "escape" (between begin_escape and end_escape) is captured
-and put in a table (with only one component). Indeed, we have decided that a capture
-which is encapsulated in a table must be transmitted to TeX with the normal TeX catcodes.]]
-
-local Escape = P(begin_escape)
-               * Ct ( C ( ( 1 - P(end_escape) ) ^ 1 ) )
-               * P(end_escape)
-
-
-lpeg.locale(lpeg) -- mandatory
-
-local alpha , digit , space , punct = lpeg.alpha , lpeg.digit , lpeg.space , lpeg.punct
-
--- Remember that à, â, ç, etc. are strings of length 2 (2 bytes)
+%    \end{macrocode}
+% The formatting commands in a given \pkg{piton} style (eg. the style |Keyword|)
+% may be semi-global declarations (such as |\bfseries| or |\slshape|) or LaTeX
+% macros with an argument (such as |\fbox| or |\colorbox{yellow}|). In order to
+% deal with both syntaxes, we have used two pairs of braces: 
+% |{\PitonStyle{Keyword}{|\texttt{\slshape text to format}|}}|.
+% 
+% \bigskip
+% The following \textsc{lpeg} catches the Python chunks which are in LaTeX
+% escapes (and that chunks will be considered as normal LaTeX constructions). We
+% recall that |begin_espace| and |end_escape| are Lua strings corresponding to
+% the key |escape-inside|\footnote{The \pkg{piton} key |escape-inside| is
+% available at load-time only.}. Since the elements that will be catched must be
+% sent to LaTeX with standard LaTeX catcodes, we put the capture (done by the
+% function |C|) in a table (by using |Ct|, which is an alias for |lpeg.Ct|)
+% without number of catcode table at the first component of the table.
+%    \begin{macrocode}
+local Escape = 
+  P(begin_escape)
+  * L ( ( 1 - P(end_escape) ) ^ 1 ) 
+  * P(end_escape)
+%    \end{macrocode}
+% 
+% \vspace{1cm}
+% The following line is mandatory.
+%    \begin{macrocode}
+lpeg.locale(lpeg)
+%    \end{macrocode}
+%
+% \bigskip
+% \subsubsection{The LPEG SyntaxPython}
+%
+%    \begin{macrocode}
+local alpha, digit, space = lpeg.alpha, lpeg.digit, lpeg.space
+%    \end{macrocode}
+% 
+% Remember that, for \textsc{lpeg}, the Unicode characters such as |à|, |â|,
+% |ç|, etc. are in fact strings of length 2 (2 bytes) because \pkg{lpeg} is not
+% Unicode-aware. 
+%    \begin{macrocode}
 local letter = alpha + P "_" 
- + P "â" + P "à" + P "ç" + P "é" + P "è" + P "ê" + P "ë" + P "ï" + P "î" + P "ô" + P "û" + P "ü" + 
-     P "Â" + P "À" + P "Ç" + P "É" + P "È" + P "Ê" + P "Ë" + P "Ï" + P "Î" + P "Ô" + P "Û" + P "Ü" 
+  + P "â" + P "à" + P "ç" + P "é" + P "è" + P "ê" + P "ë" + P "ï" + P "î" 
+  + P "ô" + P "û" + P "ü" + P "Â" + P "À" + P "Ç" + P "É" + P "È" + P "Ê" 
+  + P "Ë" + P "Ï" + P "Î" + P "Ô" + P "Û" + P "Ü" 
 
 local alphanum = letter + digit
-
+%    \end{macrocode}
+% 
+% \bigskip
+% The following \textsc{lpeg} |identifier| is a mere pattern (that is to say
+% more or less a regular expression) which matches the Python identifiers (hence
+% the name).
+%    \begin{macrocode}
 local identifier = letter * alphanum ^ 0
-
-local Identifier = C ( identifier )
-
-local Space = C ( ( space - P "\r" ) ^ 1 )
-
-local SkipSpace = C ( ( space - P "\r" ) ^ 0 )
-
-local Punct = C ( punct )
-
-
-local EOL = ( P "\r" )
-            *
-            (
-              ( space^0 * -1 )
-              + 
-              Cc (
-                   { 
-                     luatexbase.catcodetables.expl , 
-                     '\\__piton_end_line: \\bool_if:NT \\l__piton_splittable_bool \\newline \\__piton_begin_line:'
-                   } 
-                 )
-            ) 
-
-
+%    \end{macrocode}
+% 
+% \medskip
+% On the other hand, the \textsc{lpeg} |Identifier| (with a capital) also return
+% a \emph{capture}. Since no special LaTeX formatting will be applied to the
+% Python identifiers, we use the function~|Q| and not the function~|K|. For
+% elements which require formatting, we will usually use our function~|K|
+% instead of the function~|C|. See just below for an example of use of the
+% function~|K|.
+%    \begin{macrocode}
+local Identifier = Q ( identifier )
+%    \end{macrocode}
+%
+% \bigskip
+% By convention, we will use names with an initial capital for \textsc{lpeg}
+% which return captures.
+%
+% 
+% \bigskip
+% Here is the first use of our function~|K|. That function will be used to
+% construct \textsc{lpeg} which capture Python chunks for which we have a
+% dedicated \pkg{piton} style. For example, for the numbers, \pkg{piton}
+% provides a style which is called |Number|. The name of the style is provided
+% as a Lua string in the second argument of the function~|K|. By convention, we
+% use single quotes for delimiting the Lua strings which are names of
+% \pkg{piton} styles (but this is only a convention).
+%    \begin{macrocode}
 local Number =
-   K (
-       ( digit^1 * P "." * digit^0 + digit^0 * P "." * digit^1 + digit^1 )
-       * ( S "eE" * S "+-" ^ -1 * digit^1 ) ^ -1
-       + digit^1 
-       , 'Number' ) 
+  K (
+      ( digit^1 * P "." * digit^0 + digit^0 * P "." * digit^1 + digit^1 )
+      * ( S "eE" * S "+-" ^ -1 * digit^1 ) ^ -1
+      + digit^1 ,
+      'Number' 
+    ) 
+%    \end{macrocode}
+%
+% \bigskip
+% We recall that |begin_espace| and |end_escape| are Lua strings 
+% corresponding to the key |escape-inside|\footnote{The \pkg{piton} key
+% |escape-inside| is available at load-time only.}. Of course, if the final user
+% has not used the key |escape-inside|, these strings are empty.
+%    \begin{macrocode}
+if begin_escape ~= ''
+then Word = Q ( ( ( 1 - space - P(begin_escape) - P(end_escape) ) 
+                   - S "'\"\r[()]" - digit ) ^ 1 )
+else Word = Q ( ( ( 1 - space ) - S "'\"\r[()]" - digit ) ^ 1 )
+end
+%    \end{macrocode}
+%
+% \bigskip
+%    \begin{macrocode}
+local Space = Q ( ( space - P "\r" ) ^ 1 )
 
+local SkipSpace = Q ( ( space - P "\r" ) ^ 0 )
 
-local Word = C ( ( ( 1 - space ) - S "'\"\r[()]" - digit ) ^ 1 )
+local Punct = Q ( S ".,:;!" )
+%    \end{macrocode}
+%
+% \bigskip
+%    \begin{macrocode}
+local Tab = P "\t" * Lc ( '\\l_@@_tab_tl' )
+%    \end{macrocode}
+% 
+% \bigskip
+% The following \textsc{lpeg} |EOL| is for the end of lines.
+%    \begin{macrocode}
+local EOL = 
+  P "\r" 
+  *
+  (
+    ( space^0 * -1 )
+    + 
+%    \end{macrocode}
+% We recall that each line in the Python code we have to parse will be sent
+% back to LaTeX between a pair |\@@_begin_line:| --
+% |\@@_end_line:|\footnote{Remember that the \texttt{\textbackslash
+% @@\_end\_line:} must be explicit because it will be used as marker in order to
+% delimit the argument of the command \texttt{\textbackslash @@\_begin\_line:}}.
+%    \begin{macrocode}
+    Lc ( '\\@@_end_line: \\@@_newline: \\@@_begin_line:' )
+  ) 
+%    \end{macrocode}
+%
+% \bigskip
+%    \begin{macrocode}
+local Delim = Q ( S "[()]" )
 
-if begin_escape ~= ''
-then  Word = C ( ( ( 1 - space - P(begin_escape) - P(end_escape) ) - S "'\"\r[()]" - digit ) ^ 1 )
-end
+local Operator = 
+  K ( P "!=" + P "==" + P "<<" + P ">>" + S "-~+/*%=<>&.@|" , 'Operator')
 
-local Delim = C ( S "[()]" )
+local OperatorWord = 
+  K ( P "in" + P "is" + P "and" + P "or" + P "not" , 'Operator.Word')
 
-
 local Keyword = 
-   K ( P "assert" + P "break" + P "case" + P "continue" + P "del"
-       + P "elif" + P "else" + P "except" + P "exec" + P "finally" + P "for" + P "from" 
-       + P "global" + P "if" + P "import" + P "lambda" + P "non local"
-       + P "pass" + P "return" + P "try" + P "while"
-       + P "with" + P "yield" + P "yield from" ,
-   'Keyword' )
-   + K ( P "True" + P "False" + P "None" , 'Keyword.Constant' ) 
+  K ( P "as" + P "assert" + P "break" + P "case" + P "class" + P "continue" 
+      + P "def" + P "del" + P "elif" + P "else" + P "except" + P "exec" 
+      + P "finally" + P "for" + P "from" + P "global" + P "if" + P "import" 
+      + P "lambda" + P "non local" + P "pass" + P "return" + P "try" 
+      + P "while" + P "with" + P "yield" + P "yield from" ,
+  'Keyword' )
+  + K ( P "True" + P "False" + P "None" , 'Keyword.Constant' ) 
 
-
 local Builtin = 
-   K ( P "__import__" + P "abs" + P "all" + P "any" + P "bin" + P "bool" + P "bytearray"
-     + P "bytes" + P "chr" + P "classmethod" + P "compile" + P "complex" + P "delattr"
-     + P "dict" + P "dir" + P "divmod" + P "enumerate" + P "eval" + P "filter"
-     + P "float" + P "format" + P "frozenset" + P "getattr" + P "globals" + P "hasattr"
-     + P "hash" + P "hex" + P "id" + P "input" + P "int" + P "isinstance" + P "issubclass"
-     + P "iter" + P "len" + P "list" + P "locals" + P "map" + P "max" + P "memoryview" + P "min"
-     + P "next" + P "object" + P "oct" + P "open" + P "ord" + P "pow" + P "print" + P "property"
-     + P "range" + P "repr" + P "reversed" + P "round" + P "set" + P "setattr" + P "slice"
-     + P "sorted" + P "staticmethod" + P "str" + P "sum" + P "super" + P "tuple" + P "type"
-     + P "vars" + P "zip" ,
-   'Name.Builtin' )
+  K ( P "__import__" + P "abs" + P "all" + P "any" + P "bin" + P "bool" 
+    + P "bytearray" + P "bytes" + P "chr" + P "classmethod" + P "compile" 
+    + P "complex" + P "delattr" + P "dict" + P "dir" + P "divmod" 
+    + P "enumerate" + P "eval" + P "filter" + P "float" + P "format" 
+    + P "frozenset" + P "getattr" + P "globals" + P "hasattr" + P "hash" 
+    + P "hex" + P "id" + P "input" + P "int" + P "isinstance" + P "issubclass"
+    + P "iter" + P "len" + P "list" + P "locals" + P "map" + P "max" 
+    + P "memoryview" + P "min" + P "next" + P "object" + P "oct" + P "open" 
+    + P "ord" + P "pow" + P "print" + P "property" + P "range" + P "repr" 
+    + P "reversed" + P "round" + P "set" + P "setattr" + P "slice" + P "sorted" 
+    + P "staticmethod" + P "str" + P "sum" + P "super" + P "tuple" + P "type"
+    + P "vars" + P "zip" ,
+  'Name.Builtin' )
 
-
 local Exception =
-   K ( "ArithmeticError" + P "AssertionError" + P "AttributeError"
-    + P "BaseException" + P "BufferError" + P "BytesWarning" + P "DeprecationWarning"
-    + P "EOFError" + P "EnvironmentError" + P "Exception" + P "FloatingPointError"
-    + P "FutureWarning" + P "GeneratorExit" + P "IOError" + P "ImportError"
-    + P "ImportWarning" + P "IndentationError" + P "IndexError" + P "KeyError"
-    + P "KeyboardInterrupt" + P "LookupError" + P "MemoryError" + P "NameError"
-    + P "NotImplementedError" + P "OSError" + P "OverflowError"
-    + P "PendingDeprecationWarning" + P "ReferenceError" + P "ResourceWarning"
-    + P "RuntimeError" + P "RuntimeWarning" + P "StopIteration"
-    + P "SyntaxError" + P "SyntaxWarning" + P "SystemError" + P "SystemExit"
-    + P "TabError" + P "TypeError" + P "UnboundLocalError" + P "UnicodeDecodeError"
-    + P "UnicodeEncodeError" + P "UnicodeError" + P "UnicodeTranslateError"
-    + P "UnicodeWarning" + P "UserWarning" + P "ValueError" + P "VMSError"
-    + P "Warning" + P "WindowsError" + P "ZeroDivisionError"
-    + P "BlockingIOError" + P "ChildProcessError" + P "ConnectionError"
-    + P "BrokenPipeError" + P "ConnectionAbortedError" + P "ConnectionRefusedError"
-    + P "ConnectionResetError" + P "FileExistsError" + P "FileNotFoundError"
-    + P "InterruptedError" + P "IsADirectoryError" + P "NotADirectoryError"
-    + P "PermissionError" + P "ProcessLookupError" + P "TimeoutError"
-    + P "StopAsyncIteration" + P "ModuleNotFoundError" + P "RecursionError" ,
-   'Exception' )
+  K ( "ArithmeticError" + P "AssertionError" + P "AttributeError"
+   + P "BaseException" + P "BufferError" + P "BytesWarning" + P "DeprecationWarning"
+   + P "EOFError" + P "EnvironmentError" + P "Exception" + P "FloatingPointError"
+   + P "FutureWarning" + P "GeneratorExit" + P "IOError" + P "ImportError"
+   + P "ImportWarning" + P "IndentationError" + P "IndexError" + P "KeyError"
+   + P "KeyboardInterrupt" + P "LookupError" + P "MemoryError" + P "NameError"
+   + P "NotImplementedError" + P "OSError" + P "OverflowError"
+   + P "PendingDeprecationWarning" + P "ReferenceError" + P "ResourceWarning"
+   + P "RuntimeError" + P "RuntimeWarning" + P "StopIteration"
+   + P "SyntaxError" + P "SyntaxWarning" + P "SystemError" + P "SystemExit"
+   + P "TabError" + P "TypeError" + P "UnboundLocalError" + P "UnicodeDecodeError"
+   + P "UnicodeEncodeError" + P "UnicodeError" + P "UnicodeTranslateError"
+   + P "UnicodeWarning" + P "UserWarning" + P "ValueError" + P "VMSError"
+   + P "Warning" + P "WindowsError" + P "ZeroDivisionError"
+   + P "BlockingIOError" + P "ChildProcessError" + P "ConnectionError"
+   + P "BrokenPipeError" + P "ConnectionAbortedError" + P "ConnectionRefusedError"
+   + P "ConnectionResetError" + P "FileExistsError" + P "FileNotFoundError"
+   + P "InterruptedError" + P "IsADirectoryError" + P "NotADirectoryError"
+   + P "PermissionError" + P "ProcessLookupError" + P "TimeoutError"
+   + P "StopAsyncIteration" + P "ModuleNotFoundError" + P "RecursionError" ,
+  'Exception' )
 
-local RaiseException = K ( P "raise" , 'Keyword' ) * SkipSpace * Exception * C ( P "(" ) 
+local RaiseException = K ( P "raise" , 'Keyword' ) * SkipSpace * Exception * Q ( P "(" ) 
 
-local ExceptionInConsole = Exception *  C ( ( 1 - P "\r" ) ^ 0 ) * EOL  
-
-
-local Namespace =
-   K ( P "from" , 'Keyword' ) * Space * K ( alphanum^1 , 'Name.Namespace' )
-   *  ( Space * K ( P "import" , 'Keyword' ) ) ^ -1
-
-
-local ImportAs = K ( P "import" , 'Keyword' )
-                  * Space
-                  * K ( identifier , 'Name.Namespace' )
-                  * ( SkipSpace * C ( P "," ) * SkipSpace * K ( identifier , 'Name.Namespace' ) ) ^ 0 
-                  * (
-                      Space * K ( P "as" , 'Keyword' ) * Space * K ( identifier , 'Name.Namespace' )
-                    ) ^ 0 
-
-local Class = K ( P "class" , 'Keyword' ) 
-                * ( Space * K ( identifier , 'Name.Class' ) ) ^ -1 
-
+local ExceptionInConsole = Exception *  Q ( ( 1 - P "\r" ) ^ 0 ) * EOL  
+%    \end{macrocode}
+%
+% \bigskip
+% In Python, a ``decorator'' is a statement whose begins by |@| which patches
+% the function defined in the following statement.
+%    \begin{macrocode}
 local Decorator = K ( P "@" * letter^1 , 'Name.Decorator' ) 
-
+%    \end{macrocode}
+% 
+% \bigskip
+% The following \textsc{lpeg} |DefClass| will be used to detect the definition of a
+% new class (the name of that new class will be formatted with the \pkg{piton}
+% style |Name.Class|). 
+%
+% \smallskip
+% Example:\enskip \piton|class myclass:|
+%    \begin{macrocode}
+local DefClass = 
+  K ( P "class" , 'Keyword' ) * Space * K ( identifier , 'Name.Class' ) 
+%    \end{macrocode}
+% 
+% If the word |class| is not followed by a identifier, it will be catched as
+% keyword by the \textsc{lpeg} |Keyword| (useful if we want to type a
+% list of keywords).
+%
+%
+% \bigskip
+% The following \textsc{lpeg} |ImportAs| is used for the lines beginning by |import|.
+% % We have to detect the potential keyword |as| because both the name of the
+% module and its alias must be formatted with the \pkg{piton} style |Name.Namespace|.
+%
+% \smallskip
+% Example:\enskip \piton|import numpy as np|
+%
+% \smallskip
+% Moreover, after the keyword |import|, it's possible to have a comma-separated
+% list of modules (if the keyword |as| is not used).
+%
+% \smallskip
+% Example:\enskip \piton|import math, numpy|
+%    \begin{macrocode}
+local ImportAs = 
+  K ( P "import" , 'Keyword' )
+   * Space 
+   * K ( identifier * ( P "." * identifier ) ^ 0 , 
+         'Name.Namespace' 
+       )
+   * ( 
+       ( Space * K ( P "as" , 'Keyword' ) * Space * K ( identifier , 'Name.Namespace' ) )
+       + 
+       ( SkipSpace * Q ( P "," ) * SkipSpace * K ( identifier , 'Name.Namespace' ) ) ^ 0 
+     ) 
+%    \end{macrocode}
+% Be careful: there is no commutativity of |+| in the previous expression.
+%
+% \bigskip
+% The \textsc{lpeg} |FromImport| is used for the lines beginning by |from|. We
+% need a special treatment because the identifier following the keyword |from|
+% must be formatted with the \pkg{piton} style |Name.Namespace| and the
+% following keyword |import| must be formatted with the \pkg{piton} style
+% |Keyword| and must \emph{not} be catched by the \textsc{lpeg} |ImportAs|.
+%
+% \smallskip
+% Example:\enskip \piton|from math import pi|
+%
+% \smallskip
+%    \begin{macrocode}
+local FromImport =
+  K ( P "from" , 'Keyword' ) 
+    * Space * K ( identifier , 'Name.Namespace' )
+    * Space * K ( P "import" , 'Keyword' ) 
+%    \end{macrocode}
+%
+% \bigskip
+% \paragraph{The strings of Python}
+%
+% For the strings in Python, there are four categories of delimiters (without
+% counting the prefixes for f-strings and raw strings). We will use, in the
+% names of our \textsc{lpeg}, prefixes to distinguish the \textsc{lpeg} dealing
+% with that categories of strings, as presented in the following tabular.
+% \begin{center}
+% \begin{tabular}{ccc}
+% \toprule
+%         & |Single| & |Double| \\
+% \midrule
+% |Short| & |'text'|     & |"text"| \\
+% |Long|  & |'''test'''| & |"""text"""| \\
+% \bottomrule
+% \end{tabular}
+% \end{center}
+%
+% 
+% \bigskip
+% First, we define \textsc{lpeg} for the interpolations in the f-strings. Here
+% is an example of a f-string with an interpolation and a format
+% instruction\footnote{There is no special \pkg{piton} style for the formatting
+% instruction (after the comma): the style which will be applied will be the
+% style of the encompassing string, that is to say |String.Short| or
+% |String.Long|.} in that interpolation:
+%
+% \piton|f'Total price: {total+1:.2f} €'|
+%
+% \medskip
+% The following \textsc{lpeg} |SingleShortInterpol| (and the three variants)
+% will catch the whole interpolation, included the braces, that is to say, in
+% the previous example:\enskip |{total+1:.2f}|
+% 
+% \medskip
+%   \begin{macrocode}
 local SingleShortInterpol =
     K ( P "{" , 'String.Interpol')
   * K ( ( 1 - S "}':" ) ^ 0 , 'Interpol.Inside' )
-  * C ( P ":" * (1 - S "}:'") ^ 0 ) ^ -1
+  * Q ( P ":" * (1 - S "}:'") ^ 0 ) ^ -1
   * K ( P "}" , 'String.Interpol' )
 
 local DoubleShortInterpol =
     K ( P "{" , 'String.Interpol' )
   * K ( ( 1 - S "}\":" ) ^ 0 , 'Interpol.Inside' )
-  * ( K ( P ":" , 'String.Interpol' ) * C ( (1 - S "}:\"") ^ 0 ) ) ^ -1
+  * ( K ( P ":" , 'String.Interpol' ) * Q ( (1 - S "}:\"") ^ 0 ) ) ^ -1
   * K ( P "}" , 'String.Interpol' )
 
 local SingleLongInterpol =
     K ( P "{" , 'String.Interpol' )
   * K ( ( 1 - S "}:\r" - P "'''" ) ^ 0 , 'Interpol.Inside' )
-  * C ( P ":" * (1 - S "}:\r" - P "'''" ) ^ 0 ) ^ -1
+  * Q ( P ":" * (1 - S "}:\r" - P "'''" ) ^ 0 ) ^ -1
   * K ( P "}" , 'String.Interpol' )
 
 local DoubleLongInterpol =
     K ( P "{" , 'String.Interpol' )
   * K ( ( 1 - S "}:\r" - P "\"\"\"" ) ^ 0 , 'Interpol.Inside' )
-  * C ( P ":" * (1 - S "}:\r" - P "\"\"\"" ) ^ 0 ) ^ -1
+  * Q ( P ":" * (1 - S "}:\r" - P "\"\"\"" ) ^ 0 ) ^ -1
   * K ( P "}" , 'String.Interpol' )
+%    \end{macrocode}
+%
+% \bigskip
+% Now, we define \textsc{lpeg} for the parts of the strings which are \emph{not}
+% in the interpolations.
+%    \begin{macrocode}
+local SingleShortPureString = 
+  Q ( ( P "\\'" + P "{{" + P "}}" + 1 - S "{}'" ) ^ 1 )
 
-local SingleShortPureString = C ( ( P "\\'" + P "{{" + P "}}" + 1 - S "{}'" ) ^ 1 )
+local DoubleShortPureString = 
+  Q ( ( P "\\\"" + P "{{" + P "}}" + 1 - S "{}\"" ) ^ 1 )
 
-local DoubleShortPureString = C ( ( P "\\\"" + P "{{" + P "}}" + 1 - S "{}\"" ) ^ 1 )
+local SingleLongPureString = 
+  Q ( ( 1 - P "'''" - S "{}'\r" ) ^ 1 )
 
-local SingleLongPureString = C ( ( 1 - P "'''" - S "{}'\r" ) ^ 1 )
-
-local DoubleLongPureString = C ( ( 1 - P "\"\"\"" - S "{}\"\r" ) ^ 1 ) 
-
+local DoubleLongPureString = 
+  Q ( ( 1 - P "\"\"\"" - S "{}\"\r" ) ^ 1 ) 
+%    \end{macrocode}
+% 
+% \bigskip
+% We can now define the \textsc{lpeg} for the four kinds of strings. It's not
+% possible to use our function~|K| because of the interpolations which must be
+% formatted with another \pkg{piton} style that the rest of the
+% string.\footnote{The interpolations are formatted with the \pkg{piton} style
+% |Interpol.Inside|. The initial value of that style is \texttt{\textbackslash
+% @@\_piton:n} wich means that the interpolations are parsed once again by \pkg{piton}.}
+%    \begin{macrocode}
 local SingleShortString =
-    L ( "{\\PitonStyle{String.Short}{" )
-     * (
-           C ( P "f'" + P "F'" ) 
-           * ( SingleShortInterpol + SingleShortPureString ) ^ 0 
-           * C ( P "'" )
-         + 
-           C ( ( P "'" + P "r'" + P "R'" ) * ( P "\\'" + 1 - S "'\r" ) ^ 0 * P "'" )
-       )
-     * L ( "}}" ) 
+  Lc ( "{\\PitonStyle{String.Short}{" )
+   * (
+%    \end{macrocode}
+% First, we deal with the f-strings of Python, which are prefixed by |f| or |F|.
+%    \begin{macrocode}
+         Q ( P "f'" + P "F'" ) 
+         * ( SingleShortInterpol + SingleShortPureString ) ^ 0 
+         * Q ( P "'" )
+       + 
+%    \end{macrocode}
+% Now, we deal with the standard strings of Python, but also the ``raw strings''.
+%    \begin{macrocode}
+         Q ( ( P "'" + P "r'" + P "R'" ) 
+         * ( P "\\'" + 1 - S "'\r" ) ^ 0 * P "'" )
+     )
+   * Lc ( "}}" ) 
 
 local DoubleShortString =
-    L ( "{\\PitonStyle{String.Short}{" )
-     * (
-           C ( P "f\"" + P "F\"" ) 
-           * ( DoubleShortInterpol + DoubleShortPureString ) ^ 0 
-           * C ( P "\"" )
-         +
-           C ( ( P "\"" + P "r\"" + P "R\"" ) * ( P "\\\"" + 1 - S "\"\r" ) ^ 0 * P "\"" )
-       )
-     * L ( "}}" ) 
+  Lc ( "{\\PitonStyle{String.Short}{" )
+   * (
+         Q ( P "f\"" + P "F\"" ) 
+         * ( DoubleShortInterpol + DoubleShortPureString ) ^ 0 
+         * Q ( P "\"" )
+       +
+         Q ( ( P "\"" + P "r\"" + P "R\"" ) 
+         * ( P "\\\"" + 1 - S "\"\r" ) ^ 0 * P "\"" )
+     )
+   * Lc ( "}}" ) 
 
 
 local ShortString = SingleShortString + DoubleShortString
-
-
+%    \end{macrocode}
+% 
+% \bigskip
+% Of course, it's more complicated for ``longs strings'' because, by definition,
+% in Python, those strings may be broken by an end on line (which is catched by
+% the \textsc{lpeg} |EOL|).
+%    \begin{macrocode}
 local SingleLongString =
-      L "{\\PitonStyle{String.Long}{" 
-       * (
-             C ( S "fF" * P "'''" )
+  Lc "{\\PitonStyle{String.Long}{" 
+   * (
+         Q ( S "fF" * P "'''" )
+         * ( SingleLongInterpol + SingleLongPureString ) ^ 0
+         * Lc "}}" 
+         * (
+             EOL
+             + 
+             Lc "{\\PitonStyle{String.Long}{" 
              * ( SingleLongInterpol + SingleLongPureString ) ^ 0
-             * L "}}" 
-             * (
-                 EOL
-                 + 
-                 L "{\\PitonStyle{String.Long}{" 
-                 * ( SingleLongInterpol + SingleLongPureString ) ^ 0
-                 * L "}}"
-                 * EOL
-               ) ^ 0 
-             * L "{\\PitonStyle{String.Long}{" 
-             * ( SingleLongInterpol + SingleLongPureString ) ^ 0
-           +
-             C ( ( S "rR" ) ^ -1  * P "'''" * ( 1 - P "'''" - P "\r" ) ^ 0 ) 
-             * L "}}" 
-             * (
-                 L "{\\PitonStyle{String.Long}{" 
-                 * C ( ( 1 - P "'''" - P "\r" ) ^ 0 )
-                 * L "}}" 
-                 * EOL
-               ) ^ 0 
-             * L "{\\PitonStyle{String.Long}{" 
-             * C ( ( 1 - P "'''" - P "\r" ) ^ 0 )
-          )
-       * C ( P "'''" )
-       * L "}}" 
+             * Lc "}}"
+             * EOL
+           ) ^ 0 
+         * Lc "{\\PitonStyle{String.Long}{" 
+         * ( SingleLongInterpol + SingleLongPureString ) ^ 0
+       +
+         Q ( ( S "rR" ) ^ -1  * P "'''" 
+             * ( 1 - P "'''" - P "\r" ) ^ 0 ) 
+         * Lc "}}" 
+         * (
+             Lc "{\\PitonStyle{String.Long}{" 
+             * Q ( ( 1 - P "'''" - P "\r" ) ^ 0 )
+             * Lc "}}" 
+             * EOL
+           ) ^ 0 
+         * Lc "{\\PitonStyle{String.Long}{" 
+         * Q ( ( 1 - P "'''" - P "\r" ) ^ 0 )
+      )
+   * Q ( P "'''" )
+   * Lc "}}" 
 
 
 local DoubleLongString =
-      L "{\\PitonStyle{String.Long}{" 
-       * (
-             C ( S "fF" * P "\"\"\"" )
+  Lc "{\\PitonStyle{String.Long}{" 
+   * (
+         Q ( S "fF" * P "\"\"\"" )
+         * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
+         * Lc "}}" 
+         * (
+             EOL
+             + 
+             Lc "{\\PitonStyle{String.Long}{" 
              * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
-             * L "}}" 
-             * (
-                 EOL
-                 + 
-                 L "{\\PitonStyle{String.Long}{" 
-                 * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
-                 * L "}}" 
-                 * EOL
-               ) ^ 0 
-             * L "{\\PitonStyle{String.Long}{" 
-             * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
-           +
-             C ( ( S "rR" ) ^ -1  * P "\"\"\"" * ( 1 - P "\"\"\"" - P "\r" ) ^ 0 ) 
-             * L "}}" 
-             * (
-                 L "{\\PitonStyle{String.Long}{" 
-                 * C ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
-                 * L "}}" 
-                 * EOL
-               ) ^ 0 
-             * L "{\\PitonStyle{String.Long}{" 
-             * C ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
-          )
-       * C ( P "\"\"\"" )
-       * L "}}" 
-
-
-
+             * Lc "}}" 
+             * EOL
+           ) ^ 0 
+         * Lc "{\\PitonStyle{String.Long}{" 
+         * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
+       +
+         Q ( ( S "rR" ) ^ -1  * P "\"\"\""
+              * ( 1 - P "\"\"\"" - P "\r" ) ^ 0 ) 
+         * Lc "}}" 
+         * (
+             Lc "{\\PitonStyle{String.Long}{" 
+             * Q ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
+             * Lc "}}" 
+             * EOL
+           ) ^ 0 
+         * Lc "{\\PitonStyle{String.Long}{" 
+         * Q ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
+      )
+   * Q ( P "\"\"\"" )
+   * Lc "}}" 
+%    \end{macrocode}
+%
+%    \begin{macrocode}
 local LongString = SingleLongString + DoubleLongString
+%    \end{macrocode}
+%
+% \bigskip
+% We have a \textsc{lpeg} for the Python docstrings. That \textsc{lpeg} will
+% be used in the \textsc{lpeg} |DefFunction| which deals with the whole preamble
+% of a function definition (which begins with |def|).
+%    \begin{macrocode}
+local StringDoc = 
+    K ( P "\"\"\"" , 'String.Doc' )
+      * ( K ( (1 - P "\"\"\"" - P "\r" ) ^ 0 , 'String.Doc' ) * EOL * Tab ^0 ) ^ 0
+      * K ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 * P "\"\"\"" , 'String.Doc' )
+%    \end{macrocode}
+%
+% \bigskip
+% \paragraph{The comments in the Python listings}
+%
+% We define different \textsc{lpeg} dealing with comments in the Python
+% listings.
+%    \begin{macrocode}
+local CommentMath = 
+  P "$" * K ( ( 1 - S "$\r" ) ^ 1 , 'Comment.Math' ) * P "$"
 
+local Comment = 
+  Lc ( "{\\PitonStyle{Comment}{" ) 
+  * Q ( P "#" ) 
+  * ( CommentMath + Q ( ( 1 - S "$\r" ) ^ 1 ) ) ^ 0
+  * Lc ( "}}" )
+  * ( EOL + -1 )
+%    \end{macrocode}
+% 
 
-local Expression =
-    P { "E" ,
-         E = ( 1 - S "{}()[]\r," ) ^ 0
-             *  (
-                  (   P "{" * V "F" * P "}"
-                    + P "(" * V "F" * P ")"
-                    + P "[" * V "F" * P "]" ) * ( 1 - S "{}()[]\r," ) ^ 0
-                ) ^ 0 ,
-         F = ( 1 - S "{}()[]\r\"'" ) ^ 0
-             * ( (
-                     P "'" * (P "\\'" + 1 - S"'\r" )^0 * P "'"
-                   + P "\"" * (P "\\\"" + 1 - S"\"\r" )^0 * P "\""
-                   + P "{"  * V "F" * P "}"
-                   + P "(" * V "F" * P ")"
-                   + P "[" * V "F" * P "]"
-                 ) * ( 1 - S "{}()[]\r\"'" ) ^ 0 ) ^ 0 , 
-      }
 
-local Param = SkipSpace * K ( identifier , '' ) * SkipSpace
-                     *  ( K ( P "=" * Expression , 'InitialValues' )
-                          +  K ( P ":" , '' ) * SkipSpace * K ( letter^1 , 'Name.Type' ))
-                  + SkipSpace * K ( alphanum ^ 1 , '' ) * SkipSpace 
-
-local Params =  Param * (  K ( P "," , '' ) * Param ) ^ 0
-
-local StringDoc = K ( P "\"\"\"" , 'String.Doc' )
-                      * ( K ( (1 - P "\"\"\"" - P "\r" ) ^ 0 , 'String.Doc' ) * EOL ) ^ 0
-                      * K ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 * P "\"\"\"" , 'String.Doc' )
-                  + K ( P "'''" , 'String.Doc' )
-                      * ( K ( (1 - P "'''" - P "\r")^0 , 'String.Doc' ) * EOL ) ^ 0
-                      * K ( ( 1 - P "'''" - P "\r")^0 * P "'''" , 'String.Doc' )
-
-local CommentMath = P "$" * K ( ( 1 - S "$\r" ) ^ 1 , 'Comment.Math' ) * P "$"
-
-
-local Comment = L ( "{\\PitonStyle{Comment}{" ) 
-                * C ( P "#" ) * ( CommentMath + C ( ( 1 - S "$\r" ) ^ 1 ) ) ^ 0
-                * L ( "}}" )
-                * ( EOL + -1 )
-
+% \bigskip
+% The following \textsc{lpeg} |CommentLaTeX| is for what is called in that
+% document the ``LaTeX comments''. Since the elements that will be catched must
+% be sent to LaTeX with standard LaTeX catcodes, we put the capture (done by
+% the function~|C|) in a table (by using~|Ct|, which is an alias for |lpeg.Ct|).
+%    \begin{macrocode}
 local CommentLaTeX =
-   P "##"
-   * L "{\\PitonStyle{Comment.LaTeX}{\\ignorespaces" 
-   * Ct ( C ( ( 1 - P "\r" ) ^ 0 ) ) 
-   * L "}}" 
-   * ( EOL + -1 ) 
-
+  P "##"
+  * Lc "{\\PitonStyle{Comment.LaTeX}{\\ignorespaces" 
+  * L ( ( 1 - P "\r" ) ^ 0 ) 
+  * Lc "}}" 
+  * ( EOL + -1 ) 
+%    \end{macrocode}
+% 
+% \bigskip
+% \paragraph{DefFunction}
+%
+% The following \textsc{lpeg} |Expression| will be used for the parameters in
+% the \emph{argspec} of a Python function. It's necessary to use a \emph{grammar}
+% because that pattern mainly checks the correct nesting of the delimiters
+% (and it's known in the theory of formal languages that this can't be done with
+% regular expressions \emph{stricto sensu} only). 
+%    \begin{macrocode}
+local Expression =
+  P { "E" ,
+       E = ( 1 - S "{}()[]\r," ) ^ 0
+           *  (
+                (   P "{" * V "F" * P "}"
+                  + P "(" * V "F" * P ")"
+                  + P "[" * V "F" * P "]" ) * ( 1 - S "{}()[]\r," ) ^ 0
+              ) ^ 0 ,
+       F = ( 1 - S "{}()[]\r\"'" ) ^ 0
+           * ( (
+                   P "'" * (P "\\'" + 1 - S"'\r" )^0 * P "'"
+                 + P "\"" * (P "\\\"" + 1 - S"\"\r" )^0 * P "\""
+                 + P "{"  * V "F" * P "}"
+                 + P "(" * V "F" * P ")"
+                 + P "[" * V "F" * P "]"
+               ) * ( 1 - S "{}()[]\r\"'" ) ^ 0 ) ^ 0 , 
+    }
+%    \end{macrocode}
+%
+% \bigskip 
+% We will now define a \textsc{lpeg} |Params| that will catch the list of
+% parameters (that is to say the \emph{argspec}) in the definition of a Python
+% function. For example, in the line of code
+% \begin{center}
+% \piton|def MyFunction(a,b,x=10,n:int): return n|
+% \end{center}
+% the \textsc{lpeg} |Params| will be used to catch the chunk\enskip |a,b,x=10,n:int|.
+% 
+% Or course, a |Params| is simply a comma-separated list of |Param|, and that's
+% why we define first the \textsc{lpeg} |Param|.
+%
+% \medskip
+%    \begin{macrocode}
+local Param = 
+  SkipSpace * Identifier * SkipSpace
+   * ( 
+         K ( P "=" * Expression , 'InitialValues' )
+       + Q ( P ":" ) * SkipSpace * K ( letter^1 , 'Name.Type' ) 
+     ) ^ -1
+%    \end{macrocode}
+% 
+% \medskip
+%    \begin{macrocode}
+local Params = ( Param * ( Q "," * Param ) ^ 0 ) ^ -1
+%    \end{macrocode}
+% 
+% \bigskip
+% The following \textsc{lpeg} |DefFunction| catches a keyword |def| and the
+% following name of function \emph{but also everything else until a potential
+% docstring}. That's why this definition of \textsc{lpeg} must occur (in the file
+% |piton.sty|) after the definition of several other \textsc{lpeg} such as
+% |Comment|, |CommentLaTeX|, |Params|, |StringDoc|...
+%    \begin{macrocode}
 local DefFunction =
-   K ( P "def" , 'Keyword' )
-   * ( Space
-       * K ( identifier , 'Name.Function' ) 
-       * ( SkipSpace * K ( P "(" , '' ) * Params * K ( P ")" , '' ) ) ^ -1
-       * ( SkipSpace
-            * K ( ( 1 - S ":\r" )^0 , 'Post.Function' ) 
-            * K ( P ":" , 'Keyword' )
-            * SkipSpace
-            * ( EOL + CommentLaTeX + Comment )
-            * SkipSpace
-            * StringDoc ) ^ -1
-      ) ^ -1 
+  K ( P "def" , 'Keyword' )
+  * Space
+  * K ( identifier , 'Name.Function' ) 
+  * SkipSpace 
+  * Q ( P "(" ) * Params * Q ( P ")" ) 
+  * SkipSpace
+%    \end{macrocode}
+% Here, we need a \pkg{piton} style |Post.Function| which will be linked to
+% |\@@_piton:n| (that means that the capture will be parsed once again by
+% \pkg{piton}). We could avoid that kind of trick by using a non-terminal of a
+% grammar but we have probably here a better legibility.
+%    \begin{macrocode}
+  * K ( ( 1 - S ":\r" )^0 , 'Post.Function' ) 
+  * Q ( P ":" )
+  * ( SkipSpace
+      * ( EOL + CommentLaTeX + Comment ) -- in all cases, that contains an EOL
+      * Tab ^ 0 
+      * SkipSpace
+      * StringDoc ^ 0 -- there may be additionnal docstrings 
+    ) ^ -1
+%    \end{macrocode}
+% Remark that, in the previous code, |CommentLaTeX| \emph{must} appear
+% before |Comment|: there is no commutativity of the addition for the
+% \emph{parsing expression grammars} (\textsc{peg}).
+% 
+% \smallskip 
+% If the word |def| is not followed by a identifier and parenthesis, it will be
+% catched as keyword by the \textsc{lpeg} |Keyword| (useful if, for example, the
+% final user wants to speak of the keyword \piton{def}).
+%
+% \bigskip
+% \paragraph{The dictionaries of Python}
+%
+% We have \textsc{lpeg} dealings with dictionaries of Python because, in
+% typesettings of explicit Python dictionnaries, one may prefer to have all the
+% values formatted in black (in order to see more clearly the keys which are
+% usually Python strings). That's why we have a \pkg{piton} style |Dict.Value|.
+%
+% The initial value of that \pkg{piton} style is |\@@_piton:n|, which means that
+% the value of the entry of the dictionary is parsed once again by \pkg{piton}
+% (and nothing special is done for the dictionary). In the following example, we
+% have set the \pkg{piton} style |Dict.Value| to |\color{black}|:
+%
+% \medskip
+% \begingroup
+% \SetPitonStyle{Dict.Value = \color{black}}
+% \piton|mydict = { 'name' : 'Paul', 'sex' : 'male', 'age' : 31 } |
+% \endgroup
+%
+% \medskip
+% At this time, this mechanism works only for explicit dictionaries on a single line!
+%
+%    \begin{macrocode}
+local ItemDict = 
+  ShortString * SkipSpace * Q ( P ":" ) * K ( Expression , 'Dict.Value' ) 
 
-local ItemDict = ShortString * SkipSpace * C ( P ":" ) * K ( Expression , 'Dict.Value' ) 
-
 local ItemOfSet = SkipSpace * ( ItemDict + ShortString ) * SkipSpace 
 
-local Set = C ( P "{" ) 
-            * ItemOfSet * ( C ( P "," ) * ItemOfSet )  ^ 0 
-            * C ( P "}" )
+local Set = 
+  Q ( P "{" ) 
+  * ItemOfSet * ( Q ( P "," ) * ItemOfSet )  ^ 0 
+  * Q ( P "}" )
 
-local Operator = K ( P "!=" + P "==" + P "<<" + P ">>" + S "-~+/*%=<>&.@|" , 'Operator')
-
-local OperatorWord = K ( P "in" + P "is" + P "and" + P "or" + P "not" , 'Operator.Word')
-
-local SyntaxPython =
+%    \end{macrocode}
+%
+%
+% \bigskip
+% \paragraph{The main LPEG}
+%
+% |SyntaxPython| is the main \textsc{lpeg} of the package \pkg{piton}. We have
+% written an auxiliary \textsc{lpeg} |SyntaxPythonAux| only for legibility.
+%    \begin{macrocode}
+local SyntaxPythonAux =
+%    \end{macrocode}
+% We recall that each line in the Python code to parse will be sent back to
+% LaTeX between a pair |\@@_begin_line:| -- |\@@_end_line:|\footnote{Remember
+% that the \texttt{\textbackslash @@\_end\_line:} must be explicit because it
+% will be used as marker in order to delimit the argument of the command
+% \texttt{\textbackslash @@\_begin\_line:}}.
+%    \begin{macrocode}
+      Lc ( '\\@@_begin_line:' ) *
       ( ( space - P "\r" ) ^0 * P "\r" ) ^ -1 * 
       (  ( space^1 * -1 ) 
          + EOL
+         + Tab
          + Space
          + Escape 
          + CommentLaTeX
@@ -1976,117 +2566,183 @@
          + ExceptionInConsole
          + Set 
          + Delim
-         + Class * ( Space + Punct + EOL )
-         + Namespace * ( Space + Punct + EOL )
+%    \end{macrocode}
+% |Operator| must be before |Punct|.
+%    \begin{macrocode}
+         + Operator
+         + ShortString
+         + Punct
+         + FromImport
          + ImportAs
          + RaiseException 
-         + Keyword * ( Space + Punct + EOL )
          + DefFunction
-         + ShortString
-         + Decorator * ( Space + Punct + EOL )
-         + Operator
-         + OperatorWord * ( Space + Punct + EOL ) 
-         + Builtin * ( Space + Punct + EOL )
+         + DefClass 
+         + Keyword * ( Space + Punct + Delim + EOL + -1) 
+         + Decorator
+         + OperatorWord 
+         + Builtin * ( Space + Punct + Delim + EOL + -1) 
          + Identifier 
          + Number
          + Word
-      ) ^0 * -1 
-
-
-local MinimalSyntax =
-  P { "S" ;
-      S = K ( (1 - P "\r" ) ^ 0 , '') + EOL * S
-    }
-
-
+      ) ^0 * -1 * Lc ( '\\@@_end_line:' )
+%    \end{macrocode}
+%
+% We have written a auxiliary \textsc{lpeg} |SyntaxPythonAux| for legibility only.
+%    \begin{macrocode}
+local SyntaxPython = Ct ( SyntaxPythonAux ) 
+%    \end{macrocode}
+% 
+% \bigskip
+% \subsubsection{The function Parse}
+%
+% The function |Parse| is the main function of the package \pkg{piton}.
+% It parses its argument and sends back to LaTeX the code with interlaced
+% formatting LaTeX instructions. In fact, everything is done by the
+% \textsc{lpeg} |SyntaxPython| which returns as capture a Lua table containing
+% data to send to LaTeX.
+% 
+% \bigskip
+%    \begin{macrocode}
 function Parse(code)
-  local t = Ct(SyntaxPython) : match(code)
-  tex.sprint( luatexbase.catcodetables.expl , '\\__piton_begin_line:' )
-  if t then else t = Ct(MinimalSyntax) : match(code) end   
-  for i = 1 , #t do
-    if type(t[i]) == 'string'
-    then
-      tex.sprint(luatexbase.catcodetables.CatcodeTableOther, t[i])
-    else
-      tex.tprint(t[i]) 
-    end 
-  end
-  tex.sprint( luatexbase.catcodetables.expl , '\\__piton_end_line:' )
+  local t = SyntaxPython : match ( code )
+  for _ , s in ipairs(t) do tex.tprint(s) end 
 end
-
+%    \end{macrocode}
+%
+% \bigskip
+% The function |ParseFile| will be used by the LaTeX command |\PitonInputFile|.
+% That function merely reads the whole file (that is to say all its lines) and
+% then apply the function~|Parse| to the resulting Lua string.
+%    \begin{macrocode}
 function ParseFile(name)
   s = ''
   for line in io.lines(name) do s = s .. '\r' .. line end
   Parse(s) 
 end
-
-
-function define_gobble_syntax(n) 
-  GobbleSyntax = ( 1 - P "\r" ) ^ (-n)  * C ( ( 1 - P "\r" ) ^0 ) 
-               * ( C ( P "\r" )
-                   * ( 1 - P "\r" ) ^ (-n)
-                   * C ( ( 1 - P "\r" ) ^0 )
-                 ) ^ 0
-end 
-
-function GobbleParse(code)
-  local t = Ct(GobbleSyntax):match(code)
-  local new_code = ""
-  for i = 1 , #t do
-    new_code = new_code .. t[i]
+%    \end{macrocode}
+%
+% \bigskip
+% \subsubsection{The preprocessors of the function Parse}
+%
+% We deal now with preprocessors of the function |Parse| which are needed when
+% the ``gobble mechanism'' is used.
+% 
+%
+% \bigskip
+% The function |gobble| gobbles $n$ characters on the left of the code. It uses
+% a \textsc{lpeg} that we have to compute dynamically because if depends on the
+% value of~$n$.
+%    \begin{macrocode}
+function gobble(n,code)
+  function concat(acc,new_value)
+    return acc .. new_value
+  end 
+  if n==0 
+  then return code
+  else 
+       return Cf ( 
+                   Cc ( "" ) * 
+                   ( 1 - P "\r" ) ^ (-n)  * C ( ( 1 - P "\r" ) ^ 0 ) 
+                     * ( C ( P "\r" )
+                     * ( 1 - P "\r" ) ^ (-n)
+                     * C ( ( 1 - P "\r" ) ^ 0 )
+                    ) ^ 0 ,
+                    concat 
+                 ) : match ( code ) 
   end
-  Parse(new_code)
 end
-
+%    \end{macrocode}
+%
+% \bigskip
+%    \begin{macrocode}
+function GobbleParse(n,code)
+  if n==-1 
+  then n = AutoGobbleLPEG : match(code)
+  else if n==-2 
+       then n = EnvGobbleLPEG : match(code)
+       end
+  end
+  Parse(gobble(n,code))
+end 
+%    \end{macrocode}
+% 
+% 
+% \bigskip
+% The following function |add| will be used in the following \textsc{lpeg}
+% |AutoGobbleLPEG| and |EnvGobbleLPEG|.
+%    \begin{macrocode}
 function add(acc,new_value)
   return acc + new_value
 end 
-
-
-
-
---[[ The following LPEG returns as capture the minimal number of spaces at
-the beginning of the lines of code]]
-AutoGobbleSyntax = 
+%    \end{macrocode}
+% 
+% \bigskip
+% The following \textsc{lpeg} returns as capture the minimal number of spaces at
+% the beginning of the lines of code. The main work is done by two \emph{fold
+% captures} (|lpeg.Cf|), one using |add| and the other (encompassing the
+% previous one) using |math.min| as folding operator.
+%    \begin{macrocode}
+AutoGobbleLPEG = 
     ( space ^ 0 * P "\r" ) ^ -1 
     * Cf (
            (
+%    \end{macrocode}
+% We don't take into account the empty lines (with only spaces).
+%    \begin{macrocode}
              ( P " " ) ^ 0 * P "\r"
              + 
              Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add )
              * ( 1 - P " " ) * ( 1 - P "\r" ) ^ 0 * P "\r"
            ) ^ 0
+%    \end{macrocode}
+% Now for the last line of the Python code...
+%    \begin{macrocode}
            *
            ( Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add )
            * ( 1 - P " " ) * ( 1 - P "\r" ) ^ 0 ) ^ -1 ,
            math.min
          )
-
-function AutoGobbleParse(code)
-  local n = AutoGobbleSyntax:match(code)
-  if n==0
-  then Parse(code)
-  else define_gobble_syntax(n)
-       GobbleParse(code) 
-  end 
-end
-
-
---[[ The following LPEG returns as capture the number of spaces at the last line,
-that is to say begin the \end{Piton} ]]
-
-EnvGobbleSyntax =
-       ( ( 1 - P "\r" ) ^ 0 * P "\r" ) ^ 0 
-         * Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add ) * -1
-
-function EnvGobbleParse(code)
-  local n = EnvGobbleSyntax:match(code)
-  if n==0
-  then Parse(code)
-  else define_gobble_syntax(n)
-       GobbleParse(code) 
-  end 
-end
+%    \end{macrocode}
+%
+% \bigskip
+% The following \textsc{lpeg} returns as capture the number of spaces at the
+% last line, that is to say before the |\end{Piton}| (and usually it's also the
+% number of spaces before the corresponding |\begin{Piton}| because that's the
+% traditionnal way to indent in LaTeX). The main work is done by a \emph{fold
+% capture} (|lpeg.Cf|) using the function |add| as folding operator.
+%    \begin{macrocode}
+EnvGobbleLPEG =
+  ( ( 1 - P "\r" ) ^ 0 * P "\r" ) ^ 0 
+    * Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add ) * -1
+%    \end{macrocode}
+% 
+% \bigskip
+% \subsubsection{To count the number of lines}
+%
+% \medskip
+%    \begin{macrocode}
+function CountLines(code)
+  local count = 0 
+  for i in code:gmatch("\r") do count = count + 1 end 
+  tex.sprint( 
+      luatexbase.catcodetables.expl , 
+      '\\int_set:Nn \\l_@@_nb_lines_int {' .. count .. '}' )
+end 
+%    \end{macrocode}
+%
+% \bigskip
+%    \begin{macrocode}
+function CountLinesFile(name)
+  local count = 0 
+  for line in io.lines(name) do count = count + 1 end
+  tex.sprint( 
+      luatexbase.catcodetables.expl , 
+      '\\int_set:Nn \\l_@@_nb_lines_int {' .. count .. '}' )
+end 
+%    \end{macrocode}
+% 
+% \bigskip
+%    \begin{macrocode}
 \end{luacode*}
 %    \end{macrocode}
 %
@@ -2107,6 +2763,12 @@
 %
 % New key |left-margin|.
 %
+% \subsection*{Changes between versions 0.8 and 0.9}
+%
+% New key |tab-size|.
+%
+% Integer value for the key |splittable|.
+%
 % \end{document}
 % 
 %

Modified: trunk/Master/texmf-dist/tex/lualatex/piton/piton.sty
===================================================================
--- trunk/Master/texmf-dist/tex/lualatex/piton/piton.sty	2022-10-31 19:48:53 UTC (rev 64882)
+++ trunk/Master/texmf-dist/tex/lualatex/piton/piton.sty	2022-10-31 19:49:19 UTC (rev 64883)
@@ -18,8 +18,8 @@
 %% and version 1.3 or later is part of all distributions of
 %% LaTeX version 2005/12/01 or later.
 %% 
-\def\myfileversion{0.8}
-\def\myfiledate{2022/10/23}
+\def\myfileversion{0.9}
+\def\myfiledate{2022/10/31}
 
 
 
@@ -116,21 +116,44 @@
         \bool_set_true:N \c__piton_footnote_bool
       }
   }
+\int_new:N \l__piton_nb_lines_int
+\int_new:N \g__piton_line_int
 \tl_new:N \g__piton_aux_tl
-\bool_new:N \l__piton_splittable_bool
+\int_new:N \l__piton_splittable_int
+\int_set:Nn \l__piton_splittable_int { 100 }
 \str_new:N \l__piton_background_color_str
-\dim_new:N \l__piton_width_dim
+\dim_new:N \g__piton_width_dim
 \dim_new:N \l__piton_width_on_aux_dim
 \int_new:N \g__piton_env_int
 \bool_new:N \l__piton_slim_bool
 \dim_new:N \l__piton_left_margin_dim
+\tl_new:N \l__piton_tab_tl
+\cs_new_protected:Npn \__piton_set_tab_tl:n #1
+  {
+    \tl_clear:N \l__piton_tab_tl
+    \prg_replicate:nn { #1 }
+      { \tl_put_right:Nn \l__piton_tab_tl { ~ } }
+  }
+\__piton_set_tab_tl:n { 4 }
+\cs_new_protected:Npn \__piton_newline:
+  {
+    \int_gincr:N \g__piton_line_int
+    \int_compare:nNnT \g__piton_line_int > { \l__piton_splittable_int - 1 }
+      {
+        \int_compare:nNnT
+          { \l__piton_nb_lines_int - \g__piton_line_int } > \l__piton_splittable_int
+          {
+            \egroup
+            \bool_if:NT \c__piton_footnote_bool { \end { savenotes } }
+            \newline
+            \bool_if:NT \c__piton_footnote_bool { \begin { savenotes } }
+            \vtop \bgroup
+          }
+     }
+  }
 \int_new:N \l__piton_gobble_int
-\cs_new_protected:Npn \__piton_define_gobble_syntax:n #1
-  { \lua_now:n { define_gobble_syntax(#1) } }
 \cs_set_protected:Npn \__piton_begin_line: #1 \__piton_end_line:
   {
-    \bool_lazy_and:nnT \l__piton_splittable_bool \c__piton_footnote_bool
-      { \begin { savenotes } }
     \bool_if:NTF \l__piton_slim_bool
       { \hbox_set:Nn \l_tmpa_box }
       {
@@ -153,8 +176,8 @@
         \str_if_empty:NF \l__piton_background_color_str \space
         #1 \hfil
       }
-    \dim_compare:nNnT { \box_wd:N \l_tmpa_box } > \l__piton_width_dim
-      { \dim_set:Nn \l__piton_width_dim { \box_wd:N \l_tmpa_box } }
+    \dim_compare:nNnT { \box_wd:N \l_tmpa_box } > \g__piton_width_dim
+      { \dim_gset:Nn \g__piton_width_dim { \box_wd:N \l_tmpa_box } }
     \box_set_dp:Nn \l_tmpa_box { \box_dp:N \l_tmpa_box + 1.25 pt }
     \box_set_ht:Nn \l_tmpa_box { \box_ht:N \l_tmpa_box + 1.25 pt }
     \tl_if_empty:NTF \l__piton_background_color_str
@@ -174,8 +197,6 @@
             \box_use_drop:N \l_tmpa_box
           }
       }
-    \bool_lazy_and:nnT \l__piton_splittable_bool \c__piton_footnote_bool
-      { \end { savenotes } }
     \vspace { - 2.5 pt }
   }
 \bool_new:N \l__piton_line_numbers_bool
@@ -197,8 +218,8 @@
     all-line-numbers .value_forbidden:n = true  ,
     resume           .bool_set:N        = \l__piton_resume_bool ,
     resume           .value_forbidden:n = true ,
-    splittable       .bool_set:N        = \l__piton_splittable_bool ,
-    splittable       .default:n         = true ,
+    splittable       .int_set:N         = \l__piton_splittable_int ,
+    splittable       .default:n         = 1 ,
     background-color .str_set:N         = \l__piton_background_color_str ,
     background-color .value_required:n  = true ,
     slim             .bool_set:N        = \l__piton_slim_bool ,
@@ -205,6 +226,8 @@
     slim             .default:n         = true ,
     left-margin      .dim_set:N         = \l__piton_left_margin_dim ,
     left-margin      .value_required:n  = true ,
+    tab-size         .code:n            = \__piton_set_tab_tl:n { #1 } ,
+    tab-size         .value_required:n  = true ,
     unknown          .code:n =
       \msg_error:nn { piton } { Unknown~key~for~PitonOptions }
   }
@@ -213,17 +236,17 @@
     Unknown~key. \\
     The~key~'\l_keys_key_str'~is~unknown~for~\token_to_str:N \PitonOptions.~The~
     available~keys~are:~all-line-numbers,~auto-gobble,~env-gobble,~gobble,~
-    left-margin,~line-numbers,~resume,~slim~and~splittable.\\
+    left-margin,~line-numbers,~resume,~slim~splittable~and~tab-size.\\
     If~you~go~on,~that~key~will~be~ignored.
   }
 \NewDocumentCommand \PitonOptions { } { \keys_set:nn { PitonOptions } }
-\int_new:N \g__piton_line_int
+\int_new:N \g__piton_visual_line_int
 \cs_new_protected:Npn \__piton_print_number:
   {
-    \int_gincr:N \g__piton_line_int
+    \int_gincr:N \g__piton_visual_line_int
     \hbox_overlap_left:n
       {
-        { \color { gray } \footnotesize \int_to_arabic:n \g__piton_line_int }
+        { \color { gray } \footnotesize \int_to_arabic:n \g__piton_visual_line_int }
         \skip_horizontal:n { 0.4 em }
       }
   }
@@ -250,7 +273,7 @@
             \tl_gput_right:Nx \g__piton_aux_tl
               {
                 \dim_set:Nn \l__piton_width_on_aux_dim
-                  { \dim_use:N \l__piton_width_dim + 0.5 em }
+                  { \dim_eval:n { \g__piton_width_dim + 0.5 em } }
               }
           }
       }
@@ -272,26 +295,35 @@
       \lua_now:e { Parse(token.scan_argument()) } { #1 }
     \group_end:
   }
-\NewDocumentCommand { \PitonInputFile } { m }
+\cs_new:Npn \__piton_pre_env:
   {
     \int_gincr:N \g__piton_env_int
     \tl_gclear:N \g__piton_aux_tl
     \tl_if_exist:cT { c__piton _ \int_use:N \g__piton_env_int _ tl }
       { \use:c { c__piton _ \int_use:N \g__piton_env_int _ tl } }
-    \bool_if:NF \l__piton_splittable_bool
-      { \bool_if:NT \c__piton_footnote_bool { \begin { savenotes } } }
     \dim_compare:nNnT \l__piton_width_on_aux_dim = \c_zero_dim
       { \dim_set_eq:NN \l__piton_width_on_aux_dim \linewidth }
-    \bool_if:NF \l__piton_resume_bool { \int_gzero:N \g__piton_line_int }
+    \bool_if:NF \l__piton_resume_bool { \int_gzero:N \g__piton_visual_line_int }
+    \dim_gzero:N \g__piton_width_dim
+    \int_gzero:N \g__piton_line_int
+    \dim_zero:N \parindent
+    \dim_zero:N \lineskip
+  }
+\NewDocumentCommand { \PitonInputFile } { m }
+  {
     \group_begin:
-      \dim_set_eq:NN \parindent \c_zero_dim
+      \__piton_pre_env:
+      \mode_if_vertical:TF \mode_leave_vertical: \newline
+      \lua_now:n { CountLinesFile(token.scan_argument()) } { #1 }
       \ttfamily
-      \lua_now:e { ParseFile(token.scan_argument()) } { #1 }
-    \__piton_width_to_aux:
+      \bool_if:NT \c__piton_footnote_bool { \begin { savenotes } }
+      \vtop \bgroup
+      \lua_now:n { ParseFile(token.scan_argument()) } { #1 }
+      \egroup
+      \bool_if:NT \c__piton_footnote_bool { \end { savenotes } }
+      \__piton_width_to_aux:
     \group_end:
     \__piton_write_aux:
-    \bool_if:NF \l__piton_splittable_bool
-      { \bool_if:NT \c__piton_footnote_bool { \end { savenotes } } }
   }
 \NewDocumentCommand { \NewPitonEnvironment } { m m m m }
   {
@@ -305,36 +337,21 @@
       }
           {
             \group_end:
-            \mode_if_vertical:TF
-              { \mode_leave_vertical: }
-              \newline
-            \bool_if:NF \l__piton_splittable_bool
+            \mode_if_vertical:TF \mode_leave_vertical: \newline
+            \lua_now:n { CountLines(token.scan_argument()) } { ##1 }
+            \ttfamily
+            \bool_if:NT \c__piton_footnote_bool { \begin { savenotes } }
+            \vtop \bgroup
+            \lua_now:e
               {
-                \bool_if:NT \c__piton_footnote_bool { \begin { savenotes } }
-                \vbox_top:n
+                GobbleParse
+                  ( \int_use:N \l__piton_gobble_int , token.scan_argument() )
               }
-              {
-                \ttfamily
-                \dim_zero:N \lineskip
-                \int_case:nnF \l__piton_gobble_int
-                  {
-                    0
-                    { \lua_now:n { Parse(token.scan_argument()) } }
-                    { -1 }
-                    { \lua_now:n { AutoGobbleParse(token.scan_argument()) } }
-                    { -2 }
-                    { \lua_now:n { EnvGobbleParse(token.scan_argument()) } }
-                  }
-                  {
-                     \exp_args:NV \__piton_define_gobble_syntax:n \l__piton_gobble_int
-                     \lua_now:n { GobbleParse(token.scan_argument()) }
-                  }
-                { ##1 }
-                \vspace { 2.5 pt }
-                \__piton_width_to_aux:
-              }
-            \bool_if:NF \l__piton_splittable_bool
-              { \bool_if:NT \c__piton_footnote_bool { \end { savenotes } } }
+              { ##1 }
+            \vspace { 2.5 pt }
+            \egroup
+            \bool_if:NT \c__piton_footnote_bool { \end { savenotes } }
+            \__piton_width_to_aux:
             \end { #1 }
             \__piton_write_aux:
           }
@@ -341,17 +358,10 @@
     \NewDocumentEnvironment { #1 } { #2 }
       {
         #3
-        \int_gincr:N \g__piton_env_int
-        \tl_gclear:N \g__piton_aux_tl
-        \tl_if_exist:cT { c__piton _ \int_use:N \g__piton_env_int _ tl }
-          { \use:c { c__piton _ \int_use:N \g__piton_env_int _ tl } }
-        \dim_compare:nNnT \l__piton_width_on_aux_dim = \c_zero_dim
-          { \dim_set_eq:NN \l__piton_width_on_aux_dim \linewidth }
-        \bool_if:NF \l__piton_resume_bool { \int_gzero:N \g__piton_line_int }
+        \__piton_pre_env:
         \group_begin:
-        \box_clear:N \l_tmpa_box
         \tl_map_function:nN
-          { \ \\ \{ \} \$ \& \# \^ \_ \% \~ }
+          { \ \\ \{ \} \$ \& \# \^ \_ \% \~ \^^I }
           \char_set_catcode_other:N
         \use:c { __piton_collect_ #1 :w }
       }
@@ -465,8 +475,8 @@
     Name.Type        = \color[HTML]{336666} ,
     InitialValues    = \__piton_piton:n ,
     Dict.Value       = \__piton_piton:n ,
+    Interpol.Inside  = \color{black}\__piton_piton:n ,
     Post.Function    = \__piton_piton:n ,
-    Interpol.Inside  = \color{black}\__piton_piton:n ,
   }
 \AddToHook { env / piton / begin }
    { \msg_fatal:nn { piton } { No~environment~piton } }
@@ -481,342 +491,336 @@
 \ExplSyntaxOff
 \RequirePackage{luacode}
 \begin{luacode*}
-local P, S, V , C , Ct, Cc, Cf = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cf
-
---[[ By convention, a capture which provides as value a table (and not a string), provides, in fact,
-a string (the first element of the table) which is a formatting LaTeX instruction (it will be
-thrown back to TeX with normal catcodes (ant not ``other'' catcode for everybody).]]
-
-local function L(string)
-  return Cc ( { string } )
+local P, S, V, C, Ct, Cc, Cf = lpeg.P, lpeg.S, lpeg.V, lpeg.C, lpeg.Ct, lpeg.Cc, lpeg.Cf
+local function Q(pattern)
+  return Ct ( Cc ( luatexbase.catcodetables.CatcodeTableOther ) * C ( pattern ) )
 end
-
+local function L(pattern)
+  return Ct ( C ( pattern ) )
+end
+local function Lc(string)
+  return Cc ( { luatexbase.catcodetables.expl , string } )
+end
 local function K(pattern, style)
   return
-     L ( "{\\PitonStyle{" .. style .. "}{" )
-     * C(pattern)
-     * L ( "}}" )
+     Lc ( "{\\PitonStyle{" .. style .. "}{" )
+     * Q ( pattern )
+     * Lc ( "}}" )
 end
-
---[[ The text in "escape" (between begin_escape and end_escape) is captured
-and put in a table (with only one component). Indeed, we have decided that a capture
-which is encapsulated in a table must be transmitted to TeX with the normal TeX catcodes.]]
-
-local Escape = P(begin_escape)
-               * Ct ( C ( ( 1 - P(end_escape) ) ^ 1 ) )
-               * P(end_escape)
-
-lpeg.locale(lpeg) -- mandatory
-
-local alpha , digit , space , punct = lpeg.alpha , lpeg.digit , lpeg.space , lpeg.punct
-
--- Remember that à, â, ç, etc. are strings of length 2 (2 bytes)
+local Escape =
+  P(begin_escape)
+  * L ( ( 1 - P(end_escape) ) ^ 1 )
+  * P(end_escape)
+lpeg.locale(lpeg)
+local alpha, digit, space = lpeg.alpha, lpeg.digit, lpeg.space
 local letter = alpha + P "_"
- + P "â" + P "à" + P "ç" + P "é" + P "è" + P "ê" + P "ë" + P "ï" + P "î" + P "ô" + P "û" + P "ü" +
-     P "Â" + P "À" + P "Ç" + P "É" + P "È" + P "Ê" + P "Ë" + P "Ï" + P "Î" + P "Ô" + P "Û" + P "Ü"
+  + P "â" + P "à" + P "ç" + P "é" + P "è" + P "ê" + P "ë" + P "ï" + P "î"
+  + P "ô" + P "û" + P "ü" + P "Â" + P "À" + P "Ç" + P "É" + P "È" + P "Ê"
+  + P "Ë" + P "Ï" + P "Î" + P "Ô" + P "Û" + P "Ü"
 
 local alphanum = letter + digit
-
 local identifier = letter * alphanum ^ 0
+local Identifier = Q ( identifier )
+local Number =
+  K (
+      ( digit^1 * P "." * digit^0 + digit^0 * P "." * digit^1 + digit^1 )
+      * ( S "eE" * S "+-" ^ -1 * digit^1 ) ^ -1
+      + digit^1 ,
+      'Number'
+    )
+if begin_escape ~= ''
+then Word = Q ( ( ( 1 - space - P(begin_escape) - P(end_escape) )
+                   - S "'\"\r[()]" - digit ) ^ 1 )
+else Word = Q ( ( ( 1 - space ) - S "'\"\r[()]" - digit ) ^ 1 )
+end
+local Space = Q ( ( space - P "\r" ) ^ 1 )
 
-local Identifier = C ( identifier )
+local SkipSpace = Q ( ( space - P "\r" ) ^ 0 )
 
-local Space = C ( ( space - P "\r" ) ^ 1 )
+local Punct = Q ( S ".,:;!" )
+local Tab = P "\t" * Lc ( '\\l__piton_tab_tl' )
+local EOL =
+  P "\r"
+  *
+  (
+    ( space^0 * -1 )
+    +
+    Lc ( '\\__piton_end_line: \\__piton_newline: \\__piton_begin_line:' )
+  )
+local Delim = Q ( S "[()]" )
 
-local SkipSpace = C ( ( space - P "\r" ) ^ 0 )
+local Operator =
+  K ( P "!=" + P "==" + P "<<" + P ">>" + S "-~+/*%=<>&.@|" , 'Operator')
 
-local Punct = C ( punct )
+local OperatorWord =
+  K ( P "in" + P "is" + P "and" + P "or" + P "not" , 'Operator.Word')
 
-local EOL = ( P "\r" )
-            *
-            (
-              ( space^0 * -1 )
-              +
-              Cc (
-                   {
-                     luatexbase.catcodetables.expl ,
-                     '\\__piton_end_line: \\bool_if:NT \\l__piton_splittable_bool \\newline \\__piton_begin_line:'
-                   }
-                 )
-            )
-
-local Number =
-   K (
-       ( digit^1 * P "." * digit^0 + digit^0 * P "." * digit^1 + digit^1 )
-       * ( S "eE" * S "+-" ^ -1 * digit^1 ) ^ -1
-       + digit^1
-       , 'Number' )
-
-local Word = C ( ( ( 1 - space ) - S "'\"\r[()]" - digit ) ^ 1 )
-
-if begin_escape ~= ''
-then  Word = C ( ( ( 1 - space - P(begin_escape) - P(end_escape) ) - S "'\"\r[()]" - digit ) ^ 1 )
-end
-
-local Delim = C ( S "[()]" )
-
 local Keyword =
-   K ( P "assert" + P "break" + P "case" + P "continue" + P "del"
-       + P "elif" + P "else" + P "except" + P "exec" + P "finally" + P "for" + P "from"
-       + P "global" + P "if" + P "import" + P "lambda" + P "non local"
-       + P "pass" + P "return" + P "try" + P "while"
-       + P "with" + P "yield" + P "yield from" ,
-   'Keyword' )
-   + K ( P "True" + P "False" + P "None" , 'Keyword.Constant' )
+  K ( P "as" + P "assert" + P "break" + P "case" + P "class" + P "continue"
+      + P "def" + P "del" + P "elif" + P "else" + P "except" + P "exec"
+      + P "finally" + P "for" + P "from" + P "global" + P "if" + P "import"
+      + P "lambda" + P "non local" + P "pass" + P "return" + P "try"
+      + P "while" + P "with" + P "yield" + P "yield from" ,
+  'Keyword' )
+  + K ( P "True" + P "False" + P "None" , 'Keyword.Constant' )
 
 local Builtin =
-   K ( P "__import__" + P "abs" + P "all" + P "any" + P "bin" + P "bool" + P "bytearray"
-     + P "bytes" + P "chr" + P "classmethod" + P "compile" + P "complex" + P "delattr"
-     + P "dict" + P "dir" + P "divmod" + P "enumerate" + P "eval" + P "filter"
-     + P "float" + P "format" + P "frozenset" + P "getattr" + P "globals" + P "hasattr"
-     + P "hash" + P "hex" + P "id" + P "input" + P "int" + P "isinstance" + P "issubclass"
-     + P "iter" + P "len" + P "list" + P "locals" + P "map" + P "max" + P "memoryview" + P "min"
-     + P "next" + P "object" + P "oct" + P "open" + P "ord" + P "pow" + P "print" + P "property"
-     + P "range" + P "repr" + P "reversed" + P "round" + P "set" + P "setattr" + P "slice"
-     + P "sorted" + P "staticmethod" + P "str" + P "sum" + P "super" + P "tuple" + P "type"
-     + P "vars" + P "zip" ,
-   'Name.Builtin' )
+  K ( P "__import__" + P "abs" + P "all" + P "any" + P "bin" + P "bool"
+    + P "bytearray" + P "bytes" + P "chr" + P "classmethod" + P "compile"
+    + P "complex" + P "delattr" + P "dict" + P "dir" + P "divmod"
+    + P "enumerate" + P "eval" + P "filter" + P "float" + P "format"
+    + P "frozenset" + P "getattr" + P "globals" + P "hasattr" + P "hash"
+    + P "hex" + P "id" + P "input" + P "int" + P "isinstance" + P "issubclass"
+    + P "iter" + P "len" + P "list" + P "locals" + P "map" + P "max"
+    + P "memoryview" + P "min" + P "next" + P "object" + P "oct" + P "open"
+    + P "ord" + P "pow" + P "print" + P "property" + P "range" + P "repr"
+    + P "reversed" + P "round" + P "set" + P "setattr" + P "slice" + P "sorted"
+    + P "staticmethod" + P "str" + P "sum" + P "super" + P "tuple" + P "type"
+    + P "vars" + P "zip" ,
+  'Name.Builtin' )
 
 local Exception =
-   K ( "ArithmeticError" + P "AssertionError" + P "AttributeError"
-    + P "BaseException" + P "BufferError" + P "BytesWarning" + P "DeprecationWarning"
-    + P "EOFError" + P "EnvironmentError" + P "Exception" + P "FloatingPointError"
-    + P "FutureWarning" + P "GeneratorExit" + P "IOError" + P "ImportError"
-    + P "ImportWarning" + P "IndentationError" + P "IndexError" + P "KeyError"
-    + P "KeyboardInterrupt" + P "LookupError" + P "MemoryError" + P "NameError"
-    + P "NotImplementedError" + P "OSError" + P "OverflowError"
-    + P "PendingDeprecationWarning" + P "ReferenceError" + P "ResourceWarning"
-    + P "RuntimeError" + P "RuntimeWarning" + P "StopIteration"
-    + P "SyntaxError" + P "SyntaxWarning" + P "SystemError" + P "SystemExit"
-    + P "TabError" + P "TypeError" + P "UnboundLocalError" + P "UnicodeDecodeError"
-    + P "UnicodeEncodeError" + P "UnicodeError" + P "UnicodeTranslateError"
-    + P "UnicodeWarning" + P "UserWarning" + P "ValueError" + P "VMSError"
-    + P "Warning" + P "WindowsError" + P "ZeroDivisionError"
-    + P "BlockingIOError" + P "ChildProcessError" + P "ConnectionError"
-    + P "BrokenPipeError" + P "ConnectionAbortedError" + P "ConnectionRefusedError"
-    + P "ConnectionResetError" + P "FileExistsError" + P "FileNotFoundError"
-    + P "InterruptedError" + P "IsADirectoryError" + P "NotADirectoryError"
-    + P "PermissionError" + P "ProcessLookupError" + P "TimeoutError"
-    + P "StopAsyncIteration" + P "ModuleNotFoundError" + P "RecursionError" ,
-   'Exception' )
+  K ( "ArithmeticError" + P "AssertionError" + P "AttributeError"
+   + P "BaseException" + P "BufferError" + P "BytesWarning" + P "DeprecationWarning"
+   + P "EOFError" + P "EnvironmentError" + P "Exception" + P "FloatingPointError"
+   + P "FutureWarning" + P "GeneratorExit" + P "IOError" + P "ImportError"
+   + P "ImportWarning" + P "IndentationError" + P "IndexError" + P "KeyError"
+   + P "KeyboardInterrupt" + P "LookupError" + P "MemoryError" + P "NameError"
+   + P "NotImplementedError" + P "OSError" + P "OverflowError"
+   + P "PendingDeprecationWarning" + P "ReferenceError" + P "ResourceWarning"
+   + P "RuntimeError" + P "RuntimeWarning" + P "StopIteration"
+   + P "SyntaxError" + P "SyntaxWarning" + P "SystemError" + P "SystemExit"
+   + P "TabError" + P "TypeError" + P "UnboundLocalError" + P "UnicodeDecodeError"
+   + P "UnicodeEncodeError" + P "UnicodeError" + P "UnicodeTranslateError"
+   + P "UnicodeWarning" + P "UserWarning" + P "ValueError" + P "VMSError"
+   + P "Warning" + P "WindowsError" + P "ZeroDivisionError"
+   + P "BlockingIOError" + P "ChildProcessError" + P "ConnectionError"
+   + P "BrokenPipeError" + P "ConnectionAbortedError" + P "ConnectionRefusedError"
+   + P "ConnectionResetError" + P "FileExistsError" + P "FileNotFoundError"
+   + P "InterruptedError" + P "IsADirectoryError" + P "NotADirectoryError"
+   + P "PermissionError" + P "ProcessLookupError" + P "TimeoutError"
+   + P "StopAsyncIteration" + P "ModuleNotFoundError" + P "RecursionError" ,
+  'Exception' )
 
-local RaiseException = K ( P "raise" , 'Keyword' ) * SkipSpace * Exception * C ( P "(" )
+local RaiseException = K ( P "raise" , 'Keyword' ) * SkipSpace * Exception * Q ( P "(" )
 
-local ExceptionInConsole = Exception *  C ( ( 1 - P "\r" ) ^ 0 ) * EOL
-
-local Namespace =
-   K ( P "from" , 'Keyword' ) * Space * K ( alphanum^1 , 'Name.Namespace' )
-   *  ( Space * K ( P "import" , 'Keyword' ) ) ^ -1
-
-local ImportAs = K ( P "import" , 'Keyword' )
-                  * Space
-                  * K ( identifier , 'Name.Namespace' )
-                  * ( SkipSpace * C ( P "," ) * SkipSpace * K ( identifier , 'Name.Namespace' ) ) ^ 0
-                  * (
-                      Space * K ( P "as" , 'Keyword' ) * Space * K ( identifier , 'Name.Namespace' )
-                    ) ^ 0
-
-local Class = K ( P "class" , 'Keyword' )
-                * ( Space * K ( identifier , 'Name.Class' ) ) ^ -1
-
+local ExceptionInConsole = Exception *  Q ( ( 1 - P "\r" ) ^ 0 ) * EOL
 local Decorator = K ( P "@" * letter^1 , 'Name.Decorator' )
-
+local DefClass =
+  K ( P "class" , 'Keyword' ) * Space * K ( identifier , 'Name.Class' )
+local ImportAs =
+  K ( P "import" , 'Keyword' )
+   * Space
+   * K ( identifier * ( P "." * identifier ) ^ 0 ,
+         'Name.Namespace'
+       )
+   * (
+       ( Space * K ( P "as" , 'Keyword' ) * Space * K ( identifier , 'Name.Namespace' ) )
+       +
+       ( SkipSpace * Q ( P "," ) * SkipSpace * K ( identifier , 'Name.Namespace' ) ) ^ 0
+     )
+local FromImport =
+  K ( P "from" , 'Keyword' )
+    * Space * K ( identifier , 'Name.Namespace' )
+    * Space * K ( P "import" , 'Keyword' )
 local SingleShortInterpol =
     K ( P "{" , 'String.Interpol')
   * K ( ( 1 - S "}':" ) ^ 0 , 'Interpol.Inside' )
-  * C ( P ":" * (1 - S "}:'") ^ 0 ) ^ -1
+  * Q ( P ":" * (1 - S "}:'") ^ 0 ) ^ -1
   * K ( P "}" , 'String.Interpol' )
 
 local DoubleShortInterpol =
     K ( P "{" , 'String.Interpol' )
   * K ( ( 1 - S "}\":" ) ^ 0 , 'Interpol.Inside' )
-  * ( K ( P ":" , 'String.Interpol' ) * C ( (1 - S "}:\"") ^ 0 ) ) ^ -1
+  * ( K ( P ":" , 'String.Interpol' ) * Q ( (1 - S "}:\"") ^ 0 ) ) ^ -1
   * K ( P "}" , 'String.Interpol' )
 
 local SingleLongInterpol =
     K ( P "{" , 'String.Interpol' )
   * K ( ( 1 - S "}:\r" - P "'''" ) ^ 0 , 'Interpol.Inside' )
-  * C ( P ":" * (1 - S "}:\r" - P "'''" ) ^ 0 ) ^ -1
+  * Q ( P ":" * (1 - S "}:\r" - P "'''" ) ^ 0 ) ^ -1
   * K ( P "}" , 'String.Interpol' )
 
 local DoubleLongInterpol =
     K ( P "{" , 'String.Interpol' )
   * K ( ( 1 - S "}:\r" - P "\"\"\"" ) ^ 0 , 'Interpol.Inside' )
-  * C ( P ":" * (1 - S "}:\r" - P "\"\"\"" ) ^ 0 ) ^ -1
+  * Q ( P ":" * (1 - S "}:\r" - P "\"\"\"" ) ^ 0 ) ^ -1
   * K ( P "}" , 'String.Interpol' )
+local SingleShortPureString =
+  Q ( ( P "\\'" + P "{{" + P "}}" + 1 - S "{}'" ) ^ 1 )
 
-local SingleShortPureString = C ( ( P "\\'" + P "{{" + P "}}" + 1 - S "{}'" ) ^ 1 )
+local DoubleShortPureString =
+  Q ( ( P "\\\"" + P "{{" + P "}}" + 1 - S "{}\"" ) ^ 1 )
 
-local DoubleShortPureString = C ( ( P "\\\"" + P "{{" + P "}}" + 1 - S "{}\"" ) ^ 1 )
+local SingleLongPureString =
+  Q ( ( 1 - P "'''" - S "{}'\r" ) ^ 1 )
 
-local SingleLongPureString = C ( ( 1 - P "'''" - S "{}'\r" ) ^ 1 )
-
-local DoubleLongPureString = C ( ( 1 - P "\"\"\"" - S "{}\"\r" ) ^ 1 )
-
+local DoubleLongPureString =
+  Q ( ( 1 - P "\"\"\"" - S "{}\"\r" ) ^ 1 )
 local SingleShortString =
-    L ( "{\\PitonStyle{String.Short}{" )
-     * (
-           C ( P "f'" + P "F'" )
-           * ( SingleShortInterpol + SingleShortPureString ) ^ 0
-           * C ( P "'" )
-         +
-           C ( ( P "'" + P "r'" + P "R'" ) * ( P "\\'" + 1 - S "'\r" ) ^ 0 * P "'" )
-       )
-     * L ( "}}" )
+  Lc ( "{\\PitonStyle{String.Short}{" )
+   * (
+         Q ( P "f'" + P "F'" )
+         * ( SingleShortInterpol + SingleShortPureString ) ^ 0
+         * Q ( P "'" )
+       +
+         Q ( ( P "'" + P "r'" + P "R'" )
+         * ( P "\\'" + 1 - S "'\r" ) ^ 0 * P "'" )
+     )
+   * Lc ( "}}" )
 
 local DoubleShortString =
-    L ( "{\\PitonStyle{String.Short}{" )
-     * (
-           C ( P "f\"" + P "F\"" )
-           * ( DoubleShortInterpol + DoubleShortPureString ) ^ 0
-           * C ( P "\"" )
-         +
-           C ( ( P "\"" + P "r\"" + P "R\"" ) * ( P "\\\"" + 1 - S "\"\r" ) ^ 0 * P "\"" )
-       )
-     * L ( "}}" )
+  Lc ( "{\\PitonStyle{String.Short}{" )
+   * (
+         Q ( P "f\"" + P "F\"" )
+         * ( DoubleShortInterpol + DoubleShortPureString ) ^ 0
+         * Q ( P "\"" )
+       +
+         Q ( ( P "\"" + P "r\"" + P "R\"" )
+         * ( P "\\\"" + 1 - S "\"\r" ) ^ 0 * P "\"" )
+     )
+   * Lc ( "}}" )
 
 local ShortString = SingleShortString + DoubleShortString
-
 local SingleLongString =
-      L "{\\PitonStyle{String.Long}{"
-       * (
-             C ( S "fF" * P "'''" )
+  Lc "{\\PitonStyle{String.Long}{"
+   * (
+         Q ( S "fF" * P "'''" )
+         * ( SingleLongInterpol + SingleLongPureString ) ^ 0
+         * Lc "}}"
+         * (
+             EOL
+             +
+             Lc "{\\PitonStyle{String.Long}{"
              * ( SingleLongInterpol + SingleLongPureString ) ^ 0
-             * L "}}"
-             * (
-                 EOL
-                 +
-                 L "{\\PitonStyle{String.Long}{"
-                 * ( SingleLongInterpol + SingleLongPureString ) ^ 0
-                 * L "}}"
-                 * EOL
-               ) ^ 0
-             * L "{\\PitonStyle{String.Long}{"
-             * ( SingleLongInterpol + SingleLongPureString ) ^ 0
-           +
-             C ( ( S "rR" ) ^ -1  * P "'''" * ( 1 - P "'''" - P "\r" ) ^ 0 )
-             * L "}}"
-             * (
-                 L "{\\PitonStyle{String.Long}{"
-                 * C ( ( 1 - P "'''" - P "\r" ) ^ 0 )
-                 * L "}}"
-                 * EOL
-               ) ^ 0
-             * L "{\\PitonStyle{String.Long}{"
-             * C ( ( 1 - P "'''" - P "\r" ) ^ 0 )
-          )
-       * C ( P "'''" )
-       * L "}}"
+             * Lc "}}"
+             * EOL
+           ) ^ 0
+         * Lc "{\\PitonStyle{String.Long}{"
+         * ( SingleLongInterpol + SingleLongPureString ) ^ 0
+       +
+         Q ( ( S "rR" ) ^ -1  * P "'''"
+             * ( 1 - P "'''" - P "\r" ) ^ 0 )
+         * Lc "}}"
+         * (
+             Lc "{\\PitonStyle{String.Long}{"
+             * Q ( ( 1 - P "'''" - P "\r" ) ^ 0 )
+             * Lc "}}"
+             * EOL
+           ) ^ 0
+         * Lc "{\\PitonStyle{String.Long}{"
+         * Q ( ( 1 - P "'''" - P "\r" ) ^ 0 )
+      )
+   * Q ( P "'''" )
+   * Lc "}}"
 
 local DoubleLongString =
-      L "{\\PitonStyle{String.Long}{"
-       * (
-             C ( S "fF" * P "\"\"\"" )
+  Lc "{\\PitonStyle{String.Long}{"
+   * (
+         Q ( S "fF" * P "\"\"\"" )
+         * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
+         * Lc "}}"
+         * (
+             EOL
+             +
+             Lc "{\\PitonStyle{String.Long}{"
              * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
-             * L "}}"
-             * (
-                 EOL
-                 +
-                 L "{\\PitonStyle{String.Long}{"
-                 * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
-                 * L "}}"
-                 * EOL
-               ) ^ 0
-             * L "{\\PitonStyle{String.Long}{"
-             * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
-           +
-             C ( ( S "rR" ) ^ -1  * P "\"\"\"" * ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
-             * L "}}"
-             * (
-                 L "{\\PitonStyle{String.Long}{"
-                 * C ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
-                 * L "}}"
-                 * EOL
-               ) ^ 0
-             * L "{\\PitonStyle{String.Long}{"
-             * C ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
-          )
-       * C ( P "\"\"\"" )
-       * L "}}"
-
+             * Lc "}}"
+             * EOL
+           ) ^ 0
+         * Lc "{\\PitonStyle{String.Long}{"
+         * ( DoubleLongInterpol + DoubleLongPureString ) ^ 0
+       +
+         Q ( ( S "rR" ) ^ -1  * P "\"\"\""
+              * ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
+         * Lc "}}"
+         * (
+             Lc "{\\PitonStyle{String.Long}{"
+             * Q ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
+             * Lc "}}"
+             * EOL
+           ) ^ 0
+         * Lc "{\\PitonStyle{String.Long}{"
+         * Q ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 )
+      )
+   * Q ( P "\"\"\"" )
+   * Lc "}}"
 local LongString = SingleLongString + DoubleLongString
+local StringDoc =
+    K ( P "\"\"\"" , 'String.Doc' )
+      * ( K ( (1 - P "\"\"\"" - P "\r" ) ^ 0 , 'String.Doc' ) * EOL * Tab ^0 ) ^ 0
+      * K ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 * P "\"\"\"" , 'String.Doc' )
+local CommentMath =
+  P "$" * K ( ( 1 - S "$\r" ) ^ 1 , 'Comment.Math' ) * P "$"
 
-local Expression =
-    P { "E" ,
-         E = ( 1 - S "{}()[]\r," ) ^ 0
-             *  (
-                  (   P "{" * V "F" * P "}"
-                    + P "(" * V "F" * P ")"
-                    + P "[" * V "F" * P "]" ) * ( 1 - S "{}()[]\r," ) ^ 0
-                ) ^ 0 ,
-         F = ( 1 - S "{}()[]\r\"'" ) ^ 0
-             * ( (
-                     P "'" * (P "\\'" + 1 - S"'\r" )^0 * P "'"
-                   + P "\"" * (P "\\\"" + 1 - S"\"\r" )^0 * P "\""
-                   + P "{"  * V "F" * P "}"
-                   + P "(" * V "F" * P ")"
-                   + P "[" * V "F" * P "]"
-                 ) * ( 1 - S "{}()[]\r\"'" ) ^ 0 ) ^ 0 ,
-      }
+local Comment =
+  Lc ( "{\\PitonStyle{Comment}{" )
+  * Q ( P "#" )
+  * ( CommentMath + Q ( ( 1 - S "$\r" ) ^ 1 ) ) ^ 0
+  * Lc ( "}}" )
+  * ( EOL + -1 )
 
-local Param = SkipSpace * K ( identifier , '' ) * SkipSpace
-                     *  ( K ( P "=" * Expression , 'InitialValues' )
-                          +  K ( P ":" , '' ) * SkipSpace * K ( letter^1 , 'Name.Type' ))
-                  + SkipSpace * K ( alphanum ^ 1 , '' ) * SkipSpace
-
-local Params =  Param * (  K ( P "," , '' ) * Param ) ^ 0
-
-local StringDoc = K ( P "\"\"\"" , 'String.Doc' )
-                      * ( K ( (1 - P "\"\"\"" - P "\r" ) ^ 0 , 'String.Doc' ) * EOL ) ^ 0
-                      * K ( ( 1 - P "\"\"\"" - P "\r" ) ^ 0 * P "\"\"\"" , 'String.Doc' )
-                  + K ( P "'''" , 'String.Doc' )
-                      * ( K ( (1 - P "'''" - P "\r")^0 , 'String.Doc' ) * EOL ) ^ 0
-                      * K ( ( 1 - P "'''" - P "\r")^0 * P "'''" , 'String.Doc' )
-
-local CommentMath = P "$" * K ( ( 1 - S "$\r" ) ^ 1 , 'Comment.Math' ) * P "$"
-
-local Comment = L ( "{\\PitonStyle{Comment}{" )
-                * C ( P "#" ) * ( CommentMath + C ( ( 1 - S "$\r" ) ^ 1 ) ) ^ 0
-                * L ( "}}" )
-                * ( EOL + -1 )
-
 local CommentLaTeX =
-   P "##"
-   * L "{\\PitonStyle{Comment.LaTeX}{\\ignorespaces"
-   * Ct ( C ( ( 1 - P "\r" ) ^ 0 ) )
-   * L "}}"
-   * ( EOL + -1 )
-
+  P "##"
+  * Lc "{\\PitonStyle{Comment.LaTeX}{\\ignorespaces"
+  * L ( ( 1 - P "\r" ) ^ 0 )
+  * Lc "}}"
+  * ( EOL + -1 )
+local Expression =
+  P { "E" ,
+       E = ( 1 - S "{}()[]\r," ) ^ 0
+           *  (
+                (   P "{" * V "F" * P "}"
+                  + P "(" * V "F" * P ")"
+                  + P "[" * V "F" * P "]" ) * ( 1 - S "{}()[]\r," ) ^ 0
+              ) ^ 0 ,
+       F = ( 1 - S "{}()[]\r\"'" ) ^ 0
+           * ( (
+                   P "'" * (P "\\'" + 1 - S"'\r" )^0 * P "'"
+                 + P "\"" * (P "\\\"" + 1 - S"\"\r" )^0 * P "\""
+                 + P "{"  * V "F" * P "}"
+                 + P "(" * V "F" * P ")"
+                 + P "[" * V "F" * P "]"
+               ) * ( 1 - S "{}()[]\r\"'" ) ^ 0 ) ^ 0 ,
+    }
+local Param =
+  SkipSpace * Identifier * SkipSpace
+   * (
+         K ( P "=" * Expression , 'InitialValues' )
+       + Q ( P ":" ) * SkipSpace * K ( letter^1 , 'Name.Type' )
+     ) ^ -1
+local Params = ( Param * ( Q "," * Param ) ^ 0 ) ^ -1
 local DefFunction =
-   K ( P "def" , 'Keyword' )
-   * ( Space
-       * K ( identifier , 'Name.Function' )
-       * ( SkipSpace * K ( P "(" , '' ) * Params * K ( P ")" , '' ) ) ^ -1
-       * ( SkipSpace
-            * K ( ( 1 - S ":\r" )^0 , 'Post.Function' )
-            * K ( P ":" , 'Keyword' )
-            * SkipSpace
-            * ( EOL + CommentLaTeX + Comment )
-            * SkipSpace
-            * StringDoc ) ^ -1
-      ) ^ -1
+  K ( P "def" , 'Keyword' )
+  * Space
+  * K ( identifier , 'Name.Function' )
+  * SkipSpace
+  * Q ( P "(" ) * Params * Q ( P ")" )
+  * SkipSpace
+  * K ( ( 1 - S ":\r" )^0 , 'Post.Function' )
+  * Q ( P ":" )
+  * ( SkipSpace
+      * ( EOL + CommentLaTeX + Comment ) -- in all cases, that contains an EOL
+      * Tab ^ 0
+      * SkipSpace
+      * StringDoc ^ 0 -- there may be additionnal docstrings
+    ) ^ -1
+local ItemDict =
+  ShortString * SkipSpace * Q ( P ":" ) * K ( Expression , 'Dict.Value' )
 
-local ItemDict = ShortString * SkipSpace * C ( P ":" ) * K ( Expression , 'Dict.Value' )
-
 local ItemOfSet = SkipSpace * ( ItemDict + ShortString ) * SkipSpace
 
-local Set = C ( P "{" )
-            * ItemOfSet * ( C ( P "," ) * ItemOfSet )  ^ 0
-            * C ( P "}" )
+local Set =
+  Q ( P "{" )
+  * ItemOfSet * ( Q ( P "," ) * ItemOfSet )  ^ 0
+  * Q ( P "}" )
 
-local Operator = K ( P "!=" + P "==" + P "<<" + P ">>" + S "-~+/*%=<>&.@|" , 'Operator')
-
-local OperatorWord = K ( P "in" + P "is" + P "and" + P "or" + P "not" , 'Operator.Word')
-
-local SyntaxPython =
+local SyntaxPythonAux =
+      Lc ( '\\__piton_begin_line:' ) *
       ( ( space - P "\r" ) ^0 * P "\r" ) ^ -1 *
       (  ( space^1 * -1 )
          + EOL
+         + Tab
          + Space
          + Escape
          + CommentLaTeX
@@ -825,72 +829,63 @@
          + ExceptionInConsole
          + Set
          + Delim
-         + Class * ( Space + Punct + EOL )
-         + Namespace * ( Space + Punct + EOL )
+         + Operator
+         + ShortString
+         + Punct
+         + FromImport
          + ImportAs
          + RaiseException
-         + Keyword * ( Space + Punct + EOL )
          + DefFunction
-         + ShortString
-         + Decorator * ( Space + Punct + EOL )
-         + Operator
-         + OperatorWord * ( Space + Punct + EOL )
-         + Builtin * ( Space + Punct + EOL )
+         + DefClass
+         + Keyword * ( Space + Punct + Delim + EOL + -1)
+         + Decorator
+         + OperatorWord
+         + Builtin * ( Space + Punct + Delim + EOL + -1)
          + Identifier
          + Number
          + Word
-      ) ^0 * -1
-
-local MinimalSyntax =
-  P { "S" ;
-      S = K ( (1 - P "\r" ) ^ 0 , '') + EOL * S
-    }
-
+      ) ^0 * -1 * Lc ( '\\__piton_end_line:' )
+local SyntaxPython = Ct ( SyntaxPythonAux )
 function Parse(code)
-  local t = Ct(SyntaxPython) : match(code)
-  tex.sprint( luatexbase.catcodetables.expl , '\\__piton_begin_line:' )
-  if t then else t = Ct(MinimalSyntax) : match(code) end
-  for i = 1 , #t do
-    if type(t[i]) == 'string'
-    then
-      tex.sprint(luatexbase.catcodetables.CatcodeTableOther, t[i])
-    else
-      tex.tprint(t[i])
-    end
-  end
-  tex.sprint( luatexbase.catcodetables.expl , '\\__piton_end_line:' )
+  local t = SyntaxPython : match ( code )
+  for _ , s in ipairs(t) do tex.tprint(s) end
 end
-
 function ParseFile(name)
   s = ''
   for line in io.lines(name) do s = s .. '\r' .. line end
   Parse(s)
 end
-
-function define_gobble_syntax(n)
-  GobbleSyntax = ( 1 - P "\r" ) ^ (-n)  * C ( ( 1 - P "\r" ) ^0 )
-               * ( C ( P "\r" )
-                   * ( 1 - P "\r" ) ^ (-n)
-                   * C ( ( 1 - P "\r" ) ^0 )
-                 ) ^ 0
+function gobble(n,code)
+  function concat(acc,new_value)
+    return acc .. new_value
+  end
+  if n==0
+  then return code
+  else
+       return Cf (
+                   Cc ( "" ) *
+                   ( 1 - P "\r" ) ^ (-n)  * C ( ( 1 - P "\r" ) ^ 0 )
+                     * ( C ( P "\r" )
+                     * ( 1 - P "\r" ) ^ (-n)
+                     * C ( ( 1 - P "\r" ) ^ 0 )
+                    ) ^ 0 ,
+                    concat
+                 ) : match ( code )
+  end
 end
-
-function GobbleParse(code)
-  local t = Ct(GobbleSyntax):match(code)
-  local new_code = ""
-  for i = 1 , #t do
-    new_code = new_code .. t[i]
+function GobbleParse(n,code)
+  if n==-1
+  then n = AutoGobbleLPEG : match(code)
+  else if n==-2
+       then n = EnvGobbleLPEG : match(code)
+       end
   end
-  Parse(new_code)
+  Parse(gobble(n,code))
 end
-
 function add(acc,new_value)
   return acc + new_value
 end
-
---[[ The following LPEG returns as capture the minimal number of spaces at
-the beginning of the lines of code]]
-AutoGobbleSyntax =
+AutoGobbleLPEG =
     ( space ^ 0 * P "\r" ) ^ -1
     * Cf (
            (
@@ -904,30 +899,22 @@
            * ( 1 - P " " ) * ( 1 - P "\r" ) ^ 0 ) ^ -1 ,
            math.min
          )
-
-function AutoGobbleParse(code)
-  local n = AutoGobbleSyntax:match(code)
-  if n==0
-  then Parse(code)
-  else define_gobble_syntax(n)
-       GobbleParse(code)
-  end
+EnvGobbleLPEG =
+  ( ( 1 - P "\r" ) ^ 0 * P "\r" ) ^ 0
+    * Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add ) * -1
+function CountLines(code)
+  local count = 0
+  for i in code:gmatch("\r") do count = count + 1 end
+  tex.sprint(
+      luatexbase.catcodetables.expl ,
+      '\\int_set:Nn \\l__piton_nb_lines_int {' .. count .. '}' )
 end
-
---[[ The following LPEG returns as capture the number of spaces at the last line,
-that is to say begin the \end{Piton} ]]
-
-EnvGobbleSyntax =
-       ( ( 1 - P "\r" ) ^ 0 * P "\r" ) ^ 0
-         * Cf ( Cc(0) * ( P " " * Cc(1) ) ^ 0 , add ) * -1
-
-function EnvGobbleParse(code)
-  local n = EnvGobbleSyntax:match(code)
-  if n==0
-  then Parse(code)
-  else define_gobble_syntax(n)
-       GobbleParse(code)
-  end
+function CountLinesFile(name)
+  local count = 0
+  for line in io.lines(name) do count = count + 1 end
+  tex.sprint(
+      luatexbase.catcodetables.expl ,
+      '\\int_set:Nn \\l__piton_nb_lines_int {' .. count .. '}' )
 end
 \end{luacode*}
 



More information about the tex-live-commits mailing list.