texlive[73192] Master/texmf-dist: expltools (23dec24)
commits+karl at tug.org
commits+karl at tug.org
Mon Dec 23 22:15:44 CET 2024
Revision: 73192
https://tug.org/svn/texlive?view=revision&revision=73192
Author: karl
Date: 2024-12-23 22:15:44 +0100 (Mon, 23 Dec 2024)
Log Message:
-----------
expltools (23dec24)
Modified Paths:
--------------
trunk/Master/texmf-dist/doc/support/expltools/CHANGES.md
trunk/Master/texmf-dist/doc/support/expltools/README.md
trunk/Master/texmf-dist/doc/support/expltools/project-proposal.pdf
trunk/Master/texmf-dist/doc/support/expltools/warnings-and-errors-01-preprocessing.md
trunk/Master/texmf-dist/doc/support/expltools/warnings-and-errors.pdf
trunk/Master/texmf-dist/scripts/expltools/explcheck-cli.lua
trunk/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua
Added Paths:
-----------
trunk/Master/texmf-dist/scripts/expltools/explcheck-config.lua
trunk/Master/texmf-dist/scripts/expltools/explcheck-preprocessing-comments.lua
Modified: trunk/Master/texmf-dist/doc/support/expltools/CHANGES.md
===================================================================
--- trunk/Master/texmf-dist/doc/support/expltools/CHANGES.md 2024-12-23 00:41:46 UTC (rev 73191)
+++ trunk/Master/texmf-dist/doc/support/expltools/CHANGES.md 2024-12-23 21:15:44 UTC (rev 73192)
@@ -1,5 +1,47 @@
# Changes
+## expltools 2024-12-23
+
+### explcheck v0.3.0
+
+#### Development
+
+- Add option `--expect-expl3-everywhere` to ignore \ExplSyntaxOn and Off.
+ (discussed with @muzimuzhi in #17, added in #19)
+
+- Add short-hand command-line option `-p` for `--porcelain`.
+ (suggested by @FrankMittelbach in #8, added in #19)
+
+- Add file `explcheck-config.lua` with the default configuration of explcheck. (#19)
+
+ You may place a file named `explcheck-config.lua` with your own configuration
+ in your repository to control the behavior of explcheck.
+
+ Note that the configuration options are provisional and may be changed or
+ removed before version 1.0.0. Furthermore, support for configuration YAML
+ files that will allow you to specify different configuration for different
+ .tex files is envisioned for a future release and will be the recommended way
+ to configure explcheck.
+
+#### Fixes
+
+- Make the detection of error E102 (expl3 material in non-expl3 parts) more precise.
+ (discussed with @cfr42 in #18, fixed in #19)
+
+- Use a less naïve parser of TeX comments to improve the detection of issues
+ W100 and E102. (reported by @FrankMittelbach in #8, fixed in #16)
+
+#### Documentation
+
+- State in the output of `explcheck --help` that command-line options are
+ provisional and subject to change. (discussed with @FrankMittelbach and
+ @muzimuzhi in #8 and #17, added in #19)
+
+- Display the default maximum line length in the output of `explcheck --help`. (#19)
+
+- Rename E102 to "expl3 material in non-expl3 parts".
+ (discussed with @cfr42 in #18, added in #19)
+
## expltools 2024-12-13
### explcheck v0.2.0
@@ -7,7 +49,7 @@
#### Development
- Add a command-line option `--porcelain` for machine-readable output.
- (suggested by @FrankMittelbach in #8, added in #14)
+ (suggested by @FrankMittelbach in #8, added in #15)
See <https://github.com/Witiko/expltools/pull/15#issuecomment-2542418484>
and below for a demonstration of how you might set up your text editor, so
Modified: trunk/Master/texmf-dist/doc/support/expltools/README.md
===================================================================
--- trunk/Master/texmf-dist/doc/support/expltools/README.md 2024-12-23 00:41:46 UTC (rev 73191)
+++ trunk/Master/texmf-dist/doc/support/expltools/README.md 2024-12-23 21:15:44 UTC (rev 73192)
@@ -6,11 +6,15 @@
2. [Requirements][2]
3. [Related work][3]
4. [Design][4]
-5. [First public release][5]
-On September 6, 2024, the tool has been [accepted for funding][6] by [the TeX Development Fund][7].
-The full text of the project proposal, which summarizes devlog posts 1–4 is available [here][8].
+On September 6, 2024, the tool has been [accepted for funding][5] by [the TeX Development Fund][6].
+The full text of the project proposal, which summarizes devlog posts 1–4 is available [here][7].
+These devlog posts chronicle the latest updates and progress in the ongoing development of the tool:
+
+5. [Frank Mittelbach in Brno, the first public release of explcheck, and expl3 usage statistics][8] from December 5, 2024
+6. [A flurry of releases, CSTUG talk, and what's next][9] from December 19, 2024
+
In the future, this repository may also contain the code of other useful development tools for expl3 programmers, such as a command-line utility similar to `grep` that will ignore whitespaces and newlines as well as other tools.
[1]: https://witiko.github.io/Expl3-Linter-1/
@@ -17,10 +21,11 @@
[2]: https://witiko.github.io/Expl3-Linter-2/
[3]: https://witiko.github.io/Expl3-Linter-3/
[4]: https://witiko.github.io/Expl3-Linter-4/
- [5]: https://witiko.github.io/Expl3-Linter-5/
- [6]: https://tug.org/tc/devfund/grants.html
- [7]: https://tug.org/tc/devfund/application.html
- [8]: https://tug.org/tc/devfund/documents/2024-09-expltools.pdf
+ [5]: https://tug.org/tc/devfund/grants.html
+ [6]: https://tug.org/tc/devfund/application.html
+ [7]: https://tug.org/tc/devfund/documents/2024-09-expltools.pdf
+ [8]: https://witiko.github.io/Expl3-Linter-5/
+ [9]: https://witiko.github.io/Expl3-Linter-6/
## Usage
Modified: trunk/Master/texmf-dist/doc/support/expltools/project-proposal.pdf
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/doc/support/expltools/warnings-and-errors-01-preprocessing.md
===================================================================
--- trunk/Master/texmf-dist/doc/support/expltools/warnings-and-errors-01-preprocessing.md 2024-12-23 00:41:46 UTC (rev 73191)
+++ trunk/Master/texmf-dist/doc/support/expltools/warnings-and-errors-01-preprocessing.md 2024-12-23 21:15:44 UTC (rev 73192)
@@ -11,8 +11,8 @@
/w101.tex
-## Expl3 control sequences in non-expl3 parts {.e label=e102}
-An input file contains what looks like expl3 control sequences [@latexteam2024interfaces, Section 1.1] in non-expl3 parts.
+## Expl3 material in non-expl3 parts {.e label=e102}
+An input file contains what looks like expl3 material [@latexteam2024interfaces, Section 1.1] in non-expl3 parts.
/e102.tex
Modified: trunk/Master/texmf-dist/doc/support/expltools/warnings-and-errors.pdf
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/scripts/expltools/explcheck-cli.lua
===================================================================
--- trunk/Master/texmf-dist/scripts/expltools/explcheck-cli.lua 2024-12-23 00:41:46 UTC (rev 73191)
+++ trunk/Master/texmf-dist/scripts/expltools/explcheck-cli.lua 2024-12-23 21:15:44 UTC (rev 73192)
@@ -1,5 +1,6 @@
-- A command-line interface for the static analyzer explcheck.
+local config = require("explcheck-config")
local new_issues = require("explcheck-issues")
local format = require("explcheck-format")
@@ -82,11 +83,11 @@
end
-- Process all input files.
-local function main(pathnames, warnings_are_errors, max_line_length, porcelain)
+local function main(pathnames, options)
local num_warnings = 0
local num_errors = 0
- if not porcelain then
+ if not options.porcelain then
print("Checking " .. #pathnames .. " " .. format.pluralize("file", #pathnames))
end
@@ -99,7 +100,10 @@
local issues = new_issues()
-- Run all processing steps.
- local line_starting_byte_numbers, _ = preprocessing(issues, content, max_line_length)
+ local line_starting_byte_numbers, _ = preprocessing(issues, content, {
+ expect_expl3_everywhere = options.expect_expl3_everywhere,
+ max_line_length = options.max_line_length,
+ })
if #issues.errors > 0 then
goto continue
end
@@ -112,17 +116,17 @@
::continue::
num_warnings = num_warnings + #issues.warnings
num_errors = num_errors + #issues.errors
- format.print_results(pathname, issues, line_starting_byte_numbers, pathname_number == #pathnames, porcelain)
+ format.print_results(pathname, issues, line_starting_byte_numbers, pathname_number == #pathnames, options.porcelain)
end
-- Print a summary.
- if not porcelain then
- format.print_summary(#pathnames, num_warnings, num_errors, porcelain)
+ if not options.porcelain then
+ format.print_summary(#pathnames, num_warnings, num_errors, options.porcelain)
end
if(num_errors > 0) then
return 1
- elseif(warnings_are_errors and num_warnings > 0) then
+ elseif(options.warnings_are_errors and num_warnings > 0) then
return 2
else
return 0
@@ -132,14 +136,21 @@
local function print_usage()
print("Usage: " .. arg[0] .. " [OPTIONS] FILENAMES\n")
print("Run static analysis on expl3 files.\n")
- print("Options:")
- print("\t--max-line-length=N The maximum line length before the warning S103 (Line too long) is produced.")
- print("\t--porcelain Produce machine-readable output.")
- print("\t--warnings-are-errors Produce a non-zero exit code if any warnings are produced by the analysis.")
+ local max_line_length = tostring(config.max_line_length)
+ print(
+ "Options:\n\n"
+ .. "\t--expect-expl3-everywhere Expect that the whole files are in expl3, ignoring \\ExplSyntaxOn and Off.\n"
+ .. "\t This prevents the error E102 (expl3 material in non-expl3 parts).\n\n"
+ .. "\t--max-line-length=N The maximum line length before the warning S103 (Line too long) is produced.\n"
+ .. "\t The default maximum line length is N=" .. max_line_length .. " characters.\n\n"
+ .. "\t--porcelain, -p Produce machine-readable output.\n\n"
+ .. "\t--warnings-are-errors Produce a non-zero exit code if any warnings are produced by the analysis.\n"
+ )
+ print("The options are provisional and may be changed or removed before version 1.0.0.")
end
local function print_version()
- print("explcheck (expltools 2024-12-13) v0.2.0")
+ print("explcheck (expltools 2024-12-23) v0.3.0")
print("Copyright (c) 2024 Vít Starý Novotný")
print("Licenses: LPPL 1.3 or later, GNU GPL v2 or later")
end
@@ -150,10 +161,8 @@
else
-- Collect arguments.
local pathnames = {}
- local warnings_are_errors = false
local only_pathnames_from_now_on = false
- local max_line_length = nil
- local porcelain = false
+ local options = {}
for _, argument in ipairs(arg) do
if only_pathnames_from_now_on then
table.insert(pathnames, argument)
@@ -165,12 +174,14 @@
elseif argument == "--version" or argument == "-v" then
print_version()
os.exit(0)
+ elseif argument == "--expect-expl3-everywhere" then
+ options.expect_expl3_everywhere = true
+ elseif argument:sub(1, 18) == "--max-line-length=" then
+ options.max_line_length = tonumber(argument:sub(19))
+ elseif argument == "--porcelain" or argument == "-p" then
+ options.porcelain = true
elseif argument == "--warnings-are-errors" then
- warnings_are_errors = true
- elseif argument == "--porcelain" then
- porcelain = true
- elseif argument:sub(1, 18) == "--max-line-length=" then
- max_line_length = tonumber(argument:sub(19))
+ options.warnings_are_errors = true
elseif argument:sub(1, 2) == "--" then
-- An unknown argument
print_usage()
@@ -191,6 +202,6 @@
end
-- Run the analysis.
- local exit_code = main(pathnames, warnings_are_errors, max_line_length, porcelain)
+ local exit_code = main(pathnames, options)
os.exit(exit_code)
end
Added: trunk/Master/texmf-dist/scripts/expltools/explcheck-config.lua
===================================================================
--- trunk/Master/texmf-dist/scripts/expltools/explcheck-config.lua (rev 0)
+++ trunk/Master/texmf-dist/scripts/expltools/explcheck-config.lua 2024-12-23 21:15:44 UTC (rev 73192)
@@ -0,0 +1,8 @@
+-- The default configuration for the static analyzer explcheck.
+
+return {
+ expect_expl3_everywhere = false,
+ max_line_length = 80,
+ porcelain = false,
+ warnings_are_errors = false,
+}
Property changes on: trunk/Master/texmf-dist/scripts/expltools/explcheck-config.lua
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
Added: trunk/Master/texmf-dist/scripts/expltools/explcheck-preprocessing-comments.lua
===================================================================
--- trunk/Master/texmf-dist/scripts/expltools/explcheck-preprocessing-comments.lua (rev 0)
+++ trunk/Master/texmf-dist/scripts/expltools/explcheck-preprocessing-comments.lua 2024-12-23 21:15:44 UTC (rev 73192)
@@ -0,0 +1,108 @@
+-- The TeX comment removal part for the preprocessing step of static analysis.
+
+local lpeg = require("lpeg")
+local P, S, Cp, Ct = lpeg.P, lpeg.S, lpeg.Cp, lpeg.Ct
+
+-- Define base parsers.
+---- Generic
+local any = P(1)
+
+---- Tokens
+local percent_sign = P("%")
+local backslash = P([[\]])
+
+---- Spacing
+local spacechar = S("\t ")
+local optional_spaces = spacechar^0
+local newline = (
+ P("\n")
+ + P("\r\n")
+ + P("\r")
+)
+local linechar = any - newline
+local blank_line = optional_spaces * newline
+
+-- Define intermediate parsers.
+local commented_line_letter = (
+ linechar
+ + newline
+ - backslash
+ - percent_sign
+)
+local commented_line = (
+ (
+ (
+ commented_line_letter
+ - newline
+ )^1 -- initial state
+ + (
+ backslash -- even backslash
+ * (
+ backslash
+ + #newline
+ )
+ )^1
+ + (
+ backslash
+ * (
+ percent_sign
+ + commented_line_letter
+ )
+ )
+ )^0
+ * (
+ #percent_sign
+ * Cp()
+ * (
+ (
+ percent_sign -- comment
+ * linechar^0
+ * Cp()
+ * newline
+ * #blank_line -- blank line
+ )
+ + percent_sign -- comment
+ * linechar^0
+ * Cp()
+ * newline
+ * optional_spaces -- leading spaces
+ )
+ + newline
+ )
+)
+
+-- Strip TeX comments from a text. Besides the transformed text, also return
+-- a function that maps positions in the transformed text back to the original
+-- text.
+local function strip_comments(text)
+ local transformed_index = 0
+ local numbers_of_bytes_removed = {}
+ local transformed_text_table = {}
+ for index, text_position in ipairs(lpeg.match(Ct(commented_line^1), text)) do
+ local span_size = text_position - transformed_index - 1
+ if span_size > 0 then
+ if index % 2 == 1 then -- chunk of text
+ table.insert(transformed_text_table, text:sub(transformed_index + 1, text_position - 1))
+ else -- comment
+ table.insert(numbers_of_bytes_removed, {transformed_index, span_size})
+ end
+ transformed_index = transformed_index + span_size
+ end
+ end
+ table.insert(transformed_text_table, text:sub(transformed_index + 1, -1))
+ local transformed_text = table.concat(transformed_text_table, "")
+ local function map_back(index)
+ for _, where_and_number_of_bytes_removed in ipairs(numbers_of_bytes_removed) do
+ local where, number_of_bytes_removed = table.unpack(where_and_number_of_bytes_removed)
+ if index > where then
+ index = index + number_of_bytes_removed
+ else
+ break
+ end
+ end
+ return index
+ end
+ return transformed_text, map_back
+end
+
+return strip_comments
Property changes on: trunk/Master/texmf-dist/scripts/expltools/explcheck-preprocessing-comments.lua
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:executable
## -0,0 +1 ##
+*
\ No newline at end of property
Modified: trunk/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua
===================================================================
--- trunk/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua 2024-12-23 00:41:46 UTC (rev 73191)
+++ trunk/Master/texmf-dist/scripts/expltools/explcheck-preprocessing.lua 2024-12-23 21:15:44 UTC (rev 73192)
@@ -1,5 +1,8 @@
-- The preprocessing step of static analysis determines which parts of the input files contain expl3 code.
+local config = require("explcheck-config")
+local strip_comments = require("explcheck-preprocessing-comments")
+
local lpeg = require("lpeg")
local Cp, P, R, S, V = lpeg.Cp, lpeg.P, lpeg.R, lpeg.S, lpeg.V
@@ -7,11 +10,11 @@
---- Generic
local any = P(1)
local eof = -any
+local fail = P(false)
---- Tokens
local lbrace = P("{")
local rbrace = P("}")
-local percent_sign = P("%")
local backslash = P([[\]])
local letter = R("AZ","az")
local underscore = P("_")
@@ -36,26 +39,39 @@
-- Define intermediate parsers.
---- Parts of TeX syntax
-local comment = (
- percent_sign
- * linechar^0
- * newline
- * optional_spaces
-)
local argument = (
lbrace
- * (
- comment
- + (any - rbrace)
- )^0
+ * (any - rbrace)^0
* rbrace
)
-local expl3like_control_sequence = (
+
+local expl3_function = (
backslash
- * (letter - underscore - colon)^1
- * (underscore + colon)
- * letter^1
+ * (underscore * underscore)^-1 * letter^1 -- module
+ * underscore
+ * letter^1 -- description
+ * colon
+ * S("NncVvoxefTFpwD")^1 -- argspec
+ * (eof + -letter)
)
+local expl3_variable_or_constant = (
+ backslash
+ * S("cgl") -- scope
+ * underscore
+ * (
+ letter^1 -- just description
+ + underscore^-1 * letter^1 -- module
+ * underscore
+ * letter^1 -- description
+ )
+ * underscore
+ * letter^1 -- type
+ * (eof + -letter)
+)
+local expl3like_material = (
+ expl3_function
+ + expl3_variable_or_constant
+)
---- Standard delimiters
local provides = (
@@ -66,40 +82,45 @@
+ P("File")
)
* optional_spaces_and_newline
- * comment^0
* argument
* optional_spaces_and_newline
- * comment^0
* argument
* optional_spaces_and_newline
- * comment^0
* argument
* optional_spaces_and_newline
- * comment^0
* argument
)
local expl_syntax_on = P([[\ExplSyntaxOn]])
local expl_syntax_off = P([[\ExplSyntaxOff]])
-local function preprocessing(issues, content, max_line_length)
- if max_line_length == nil then
- max_line_length = 80
+-- Get the value of an option or the default value if unspecified.
+local function get_option(options, key)
+ if options == nil or options[key] == nil then
+ return config[key]
end
+ return options[key]
+end
+-- Preprocess the content and register any issues.
+local function preprocessing(issues, content, options)
+
-- Determine the bytes where lines begin.
local line_starting_byte_numbers = {}
+
local function record_line(line_start)
table.insert(line_starting_byte_numbers, line_start)
end
+
local function line_too_long(range_start, range_end)
issues:add('s103', 'line too long', range_start, range_end + 1)
end
+
local line_numbers_grammar = (
Cp() / record_line
* (
(
(
- Cp() * linechar^(max_line_length + 1) * Cp() / line_too_long
+ Cp() * linechar^(get_option(options, 'max_line_length') + 1) * Cp() / line_too_long
+ linechar^0
)
* newline
@@ -108,19 +129,49 @@
)^0
)
lpeg.match(line_numbers_grammar, content)
+
+ -- Strip TeX comments before further analysis.
+ local transformed_content, map_back = strip_comments(content)
+
-- Determine which parts of the input files contain expl3 code.
local expl_ranges = {}
+
local function capture_range(range_start, range_end)
+ range_start, range_end = map_back(range_start), map_back(range_end)
table.insert(expl_ranges, {range_start, range_end + 1})
end
+
local function unexpected_pattern(pattern, code, message, test)
return Cp() * pattern * Cp() / function(range_start, range_end)
+ range_start, range_end = map_back(range_start), map_back(range_end)
if test == nil or test() then
issues:add(code, message, range_start, range_end + 1)
end
end
end
+
local num_provides = 0
+ local Opener = unexpected_pattern(
+ provides,
+ "e104",
+ [[multiple delimiters `\ProvidesExpl*` in a single file]],
+ function()
+ num_provides = num_provides + 1
+ return num_provides > 1
+ end
+ )
+ local Closer = fail
+ if not get_option(options, 'expect_expl3_everywhere') then
+ Opener = (
+ expl_syntax_on
+ + Opener
+ )
+ Closer = (
+ expl_syntax_off
+ + Closer
+ )
+ end
+
local analysis_grammar = P{
"Root";
Root = (
@@ -138,9 +189,9 @@
"unexpected delimiters"
)
+ unexpected_pattern(
- expl3like_control_sequence,
+ expl3like_material,
"e102",
- "expl3 control sequences in non-expl3 parts"
+ "expl3 material in non-expl3 parts"
)
+ (any - V"Opener")
)^0
@@ -159,26 +210,18 @@
* Cp()
* (V"Closer" + eof)
),
- Opener = (
- expl_syntax_on
- + unexpected_pattern(
- provides,
- "e104",
- [[multiple delimiters `\ProvidesExpl*` in a single file]],
- function()
- num_provides = num_provides + 1
- return num_provides > 1
- end
- )
- ),
- Closer = expl_syntax_off,
+ Opener = Opener,
+ Closer = Closer,
}
- lpeg.match(analysis_grammar, content)
+ lpeg.match(analysis_grammar, transformed_content)
+
-- If no parts were detected, assume that the whole input file is in expl3.
if(#expl_ranges == 0 and #content > 0) then
table.insert(expl_ranges, {0, #content})
- issues:add('w100', 'no standard delimiters')
- issues:ignore('e102')
+ if not get_option(options, 'expect_expl3_everywhere') then
+ issues:add('w100', 'no standard delimiters')
+ issues:ignore('e102')
+ end
end
return line_starting_byte_numbers, expl_ranges
end
More information about the tex-live-commits
mailing list.