texlive[61494] trunk: texlogsieve (4jan22)

commits+karl at tug.org commits+karl at tug.org
Tue Jan 4 23:29:24 CET 2022


Revision: 61494
          http://tug.org/svn/texlive?view=revision&revision=61494
Author:   karl
Date:     2022-01-04 23:29:24 +0100 (Tue, 04 Jan 2022)
Log Message:
-----------
texlogsieve (4jan22)

Modified Paths:
--------------
    trunk/Build/source/texk/texlive/linked_scripts/texlogsieve/texlogsieve
    trunk/Master/texmf-dist/doc/man/man1/texlogsieve.1
    trunk/Master/texmf-dist/doc/man/man1/texlogsieve.man1.pdf
    trunk/Master/texmf-dist/doc/support/texlogsieve/README.md
    trunk/Master/texmf-dist/doc/support/texlogsieve/texlogsieve.pdf
    trunk/Master/texmf-dist/doc/support/texlogsieve/texlogsieve.tex
    trunk/Master/texmf-dist/scripts/texlogsieve/texlogsieve

Modified: trunk/Build/source/texk/texlive/linked_scripts/texlogsieve/texlogsieve
===================================================================
--- trunk/Build/source/texk/texlive/linked_scripts/texlogsieve/texlogsieve	2022-01-04 22:28:55 UTC (rev 61493)
+++ trunk/Build/source/texk/texlive/linked_scripts/texlogsieve/texlogsieve	2022-01-04 22:29:24 UTC (rev 61494)
@@ -2,7 +2,7 @@
 
 -- texlogsieve - filter and summarize LaTeX log files
 --
--- Copyright (C) 2021 Nelson Lago <lago at ime.usp.br>
+-- Copyright (C) 2021, 2022 Nelson Lago <lago at ime.usp.br>
 --
 -- This program is free software: you can redistribute it and/or modify
 -- it under the terms of the GNU General Public License as published by
@@ -332,6 +332,7 @@
       local line
       if position == nil then position = 0 end
       line = Lines:get(position)
+      if line == nil then return false, {} end
 
       local first, last = string.find(line, self.pattern)
       if first == nil then return false
@@ -525,9 +526,9 @@
  UNKNOWN = 4
 
 function main(arg)
+  initializeKpse()
   processCommandLine(arg)
   initializeGlobals()
-  initializeKpse()
   registerHandlers()
   registerSummaries()
   convertFilterStringsToPatterns()
@@ -552,7 +553,7 @@
   -- if there is remaining data from the previous iteration,
   -- we leave everything as-is for it to be processed now
   local tmp = Lines.current
-  if tmp ~= nil and string.len(tmp) > 0 then return true end
+  if tmp ~= nil and tmp ~= "" then return true end
 
   -- Refill the buffer. A simple experiment suggests 8 lines
   -- is enough, but why not use a higher value?
@@ -562,7 +563,7 @@
       -- We *need* to remove blank lines here because
       -- sometimes a wrapped line is followed by a blank
       -- line, which messes our detection of wrapped lines.
-      if string.len(tmp) > 0 then Lines:append(tmp) end
+      if tmp ~= "" then Lines:append(tmp) end
   end
 
   -- proceed to the next line
@@ -655,23 +656,21 @@
   -- map physicalPage (from numShipouts) to latexPage (LaTeX counter)
   latexPages = {}
 
-  -- After printing each message, the output coroutine stores them
-  -- in currentPageMessages. When it receives a shipout message,
-  -- it traverses currentPageMessages adding the page number it
-  -- just learned about to each of the messages, calls :toSummary()
-  -- for each of them and and clears currentPageMessages (we do this
-  -- so the data in the summaries may include the page numbers). The
-  -- objects representing the summary for each kind of message are
-  -- stored in summaries, so after all messages we can just traverse
-  -- this list calling :toString() and get all the summaries. The
-  -- summaries table is populated by registerSummaries().
+  -- After printing each message, the output coroutine stores them in
+  -- currentPageMessages. When it receives a shipout message, it traverses
+  -- currentPageMessages adding the page number it just learned about to
+  -- each of the messages and clears currentPageMessages. This serves two
+  -- purposes: it allows us to include the page numbers in the summaries
+  -- and it allows us to include the page number in page-delay mode.
   currentPageMessages = {}
+
+  -- The objects representing the summary for each kind of message are
+  -- stored in summaries, so after all messages are processed we can just
+  -- traverse this list calling :toString() and get all the summaries. The
+  -- summaries are also used to suppress repeated messages. This table is
+  -- populated by registerSummaries().
   summaries = {}
 
-  -- When the same message appears several times, we only output it
-  -- once, thanks to this table. This is used by showMessage()
-  alreadySeen = {}
-
   -- All handlers should be in either of these. They are populated by
   -- registerHandlers().
   beginningOfLineHandlers = {}
@@ -717,9 +716,11 @@
   table.insert(beginningOfLineHandlers, beginningOfLineDebugStringsHandler)
   table.insert(beginningOfLineHandlers, beginningOfLineInfoStringsHandler)
   table.insert(beginningOfLineHandlers, beginningOfLineWarningStringsHandler)
+  table.insert(beginningOfLineHandlers, beginningOfLineCriticalStringsHandler)
   table.insert(anywhereHandlers, anywhereDebugStringsHandler)
   table.insert(anywhereHandlers, anywhereInfoStringsHandler)
   table.insert(anywhereHandlers, anywhereWarningStringsHandler)
+  table.insert(anywhereHandlers, anywhereCriticalStringsHandler)
   table.insert(anywhereHandlers, openParensHandler)
   table.insert(anywhereHandlers, closeParensHandler)
   table.insert(anywhereHandlers, openSquareBracketHandler)
@@ -769,6 +770,34 @@
       table.insert(tmp, pat)
   end
   SILENCE_PKGS = tmp
+
+  tmp = {}
+  for _, str in ipairs(FORCED_DEBUG) do
+      local pat = stringToPattern(str)
+      table.insert(tmp, pat)
+  end
+  FORCED_DEBUG = tmp
+
+  tmp = {}
+  for _, str in ipairs(FORCED_INFO) do
+      local pat = stringToPattern(str)
+      table.insert(tmp, pat)
+  end
+  FORCED_INFO = tmp
+
+  tmp = {}
+  for _, str in ipairs(FORCED_WARNING) do
+      local pat = stringToPattern(str)
+      table.insert(tmp, pat)
+  end
+  FORCED_WARNING = tmp
+
+  tmp = {}
+  for _, str in ipairs(FORCED_CRITICAL) do
+      local pat = stringToPattern(str)
+      table.insert(tmp, pat)
+  end
+  FORCED_CRITICAL = tmp
 end
 
 function processCommandLine(args)
@@ -780,6 +809,8 @@
   RAW = false
   SILENCE_REPETITIONS = true
   MINLEVEL = WARNING
+  BE_REDUNDANT = false
+  DETAILED_UNDEROVER_SUMMARY = true
 
   SILENCE_STRINGS = {}
   SILENCE_PKGS = {} -- just the package names
@@ -786,6 +817,11 @@
   SEMISILENCE_FILES = {} -- filenames (without leading path), file globs work
   SILENCE_FILES_RECURSIVE = {} -- same
 
+  -- The user may redefine the severity level of some messages.
+  FORCED_DEBUG = {}
+  FORCED_INFO = {}
+  FORCED_WARNING = {}
+  FORCED_CRITICAL = {}
 
   -- "-l level -c configFile"
   local optionsWithArgs = "lc"
@@ -806,6 +842,10 @@
   --only-summary                       no filtering, only final summary
   --shipouts, --no-shipouts            enable/disable reporting shipouts
   --repetitions, --no-repetitions      allow/prevent repeated messages
+  --be-redundant, --no-be-redundant    present/suppress ordinary messages
+                                       that will also appear in the summary
+  --box-detail, --no-box-detail        include/exclude full under/overfull
+                                       boxes information in the summary
   --heartbeat, --no-heartbeat          enable/disable progress gauge
   -l LEVEL, --minlevel=LEVEL           filter out messages with severity
                                        level lower than [LEVEL]. Valid
@@ -825,6 +865,19 @@
   --add-debug-message=MESSAGE          add new recognizable debug message
   --add-info-message=MESSAGE           add new recognizable info message
   --add-warning-message=MESSAGE        add new recognizable warning message
+  --add-critical-message=MESSAGE       add new recognizable critical message
+  --set-to-level-debug=EXCERPT         reset severity of messages containing
+                                       text EXCERPT to DEBUG; can be used
+                                       multiple times
+  --set-to-level-info=EXCERPT          reset severity of messages containing
+                                       text EXCERPT to INFO; can be used
+                                       multiple times
+  --set-to-level-warning=EXCERPT       reset severity of messages containing
+                                       text EXCERPT to WARNING; can be used
+                                       multiple times
+  --set-to-level-critical=EXCERPT      reset severity of messages containing
+                                       text EXCERPT to CRITICAL; can be used
+                                       multiple times
   -c cfgfile, --config-file=cfgfile    read options from config file
   -h, --help                           give this help list
   --version                            print program version]]
@@ -835,8 +888,8 @@
 
   --version
   if vars.version then
-      print("texlogsieve 1.0.0-beta-1")
-      print("Copyright (C) 2021 Nelson Lago <lago at ime.usp.br>")
+      print("texlogsieve 1.0.0-beta-2")
+      print("Copyright (C) 2021, 2022 Nelson Lago <lago at ime.usp.br>")
       print("License GPLv3+: GNU GPL version 3 or later "
             .. "<https://gnu.org/licenses/gpl.html>.")
       print("This is free software: you are free to change "
@@ -847,29 +900,22 @@
 
   --config-file=file
   -- "-c file"
-  local configFileName
-  if vars['config-file'] ~= nil then configFileName = vars['config-file'] end
-  if vars.c ~= nil then configFileName = vars.c end
-  if configFileName ~= nil then
-      configFileName = configFileName[1]
-      local filevars = processConfigFile(configFileName)
+  local configFileNames = vars['config-file']
+  if configFileNames == nil then configFileNames = {} end
 
-      -- merge filevars with vars; vars has precedence
-      for k, v in pairs(vars) do
-          if type(v) == "boolean" then
-              filevars[k] = v
-          elseif filevars[k] == nil then
-              filevars[k] = v
-          else
-              -- the value is a table, so append
-              for _, item in ipairs(v) do
-                  table.insert(filevars[k], item)
-              end
-          end
+  if vars.c ~= nil then
+      for _, val in ipairs(vars.c) do
+          table.insert(configFileNames, val)
       end
+  end
 
-      -- use the merged values
-      vars = filevars
+  -- Add the default config file to the beginning of the list
+  local filename = kpse.find_file('texlogsieverc')
+  if filename ~= nil then table.insert(configFileNames, 1, filename) end
+
+  for _, filename in ipairs(configFileNames) do
+      local configFile = assert(io.open(filename, "r"))
+      vars = processConfigFile(configFile, vars)
   end
 
   --unwrap-only
@@ -881,6 +927,7 @@
       PAGE_DELAY = false
       SHOW_SHIPOUTS = true
       SILENCE_REPETITIONS = false
+      BE_REDUNDANT = true
       MINLEVEL = DEBUG
   end
 
@@ -904,6 +951,7 @@
   if vars['no-summary'] or vars.summary ~= nil and not vars.summary then
       SHOW_SUMMARY = false
       SILENCE_REPETITIONS = false
+      BE_REDUNDANT = true
   end
   if vars.summary then SHOW_SUMMARY = true end
 
@@ -941,6 +989,28 @@
   end
   if vars.repetitions then SILENCE_REPETITIONS = false end
 
+  --be-redundant
+  --no-be-redundant
+  --be-redundant=true/false
+  if vars['no-be-redundant']
+                    or vars['be-redundant'] ~= nil
+                    and not vars['be-redundant'] then
+
+      BE_REDUNDANT = false
+  end
+  if vars['be-redundant'] then BE_REDUNDANT = true end
+
+  --box-detail
+  --no-box-detail
+  --box-detail=true/false
+  if vars['no-box-detail']
+                    or vars['box-detail'] ~= nil
+                    and not vars['box-detail'] then
+
+      DETAILED_UNDEROVER_SUMMARY = false
+  end
+  if vars['box-detail'] then DETAILED_UNDEROVER_SUMMARY = true end
+
   --no-heartbeat
   --heartbeat
   --heartbeat=true/false
@@ -969,7 +1039,7 @@
       for _, msg in ipairs(vars['add-debug-message']) do
           local pat = stringToPattern(msg)
           if not string.find(pat, '^', 1, true) then pat = '^%s*' .. pat end
-          pat = string.gsub(pat, '\\n', '\n')
+          pat = string.gsub(pat, '\\n', '%%s*\n')
           table.insert(anywhereDebugStringsHandler.patterns, pat)
       end
   end
@@ -978,8 +1048,8 @@
       for _, msg in ipairs(vars['add-info-message']) do
           local pat = stringToPattern(msg)
           if not string.find(pat, '^', 1, true) then pat = '^%s*' .. pat end
-          pat = string.gsub(pat, '\\n', '\n')
-          table.insert(anywhereInfoStringsHandler.patterns, msg)
+          pat = string.gsub(pat, '\\n', '%%s*\n')
+          table.insert(anywhereInfoStringsHandler.patterns, pat)
       end
   end
 
@@ -987,39 +1057,79 @@
       for _, msg in ipairs(vars['add-warning-message']) do
           local pat = stringToPattern(msg)
           if not string.find(pat, '^', 1, true) then pat = '^%s*' .. pat end
-          pat = string.gsub(pat, '\\n', '\n')
-          table.insert(anywhereWarningStringsHandler.patterns, msg)
+          pat = string.gsub(pat, '\\n', '%%s*\n')
+          table.insert(anywhereWarningStringsHandler.patterns, pat)
       end
   end
+
+  if vars['add-critical-message'] then
+      for _, msg in ipairs(vars['add-critical-message']) do
+          local pat = stringToPattern(msg)
+          if not string.find(pat, '^', 1, true) then pat = '^%s*' .. pat end
+          pat = string.gsub(pat, '\\n', '%%s*\n')
+          table.insert(anywhereCriticalStringsHandler.patterns, pat)
+      end
+  end
+
+
+  if vars['set-to-level-debug'] then
+      FORCED_DEBUG = vars['set-to-level-debug']
+  end
+
+  if vars['set-to-level-info'] then
+      FORCED_INFO = vars['set-to-level-info']
+  end
+
+  if vars['set-to-level-warning'] then
+      FORCED_WARNING = vars['set-to-level-warning']
+  end
+
+  if vars['set-to-level-critical'] then
+      FORCED_CRITICAL = vars['set-to-level-critical']
+  end
 end
 
-function processConfigFile(filename)
-    configfile = assert(io.open(filename, "r"))
-    local fileOptions = {}
+function processConfigFile(configFile, currentVars)
+  local fileVars = {}
 
-    while true do
-        local line = configfile:read("*line")
-        if line == nil then break end
+  while true do
+      local line = configFile:read("*line")
+      if line == nil then break end
 
-        line = trim(line)
-        local first = string.find(line, '^#')
-        if first == nil and string.len(line) > 0 then
+      line = trim(line)
+      local first = string.find(line, '^#')
 
-            local equals = string.find(line, '=', 1, true)
-            if equals ~= nil then
-                optname = string.sub(line, 1, equals -1)
-                optval = string.sub(line, equals +1)
-                optname = trim(optname)
-                optval = trim(optval)
-            else
-                optname = line
-                optval = true
+      if first == nil and line ~= "" then
+          local equals = string.find(line, '=', 1, true)
+          if equals ~= nil then
+              optname = string.sub(line, 1, equals -1)
+              optval = string.sub(line, equals +1)
+              optname = trim(optname)
+              optval = trim(optval)
+          else
+              optname = line
+              optval = true
+          end
+          simpleGetoptStoreVal(fileVars, optname, optval)
+      end
+  end
+
+    -- merge fileVars with currentVars; currentVars has precedence
+    for k, v in pairs(currentVars) do
+        if type(v) == "boolean" then
+            fileVars[k] = v
+        elseif fileVars[k] == nil then
+            fileVars[k] = v
+        else
+            -- the value is a table, so append
+            for _, item in ipairs(v) do
+                table.insert(fileVars[k], item)
             end
-            simpleGetoptStoreVal(fileOptions, optname, optval)
         end
     end
 
-    return fileOptions
+  -- return the merged values
+  return fileVars
 end
 
 
@@ -1039,7 +1149,107 @@
 
 dispatch = function(msg) coroutine.resume(outputCoroutine, msg) end
 
+function adjustSeverity(msg)
+  formatted = trim(msg:realToString())
+  if formatted == "" then return end
+
+  DEFAULT_FORCED_INFO = {
+    "File %b`' already exists on the system%."
+            .. "%s*Not generating it from",
+    "You have requested package %b`',"
+            .. "%s*but the package provides",
+    "Writing file %b`'",
+    "Form Feed has been converted to Blank",
+    "Tab has been converted to Blank",
+    "The morewrites package is unnecessary",
+    'Unused \\captionsetup%b[]',
+  }
+
+  DEFAULT_FORCED_CRITICAL = {
+    "Label %b`' multiply defined",
+    "Command .- invalid in math mode",
+    "Optional argument of \\twocolumn too tall on page",
+    "Marginpar on page %S- moved",
+    "Some font shapes were not available, defaults substituted%.",
+    "Font shape %b`' in size %b<> not available"
+            .. "%s+Font shape %b`' tried instead",
+    "Font shape %b`' in size %S+ not available"
+            .. "%s+external font %b`' used",
+    "Font shape %b`' undefined"
+            .. "%s+using %b`' instead",
+  }
+
+
+  -- We do things this way so that user-defined strings override these
+  -- defaults (note that there is no "return" in the first two blocks)
+  if msg.severity ~= INFO then
+      for _, val in ipairs(DEFAULT_FORCED_INFO) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = INFO
+          end
+      end
+  end
+
+  if msg.severity ~= CRITICAL then
+      for _, val in ipairs(DEFAULT_FORCED_CRITICAL) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = CRITICAL
+          end
+      end
+  end
+
+  if msg.severity ~= DEBUG then
+      for _, val in ipairs(FORCED_DEBUG) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = DEBUG
+              return
+          end
+      end
+  end
+
+  if msg.severity ~= INFO then
+      for _, val in ipairs(FORCED_INFO) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = INFO
+              return
+          end
+      end
+  end
+
+  if msg.severity ~= WARNING then
+      for _, val in ipairs(FORCED_WARNING) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = WARNING
+              return
+          end
+      end
+  end
+
+  if msg.severity ~= CRITICAL then
+      for _, val in ipairs(FORCED_CRITICAL) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = CRITICAL
+              return
+          end
+      end
+  end
+end
+
 function processMessage(msg)
+  adjustSeverity(msg)
+
   if ONLY_SUMMARY or PAGE_DELAY then
       heartbeat:tick()
   else
@@ -1054,7 +1264,9 @@
 
       for _, tmp in ipairs(currentPageMessages) do
           tmp.physicalPage = msg.physicalPage
-          tmp:toSummary()
+          -- normally, toSummary() is called by showMessage(),
+          -- but with ONLY_SUMMARY that is never called
+          if ONLY_SUMMARY then tmp:toSummary() end
       end
 
       if PAGE_DELAY and not ONLY_SUMMARY then
@@ -1069,52 +1281,42 @@
   heartbeat:stop()
 
   -- messages after the last shipout
-  if PAGE_DELAY and not ONLY_SUMMARY then
-      print("")
-      print("After last page:")
-      print("")
-      showPageMessages()
-  end
+  if PAGE_DELAY and not ONLY_SUMMARY then showRemainingMessages() end
 
-  -- now, the summaries
   if SHOW_SUMMARY then showSummary() end
 end
 
 function showMessage(msg)
   local formatted = msg:toString()
-  if string.len(trim(formatted)) == 0 then return end
+  if trim(formatted) ~= "" then
+      local pageinfo = ""
+      local spaces = ""
+      if not RAW and msg.physicalPage ~= nil then
+          pageinfo = 'pg ' .. msg.physicalPage .. ': '
+          spaces = string.rep(" ", string.len(pageinfo))
+      end
 
-  if RAW then
-      for _, line in ipairs(linesToTable(formatted)) do print(line) end
-      return
-  end
+      -- A message is a repetition if it has
+      -- already been included in some summary
+      local alreadySeen = false
+      if SILENCE_REPETITIONS then
+          for _, summary in ipairs(summaries) do
+              if summary:alreadySeen(msg) then
+                  alreadySeen = true
+                  break
+              end
+          end
+      end
 
-  local pageinfo = ""
-  if msg.physicalPage ~= nil then
-      pageinfo = 'pg ' .. msg.physicalPage .. ': '
-  end
-  local spaces = string.rep(" ", string.len(pageinfo))
-
-  if not SILENCE_REPETITIONS then
-      local lines = linesToTable(formatted)
-      for _, line in ipairs(lines) do
-          print(pageinfo .. line)
-          pageinfo = spaces
+      if not SILENCE_REPETITIONS or not alreadySeen then
+          for _, line in ipairs(linesToTable(formatted)) do
+              print(pageinfo .. line)
+              pageinfo = spaces
+          end
       end
-
-      return
   end
 
-  if alreadySeen[formatted] == nil then
-      alreadySeen[formatted] = {msg}
-      local lines = linesToTable(formatted)
-      for _, line in ipairs(lines) do
-          print(pageinfo .. line)
-          pageinfo = spaces
-      end
-  else
-      table.insert(alreadySeen[formatted], msg)
-  end
+  msg:toSummary()
 end
 
 function showPageMessages()
@@ -1123,13 +1325,43 @@
   end
 end
 
+function showRemainingMessages()
+  local thereIsSomething = false
+  for _, msg in ipairs(currentPageMessages) do
+      if trim(msg:toString()) ~= "" then
+          thereIsSomething = true
+          break
+      end
+  end
+
+  if thereIsSomething then
+      print("")
+      print("After last page:")
+      print("")
+  end
+
+  -- we always call this, even if there is nothing to show,
+  -- because it calls :toSummary() for each message
+  showPageMessages()
+end
+
 function showSummary()
-  if not ONLY_SUMMARY then for i = 1, 5 do print("") end end
+  local thereIsSomething = false
+  for _, summary in ipairs(summaries) do
+      if trim(summary:toString()) ~= "" then
+          thereIsSomething = true
+          break
+      end
+  end
 
+  if not thereIsSomething then return end
+
+  if not ONLY_SUMMARY then for i = 1, 3 do print("") end end
+
   for _, summary in ipairs(summaries) do
       local formatted = summary:toString()
       local prefix = ""
-      if string.len(trim(formatted)) > 0 then
+      if trim(formatted) ~= "" then
           for _, line in ipairs(linesToTable(formatted)) do
               print(prefix .. line)
               prefix = '    '
@@ -1244,6 +1476,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   -- This appears in the logfile but not on stdout
   local _, last = string.find(line, "^Here is how much")
@@ -1324,6 +1557,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local basePattern = "^([UO][nv][de][e]?r)full \\(.)box (%b())"
   local first, last, underover,
@@ -1487,6 +1721,8 @@
   self.message = self:newMessage()
   self.message.severity = self.severity
 
+  -- ignore leading spaces in the first line (in the others,
+  -- they may be indentation or somehow relevant)
   local _, last = string.find(Lines.current, '^%s+')
   if last ~= nil then Lines:handledChars(last) end
 
@@ -1524,11 +1760,13 @@
       return true
   end
 
+  -- trailing spaces in any line in this kind of messsage
+  -- are safe to ignore and may appear in \message's
+  local chunk = trimRight(string.sub(Lines.current, 1, last))
   if self.patternLineNumber == 1 then
-      self.message.content = string.sub(Lines.current, 1, last)
+      self.message.content = chunk
   else
-      self.message.content = self.message.content .. '\n'
-                             .. string.sub(Lines.current, 1, last)
+      self.message.content = self.message.content .. '\n' .. chunk
   end
 
   Lines:handledChars(last)
@@ -1575,6 +1813,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   -- skip what was processed in a previous iteration/recursion
   if offset > 0 then line = string.sub(line, offset +1) end
@@ -1672,7 +1911,6 @@
   '^%s*restricted \\write18 enabled%.',
   '^%s*%%%&%-line parsing enabled%.',
   '^%*%*[%w%.]+', -- "**jobname"
-  '^file:line:error style messages enabled%.',
   '^\\[^%s=]+=[^%s=]+', -- "\c at chapter=\count174"
   "^\\openout%d+%s*=%s*`?[^']+'?%.?",
 
@@ -1713,9 +1951,6 @@
   "^Loading configuration file `" .. filepat .. "'%.",
   "^contour: Using driver file `" .. filepat .. "'%.",
 
-  '^ABD: EverySelectfont initializing macros',
-  '^ABD: EveryShipout initializing macros',
-
   '^%[Loading MPS to PDF converter %(version ' .. datepat .. '%)%.%]',
 
 
@@ -1769,8 +2004,9 @@
   '^Lua module: luaotfload%-multiscript ' .. datepat
                    .. ' [%d%.]+ luaotfload submodule / multiscript',
 
-  '^' .. string.rep('%*', 37) .. '\n%* Using libertinus math %*\n'
-                   .. string.rep('%*', 37),
+  '^' .. string.rep('%*', 37) .. '\n'
+      .. '%* Using libertinus math %*\n'
+      .. string.rep('%*', 37),
 
   '^`inconsolata%-zi4\' v%S-, ' .. datepat
                    .. ' Text macros for Inconsolata %(msharpe%)',
@@ -1789,6 +2025,13 @@
   '^%s*L3 programming layer %b<> xparse %b<>',
   '^%s*%{.*pdftex%.map%}',
 
+  '^%s*ABD: EverySelectfont initializing macros',
+  '^%s*ABD: EveryShipout initializing macros',
+
+  '^%s*' .. string.rep('%*', 65) .. '%s*\n'
+         .. 'GFS%-Solomos style file by A%. Tsolomitis%s*\n'
+         .. string.rep('%*', 65),
+
   -- <blah.jpg, id=555, [...,] 722.7pt x 722.7pt>
   '^%s*%<' .. filepat .. ', id=.- [%d%.]+pt x [%d%.]+pt%>',
   '^%s*%<use ' .. filepat .. '%>', -- <use blah.jpg>
@@ -1803,10 +2046,11 @@
 beginningOfLineInfoStringsHandler.severity = INFO
 beginningOfLineInfoStringsHandler.patterns = {
   "^Writing index file.*%.idx",
+  "^Writing glossary file.*%.glo",
   "^%*geometry%* driver:.*",
   "^%*geometry%* detected driver:.*",
   "^Driver file for pgf:.*%.def",
-  "^%s*file:line:error style messages enabled",
+  "^%s*file:line:error style messages enabled%.",
   "^Applying: %b[] float order in 2%-column on input line .-%.",
   "^Already applied: %b[] float order in 2%-column on input line .-%.",
   "^\\%S+ = a dialect from .*",
@@ -1814,6 +2058,7 @@
   "^No file .-%.aux%.",
   "^No file .-%.ind%.",
   "^No file .-%.bbl%.",
+  "^No file .-%.gls%.",
   "^reledmac reminder:%s*\n"
     .. "%s*The number of the footnotes in this section "
     .. "has changed since the last run.\n"
@@ -1864,7 +2109,7 @@
   '^luaotfload | aux : no font with id %d+',
 
   "^warning  %(pdf backend%): ignoring duplicate destination "
-                            .. "with the name '%-%.'",
+                            .. "with the name '.-'",
 
   "^Couldn't patch \\%S+",
   "^Invalid UTF%-8 byte or sequence at line %d+ replaced by U%+FFFD%.",
@@ -1884,6 +2129,29 @@
 }
 
 
+-- We know these messages always start at the beginning of a line
+-- Always start these patterns with "^", see lookahead().
+-- Order matters! The first match wins, so the longer ones should come first.
+beginningOfLineCriticalStringsHandler = stringsHandler:new()
+beginningOfLineCriticalStringsHandler.severity = CRITICAL
+beginningOfLineCriticalStringsHandler.patterns = {
+    "^The control sequence at the end of the top line\n"
+      .. "of your error message was never \\def'ed%. If you have\n"
+      .. "misspelled it %(e%.g%., `\\hobx'%), type `I' and the correct\n"
+      .. "spelling %(e%.g%., `I\\hbox'%)%. Otherwise just continue,\n"
+      .. "and I'll forget about whatever was undefined%.",
+}
+
+
+-- These messages may start anywhere in a line
+-- Always start these patterns with "^%s*", see lookahead().
+-- Order matters! The first match wins, so the longer ones should come first.
+anywhereCriticalStringsHandler = stringsHandler:new()
+anywhereCriticalStringsHandler.severity = CRITICAL
+anywhereCriticalStringsHandler.patterns = {
+}
+
+
 -------------------------------------------------------------------------------
 -- missingCharHandler
 -- (from stringsHandler)
@@ -1938,6 +2206,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local last, data
 
@@ -1966,10 +2235,12 @@
   self.message.what = what
   self.message.name = name
   self.message.severity = self:parseSeverity(severity)
-  self.message.content = Lines.current
 
   self:findPrefix(last, name, what)
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 function genericLatexHandler:handleFirstLine()
@@ -1977,7 +2248,6 @@
   if not myTurn then return false end
 
   flushUnrecognizedMessages()
-  self:unwrapLines()
 
   -- erase any previous values; nil is not a good idea! If one of these
   -- is nil in a derived object, the object may grab the value of the
@@ -2017,7 +2287,6 @@
       if self.linenum ~= "" then
           self.message.linenum = self.linenum
       end
-      self:adjustSeverity()
       dispatch(self.message)
   end
 
@@ -2089,55 +2358,7 @@
   end
 end
 
--- LaTeX only uses severity "INFO" and "WARNING", which
--- is very limited. Let's demote some warnings to INFO
--- and promote some others to CRITICAL.
-function genericLatexHandler:adjustSeverity()
-  for _, pat in ipairs(self.downgradePatterns) do
-      local first = string.find(self.message.content, pat)
-      if first ~= nil then
-          self.message.severity = INFO
-          break
-      end
-  end
 
-  for _, pat in ipairs(self.upgradePatterns) do
-      local first = string.find(self.message.content, pat)
-      if first ~= nil then
-          self.message.severity = CRITICAL
-          break
-      end
-  end
-end
-
-genericLatexHandler.downgradePatterns = {
-    -- No need to include the full message
-    "File %b`' already exists on the system%."
-            .. "%s*Not generating it from",
-    "You have requested package %b`',"
-            .. "%s*but the package provides",
-    "Writing file %b`'",
-    "Form Feed has been converted to Blank",
-    "Tab has been converted to Blank",
-    "The morewrites package is unnecessary",
-    'Unused \\captionsetup%b[]',
-}
-
-genericLatexHandler.upgradePatterns = {
-    "Label %b`' multiply defined",
-    "Command .- invalid in math mode",
-    "Optional argument of \\twocolumn too tall on page",
-    "Marginpar on page %S- moved",
-    "Some font shapes were not available, defaults substituted%.",
-    "Font shape %b`' in size %b<> not available"
-            .. "%s+Font shape %b`' tried instead",
-    "Font shape %b`' in size %S+ not available"
-            .. "%s+external font %b`' used",
-    "Font shape %b`' undefined"
-            .. "%s+using %b`' instead",
-}
-
-
 -------------------------------------------------------------------------------
 -- latex23MessageHandler
 -- genericLatexVariantHandler
@@ -2160,7 +2381,6 @@
   local severity = data[3]
   self.message.severity = self:parseSeverity(severity)
   self.message.what = what
-  self.message.content = Lines.current
 
   local name
   -- When the message looks like "LaTeX...", there is no package
@@ -2169,6 +2389,9 @@
 
   self:findPrefix(last, name, what)
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 genericLatexVariantHandler = genericLatexHandler:new()
@@ -2186,10 +2409,12 @@
   self.message.what = what
   self.message.name = name
   self.message.severity = self:parseSeverity(severity)
-  self.message.content = Lines.current
 
   self:findPrefix(last, name, what)
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 
@@ -2206,8 +2431,8 @@
 citationHandler = genericLatexHandler:new()
 
 citationHandler.patterns = {
-  "^(LaTeX)%s+(Warning): (Citation) (%b`') on page (.-) undefined",
-  "^(LaTeX)%s+(Warning): (Citation) (%b`') undefined",
+  "^(LaTeX)%s+(Warning): (Citation) ('.-') on page (.-) undefined",
+  "^(LaTeX)%s+(Warning): (Citation) ('.-') undefined",
 }
 
 function citationHandler:unpackData(data)
@@ -2223,9 +2448,11 @@
   self.message.severity = self:parseSeverity(severity)
   self.message.key = key
   self.message.page = page
-  self.message.content = Lines.current
   self:findPrefix(last, nil, what)
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 function citationHandler:newMessage()
@@ -2287,7 +2514,6 @@
 
   self.message.what = what
   self.message.name = name
-  self.message.content = Lines.current
 
   -- There are no continuation lines of this kind for
   -- these messages, but the generic code still wants
@@ -2294,6 +2520,9 @@
   -- to check for the prefix.
   self.prefix = '[^%s%S]+' -- nothing matches
 
+  self:unwrapLines()
+  self.message.content = Lines.current
+
   if not Lines:empty() then
       local first = string.find(Lines:get(1), 'with kernel methods')
       if first ~= nil then
@@ -2326,9 +2555,11 @@
 
 function geometryDetailsHandler:unpackData(data)
   self.message.name = 'geometry'
-  self.message.content = Lines.current
   self.prefix = '* '
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 -------------------------------------------------------------------------------
@@ -2380,6 +2611,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first, last = string.find(line, self.pattern)
   if first == nil then return false, {} end
@@ -2434,6 +2666,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first = string.find(line, self.pattern)
   if first == nil then return false, {} end
@@ -2476,6 +2709,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first, last = string.find(line, self.pattern)
   if first == nil then return false, {} end
@@ -2529,6 +2763,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first = string.find(line, self.pattern)
   if first == nil then return false, {} end
@@ -2603,6 +2838,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first, encoding
   if self.doit == self.handleFirstLine then
@@ -2757,12 +2993,22 @@
 
 Message.severity = UNKNOWN
 
-function Message:toString()
-    if self.mute then return "" end
+function Message:toString(noSilence)
+    -- noSilence may be useful when generating summaries
+    if noSilence then
+        local formatted = self:realToString()
+        if trim(formatted) == "" then return "" else return formatted end
+    end
 
     -- If we've already been here, just output the previous result
     if self.formatted ~= nil then return self.formatted end
 
+    if self.mute then self.formatted = "" return "" end
+
+    if self.severity < MINLEVEL then self.formatted = "" return "" end
+
+    if self:redundant() then self.formatted = "" return "" end
+
     self.formatted = self:realToString()
     if trim(self.formatted) == "" then self.formatted = "" return "" end
 
@@ -2778,8 +3024,6 @@
         end
     end
 
-    if self.severity < MINLEVEL then self.formatted = "" return "" end
-
     return self.formatted
 end
 
@@ -2794,10 +3038,20 @@
   return msg
 end
 
+function Message:redundant()
+  return false
+end
+
 function Message:toSummary()
   local formatted = self:toString()
-  if string.len(trim(formatted)) == 0 then return end
+  if trim(formatted) == "" then return end
 
+  -- In the rare event that one of these is sent out as
+  -- an unrecognizedMessage with no other text, allow for
+  -- repetitions
+  local first = string.find(trim(formatted), '^[%(%)%[%]]$')
+  if first ~= nil then return end
+
   repetitionsSummary:add(self)
 end
 
@@ -2865,7 +3119,11 @@
   return Message.realToString(self)
 end
 
+-- We never want to suppress these repetitions
+function openFileMessage:toSummary()
+end
 
+
 closeFileMessage = Message:new()
 function closeFileMessage:realToString()
   if RAW then return ")" end
@@ -2873,9 +3131,18 @@
   return Message.realToString(self)
 end
 
+-- We never want to suppress these repetitions
+function closeFileMessage:toSummary()
+end
 
+
 underOverMessage = Message:new()
 underOverMessage.severity = WARNING
+
+function underOverMessage:redundant()
+  return not BE_REDUNDANT
+end
+
 function underOverMessage:realToString()
     local tmp = self.content
     if self.failedText ~= nil then
@@ -2898,6 +3165,10 @@
 
 missingCharMessage = Message:new()
 
+function missingCharMessage:redundant()
+  return not BE_REDUNDANT
+end
+
 -- This is a hack: it would be too painful to define
 -- pattern captures in the handler, so we do this here
 function missingCharMessage:realToString()
@@ -2915,16 +3186,23 @@
 
 
 citationMessage = Message:new()
+
+function citationMessage:redundant()
+  return not BE_REDUNDANT
+end
+
 function citationMessage:toSummary()
   citationsSummary:add(self)
 end
 
-referenceMessage = Message:new()
+referenceMessage = citationMessage:new()
+
 function referenceMessage:toSummary()
   referencesSummary:add(self)
 end
 
-labelMessage = Message:new()
+labelMessage = citationMessage:new()
+
 function labelMessage:toSummary()
   labelsSummary:add(self)
 end
@@ -2955,6 +3233,7 @@
 function SummaryPrototype:add(msg)
   -- group messages by message content
   local formatted = msg:toString()
+  if trim(formatted) == "" then return end
 
   if self.messages[formatted] == nil then
       self.messages[formatted] = {}
@@ -2963,6 +3242,12 @@
   table.insert(self.messages[formatted], msg)
 end
 
+function SummaryPrototype:alreadySeen(msg)
+  local formatted = msg:toString()
+  if trim(formatted) == "" then return false end
+  return self.messages[formatted] ~= nil
+end
+
 function SummaryPrototype:toString()
   -- check if the table is empty - https://stackoverflow.com/a/1252776
   if next(self.messages) == nil then return "" end
@@ -3008,7 +3293,9 @@
     local pages = {}
     local files = {}
     for _, msg in ipairs(messages) do
-        pages[msg.physicalPage] = true
+        if msg.physicalPage ~= nil then
+            pages[msg.physicalPage] = true
+        end
         if msg.filename ~= nil then
             files[msg.filename] = true
         end
@@ -3065,12 +3352,23 @@
   local text = ""
   if #messages > 1 then
       local pages, files = self:pageAndFileList(messages)
+
+      local where
+      if pages ~= "" and files ~= "" then
+          where = 'in pages ' .. pages .. ' (files ' .. files .. ') - '
+      elseif pages == "" and files ~= "" then
+          where = 'in files ' .. files .. ' - '
+      elseif pages ~= "" and files == "" then
+          where = 'in pages ' .. pages .. ' - '
+      end
+
       local content = messages[1]:toString()
+      if trim(content) ~= "" then
 
-      text = content .. '\n'
-             .. 'in pages ' .. pages
-             .. " (files " .. files .. ") - "
-             .. #messages .. ' repetitions'
+          text = content .. '\n'
+                 .. where
+                 .. #messages .. ' repetitions'
+      end
   end
 
   return text
@@ -3101,6 +3399,7 @@
   -- here because some messages may include the page number, making
   -- messages that are otherwise the same appear to be different.
   local key = msg.key
+  if key == "" then key = '???' end
 
   if self.messages[key] == nil then
       self.messages[key] = {}
@@ -3109,10 +3408,18 @@
   table.insert(self.messages[key], msg)
 end
 
+function citationsSummary:alreadySeen(msg)
+  local key = msg.key
+  if key == "" then key = '???' end
+
+  return self.messages[key] ~= nil
+end
+
 function citationsSummary:processSingleMessageList(messages)
   local text = ""
   local pages, files = self:pageAndFileList(messages)
   local key = messages[1].key
+  if key == "" then key = '???' end
 
   text = key .. '\n'
          .. 'in pages ' .. pages .. " (files " .. files .. ")"
@@ -3144,8 +3451,18 @@
 
   local pages, files = self:pageAndFileList(self.messages)
 
-  return "Under/overfull boxes in pages "
+  if DETAILED_UNDEROVER_SUMMARY then
+      local output = "Under/overfull boxes:"
+      for _, msg in ipairs(self.messages) do
+          output = output .. '\npage ' .. msg.physicalPage
+                   .. ' (file ' .. msg.filename .. '):\n'
+          output = output .. msg:toString(true)
+      end
+      return output
+  else
+      return "Under/overfull boxes in pages "
              .. pages .. " (files " .. files .. ")"
+  end
 end
 
 
@@ -3185,6 +3502,8 @@
 
 function trim(s) return (string.gsub(s, '^%s*(.-)%s*$', '%1')) end
 
+function trimRight(s) return (string.gsub(s, '^(.-)%s*$', '%1')) end
+
 function stringToPattern(s)
   local first, _ = string.find(s, '^////')
   local pat
@@ -3204,7 +3523,7 @@
   local size = string.len(s)
   local i = 1
   local lines = {}
-  while i < size do
+  while i <= size do
       -- check \r in case the user added to this file a pattern
       -- with an embedded dos-style "CR LF" sequence.
       local first, last, line = string.find(s, '(.-)[\r]?\n', i)
@@ -3211,7 +3530,7 @@
 
       if first == nil then
           table.insert(lines, string.sub(s, i))
-          i = size
+          i = size +1
       else
           table.insert(lines, line)
           i = last +1
@@ -3824,6 +4143,9 @@
   -- but it saves us from problems with a couple of messages that use
   -- "filepat" at the end of the pattern.
 
+  -- Nothing to do if the next line does not exist
+  if Lines:get(1) == nil then return last, matches end
+
   -- these might fit in 2 chars, so do not consider lines with that
   local realMessage = string.find(Lines:get(1), '[%)%[%]]')
   if string.len(Lines:get(1)) < 3 and not realMessage then
@@ -3915,6 +4237,17 @@
   local _, last = string.find(line, '^%s*%[')
   if last ~= nil then line = string.sub(line, last +1) end
 
+  -- If this is a 79-chars line that ends with ...[NUMBER, we will try
+  -- to unwrap. With that, we may find ...[LONGERNUMBER, which is what
+  -- we want. If, however, we find [NUMBERLETTER, we give up and consider
+  -- this not to be a shipout. Should we stick with [NUMBER instead?
+  --
+  -- No.
+  --
+  -- Either way, we have no way of knowing whether we are right or wrong,
+  -- but giving up is the lesser evil: the user may register the unknown
+  -- message with --add-[debug|info|warning]-message and solve the problem.
+  local page
   while true do
       for _, pattern in ipairs(patterns) do
           _, _, page = string.find(line, pattern)
@@ -3940,7 +4273,7 @@
   -- luatex puts quotes around filenames with spaces, which makes things easier
   while true do
       local _, last = string.find(line, '^"')
-      if string.len(line) > 0 and last == nil then return false end -- no quotes
+      if line ~= "" and last == nil then return false end -- no quotes
 
       last = string.find(line, '^"[%.]?/') -- relative path or unix-style path
       if last == nil then
@@ -3979,6 +4312,7 @@
   -- we already checked in the previous ones.
   local alreadyCheckedIdx = 0
 
+  local filename
   while true do
       local first = string.find(line, '^[%.]?/') -- relative or unix-style path
       if first == nil then
@@ -3996,18 +4330,23 @@
       -- From longest to shortest, to avoid problems if there is a
       -- substring in the filename that matches some other filename.
       for i = longest, alreadyCheckedIdx +1, -1 do
-          local filename = string.sub(line, 1, i)
-          if kpse.find_file(filename, 'other text files') ~= nil then
-              return filename
+          local candidate = string.sub(line, 1, i)
+          if kpse.find_file(candidate, 'other text files') ~= nil then
+              filename = candidate
+              break
           end
       end
 
-      -- we did not find the filename; can we unwrap this line?
-      if not Lines:seemsWrapped(position) then return nil end
+      -- We may or may not have found the filename. Either way, if we
+      -- can unwrap this line, we should:
+      -- 1. If we did not find the filename, we might
+      -- 2. If we did find the filename, we might find a different
+      --    (longer) filename, in which case we should stick with
+      --    it (yes, I have seen it happen!)
+      if not Lines:seemsWrapped(position) then return filename end
       first = string.find(line, "[%)%(%[%]]")
-      if first ~= nil then return nil end
+      if first ~= nil then return filename end
 
-      -- yep!
       alreadyCheckedIdx = longest
       line = line .. Lines:get(position +1)
       position = position +1

Modified: trunk/Master/texmf-dist/doc/man/man1/texlogsieve.1
===================================================================
--- trunk/Master/texmf-dist/doc/man/man1/texlogsieve.1	2022-01-04 22:28:55 UTC (rev 61493)
+++ trunk/Master/texmf-dist/doc/man/man1/texlogsieve.1	2022-01-04 22:29:24 UTC (rev 61494)
@@ -1,4 +1,4 @@
-.TH TEXLOGSIEVE "1" "December 2021" "texlogsieve 1.0.0-beta-1" "User Commands"
+.TH TEXLOGSIEVE "1" "January 2022" "texlogsieve 1.0.0-beta-2" "User Commands"
 
 .SH NAME
 
@@ -16,7 +16,9 @@
 
 texlogsieve \fBmust\fR be run from the same directory as [pdf|lua|xe]latex,
 because it searches for the files used during compilation (packages loaded
-from the current directory, files included with \einput etc.).
+from the current directory, files included with \einput etc.). Also, since
+it cannot detect if LaTeX stops for user input, you should \fBreally\fR run
+LaTeX in \fI\,nonstopmode\/\fR when texlogsieve is reading from a pipe.
 
 The program goes to great lengths to correctly handle TeX line wrapping. It
 understands the \fI\,max_print_line\/\fR TeX configuration variable and reads
@@ -27,10 +29,18 @@
 
 .RS 3
 .EX
-[pdf|lua|xe]latex myfile.tex | texlogsieve
+[pdf|lua|xe]latex \-interaction nonstopmode myfile.tex | texlogsieve
 .EE
 .RE
 
+or
+
+.RS 3
+.EX
+texlogsieve myfile.log
+.EE
+.RE
+
 and be satisfied with the result.
 
 .SH OPTIONS
@@ -61,6 +71,19 @@
 are supressed).
 
 .TP
+\fB\-\-be\-redundant\fR, \fB\-\-no\-be\-redundant\fR
+Present/suppress ordinary messages that will also appear in the summary.
+This affects messages that have special summaries (such as under/overfull
+boxes or undefined citations). With \-\-no\-be\-redundant (the default),
+these messages are filtered out and only appear in the final summary.
+
+.TP
+\fB\-\-box\-detail\fR, \fB\-\-no\-box\-detail\fR
+Include/exclude detailed information on under/overfull boxes in the final
+summary. With \-\-no\-box\-detail, the summary presents only a list of
+pages and files that had under/overfull boxes (default enabled).
+
+.TP
 \fB\-\-heartbeat\fR, \fB\-\-no\-heartbeat\fR
 Enable/disable progress gauge in page-delay mode (default enabled).
 
@@ -75,13 +98,14 @@
 Do not filter messages and do not output the summary, only unwrap long,
 wrapped lines. The output should be very similar (but not equal) to the
 input file, but with wrapped lines reconstructed. This activates \-l debug,
-\-\-no\-summary, \-\-no\-page\-delay, \-\-repetitions, and \-\-shipouts, and
-also supresses the verbose \[lq]open/close file\[rq] and \[lq]shipout\[rq]
-messages, simulating instead the TeX format, with parens and square brackets. \
-This is useful if you prefer the reports generated by some other tool but want
-to benefit from texlogsieve's line unwrapping algorithm; the output generated
-by this option should be parseable by other tools (but you probably need to
-coerce the other tool not to try to unwrap lines).
+\-\-no\-summary, \-\-no\-page\-delay, \-\-repetitions, \-\-be\-redundant,
+and \-\-shipouts, and also supresses the verbose \[lq]open/close file\[rq]
+and \[lq]shipout\[rq] messages, simulating instead the TeX format, with parens
+and square brackets. This is useful if you prefer the reports generated
+by some other tool but want to benefit from texlogsieve's line unwrapping
+algorithm; the output generated by this option should be parseable by other
+tools (but you probably need to coerce the other tool not to try to unwrap
+lines).
 
 .TP
 \fB\-\-silence\-package\fR=\fI\,PKGNAME\/\fR
@@ -107,8 +131,11 @@
 \fB\-\-silence\-file\fR=\fI\,FILENAME OR FILE GLOB\/\fR
 Filter out messages that have been generated while the given file was being
 processed. Do \fBnot\fR use absolute or relative paths, only filenames. Simple
-file globs, such as \[lq]*.cls\[rq], work as expected. Use this option multiple
-times to suppress messages from several different files.
+file globs, such as \[lq]*.cls\[rq], work as expected. If you are only using
+packages you already know, silencing \[lq]*.sty\[rq] may be a good idea (note
+that this does not suppress all messages from all packages, only the messages
+generated while the packages are being loaded). Use this option multiple times
+to suppress messages from several different files.
 
 .TP
 \fB\-\-semisilence\-file\fR=\fI\,FILENAME OR FILE GLOB\/\fR
@@ -123,24 +150,63 @@
 by chapter1.tex or chapter2.tex will not.
 
 .TP
-\fB\-\-add\-[debug|info|warning]\-message\fR=\fI\,MESSAGE\/\fR
+\fB\-\-add\-[debug|info|warning|critical]\-message\fR=\fI\,MESSAGE\/\fR
 Add MESSAGE to the list of messages known to the program with the given
 severity level; see Section UNRECOGNIZED MESSAGES below for more information
 about this. Like \-\-silence\-string, these should be a single line; unlike
 \-\-silence\-string, you need to embed \en explicitly to indicate line
-breaks. You may precede the string with \[lq]////\[rq] to use lua-style
-pattern matching, but embedding \en to indicate line breaks is unavoidable.
-Use these options multiple times to add many different messages.
+breaks (this is literally a backslash character followed by the letter
+\[lq]n\[rq], \fBnot\fR a linefeed character). You may precede the string
+with \[lq]////\[rq] to use lua-style pattern matching, but embedding \en
+to indicate line breaks is unavoidable. Use these options multiple times
+to add many different messages.
 
 .TP
+\fB\-\-set\-to\-level\-[debug|info|warning|critical]\fR=\fI\,EXCERPT OF MESSAGE\/\fR
+Redefine the severity level of messages that contain the provided string
+to the given level. Check the explanation for \-\-silence\-string, as this
+works in a similar way. Use these options multiple times to change the
+severity level of many different messages.
+
+.TP
 \fB\-c\fR \fI\,CFGFILE\/\fR, \fB\-\-config\-file\fR=\fI\,CFGFILE\/\fR
-Read options from the given configuration file. Options are exactly the
-same as the long command-line options, but without the preceding \[lq]\-\-\[rq]
-characters. Lines starting with a \[lq]#\[rq] sign are comments. An example
-configuration file:
+Read options from the given configuration file in addition to
+\fI\,texlogsieverc\/\fR.
 
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Show concise options description.
+
+.TP
+\fB\-\-version\fR
+Print program version.
+
+.SH UNRECOGNIZED MESSAGES
+
+texlogsieve automatically handles messages such as \[lq]Package blah
+Info:...\[rq] or \[lq]LaTeX Warning:...\[rq]. However, many messages do not
+follow this pattern. To do its thing, texlogsieve should know about these
+other messages beforehand.
+
+.PP
+While texlogsieve recognizes quite a few messages out of the box, you may run
+into a message generated by some package that it does not know about (you can
+check for this using \[lq]\-l unknown\[rq]). If that is the case, you can use
+the \-\-add\-[debug|info|warning|critical]\-message options to add it to the
+list of messages known to the program.
+
+.SH CONFIGURATION FILE
+
+texlogsieve always searches automatically for the (optional)
+\fI\,texlogsieverc\/\fR configuration file in the TeX path (i.e., it searches
+using Kpathsea). In the default configuration, the current directory is in
+the search path, so adding a config file with that name to the project
+directory is enough to make it work. Options in the config file are exactly
+the same as the long command-line options described above, but without the
+preceding \[lq]\-\-\[rq] characters. Lines starting with a \[lq]#\[rq] sign
+are comments. An example configuration file:
+
 .RS
-.RS 3
 .EX
 no\-page\-delay
 .br
@@ -158,48 +224,17 @@
 .br
 silence\-string = ////luaotfload | aux : font no %d+ %(.\-%)
 .br
-silence\-files = *.cls
+silence\-files = *.sty
 .EE
 .RE
-.RE
 
-.TP
-\fB\-h\fR, \fB\-\-help\fR
-Show concise options description.
-
-.TP
-\fB\-\-version\fR
-Print program version.
-
-.SH UNRECOGNIZED MESSAGES
-
-texlogsieve automatically handles messages such as \[lq]Package blah
-Info:...\[rq] or \[lq]LaTeX Warning:...\[rq]. However, many messages do not
-follow this pattern. To do its thing, texlogsieve should know about these
-other messages beforehand. This is important for three reasons:
-
-.IP 1. 3
-Unknown messages are given maximum priority; if you do not want to see them,
-you have to use \-\-silence\-string;
-.IP 2. 3
-If the message has more than one line, each line is treated as an independent
-message. This means you need to use \-\-silence\-string multiple times;
-.IP 3. 3
-In some rare cases, the line unwrapping algorithm may fail near an
-unrecognized message.
-
-.PP
-While texlogsieve recognizes quite a few messages out of the box, you may run
-into a message generated by some package that it does not know about (you can
-check for this using \[lq]\-l unknown\[rq]). If that is the case, you can use
-the \-\-add\-[debug|info|warning]\-message options to add it to the list of
-messages known to the program.
-
 .SH LIMITATIONS
 
 texlogsieve does not try to do anything smart about error messages (at least
 for now); if there is an error, you probably want to take a look directly
-at the log file anyway.
+at the log file anyway. It also cannot detect if LaTeX stops for user input,
+so you should \fBreally\fR run LaTeX in \fI\,nonstopmode\/\fR when texlogsieve
+is reading from a pipe.
 
 Since it needs to know what messages to expect, texlogsieve is currently
 geared towards LaTeX; I have no idea how it would work with ConTeXt or plain
@@ -219,7 +254,7 @@
 
 .SH COPYRIGHT
 
-Copyright \[co] 2021 Nelson Lago <lago at ime.usp.br>
+Copyright \[co] 2021, 2022 Nelson Lago <lago at ime.usp.br>
 .br
 License GPLv3+: GNU GPL version 3 or later
 .UR https://gnu.org/licenses/gpl.html

Modified: trunk/Master/texmf-dist/doc/man/man1/texlogsieve.man1.pdf
===================================================================
(Binary files differ)

Modified: trunk/Master/texmf-dist/doc/support/texlogsieve/README.md
===================================================================
--- trunk/Master/texmf-dist/doc/support/texlogsieve/README.md	2022-01-04 22:28:55 UTC (rev 61493)
+++ trunk/Master/texmf-dist/doc/support/texlogsieve/README.md	2022-01-04 22:29:24 UTC (rev 61494)
@@ -22,14 +22,22 @@
 
 `texlogsieve` **must** be run from the same directory as `[pdf|lua|xe]latex`,
 because it searches for the files used during compilation (packages loaded
-from the current directory, files included with `\input` etc.).
+from the current directory, files included with `\input` etc.). Also, since
+it cannot detect if LaTeX stops for user input, you should **really** run
+LaTeX in `nonstopmode` when `texlogsieve` is reading from a pipe.
 
 The defaults are reasonable; hopefully, you can just do
 
 ```
-[pdf|lua|xe]latex myfile.tex | texlogsieve
+[pdf|lua|xe]latex -interaction nonstopmode myfile.tex | texlogsieve
 ```
 
+or
+
+```
+texlogsieve myfile.log
+```
+
 and be satisfied with the result.
 
 Since it needs to know what messages to expect, `texlogsieve` is
@@ -40,94 +48,11 @@
 If you want to know more about the TeX log file and the workings of
 the program, check the initial comments in the code.
 
-## Short-term improvements
-
- * Create tests
-
-     - Synthetic unit tests: these should systematically test the code and the
-       features it implements with small "fake" log files
-
-     - Real-world files (for example, the docs for some packages or some papers
-       from arXiv): these should help us detect important log messages and
-       changes in messages whenever new versions of LaTeX and its packages are
-       released
-
- * Include the line number in the summary for messages that have it
-
- * Consider messages that only differ by "on line XXX" to be repetitions
-
- * Implement option `--collapse-lines` (show multiline messages as a single
-   line)
-
- * Implement option `--no-redundant` (makes messages that are going to be
-   summarized, such as `undefined reference blah`, not appear in the filtered
-   report, only in the summary)
-
-## Other improvements
-
- * Improve the final summary format - we should look at other tools for
-   inspiration
-
- * Filter the summary report too; look at `texloganalyser` for inspiration
-
- * Consider indicating filenames more prominently in the filtered report;
-   something like
-
-        Messages for file ./somefile.tex:
-        LaTeX Warning: File `blah.txt' already exists on the system.
-        Messages for file ./someotherfile.tex:
-        Warning: Missing character: There is no " (U+0022) in font...
-   but only when there is actually something to report for the given file
-
- * Do something smart about LaTeX errors (maybe just detecting them and saying
-   "there was an error here" is enough) - maybe get some inspiration from
-   <https://github.com/overleaf/latex-log-parser> (the parser from overleaf).
-   This can get complicated if we account for the possibility that the user
-   interactively solved whatever was the problem, but do we really want to or
-   need to handle that?
-
- * Add colors (see `texlog_extract`)
-
- * Improve error handling if/where possible
-
- * Better aggregation in the summary:
-
-     - `under/overfull boxes: p.5 (chap1.tex, L27); p.7 (chap2.tex, L37); p.19
-       (chap2.tex, L176)`
-
-     - `missing characters X, Y, Z in font blah`
-
- * Try to figure out a way to automatically extract possible messages from
-   LaTeX packages
-
- * Stuff like
-   `LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right`
-   should trigger a specific "please rerun" message in the summary
-
- * Maybe grep the .fls file first before trying to do `kpse.find_file()`; this
-   is probably more reliable and works even if the environment has changed,
-   but can we be sure it works correctly when reading the logfile from a pipe?
-
-## Refactorings to consider:
-
- * Modify `heartbeat` to use `os.difftime()`
-
- * Improve Lines:
-
-     - Eliminate `Lines.current` and instead always use `Lines.get(0)`,
-       `Lines.get(1)` etc. We might store stuff in indexes 1, 2, 3 etc. and
-       make something like `Lines:get(val) => return Lines[val +1]` (this is
-       better than actually using index 0 because the lua standard library
-       counts from one, so adhering to this convention makes using it easier)
-
-     - Reimplement the logic using <https://www.lua.org/pil/11.4.html> for a
-       small performance increase
-
 ---
 
 Code etc: <https://gitlab.com/lago/texlogsieve>
 
-Copyright 2021 Nelson Lago <lago at ime.usp.br>
+Copyright (C) 2021, 2022 Nelson Lago <lago at ime.usp.br>
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

Modified: trunk/Master/texmf-dist/doc/support/texlogsieve/texlogsieve.pdf
===================================================================
(Binary files differ)

Modified: trunk/Master/texmf-dist/doc/support/texlogsieve/texlogsieve.tex
===================================================================
--- trunk/Master/texmf-dist/doc/support/texlogsieve/texlogsieve.tex	2022-01-04 22:28:55 UTC (rev 61493)
+++ trunk/Master/texmf-dist/doc/support/texlogsieve/texlogsieve.tex	2022-01-04 22:29:24 UTC (rev 61494)
@@ -1,6 +1,6 @@
 % texlogsieve - filter and summarize LaTeX log files
 %
-% Copyright (C) 2021 Nelson Lago <lago at ime.usp.br>
+% Copyright (C) 2021, 2022 Nelson Lago <lago at ime.usp.br>
 %
 % This program is free software: you can redistribute it and/or modify
 % it under the terms of the GNU General Public License as published by
@@ -37,13 +37,26 @@
 
 \RecordChanges
 
-\changes{v1.0.0-beta-1}{2021/12/16}{first public prerelease}
+\changes{v1.0.0-beta-1}{2021/12/16}{First public prerelease}
+\changes{v1.0.0-beta-2}{2022/01/04}{Automatically read \texttt{texlogsieverc}
+                                    if it exists}
+\changes{v1.0.0-beta-2}{2022/01/04}{Add options \texttt{-\/-be-redundant}
+                                    and \texttt{-\/-box-detail}}
+\changes{v1.0.0-beta-2}{2022/01/04}{Add options
+                                    \texttt{-\/-set-to-level-[levelname]}}
+\changes{v1.0.0-beta-2}{2022/01/04}{Include silenced messages in summaries}
+\changes{v1.0.0-beta-2}{2022/01/04}{Substitute empty citation/label keys
+                                    for ``???''}
+\changes{v1.0.0-beta-2}{2022/01/04}{Fix bug that prevented
+                                    \texttt{-\/-add-[info\textbar
+                                    warning]-message} from working}
+\changes{v1.0.0-beta-2}{2022/01/04}{Misc small bugfixes}
 
 \begin{document}
 
 \title{\textsf{texlogsieve}:\thanks{This document
-corresponds to \textsf{texlogsieve}~1.0.0-beta-1,
-dated~2021-12-16.}\\[.3\baselineskip]
+corresponds to \textsf{texlogsieve}~1.0.0-beta-2,
+dated~2022-01-04.}\\[.3\baselineskip]
 {\normalsize(yet another program to)\\[-.6\baselineskip]}
 {\large filter and summarize \LaTeX\ log files}
 }
@@ -87,10 +100,14 @@
 specially important ones. It is a \texttt{texlua} script, similar in
 spirit to tools such as \texttt{texfot}, \texttt{texloganalyser},
 \texttt{rubber-info}, \texttt{textlog\_extract}, \texttt{texlogparser},
-and others. Note that it does not try to do anything smart about error
-messages (at least for now); if there is an error, you probably want to
-take a look directly at the log file anyway.
+and others.
 
+Note that it does not try to do anything smart about error messages
+(at least for now); if there is an error, you probably want to take a
+look directly at the log file anyway. It also cannot detect if \LaTeX{}
+stops for user input, so you should \textbf{really} run \LaTeX\ in
+\texttt{nonstopmode} when \texttt{texlogsieve} is reading from a pipe.
+
 \texttt{texlogsieve} \textbf{must} be run from the same directory as
 \verb/[pdf|lua|xe]latex/, because it searches for the files used during
 compilation (packages loaded from the current directory, files included
@@ -99,11 +116,15 @@
 \pagebreak[1]
 The defaults are reasonable; hopefully, you can just do
 
-\begin{quote}
-\verb/[pdf|lua|xe]latex myfile.tex | texlogsieve/
-\end{quote}
+\begin{list}{}{}
+    \small\item
+    \verb/[pdf|lua|xe]latex -interaction nonstopmode myfile.tex | texlogsieve/
+    \par
+    {\normalsize or\par}
+    \verb|texlogsieve myfile.log|
+\end{list}
 
-and be satisfied with the result.
+\noindent and be satisfied with the result.
 
 Since it needs to know what messages to expect, \texttt{texlogsieve} is
 currently geared towards \LaTeX; I have no idea how it would work with
@@ -113,7 +134,7 @@
 If you want to know more about the \TeX\ log file and the workings of the
 program, check the initial comments in the code.
 
-\section{Unwrapping long lines}
+\section{Unwrapping Long Lines}
 
 \TeX\ wraps (breaks) lines longer than \texttt{max\_print\_line} (by
 default, 79 characters). Most tools detect lines that are exactly 79
@@ -130,6 +151,60 @@
 value from the same places as \TeX. Setting \texttt{max\_print\_line} to
 a value larger than 9999 makes \texttt{texlogsieve} ignore line wrapping.
 
+\section{Unrecognized Messages}
+\label{unrecognized}
+
+\texttt{texlogsieve} automatically handles messages such as ``Package blah
+Info:\dots'' or ``LaTeX Warning:\dots''. However, many  messages do not
+follow this pattern. To do its thing, \texttt{texlogsieve} should know about
+these other messages beforehand. This is important for three reasons:
+
+\begin{enumerate}
+  \item Unknown messages are given maximum priority; if you do not want to
+        see them, you have to use \texttt{-\/-silence-string};
+  \item If the message has more than one line, each line is treated as an
+        independent message. This means you need to use
+        \texttt{-\/-silence-string} multiple times;
+  \item In some rare cases, an unrecognized message may make
+        \texttt{texlogsieve} misclassify nearby wrapped lines (if it comes
+        right after a 79 characters long line of a specific type), close
+        file messages (if it includes an unmatched close parens character),
+        or shipout messages (if it includes an unmatched close square bracket
+        character or an open square bracket character followed only by
+        numbers).
+\end{enumerate}
+
+While \texttt{texlogsieve} recognizes quite a few messages out of the box,
+you may run into a message generated by some package that it does not know
+about (you can check for this using \texttt{-l unknown}). If that is the
+case, you can use the \verb/--add-[debug|info|warning|critical]-message/
+options to add it to the list of messages known to the program.
+
+\section{Configuration File}
+
+\texttt{texlogsieve} always searches automatically for the (optional)
+\texttt{texlogsieverc} configuration file in the \TeX\ path (i.e., it
+searches using \texttt{Kpathsea}). In the default configuration, the
+current directory is in the search path, so adding a config file with that
+name to the project directory is enough to make it work. Options in the
+config file are exactly the same as the long command-line options described
+below, but without the preceding ``\texttt{-\/-}'' characters. Lines
+starting with a ``\#'' sign are comments. An example configuration file:
+
+\begin{quote}
+\begin{verbatim}
+no-page-delay
+# no-page-delay enables shipouts, but we do not want that
+no-shipouts
+silence-string = Hyperreferences in rotated content will be misplaced
+# no need to escape the "\" (or any other) character
+silence-string = Using \overbracket and \underbracket from `mathtools'
+# silence a string using lua pattern matching
+silence-string = ////luaotfload | aux : font no %d+ %(.-%)
+silence-files = *.sty
+\end{verbatim}
+\end{quote}
+
 \section{Options}
 
 \begin{description}
@@ -162,6 +237,22 @@
 \end{description}
 
 \begin{description}
+\item[\texttt{-\/-be-redundant}, \texttt{-\/-no-be-redundant}]~\\
+Present/suppress ordinary messages that will also appear in the summary.
+This affects messages that have special summaries (such as under/overfull
+boxes or undefined citations). With \texttt{-\/-no-be-redundant} (the
+default), these messages are filtered out and only appear in the final
+summary.
+\end{description}
+
+\begin{description}
+\item[\texttt{-\/-box-detail}, \texttt{-\/-no-box-detail}]~\\
+Include/exclude detailed information on under/overfull boxes in the final
+summary. With \texttt{-\/-no-box-detail}, the summary presents only a list
+of pages and files that had under/overfull boxes (default enabled).
+\end{description}
+
+\begin{description}
 \item[\texttt{-\/-heartbeat}, \texttt{-\/-no-heartbeat}]~\\
 Enable/disable progress gauge in page-delay mode (default enabled).
 \end{description}
@@ -177,14 +268,15 @@
 \item[\texttt{-u}, \texttt{-\/-unwrap-only}]~\\
 Do not filter messages and do not output the summary, only unwrap long,
 wrapped lines. The output should be very similar (but not equal) to the
-input file, but with wrapped lines reconstructed. This activates \texttt{-l
-debug}, \texttt{-\/-no-summary}, \texttt{-\/-no-page-delay},
-\texttt{-\/-repetitions}, and \texttt{-\/-shipouts}, and also supresses the
-verbose ``open/close file'' and ``shipout'' messages, simulating instead
-the \TeX{} format, with parens and square brackets. This is useful if you
-prefer the reports generated by some other tool but want to benefit from
-texlogsieve's line unwrapping algorithm; the output generated by this option
-should be parseable by other tools (but you probably need to coerce the other
+input file, but with wrapped lines reconstructed. This activates
+\texttt{-l debug}, \texttt{-\/-no-summary}, \texttt{-\/-no-page-delay},
+\texttt{-\/-repetitions}, \texttt{-\/-be-redundant}, and
+\texttt{-\/-shipouts}, and also supresses the verbose ``open/close file''
+and ``shipout'' messages, simulating instead the \TeX{} format, with
+parens and square brackets. This is useful if you prefer the reports
+generated by some other tool but want to benefit from texlogsieve's line
+unwrapping algorithm; the output generated by this option should be
+parseable by other tools (but you probably need to coerce the other
 tool not to try to unwrap lines).
 \end{description}
 
@@ -213,7 +305,10 @@
 \item[\texttt{-\/-silence-file=FILENAME OR FILE GLOB}]~\\
 Filter out messages that have been generated while the given file was being
 processed. Do \textbf{not} use absolute or relative paths, only filenames.
-Simple file globs, such as ``\texttt{*.cls}'', work as expected. Use this
+Simple file globs, such as ``\texttt{*.cls}'', work as expected. If you are
+only using packages you already know, silencing ``\texttt{*.sty}'' may be a
+good idea (note that this does not suppress all messages from all packages,
+only the messages generated while the packages are being loaded). Use this
 option multiple times to suppress messages from several different files.
 \end{description}
 
@@ -232,37 +327,32 @@
 \end{description}
 
 \begin{description}
-\item[\texttt{-\/-add-[debug\textbar info\textbar warning]-message=MESSAGE}]~\\
-Add \texttt{MESSAGE} to the list of messages known to the program with the given
-severity level; see Section \ref{unrecognized} below for more information
+\item[\texttt{-\/-add-[debug\textbar info\textbar warning\textbar
+      critical]-message=MESSAGE}]~\\
+Add \texttt{MESSAGE} to the list of messages known to the program with the
+given severity level; see Section \ref{unrecognized} for more information
 about this. Like \texttt{-\/-silence-string}, these should be a single line;
-unlike \texttt{-\/-silence-string}, you need to embed  \verb|\n|  explicitly
-to indicate line breaks. You may precede the string with ``////'' to use
-lua-style pattern matching, but embedding \verb|\n| to indicate line breaks
-is unavoidable. Use these options multiple times to add many different
-messages.
+unlike \texttt{-\/-silence-string}, you need to embed \verb|\n| explicitly
+to indicate line breaks (this is literally a backslash character followed
+by the letter ``n'', \textbf{not} a linefeed character). You may precede
+the string with ``////'' to use lua-style pattern matching, but embedding
+\verb|\n| to indicate line breaks is unavoidable. Use these options multiple
+times to add many different messages.
 \end{description}
 
 \begin{description}
+\item[\texttt{-\/-set-to-level-[debug\textbar info\textbar warning\textbar
+      critical]=EXCERPT OF MESSAGE}]~\\
+Redefine the severity level of messages that contain the provided string
+to the given level. Check the explanation for \texttt{-\/-silence-string},
+as this works in a similar way. Use these options multiple times to change
+the severity level of many different messages.
+\end{description}
+
+\begin{description}
 \item[\texttt{-c CFGFILE}, \texttt{-\/-config-file=CFGFILE}]~\\
-Read options from the given configuration file. Options are exactly the
-same as the long command-line options, but without the preceding ``-\/-''
-characters. Lines starting with a ``\#'' sign are comments. An example
-configuration file:
-
-\begin{quote}
-\begin{verbatim}
-no-page-delay
-# no-page-delay enables shipouts, but we do not want that
-no-shipouts
-silence-string = Hyperreferences in rotated content will be misplaced
-# no need to escape the "\" (or any other) character
-silence-string = Using \overbracket and \underbracket from `mathtools'
-# silence a string using lua pattern matching
-silence-string = ////luaotfload | aux : font no %d+ %(.-%)
-silence-files = *.cls
-\end{verbatim}
-\end{quote}
+Read options from the given configuration file in addition to
+\texttt{texlogsieverc}.
 \end{description}
 
 \begin{description}
@@ -275,33 +365,9 @@
 Print program version.
 \end{description}
 
-\section{Unrecognized Messages}
-\label{unrecognized}
-
-\texttt{texlogsieve} automatically handles messages such as ``Package blah
-Info:\dots'' or ``LaTeX Warning:\dots''. However, many  messages do not
-follow this pattern. To do its thing, \texttt{texlogsieve} should know about
-these other messages beforehand. This is important for three reasons:
-
-\begin{enumerate}
-  \item Unknown messages are given maximum priority; if you do not want to
-        see them, you have to use -\/-silence-string;
-  \item If the message has more than one line, each line is treated as an
-        independent message. This means you need to use -\/-silence-string
-        multiple times;
-  \item In some rare cases, the line unwrapping algorithm may fail near an
-        unrecognized message.
-\end{enumerate}
-
-While \texttt{texlogsieve} recognizes quite a few messages out of the box,
-you may run into a message generated by some package that it does not know
-about (you can check for this using \texttt{-l unknown}). If that is the
-case, you can use the \verb/--add-[debug|info|warning]-message/ options to
-add it to the list of messages known to the program.
-
 \section{License}
 
-Copyright © 2021 Nelson Lago \textless lago at ime.usp.br\textgreater\\
+Copyright © 2021, 2022 Nelson Lago \textless lago at ime.usp.br\textgreater\\
 License GPLv3+: GNU GPL version 3 or later
 \url{https://gnu.org/licenses/gpl.html}.\\
 This is free software: you are free to change and redistribute it.\\

Modified: trunk/Master/texmf-dist/scripts/texlogsieve/texlogsieve
===================================================================
--- trunk/Master/texmf-dist/scripts/texlogsieve/texlogsieve	2022-01-04 22:28:55 UTC (rev 61493)
+++ trunk/Master/texmf-dist/scripts/texlogsieve/texlogsieve	2022-01-04 22:29:24 UTC (rev 61494)
@@ -2,7 +2,7 @@
 
 -- texlogsieve - filter and summarize LaTeX log files
 --
--- Copyright (C) 2021 Nelson Lago <lago at ime.usp.br>
+-- Copyright (C) 2021, 2022 Nelson Lago <lago at ime.usp.br>
 --
 -- This program is free software: you can redistribute it and/or modify
 -- it under the terms of the GNU General Public License as published by
@@ -332,6 +332,7 @@
       local line
       if position == nil then position = 0 end
       line = Lines:get(position)
+      if line == nil then return false, {} end
 
       local first, last = string.find(line, self.pattern)
       if first == nil then return false
@@ -525,9 +526,9 @@
  UNKNOWN = 4
 
 function main(arg)
+  initializeKpse()
   processCommandLine(arg)
   initializeGlobals()
-  initializeKpse()
   registerHandlers()
   registerSummaries()
   convertFilterStringsToPatterns()
@@ -552,7 +553,7 @@
   -- if there is remaining data from the previous iteration,
   -- we leave everything as-is for it to be processed now
   local tmp = Lines.current
-  if tmp ~= nil and string.len(tmp) > 0 then return true end
+  if tmp ~= nil and tmp ~= "" then return true end
 
   -- Refill the buffer. A simple experiment suggests 8 lines
   -- is enough, but why not use a higher value?
@@ -562,7 +563,7 @@
       -- We *need* to remove blank lines here because
       -- sometimes a wrapped line is followed by a blank
       -- line, which messes our detection of wrapped lines.
-      if string.len(tmp) > 0 then Lines:append(tmp) end
+      if tmp ~= "" then Lines:append(tmp) end
   end
 
   -- proceed to the next line
@@ -655,23 +656,21 @@
   -- map physicalPage (from numShipouts) to latexPage (LaTeX counter)
   latexPages = {}
 
-  -- After printing each message, the output coroutine stores them
-  -- in currentPageMessages. When it receives a shipout message,
-  -- it traverses currentPageMessages adding the page number it
-  -- just learned about to each of the messages, calls :toSummary()
-  -- for each of them and and clears currentPageMessages (we do this
-  -- so the data in the summaries may include the page numbers). The
-  -- objects representing the summary for each kind of message are
-  -- stored in summaries, so after all messages we can just traverse
-  -- this list calling :toString() and get all the summaries. The
-  -- summaries table is populated by registerSummaries().
+  -- After printing each message, the output coroutine stores them in
+  -- currentPageMessages. When it receives a shipout message, it traverses
+  -- currentPageMessages adding the page number it just learned about to
+  -- each of the messages and clears currentPageMessages. This serves two
+  -- purposes: it allows us to include the page numbers in the summaries
+  -- and it allows us to include the page number in page-delay mode.
   currentPageMessages = {}
+
+  -- The objects representing the summary for each kind of message are
+  -- stored in summaries, so after all messages are processed we can just
+  -- traverse this list calling :toString() and get all the summaries. The
+  -- summaries are also used to suppress repeated messages. This table is
+  -- populated by registerSummaries().
   summaries = {}
 
-  -- When the same message appears several times, we only output it
-  -- once, thanks to this table. This is used by showMessage()
-  alreadySeen = {}
-
   -- All handlers should be in either of these. They are populated by
   -- registerHandlers().
   beginningOfLineHandlers = {}
@@ -717,9 +716,11 @@
   table.insert(beginningOfLineHandlers, beginningOfLineDebugStringsHandler)
   table.insert(beginningOfLineHandlers, beginningOfLineInfoStringsHandler)
   table.insert(beginningOfLineHandlers, beginningOfLineWarningStringsHandler)
+  table.insert(beginningOfLineHandlers, beginningOfLineCriticalStringsHandler)
   table.insert(anywhereHandlers, anywhereDebugStringsHandler)
   table.insert(anywhereHandlers, anywhereInfoStringsHandler)
   table.insert(anywhereHandlers, anywhereWarningStringsHandler)
+  table.insert(anywhereHandlers, anywhereCriticalStringsHandler)
   table.insert(anywhereHandlers, openParensHandler)
   table.insert(anywhereHandlers, closeParensHandler)
   table.insert(anywhereHandlers, openSquareBracketHandler)
@@ -769,6 +770,34 @@
       table.insert(tmp, pat)
   end
   SILENCE_PKGS = tmp
+
+  tmp = {}
+  for _, str in ipairs(FORCED_DEBUG) do
+      local pat = stringToPattern(str)
+      table.insert(tmp, pat)
+  end
+  FORCED_DEBUG = tmp
+
+  tmp = {}
+  for _, str in ipairs(FORCED_INFO) do
+      local pat = stringToPattern(str)
+      table.insert(tmp, pat)
+  end
+  FORCED_INFO = tmp
+
+  tmp = {}
+  for _, str in ipairs(FORCED_WARNING) do
+      local pat = stringToPattern(str)
+      table.insert(tmp, pat)
+  end
+  FORCED_WARNING = tmp
+
+  tmp = {}
+  for _, str in ipairs(FORCED_CRITICAL) do
+      local pat = stringToPattern(str)
+      table.insert(tmp, pat)
+  end
+  FORCED_CRITICAL = tmp
 end
 
 function processCommandLine(args)
@@ -780,6 +809,8 @@
   RAW = false
   SILENCE_REPETITIONS = true
   MINLEVEL = WARNING
+  BE_REDUNDANT = false
+  DETAILED_UNDEROVER_SUMMARY = true
 
   SILENCE_STRINGS = {}
   SILENCE_PKGS = {} -- just the package names
@@ -786,6 +817,11 @@
   SEMISILENCE_FILES = {} -- filenames (without leading path), file globs work
   SILENCE_FILES_RECURSIVE = {} -- same
 
+  -- The user may redefine the severity level of some messages.
+  FORCED_DEBUG = {}
+  FORCED_INFO = {}
+  FORCED_WARNING = {}
+  FORCED_CRITICAL = {}
 
   -- "-l level -c configFile"
   local optionsWithArgs = "lc"
@@ -806,6 +842,10 @@
   --only-summary                       no filtering, only final summary
   --shipouts, --no-shipouts            enable/disable reporting shipouts
   --repetitions, --no-repetitions      allow/prevent repeated messages
+  --be-redundant, --no-be-redundant    present/suppress ordinary messages
+                                       that will also appear in the summary
+  --box-detail, --no-box-detail        include/exclude full under/overfull
+                                       boxes information in the summary
   --heartbeat, --no-heartbeat          enable/disable progress gauge
   -l LEVEL, --minlevel=LEVEL           filter out messages with severity
                                        level lower than [LEVEL]. Valid
@@ -825,6 +865,19 @@
   --add-debug-message=MESSAGE          add new recognizable debug message
   --add-info-message=MESSAGE           add new recognizable info message
   --add-warning-message=MESSAGE        add new recognizable warning message
+  --add-critical-message=MESSAGE       add new recognizable critical message
+  --set-to-level-debug=EXCERPT         reset severity of messages containing
+                                       text EXCERPT to DEBUG; can be used
+                                       multiple times
+  --set-to-level-info=EXCERPT          reset severity of messages containing
+                                       text EXCERPT to INFO; can be used
+                                       multiple times
+  --set-to-level-warning=EXCERPT       reset severity of messages containing
+                                       text EXCERPT to WARNING; can be used
+                                       multiple times
+  --set-to-level-critical=EXCERPT      reset severity of messages containing
+                                       text EXCERPT to CRITICAL; can be used
+                                       multiple times
   -c cfgfile, --config-file=cfgfile    read options from config file
   -h, --help                           give this help list
   --version                            print program version]]
@@ -835,8 +888,8 @@
 
   --version
   if vars.version then
-      print("texlogsieve 1.0.0-beta-1")
-      print("Copyright (C) 2021 Nelson Lago <lago at ime.usp.br>")
+      print("texlogsieve 1.0.0-beta-2")
+      print("Copyright (C) 2021, 2022 Nelson Lago <lago at ime.usp.br>")
       print("License GPLv3+: GNU GPL version 3 or later "
             .. "<https://gnu.org/licenses/gpl.html>.")
       print("This is free software: you are free to change "
@@ -847,29 +900,22 @@
 
   --config-file=file
   -- "-c file"
-  local configFileName
-  if vars['config-file'] ~= nil then configFileName = vars['config-file'] end
-  if vars.c ~= nil then configFileName = vars.c end
-  if configFileName ~= nil then
-      configFileName = configFileName[1]
-      local filevars = processConfigFile(configFileName)
+  local configFileNames = vars['config-file']
+  if configFileNames == nil then configFileNames = {} end
 
-      -- merge filevars with vars; vars has precedence
-      for k, v in pairs(vars) do
-          if type(v) == "boolean" then
-              filevars[k] = v
-          elseif filevars[k] == nil then
-              filevars[k] = v
-          else
-              -- the value is a table, so append
-              for _, item in ipairs(v) do
-                  table.insert(filevars[k], item)
-              end
-          end
+  if vars.c ~= nil then
+      for _, val in ipairs(vars.c) do
+          table.insert(configFileNames, val)
       end
+  end
 
-      -- use the merged values
-      vars = filevars
+  -- Add the default config file to the beginning of the list
+  local filename = kpse.find_file('texlogsieverc')
+  if filename ~= nil then table.insert(configFileNames, 1, filename) end
+
+  for _, filename in ipairs(configFileNames) do
+      local configFile = assert(io.open(filename, "r"))
+      vars = processConfigFile(configFile, vars)
   end
 
   --unwrap-only
@@ -881,6 +927,7 @@
       PAGE_DELAY = false
       SHOW_SHIPOUTS = true
       SILENCE_REPETITIONS = false
+      BE_REDUNDANT = true
       MINLEVEL = DEBUG
   end
 
@@ -904,6 +951,7 @@
   if vars['no-summary'] or vars.summary ~= nil and not vars.summary then
       SHOW_SUMMARY = false
       SILENCE_REPETITIONS = false
+      BE_REDUNDANT = true
   end
   if vars.summary then SHOW_SUMMARY = true end
 
@@ -941,6 +989,28 @@
   end
   if vars.repetitions then SILENCE_REPETITIONS = false end
 
+  --be-redundant
+  --no-be-redundant
+  --be-redundant=true/false
+  if vars['no-be-redundant']
+                    or vars['be-redundant'] ~= nil
+                    and not vars['be-redundant'] then
+
+      BE_REDUNDANT = false
+  end
+  if vars['be-redundant'] then BE_REDUNDANT = true end
+
+  --box-detail
+  --no-box-detail
+  --box-detail=true/false
+  if vars['no-box-detail']
+                    or vars['box-detail'] ~= nil
+                    and not vars['box-detail'] then
+
+      DETAILED_UNDEROVER_SUMMARY = false
+  end
+  if vars['box-detail'] then DETAILED_UNDEROVER_SUMMARY = true end
+
   --no-heartbeat
   --heartbeat
   --heartbeat=true/false
@@ -969,7 +1039,7 @@
       for _, msg in ipairs(vars['add-debug-message']) do
           local pat = stringToPattern(msg)
           if not string.find(pat, '^', 1, true) then pat = '^%s*' .. pat end
-          pat = string.gsub(pat, '\\n', '\n')
+          pat = string.gsub(pat, '\\n', '%%s*\n')
           table.insert(anywhereDebugStringsHandler.patterns, pat)
       end
   end
@@ -978,8 +1048,8 @@
       for _, msg in ipairs(vars['add-info-message']) do
           local pat = stringToPattern(msg)
           if not string.find(pat, '^', 1, true) then pat = '^%s*' .. pat end
-          pat = string.gsub(pat, '\\n', '\n')
-          table.insert(anywhereInfoStringsHandler.patterns, msg)
+          pat = string.gsub(pat, '\\n', '%%s*\n')
+          table.insert(anywhereInfoStringsHandler.patterns, pat)
       end
   end
 
@@ -987,39 +1057,79 @@
       for _, msg in ipairs(vars['add-warning-message']) do
           local pat = stringToPattern(msg)
           if not string.find(pat, '^', 1, true) then pat = '^%s*' .. pat end
-          pat = string.gsub(pat, '\\n', '\n')
-          table.insert(anywhereWarningStringsHandler.patterns, msg)
+          pat = string.gsub(pat, '\\n', '%%s*\n')
+          table.insert(anywhereWarningStringsHandler.patterns, pat)
       end
   end
+
+  if vars['add-critical-message'] then
+      for _, msg in ipairs(vars['add-critical-message']) do
+          local pat = stringToPattern(msg)
+          if not string.find(pat, '^', 1, true) then pat = '^%s*' .. pat end
+          pat = string.gsub(pat, '\\n', '%%s*\n')
+          table.insert(anywhereCriticalStringsHandler.patterns, pat)
+      end
+  end
+
+
+  if vars['set-to-level-debug'] then
+      FORCED_DEBUG = vars['set-to-level-debug']
+  end
+
+  if vars['set-to-level-info'] then
+      FORCED_INFO = vars['set-to-level-info']
+  end
+
+  if vars['set-to-level-warning'] then
+      FORCED_WARNING = vars['set-to-level-warning']
+  end
+
+  if vars['set-to-level-critical'] then
+      FORCED_CRITICAL = vars['set-to-level-critical']
+  end
 end
 
-function processConfigFile(filename)
-    configfile = assert(io.open(filename, "r"))
-    local fileOptions = {}
+function processConfigFile(configFile, currentVars)
+  local fileVars = {}
 
-    while true do
-        local line = configfile:read("*line")
-        if line == nil then break end
+  while true do
+      local line = configFile:read("*line")
+      if line == nil then break end
 
-        line = trim(line)
-        local first = string.find(line, '^#')
-        if first == nil and string.len(line) > 0 then
+      line = trim(line)
+      local first = string.find(line, '^#')
 
-            local equals = string.find(line, '=', 1, true)
-            if equals ~= nil then
-                optname = string.sub(line, 1, equals -1)
-                optval = string.sub(line, equals +1)
-                optname = trim(optname)
-                optval = trim(optval)
-            else
-                optname = line
-                optval = true
+      if first == nil and line ~= "" then
+          local equals = string.find(line, '=', 1, true)
+          if equals ~= nil then
+              optname = string.sub(line, 1, equals -1)
+              optval = string.sub(line, equals +1)
+              optname = trim(optname)
+              optval = trim(optval)
+          else
+              optname = line
+              optval = true
+          end
+          simpleGetoptStoreVal(fileVars, optname, optval)
+      end
+  end
+
+    -- merge fileVars with currentVars; currentVars has precedence
+    for k, v in pairs(currentVars) do
+        if type(v) == "boolean" then
+            fileVars[k] = v
+        elseif fileVars[k] == nil then
+            fileVars[k] = v
+        else
+            -- the value is a table, so append
+            for _, item in ipairs(v) do
+                table.insert(fileVars[k], item)
             end
-            simpleGetoptStoreVal(fileOptions, optname, optval)
         end
     end
 
-    return fileOptions
+  -- return the merged values
+  return fileVars
 end
 
 
@@ -1039,7 +1149,107 @@
 
 dispatch = function(msg) coroutine.resume(outputCoroutine, msg) end
 
+function adjustSeverity(msg)
+  formatted = trim(msg:realToString())
+  if formatted == "" then return end
+
+  DEFAULT_FORCED_INFO = {
+    "File %b`' already exists on the system%."
+            .. "%s*Not generating it from",
+    "You have requested package %b`',"
+            .. "%s*but the package provides",
+    "Writing file %b`'",
+    "Form Feed has been converted to Blank",
+    "Tab has been converted to Blank",
+    "The morewrites package is unnecessary",
+    'Unused \\captionsetup%b[]',
+  }
+
+  DEFAULT_FORCED_CRITICAL = {
+    "Label %b`' multiply defined",
+    "Command .- invalid in math mode",
+    "Optional argument of \\twocolumn too tall on page",
+    "Marginpar on page %S- moved",
+    "Some font shapes were not available, defaults substituted%.",
+    "Font shape %b`' in size %b<> not available"
+            .. "%s+Font shape %b`' tried instead",
+    "Font shape %b`' in size %S+ not available"
+            .. "%s+external font %b`' used",
+    "Font shape %b`' undefined"
+            .. "%s+using %b`' instead",
+  }
+
+
+  -- We do things this way so that user-defined strings override these
+  -- defaults (note that there is no "return" in the first two blocks)
+  if msg.severity ~= INFO then
+      for _, val in ipairs(DEFAULT_FORCED_INFO) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = INFO
+          end
+      end
+  end
+
+  if msg.severity ~= CRITICAL then
+      for _, val in ipairs(DEFAULT_FORCED_CRITICAL) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = CRITICAL
+          end
+      end
+  end
+
+  if msg.severity ~= DEBUG then
+      for _, val in ipairs(FORCED_DEBUG) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = DEBUG
+              return
+          end
+      end
+  end
+
+  if msg.severity ~= INFO then
+      for _, val in ipairs(FORCED_INFO) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = INFO
+              return
+          end
+      end
+  end
+
+  if msg.severity ~= WARNING then
+      for _, val in ipairs(FORCED_WARNING) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = WARNING
+              return
+          end
+      end
+  end
+
+  if msg.severity ~= CRITICAL then
+      for _, val in ipairs(FORCED_CRITICAL) do
+          local first = string.find(formatted, val)
+          local other = string.find(msg.content, val)
+          if first ~= nil or other ~= nil then
+              msg.severity = CRITICAL
+              return
+          end
+      end
+  end
+end
+
 function processMessage(msg)
+  adjustSeverity(msg)
+
   if ONLY_SUMMARY or PAGE_DELAY then
       heartbeat:tick()
   else
@@ -1054,7 +1264,9 @@
 
       for _, tmp in ipairs(currentPageMessages) do
           tmp.physicalPage = msg.physicalPage
-          tmp:toSummary()
+          -- normally, toSummary() is called by showMessage(),
+          -- but with ONLY_SUMMARY that is never called
+          if ONLY_SUMMARY then tmp:toSummary() end
       end
 
       if PAGE_DELAY and not ONLY_SUMMARY then
@@ -1069,52 +1281,42 @@
   heartbeat:stop()
 
   -- messages after the last shipout
-  if PAGE_DELAY and not ONLY_SUMMARY then
-      print("")
-      print("After last page:")
-      print("")
-      showPageMessages()
-  end
+  if PAGE_DELAY and not ONLY_SUMMARY then showRemainingMessages() end
 
-  -- now, the summaries
   if SHOW_SUMMARY then showSummary() end
 end
 
 function showMessage(msg)
   local formatted = msg:toString()
-  if string.len(trim(formatted)) == 0 then return end
+  if trim(formatted) ~= "" then
+      local pageinfo = ""
+      local spaces = ""
+      if not RAW and msg.physicalPage ~= nil then
+          pageinfo = 'pg ' .. msg.physicalPage .. ': '
+          spaces = string.rep(" ", string.len(pageinfo))
+      end
 
-  if RAW then
-      for _, line in ipairs(linesToTable(formatted)) do print(line) end
-      return
-  end
+      -- A message is a repetition if it has
+      -- already been included in some summary
+      local alreadySeen = false
+      if SILENCE_REPETITIONS then
+          for _, summary in ipairs(summaries) do
+              if summary:alreadySeen(msg) then
+                  alreadySeen = true
+                  break
+              end
+          end
+      end
 
-  local pageinfo = ""
-  if msg.physicalPage ~= nil then
-      pageinfo = 'pg ' .. msg.physicalPage .. ': '
-  end
-  local spaces = string.rep(" ", string.len(pageinfo))
-
-  if not SILENCE_REPETITIONS then
-      local lines = linesToTable(formatted)
-      for _, line in ipairs(lines) do
-          print(pageinfo .. line)
-          pageinfo = spaces
+      if not SILENCE_REPETITIONS or not alreadySeen then
+          for _, line in ipairs(linesToTable(formatted)) do
+              print(pageinfo .. line)
+              pageinfo = spaces
+          end
       end
-
-      return
   end
 
-  if alreadySeen[formatted] == nil then
-      alreadySeen[formatted] = {msg}
-      local lines = linesToTable(formatted)
-      for _, line in ipairs(lines) do
-          print(pageinfo .. line)
-          pageinfo = spaces
-      end
-  else
-      table.insert(alreadySeen[formatted], msg)
-  end
+  msg:toSummary()
 end
 
 function showPageMessages()
@@ -1123,13 +1325,43 @@
   end
 end
 
+function showRemainingMessages()
+  local thereIsSomething = false
+  for _, msg in ipairs(currentPageMessages) do
+      if trim(msg:toString()) ~= "" then
+          thereIsSomething = true
+          break
+      end
+  end
+
+  if thereIsSomething then
+      print("")
+      print("After last page:")
+      print("")
+  end
+
+  -- we always call this, even if there is nothing to show,
+  -- because it calls :toSummary() for each message
+  showPageMessages()
+end
+
 function showSummary()
-  if not ONLY_SUMMARY then for i = 1, 5 do print("") end end
+  local thereIsSomething = false
+  for _, summary in ipairs(summaries) do
+      if trim(summary:toString()) ~= "" then
+          thereIsSomething = true
+          break
+      end
+  end
 
+  if not thereIsSomething then return end
+
+  if not ONLY_SUMMARY then for i = 1, 3 do print("") end end
+
   for _, summary in ipairs(summaries) do
       local formatted = summary:toString()
       local prefix = ""
-      if string.len(trim(formatted)) > 0 then
+      if trim(formatted) ~= "" then
           for _, line in ipairs(linesToTable(formatted)) do
               print(prefix .. line)
               prefix = '    '
@@ -1244,6 +1476,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   -- This appears in the logfile but not on stdout
   local _, last = string.find(line, "^Here is how much")
@@ -1324,6 +1557,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local basePattern = "^([UO][nv][de][e]?r)full \\(.)box (%b())"
   local first, last, underover,
@@ -1487,6 +1721,8 @@
   self.message = self:newMessage()
   self.message.severity = self.severity
 
+  -- ignore leading spaces in the first line (in the others,
+  -- they may be indentation or somehow relevant)
   local _, last = string.find(Lines.current, '^%s+')
   if last ~= nil then Lines:handledChars(last) end
 
@@ -1524,11 +1760,13 @@
       return true
   end
 
+  -- trailing spaces in any line in this kind of messsage
+  -- are safe to ignore and may appear in \message's
+  local chunk = trimRight(string.sub(Lines.current, 1, last))
   if self.patternLineNumber == 1 then
-      self.message.content = string.sub(Lines.current, 1, last)
+      self.message.content = chunk
   else
-      self.message.content = self.message.content .. '\n'
-                             .. string.sub(Lines.current, 1, last)
+      self.message.content = self.message.content .. '\n' .. chunk
   end
 
   Lines:handledChars(last)
@@ -1575,6 +1813,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   -- skip what was processed in a previous iteration/recursion
   if offset > 0 then line = string.sub(line, offset +1) end
@@ -1672,7 +1911,6 @@
   '^%s*restricted \\write18 enabled%.',
   '^%s*%%%&%-line parsing enabled%.',
   '^%*%*[%w%.]+', -- "**jobname"
-  '^file:line:error style messages enabled%.',
   '^\\[^%s=]+=[^%s=]+', -- "\c at chapter=\count174"
   "^\\openout%d+%s*=%s*`?[^']+'?%.?",
 
@@ -1713,9 +1951,6 @@
   "^Loading configuration file `" .. filepat .. "'%.",
   "^contour: Using driver file `" .. filepat .. "'%.",
 
-  '^ABD: EverySelectfont initializing macros',
-  '^ABD: EveryShipout initializing macros',
-
   '^%[Loading MPS to PDF converter %(version ' .. datepat .. '%)%.%]',
 
 
@@ -1769,8 +2004,9 @@
   '^Lua module: luaotfload%-multiscript ' .. datepat
                    .. ' [%d%.]+ luaotfload submodule / multiscript',
 
-  '^' .. string.rep('%*', 37) .. '\n%* Using libertinus math %*\n'
-                   .. string.rep('%*', 37),
+  '^' .. string.rep('%*', 37) .. '\n'
+      .. '%* Using libertinus math %*\n'
+      .. string.rep('%*', 37),
 
   '^`inconsolata%-zi4\' v%S-, ' .. datepat
                    .. ' Text macros for Inconsolata %(msharpe%)',
@@ -1789,6 +2025,13 @@
   '^%s*L3 programming layer %b<> xparse %b<>',
   '^%s*%{.*pdftex%.map%}',
 
+  '^%s*ABD: EverySelectfont initializing macros',
+  '^%s*ABD: EveryShipout initializing macros',
+
+  '^%s*' .. string.rep('%*', 65) .. '%s*\n'
+         .. 'GFS%-Solomos style file by A%. Tsolomitis%s*\n'
+         .. string.rep('%*', 65),
+
   -- <blah.jpg, id=555, [...,] 722.7pt x 722.7pt>
   '^%s*%<' .. filepat .. ', id=.- [%d%.]+pt x [%d%.]+pt%>',
   '^%s*%<use ' .. filepat .. '%>', -- <use blah.jpg>
@@ -1803,10 +2046,11 @@
 beginningOfLineInfoStringsHandler.severity = INFO
 beginningOfLineInfoStringsHandler.patterns = {
   "^Writing index file.*%.idx",
+  "^Writing glossary file.*%.glo",
   "^%*geometry%* driver:.*",
   "^%*geometry%* detected driver:.*",
   "^Driver file for pgf:.*%.def",
-  "^%s*file:line:error style messages enabled",
+  "^%s*file:line:error style messages enabled%.",
   "^Applying: %b[] float order in 2%-column on input line .-%.",
   "^Already applied: %b[] float order in 2%-column on input line .-%.",
   "^\\%S+ = a dialect from .*",
@@ -1814,6 +2058,7 @@
   "^No file .-%.aux%.",
   "^No file .-%.ind%.",
   "^No file .-%.bbl%.",
+  "^No file .-%.gls%.",
   "^reledmac reminder:%s*\n"
     .. "%s*The number of the footnotes in this section "
     .. "has changed since the last run.\n"
@@ -1864,7 +2109,7 @@
   '^luaotfload | aux : no font with id %d+',
 
   "^warning  %(pdf backend%): ignoring duplicate destination "
-                            .. "with the name '%-%.'",
+                            .. "with the name '.-'",
 
   "^Couldn't patch \\%S+",
   "^Invalid UTF%-8 byte or sequence at line %d+ replaced by U%+FFFD%.",
@@ -1884,6 +2129,29 @@
 }
 
 
+-- We know these messages always start at the beginning of a line
+-- Always start these patterns with "^", see lookahead().
+-- Order matters! The first match wins, so the longer ones should come first.
+beginningOfLineCriticalStringsHandler = stringsHandler:new()
+beginningOfLineCriticalStringsHandler.severity = CRITICAL
+beginningOfLineCriticalStringsHandler.patterns = {
+    "^The control sequence at the end of the top line\n"
+      .. "of your error message was never \\def'ed%. If you have\n"
+      .. "misspelled it %(e%.g%., `\\hobx'%), type `I' and the correct\n"
+      .. "spelling %(e%.g%., `I\\hbox'%)%. Otherwise just continue,\n"
+      .. "and I'll forget about whatever was undefined%.",
+}
+
+
+-- These messages may start anywhere in a line
+-- Always start these patterns with "^%s*", see lookahead().
+-- Order matters! The first match wins, so the longer ones should come first.
+anywhereCriticalStringsHandler = stringsHandler:new()
+anywhereCriticalStringsHandler.severity = CRITICAL
+anywhereCriticalStringsHandler.patterns = {
+}
+
+
 -------------------------------------------------------------------------------
 -- missingCharHandler
 -- (from stringsHandler)
@@ -1938,6 +2206,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local last, data
 
@@ -1966,10 +2235,12 @@
   self.message.what = what
   self.message.name = name
   self.message.severity = self:parseSeverity(severity)
-  self.message.content = Lines.current
 
   self:findPrefix(last, name, what)
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 function genericLatexHandler:handleFirstLine()
@@ -1977,7 +2248,6 @@
   if not myTurn then return false end
 
   flushUnrecognizedMessages()
-  self:unwrapLines()
 
   -- erase any previous values; nil is not a good idea! If one of these
   -- is nil in a derived object, the object may grab the value of the
@@ -2017,7 +2287,6 @@
       if self.linenum ~= "" then
           self.message.linenum = self.linenum
       end
-      self:adjustSeverity()
       dispatch(self.message)
   end
 
@@ -2089,55 +2358,7 @@
   end
 end
 
--- LaTeX only uses severity "INFO" and "WARNING", which
--- is very limited. Let's demote some warnings to INFO
--- and promote some others to CRITICAL.
-function genericLatexHandler:adjustSeverity()
-  for _, pat in ipairs(self.downgradePatterns) do
-      local first = string.find(self.message.content, pat)
-      if first ~= nil then
-          self.message.severity = INFO
-          break
-      end
-  end
 
-  for _, pat in ipairs(self.upgradePatterns) do
-      local first = string.find(self.message.content, pat)
-      if first ~= nil then
-          self.message.severity = CRITICAL
-          break
-      end
-  end
-end
-
-genericLatexHandler.downgradePatterns = {
-    -- No need to include the full message
-    "File %b`' already exists on the system%."
-            .. "%s*Not generating it from",
-    "You have requested package %b`',"
-            .. "%s*but the package provides",
-    "Writing file %b`'",
-    "Form Feed has been converted to Blank",
-    "Tab has been converted to Blank",
-    "The morewrites package is unnecessary",
-    'Unused \\captionsetup%b[]',
-}
-
-genericLatexHandler.upgradePatterns = {
-    "Label %b`' multiply defined",
-    "Command .- invalid in math mode",
-    "Optional argument of \\twocolumn too tall on page",
-    "Marginpar on page %S- moved",
-    "Some font shapes were not available, defaults substituted%.",
-    "Font shape %b`' in size %b<> not available"
-            .. "%s+Font shape %b`' tried instead",
-    "Font shape %b`' in size %S+ not available"
-            .. "%s+external font %b`' used",
-    "Font shape %b`' undefined"
-            .. "%s+using %b`' instead",
-}
-
-
 -------------------------------------------------------------------------------
 -- latex23MessageHandler
 -- genericLatexVariantHandler
@@ -2160,7 +2381,6 @@
   local severity = data[3]
   self.message.severity = self:parseSeverity(severity)
   self.message.what = what
-  self.message.content = Lines.current
 
   local name
   -- When the message looks like "LaTeX...", there is no package
@@ -2169,6 +2389,9 @@
 
   self:findPrefix(last, name, what)
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 genericLatexVariantHandler = genericLatexHandler:new()
@@ -2186,10 +2409,12 @@
   self.message.what = what
   self.message.name = name
   self.message.severity = self:parseSeverity(severity)
-  self.message.content = Lines.current
 
   self:findPrefix(last, name, what)
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 
@@ -2206,8 +2431,8 @@
 citationHandler = genericLatexHandler:new()
 
 citationHandler.patterns = {
-  "^(LaTeX)%s+(Warning): (Citation) (%b`') on page (.-) undefined",
-  "^(LaTeX)%s+(Warning): (Citation) (%b`') undefined",
+  "^(LaTeX)%s+(Warning): (Citation) ('.-') on page (.-) undefined",
+  "^(LaTeX)%s+(Warning): (Citation) ('.-') undefined",
 }
 
 function citationHandler:unpackData(data)
@@ -2223,9 +2448,11 @@
   self.message.severity = self:parseSeverity(severity)
   self.message.key = key
   self.message.page = page
-  self.message.content = Lines.current
   self:findPrefix(last, nil, what)
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 function citationHandler:newMessage()
@@ -2287,7 +2514,6 @@
 
   self.message.what = what
   self.message.name = name
-  self.message.content = Lines.current
 
   -- There are no continuation lines of this kind for
   -- these messages, but the generic code still wants
@@ -2294,6 +2520,9 @@
   -- to check for the prefix.
   self.prefix = '[^%s%S]+' -- nothing matches
 
+  self:unwrapLines()
+  self.message.content = Lines.current
+
   if not Lines:empty() then
       local first = string.find(Lines:get(1), 'with kernel methods')
       if first ~= nil then
@@ -2326,9 +2555,11 @@
 
 function geometryDetailsHandler:unpackData(data)
   self.message.name = 'geometry'
-  self.message.content = Lines.current
   self.prefix = '* '
   self.message.prefix = self.prefix
+
+  self:unwrapLines()
+  self.message.content = Lines.current
 end
 
 -------------------------------------------------------------------------------
@@ -2380,6 +2611,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first, last = string.find(line, self.pattern)
   if first == nil then return false, {} end
@@ -2434,6 +2666,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first = string.find(line, self.pattern)
   if first == nil then return false, {} end
@@ -2476,6 +2709,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first, last = string.find(line, self.pattern)
   if first == nil then return false, {} end
@@ -2529,6 +2763,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first = string.find(line, self.pattern)
   if first == nil then return false, {} end
@@ -2603,6 +2838,7 @@
   local line
   if position == nil then position = 0 end
   line = Lines:get(position)
+  if line == nil then return false, {} end
 
   local first, encoding
   if self.doit == self.handleFirstLine then
@@ -2757,12 +2993,22 @@
 
 Message.severity = UNKNOWN
 
-function Message:toString()
-    if self.mute then return "" end
+function Message:toString(noSilence)
+    -- noSilence may be useful when generating summaries
+    if noSilence then
+        local formatted = self:realToString()
+        if trim(formatted) == "" then return "" else return formatted end
+    end
 
     -- If we've already been here, just output the previous result
     if self.formatted ~= nil then return self.formatted end
 
+    if self.mute then self.formatted = "" return "" end
+
+    if self.severity < MINLEVEL then self.formatted = "" return "" end
+
+    if self:redundant() then self.formatted = "" return "" end
+
     self.formatted = self:realToString()
     if trim(self.formatted) == "" then self.formatted = "" return "" end
 
@@ -2778,8 +3024,6 @@
         end
     end
 
-    if self.severity < MINLEVEL then self.formatted = "" return "" end
-
     return self.formatted
 end
 
@@ -2794,10 +3038,20 @@
   return msg
 end
 
+function Message:redundant()
+  return false
+end
+
 function Message:toSummary()
   local formatted = self:toString()
-  if string.len(trim(formatted)) == 0 then return end
+  if trim(formatted) == "" then return end
 
+  -- In the rare event that one of these is sent out as
+  -- an unrecognizedMessage with no other text, allow for
+  -- repetitions
+  local first = string.find(trim(formatted), '^[%(%)%[%]]$')
+  if first ~= nil then return end
+
   repetitionsSummary:add(self)
 end
 
@@ -2865,7 +3119,11 @@
   return Message.realToString(self)
 end
 
+-- We never want to suppress these repetitions
+function openFileMessage:toSummary()
+end
 
+
 closeFileMessage = Message:new()
 function closeFileMessage:realToString()
   if RAW then return ")" end
@@ -2873,9 +3131,18 @@
   return Message.realToString(self)
 end
 
+-- We never want to suppress these repetitions
+function closeFileMessage:toSummary()
+end
 
+
 underOverMessage = Message:new()
 underOverMessage.severity = WARNING
+
+function underOverMessage:redundant()
+  return not BE_REDUNDANT
+end
+
 function underOverMessage:realToString()
     local tmp = self.content
     if self.failedText ~= nil then
@@ -2898,6 +3165,10 @@
 
 missingCharMessage = Message:new()
 
+function missingCharMessage:redundant()
+  return not BE_REDUNDANT
+end
+
 -- This is a hack: it would be too painful to define
 -- pattern captures in the handler, so we do this here
 function missingCharMessage:realToString()
@@ -2915,16 +3186,23 @@
 
 
 citationMessage = Message:new()
+
+function citationMessage:redundant()
+  return not BE_REDUNDANT
+end
+
 function citationMessage:toSummary()
   citationsSummary:add(self)
 end
 
-referenceMessage = Message:new()
+referenceMessage = citationMessage:new()
+
 function referenceMessage:toSummary()
   referencesSummary:add(self)
 end
 
-labelMessage = Message:new()
+labelMessage = citationMessage:new()
+
 function labelMessage:toSummary()
   labelsSummary:add(self)
 end
@@ -2955,6 +3233,7 @@
 function SummaryPrototype:add(msg)
   -- group messages by message content
   local formatted = msg:toString()
+  if trim(formatted) == "" then return end
 
   if self.messages[formatted] == nil then
       self.messages[formatted] = {}
@@ -2963,6 +3242,12 @@
   table.insert(self.messages[formatted], msg)
 end
 
+function SummaryPrototype:alreadySeen(msg)
+  local formatted = msg:toString()
+  if trim(formatted) == "" then return false end
+  return self.messages[formatted] ~= nil
+end
+
 function SummaryPrototype:toString()
   -- check if the table is empty - https://stackoverflow.com/a/1252776
   if next(self.messages) == nil then return "" end
@@ -3008,7 +3293,9 @@
     local pages = {}
     local files = {}
     for _, msg in ipairs(messages) do
-        pages[msg.physicalPage] = true
+        if msg.physicalPage ~= nil then
+            pages[msg.physicalPage] = true
+        end
         if msg.filename ~= nil then
             files[msg.filename] = true
         end
@@ -3065,12 +3352,23 @@
   local text = ""
   if #messages > 1 then
       local pages, files = self:pageAndFileList(messages)
+
+      local where
+      if pages ~= "" and files ~= "" then
+          where = 'in pages ' .. pages .. ' (files ' .. files .. ') - '
+      elseif pages == "" and files ~= "" then
+          where = 'in files ' .. files .. ' - '
+      elseif pages ~= "" and files == "" then
+          where = 'in pages ' .. pages .. ' - '
+      end
+
       local content = messages[1]:toString()
+      if trim(content) ~= "" then
 
-      text = content .. '\n'
-             .. 'in pages ' .. pages
-             .. " (files " .. files .. ") - "
-             .. #messages .. ' repetitions'
+          text = content .. '\n'
+                 .. where
+                 .. #messages .. ' repetitions'
+      end
   end
 
   return text
@@ -3101,6 +3399,7 @@
   -- here because some messages may include the page number, making
   -- messages that are otherwise the same appear to be different.
   local key = msg.key
+  if key == "" then key = '???' end
 
   if self.messages[key] == nil then
       self.messages[key] = {}
@@ -3109,10 +3408,18 @@
   table.insert(self.messages[key], msg)
 end
 
+function citationsSummary:alreadySeen(msg)
+  local key = msg.key
+  if key == "" then key = '???' end
+
+  return self.messages[key] ~= nil
+end
+
 function citationsSummary:processSingleMessageList(messages)
   local text = ""
   local pages, files = self:pageAndFileList(messages)
   local key = messages[1].key
+  if key == "" then key = '???' end
 
   text = key .. '\n'
          .. 'in pages ' .. pages .. " (files " .. files .. ")"
@@ -3144,8 +3451,18 @@
 
   local pages, files = self:pageAndFileList(self.messages)
 
-  return "Under/overfull boxes in pages "
+  if DETAILED_UNDEROVER_SUMMARY then
+      local output = "Under/overfull boxes:"
+      for _, msg in ipairs(self.messages) do
+          output = output .. '\npage ' .. msg.physicalPage
+                   .. ' (file ' .. msg.filename .. '):\n'
+          output = output .. msg:toString(true)
+      end
+      return output
+  else
+      return "Under/overfull boxes in pages "
              .. pages .. " (files " .. files .. ")"
+  end
 end
 
 
@@ -3185,6 +3502,8 @@
 
 function trim(s) return (string.gsub(s, '^%s*(.-)%s*$', '%1')) end
 
+function trimRight(s) return (string.gsub(s, '^(.-)%s*$', '%1')) end
+
 function stringToPattern(s)
   local first, _ = string.find(s, '^////')
   local pat
@@ -3204,7 +3523,7 @@
   local size = string.len(s)
   local i = 1
   local lines = {}
-  while i < size do
+  while i <= size do
       -- check \r in case the user added to this file a pattern
       -- with an embedded dos-style "CR LF" sequence.
       local first, last, line = string.find(s, '(.-)[\r]?\n', i)
@@ -3211,7 +3530,7 @@
 
       if first == nil then
           table.insert(lines, string.sub(s, i))
-          i = size
+          i = size +1
       else
           table.insert(lines, line)
           i = last +1
@@ -3824,6 +4143,9 @@
   -- but it saves us from problems with a couple of messages that use
   -- "filepat" at the end of the pattern.
 
+  -- Nothing to do if the next line does not exist
+  if Lines:get(1) == nil then return last, matches end
+
   -- these might fit in 2 chars, so do not consider lines with that
   local realMessage = string.find(Lines:get(1), '[%)%[%]]')
   if string.len(Lines:get(1)) < 3 and not realMessage then
@@ -3915,6 +4237,17 @@
   local _, last = string.find(line, '^%s*%[')
   if last ~= nil then line = string.sub(line, last +1) end
 
+  -- If this is a 79-chars line that ends with ...[NUMBER, we will try
+  -- to unwrap. With that, we may find ...[LONGERNUMBER, which is what
+  -- we want. If, however, we find [NUMBERLETTER, we give up and consider
+  -- this not to be a shipout. Should we stick with [NUMBER instead?
+  --
+  -- No.
+  --
+  -- Either way, we have no way of knowing whether we are right or wrong,
+  -- but giving up is the lesser evil: the user may register the unknown
+  -- message with --add-[debug|info|warning]-message and solve the problem.
+  local page
   while true do
       for _, pattern in ipairs(patterns) do
           _, _, page = string.find(line, pattern)
@@ -3940,7 +4273,7 @@
   -- luatex puts quotes around filenames with spaces, which makes things easier
   while true do
       local _, last = string.find(line, '^"')
-      if string.len(line) > 0 and last == nil then return false end -- no quotes
+      if line ~= "" and last == nil then return false end -- no quotes
 
       last = string.find(line, '^"[%.]?/') -- relative path or unix-style path
       if last == nil then
@@ -3979,6 +4312,7 @@
   -- we already checked in the previous ones.
   local alreadyCheckedIdx = 0
 
+  local filename
   while true do
       local first = string.find(line, '^[%.]?/') -- relative or unix-style path
       if first == nil then
@@ -3996,18 +4330,23 @@
       -- From longest to shortest, to avoid problems if there is a
       -- substring in the filename that matches some other filename.
       for i = longest, alreadyCheckedIdx +1, -1 do
-          local filename = string.sub(line, 1, i)
-          if kpse.find_file(filename, 'other text files') ~= nil then
-              return filename
+          local candidate = string.sub(line, 1, i)
+          if kpse.find_file(candidate, 'other text files') ~= nil then
+              filename = candidate
+              break
           end
       end
 
-      -- we did not find the filename; can we unwrap this line?
-      if not Lines:seemsWrapped(position) then return nil end
+      -- We may or may not have found the filename. Either way, if we
+      -- can unwrap this line, we should:
+      -- 1. If we did not find the filename, we might
+      -- 2. If we did find the filename, we might find a different
+      --    (longer) filename, in which case we should stick with
+      --    it (yes, I have seen it happen!)
+      if not Lines:seemsWrapped(position) then return filename end
       first = string.find(line, "[%)%(%[%]]")
-      if first ~= nil then return nil end
+      if first ~= nil then return filename end
 
-      -- yep!
       alreadyCheckedIdx = longest
       line = line .. Lines:get(position +1)
       position = position +1



More information about the tex-live-commits mailing list.