texlive[69285] trunk: memoize (3jan24)
commits+karl at tug.org
commits+karl at tug.org
Wed Jan 3 22:18:52 CET 2024
Revision: 69285
https://tug.org/svn/texlive?view=revision&revision=69285
Author: karl
Date: 2024-01-03 22:18:51 +0100 (Wed, 03 Jan 2024)
Log Message:
-----------
memoize (3jan24)
Modified Paths:
--------------
trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.pl
trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.py
trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.pl
trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.py
trunk/Master/texmf-dist/doc/generic/memoize/FILES
trunk/Master/texmf-dist/doc/generic/memoize/examples-src.zip
trunk/Master/texmf-dist/doc/generic/memoize/examples.zip
trunk/Master/texmf-dist/doc/generic/memoize/memoize-clean.1.md
trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.pdf
trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.sty
trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.tex
trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.sty
trunk/Master/texmf-dist/doc/generic/memoize/memoize-extract.1.md
trunk/Master/texmf-dist/doc/man/man1/memoize-clean.1
trunk/Master/texmf-dist/doc/man/man1/memoize-clean.man1.pdf
trunk/Master/texmf-dist/doc/man/man1/memoize-clean.pl.man1.pdf
trunk/Master/texmf-dist/doc/man/man1/memoize-clean.py.man1.pdf
trunk/Master/texmf-dist/doc/man/man1/memoize-extract.1
trunk/Master/texmf-dist/doc/man/man1/memoize-extract.man1.pdf
trunk/Master/texmf-dist/doc/man/man1/memoize-extract.pl.man1.pdf
trunk/Master/texmf-dist/doc/man/man1/memoize-extract.py.man1.pdf
trunk/Master/texmf-dist/scripts/memoize/memoize-clean.pl
trunk/Master/texmf-dist/scripts/memoize/memoize-clean.py
trunk/Master/texmf-dist/scripts/memoize/memoize-extract.pl
trunk/Master/texmf-dist/scripts/memoize/memoize-extract.py
trunk/Master/texmf-dist/source/generic/memoize/Makefile
trunk/Master/texmf-dist/source/generic/memoize/memoize.edtx
trunk/Master/texmf-dist/source/generic/memoize/memoize.ins
trunk/Master/texmf-dist/tex/context/third/memoize/t-memoizable.tex
trunk/Master/texmf-dist/tex/context/third/memoize/t-memoize.tex
trunk/Master/texmf-dist/tex/context/third/memoize/t-nomemoize.tex
trunk/Master/texmf-dist/tex/latex/memoize/memoizable.sty
trunk/Master/texmf-dist/tex/latex/memoize/memoize.sty
trunk/Master/texmf-dist/tex/latex/memoize/nomemoize.sty
trunk/Master/texmf-dist/tex/plain/memoize/memoizable.tex
trunk/Master/texmf-dist/tex/plain/memoize/memoize.tex
trunk/Master/texmf-dist/tex/plain/memoize/nomemoize.tex
Added Paths:
-----------
trunk/Master/texmf-dist/doc/generic/memoize/CHANGELOG.md
trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.mst
trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.pdf
trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.tex
trunk/Master/texmf-dist/tex/generic/memoize/
trunk/Master/texmf-dist/tex/generic/memoize/memoizable.code.tex
Removed Paths:
-------------
trunk/Master/texmf-dist/doc/generic/memoize/memoize.mst
trunk/Master/texmf-dist/doc/generic/memoize/memoize.pdf
trunk/Master/texmf-dist/doc/generic/memoize/memoize.tex
Modified: trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.pl
===================================================================
--- trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.pl 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.pl 2024-01-03 21:18:51 UTC (rev 69285)
@@ -18,18 +18,19 @@
# The files belonging to this work and covered by LPPL are listed in
# <texmf>/doc/generic/memoize/FILES.
+my $PROG = 'memoize-clean.pl';
+my $VERSION = '2024/01/02 v1.1.0';
+
use strict;
use Getopt::Long;
-# I intend to rewrite this script using Path::Class.
use Cwd 'realpath';
use File::Spec;
use File::Basename;
-my $VERSION = '2023/10/10 v1.0.0';
-
-my $usage = "usage: memoize-clean.py [-h] [--yes] [--all] [--quiet] [--prefix PREFIX] [mmz ...]\n";
+my $usage = "usage: $PROG [-h] [--yes] [--all] [--quiet] [--prefix PREFIX] " .
+ "[mmz ...]\n";
my $Help = <<END;
-Remove (stale) memo and extern files.
+Remove (stale) memo and extern files produced by package Memoize.
positional arguments:
mmz .mmz record files
@@ -41,7 +42,8 @@
--all, -a Remove *all* memos and externs.
--quiet, -q
--prefix PREFIX, -p PREFIX
- A path prefix to clean; this option can be specified multiple times.
+ A path prefix to clean;
+ this option can be specified multiple times.
For details, see the man page or the Memoize documentation.
END
@@ -111,8 +113,11 @@
opendir(MD, $dir) or die "Cannot open directory '$dir'";
while( (my $fn = readdir(MD)) ) {
my $path = File::Spec->catfile(($dir),$fn);
- if ($fn =~ /\Q$basename_prefix\E[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(\.memo|(?:-[0-9]+)?\.pdf|\.log)/ and ($all || !exists($keep{$path}))) {
- push @tbdeleted, $path;
+ if ($fn =~
+ /\Q$basename_prefix\E[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?#
+ (\.memo|(?:-[0-9]+)?\.pdf|\.log)/x
+ and ($all || !exists($keep{$path}))) {
+ push @tbdeleted, $path;
}
}
closedir(MD);
@@ -161,3 +166,7 @@
} elsif (!$quiet) {
print "Nothing to do, the directory seems clean.\n";
}
+
+# Local Variables:
+# after-save-hook: pl2dtx
+# End:
Modified: trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.py
===================================================================
--- trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.py 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-clean.py 2024-01-03 21:18:51 UTC (rev 69285)
@@ -18,13 +18,14 @@
# The files belonging to this work and covered by LPPL are listed in
# <texmf>/doc/generic/memoize/FILES.
-__version__ = '2023/10/10 v1.0.0'
+__version__ = '2024/01/02 v1.1.0'
import argparse, re, sys, pathlib, os
parser = argparse.ArgumentParser(
description="Remove (stale) memo and extern files.",
- epilog = "For details, see the man page or the Memoize documentation."
+ epilog = "For details, see the man page or the Memoize documentation "
+ "(https://ctan.org/pkg/memoize)."
)
parser.add_argument('--yes', '-y', action = 'store_true',
help = 'Do not ask for confirmation.')
@@ -63,7 +64,8 @@
elif endinput:
raise RuntimeError(
- r'Bailing out, \endinput is not the last line of file $mmz_fn.')
+ rf'Bailing out, '
+ rf'\endinput is not the last line of file {mmz_fn}.')
elif m := re_prefix.match(line):
prefix = m[1]
@@ -109,7 +111,8 @@
def populate_tbdeleted(folder, basename_prefix):
re_aux = re.compile(
re.escape(basename_prefix) +
- '[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(?:\.memo|(?:-[0-9]+)?\.pdf|\.log)$')
+ '[0-9A-F]{32}(?:-[0-9A-F]{32})?'
+ '(?:-[0-9]+)?(?:\.memo|(?:-[0-9]+)?\.pdf|\.log)$')
try:
for f in folder.iterdir():
if re_aux.match(f.name) and (args.all or f not in keep):
@@ -157,3 +160,8 @@
print("Bailing out.")
elif not args.quiet:
print('Nothing to do, the directory seems clean.')
+
+# Local Variables:
+# fill-column: 79
+# after-save-hook: py2dtx
+# End:
Modified: trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.pl
===================================================================
--- trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.pl 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.pl 2024-01-03 21:18:51 UTC (rev 69285)
@@ -18,188 +18,757 @@
# The files belonging to this work and covered by LPPL are listed in
# <texmf>/doc/generic/memoize/FILES.
+my $PROG = 'memoize-extract.pl';
+my $VERSION = '2024/01/02 v1.1.0';
+
use strict;
+use File::Basename qw/basename/;
use Getopt::Long;
-use Path::Class;
-use File::Spec;
-use File::Basename;
-use PDF::API2;
+use File::Spec::Functions
+ qw/splitpath catpath splitdir rootdir file_name_is_absolute/;
+use File::Path qw(make_path);
+# We will only try to import the PDF processing library once we set up the error
+# log.
-my $VERSION = '2023/10/10 v1.0.0';
+# Declare variables for command-line arguments and for |kpathsea|
+# variables. They are defined here so that they are global in the subs which use
+# them.
+our ($pdf_file, $prune, $keep, $format, $force, $quiet,
+ $pdf_library, $print_version, $mkdir, $help,
+ $openin_any, $openout_any, $texmfoutput, $texmf_output_directory);
-my $usage = "usage: memoize-extract.pl [-h] [--pdf PDF] [--prune] [--keep] [--force] [--log LOG] [--warning-template WARNING_TEMPLATE] [--quiet] [--mkdir] mmz\n";
+# \paragraph{Messages}
+
+# The messages are written both to the extraction log and the terminal (we
+# output to stdout rather than stderr so that messages on the TeX terminal and
+# document |.log| appear in chronological order). Messages are automatically
+# adapted to the TeX |--format|.
+
+# The format of the messages. It depends on the given |--format|; the last
+# entry is for t the terminal output.
+
+my %ERROR = (
+ latex => '\PackageError{memoize (perl-based extraction)}{$short}{$long}',
+ plain => '\errhelp{$long}\errmessage{memoize (perl-based extraction): $short}',
+ context => '\errhelp{$long}\errmessage{memoize (perl-based extraction): $short}',
+ '' => '$header$short. $long');
+
+my %WARNING = (
+ latex => '\PackageWarning{memoize (perl-based extraction)}{$texindent$text}',
+ plain => '\message{memoize (perl-based extraction) Warning: $texindent$text}',
+ context => '\message{memoize (perl-based extraction) Warning: $texindent$text}',
+ '' => '$header$indent$text.');
+
+my %INFO = (
+ latex => '\PackageInfo{memoize (perl-based extraction)}{$texindent$text}',
+ plain => '\message{memoize (perl-based extraction): $texindent$text}',
+ context => '\message{memoize (perl-based extraction): $texindent$text}',
+ '' => '$header$indent$text.');
+
+# Some variables used in the message routines; note that |header| will be
+# redefined once we parse the arguments.
+
+my $exit_code = 0;
+my $log;
+my $header = '';
+my $indent = '';
+my $texindent = '';
+
+# The message routines.
+
+sub error {
+ my ($short, $long) = @_;
+ if (! $quiet) {
+ $_ = $ERROR{''};
+ s/\$header/$header/;
+ s/\$short/$short/;
+ s/\$long/$long/;
+ print(STDOUT "$_\n");
+ }
+ if ($log) {
+ $long =~ s/\\/\\string\\/g;
+ $_ = $ERROR{$format};
+ s/\$short/$short/;
+ s/\$long/$long/;
+ print(LOG "$_\n");
+ }
+ $exit_code = 11;
+ endinput();
+}
+
+sub warning {
+ my $text = shift;
+ if ($log) {
+ $_ = $WARNING{$format};
+ s/\$texindent/$texindent/;
+ s/\$text/$text/;
+ print(LOG "$_\n");
+ }
+ if (! $quiet) {
+ $_ = $WARNING{''};
+ s/\$header/$header/;
+ s/\$indent/$indent/;
+ s/\$text/$text/;
+ print(STDOUT "$_\n");
+ }
+ $exit_code = 10;
+}
+
+sub info {
+ my $text = shift;
+ if ($text && ! $quiet) {
+ $_ = $INFO{''};
+ s/\$header/$header/;
+ s/\$indent/$indent/;
+ s/\$text/$text/;
+ print(STDOUT "$_\n");
+ if ($log) {
+ $_ = $INFO{$format};
+ s/\$texindent/$texindent/;
+ s/\$text/$text/;
+ print(LOG "$_\n");
+ }
+ }
+}
+
+# Mark the log as complete and exit.
+sub endinput {
+ if ($log) {
+ print(LOG "\\endinput\n");
+ close(LOG);
+ }
+ exit $exit_code;
+}
+
+sub die_handler {
+ stderr_to_warning();
+ my $text = shift;
+ chomp($text);
+ error("Perl error: $text", '');
+}
+
+sub warn_handler {
+ my $text = shift;
+ chomp($text);
+ warning("Perl warning: $text");
+}
+
+# This is used to print warning messages from PDF::Builder, which are output to
+# STDERR.
+my $stderr;
+sub stderr_to_warning {
+ if ($stderr) {
+ my $w = ' Perl info: ';
+ my $nl = '';
+ for (split(/\n/, $stderr)) {
+ /(^\s*)(.*?)(\s*)$/;
+ $w .= ($1 ? ' ' : $nl) . $2;
+ $nl = "\n";
+ }
+ warning("$w");
+ $stderr = '';
+ }
+}
+
+# \paragraph{Permission-related functions}
+
+# We will need these variables below. Note that we only support Unix and
+# Windows.
+my $on_windows = $^O eq 'MSWin32';
+my $dirsep = $on_windows ? '\\' : '/';
+
+# |paranoia_in|/|out| should work exactly as
+# |kpsewhich -safe-in-name|/|-safe-out-name|.
+sub paranoia_in {
+ my ($f, $remark) = @_;
+ error("I'm not allowed to read from '$f' (openin_any = $openin_any)",
+ $remark) unless _paranoia($f, $openin_any);
+}
+
+sub paranoia_out {
+ my ($f, $remark) = @_;
+ error("I'm not allowed to write to '$f' (openin_any = $openout_any)",
+ $remark) unless _paranoia($f, $openout_any);
+}
+
+sub _paranoia {
+ # |f| is the path to the file (it should not be empty), and |mode| is the
+ # value of |openin_any| or |openout_any|.
+ my ($f, $mode) = @_;
+ return if (! $f);
+ # We split the filename into the directory and the basename part, and the
+ # directory into components.
+ my ($volume, $dir, $basename) = splitpath($f);
+ my @dir = splitdir($dir);
+ return (
+ # In mode `any' (|a|, |y| or |1|), we may access any file.
+ $mode =~ /^[ay1]$/
+ || (
+ # Otherwise, we are at least in the restricted mode, so we should
+ # not open dot files on Unix-like systems (except file called
+ # |.tex|).
+ ! (!$on_windows && $basename =~ /^\./ && !($basename =~ /^\.tex$/))
+ && (
+ # If we are precisely in the restricted mode (|r|, |n|, |0|),
+ # then there are no further restrictions.
+ $mode =~ /^[rn0]$/
+ # Otherwise, we are in the paranoid mode (officially |p|, but
+ # any other value is interpreted as |p| as well). There are
+ # two further restrictions in the paranoid mode.
+ || (
+ # We're not allowed to go to a parent directory.
+ ! grep(/^\.\.$/, @dir) && $basename ne '..'
+ &&
+ # If the given path is absolute, is should be a descendant
+ # of either |TEXMF_OUTPUT_DIRECTORY| or |TEXMFOUTPUT|.
+ (!file_name_is_absolute($f)
+ ||
+ is_ancestor($texmf_output_directory, $f)
+ ||
+ is_ancestor($texmfoutput, $f)
+ )))));
+}
+
+# Only removes final "/"s. This is unlike |File::Spec|'s |canonpath|, which also
+# removes |.| components, collapses multiple |/| --- and unfortunately also goes
+# up for |..| on Windows.
+sub normalize_path {
+ my $path = shift;
+ my ($v, $d, $n) = splitpath($path);
+ if ($n eq '' && $d =~ /[^\Q$dirsep\E]\Q$dirsep\E+$/) {
+ $path =~ s/\Q$dirsep\E+$//;
+ }
+ return $path;
+}
+
+# On Windows, we disallow ``semi-absolute'' paths, i.e.\ paths starting with the
+# |\| but lacking the drive. |File::Spec|'s function |file_name_is_absolute|
+# returns 2 if the path is absolute with a volume, 1 if it's absolute with no
+# volume, and 0 otherwise. After a path was sanitized using this function,
+# |file_name_is_absolute| will work as we want it to.
+sub sanitize_path {
+ my $f = normalize_path(shift);
+ my ($v, $d, $n) = splitpath($f);
+ if ($on_windows) {
+ my $a = file_name_is_absolute($f);
+ if ($a == 1 || ($a == 0 && $v) ) {
+ error("\"Semi-absolute\" paths are disallowed: " . $f,
+ "The path must either both contain the drive letter and " .
+ "start with '\\', or none of these; paths like 'C:foo\\bar' " .
+ "and '\\foo\\bar' are disallowed");
+ }
+ }
+}
+
+sub access_in {
+ return -r shift;
+}
+
+sub access_out {
+ my $f = shift;
+ my $exists;
+ eval { $exists = -e $f };
+ # Presumably, we get this error when the parent directory is not executable.
+ return if ($@);
+ if ($exists) {
+ # An existing file should be writable, and if it's a directory, it
+ # should also be executable.
+ my $rw = -w $f; my $rd = -d $f; my $rx = -x $f;
+ return -w $f && (! -d $f || -x $f);
+ } else {
+ # For a non-existing file, the parent directory should be writable.
+ # (This is the only place where function |parent| is used, so it's ok
+ # that it returns the logical parent.)
+ my $p = parent($f);
+ return -w $p;
+ }
+}
+
+# This function finds the location for an input file, respecting
+# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the
+# filesystem. It returns an absolute file as-is. For a relative file, it tries
+# |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (always), and
+# |TEXMFOUTPUT| directory (if defined), in this order. The first readable file
+# found is returned; if no readable file is found, the file in the current
+# directory is returned.
+sub find_in {
+ my $f = shift;
+ sanitize_path($f);
+ return $f if file_name_is_absolute($f);
+ for my $df (
+ $texmf_output_directory ? join_paths($texmf_output_directory, $f) : undef,
+ $f,
+ $texmfoutput ? join_paths($texmfoutput, $f) : undef) {
+ return $df if $df && -r $df;
+ }
+ return $f;
+}
+
+# This function finds the location for an output file, respecting
+# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the
+# filesystem. It returns an absolute file as-is. For a relative file, it tries
+# |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (unless
+# |TEXMF_OUTPUT_DIRECTORY| is defined), and |TEXMFOUTPUT| directory (if
+# defined), in this order. The first writable file found is returned; if no
+# writable file is found, the file in either the current or the output directory
+# is returned.
+sub find_out {
+ my $f = shift;
+ sanitize_path($f);
+ return $f if file_name_is_absolute($f);
+ for my $df (
+ $texmf_output_directory ? join_paths($texmf_output_directory, $f) : undef,
+ $texmf_output_directory ? undef : $f,
+ $texmfoutput ? join_paths($texmfoutput, $f) : undef) {
+ return $df if $df && access_out($df);
+ }
+ return $texmf_output_directory ? join_paths($texmf_output_directory, $f) : $f;
+}
+
+# We next define some filename-related utilities matching what Python offers out
+# of the box. We avoid using |File::Spec|'s |canonpath|, because on Windows,
+# which has no concept of symlinks, this function resolves |..| to the parent.
+
+sub name {
+ my $path = shift;
+ my ($volume, $dir, $filename) = splitpath($path);
+ return $filename;
+}
+
+sub suffix {
+ my $path = shift;
+ my ($volume, $dir, $filename) = splitpath($path);
+ $filename =~ /\.[^.]*$/;
+ return $&;
+}
+
+sub with_suffix {
+ my ($path, $suffix) = @_;
+ my ($volume, $dir, $filename) = splitpath($path);
+ if ($filename =~ s/\.[^.]*$/$suffix/) {
+ return catpath($volume, $dir, $filename);
+ } else {
+ return catpath($volume, $dir, $filename . $suffix);
+ }
+}
+
+sub with_name {
+ my ($path, $name) = @_;
+ my ($volume, $dir, $filename) = splitpath($path);
+ my ($v,$d,$f) = splitpath($name);
+ die "Runtime error in with_name: " .
+ "'$name' should not contain the directory component"
+ unless $v eq '' && $d eq '' && $f eq $name;
+ return catpath($volume, $dir, $name);
+}
+
+sub join_paths {
+ my $path1 = normalize_path(shift);
+ my $path2 = normalize_path(shift);
+ return $path2 if !$path1 || file_name_is_absolute($path2);
+ my ($volume1, $dir1, $filename1) = splitpath($path1, 'no_file');
+ my ($volume2, $dir2, $filename2) = splitpath($path2);
+ die if $volume2;
+ return catpath($volume1,
+ join($dirsep, ($dir1 eq $dirsep ? '' : $dir1, $dir2)),
+ $filename2);
+}
+
+# The logical parent. The same as |pathlib.parent| in Python.
+sub parent {
+ my $f = normalize_path(shift);
+ my ($v, $dn, $_dummy) = splitpath($f, 1);
+ my $p_dn = $dn =~ s/[^\Q$dirsep\E]+$//r;
+ if ($p_dn eq '') {
+ $p_dn = $dn =~ /^\Q$dirsep\E/ ? $dirsep : '.';
+ }
+ my $p = catpath($v, $p_dn, '');
+ $p = normalize_path($p);
+ return $p;
+}
+
+# This function assumes that both paths are absolute; ancestor may be '',
+# signaling a non-path.
+sub is_ancestor {
+ my $ancestor = normalize_path(shift);
+ my $descendant = normalize_path(shift);
+ return if ! $ancestor;
+ $ancestor .= $dirsep unless $ancestor =~ /\Q$dirsep\E$/;
+ return $descendant =~ /^\Q$ancestor/;
+}
+
+# A paranoid |Path.mkdir|. The given folder is preprocessed by |find_out|.
+sub make_directory {
+ my $folder = find_out(shift);
+ if (! -d $folder) {
+ paranoia_out($folder);
+ # Using |make_path| is fine because we know that
+ # |TEXMF_OUTPUT_DIRECTORY|/|TEXMFOUTPUT|, if given, exists, and that
+ # ``folder'' contains no |..|.
+ make_path($folder);
+ # This does not get logged when the function is invoked via |--mkdir|,
+ # as it is not clear what the log name should be.
+ info("Created directory $folder");
+ }
+}
+
+sub unquote {
+ shift =~ s/"(.*?)"/\1/rg;
+}
+
+# \paragraph{Kpathsea}
+
+# Get the values of |openin_any|, |openout_any|, |TEXMFOUTPUT| and
+# |TEXMF_OUTPUT_DIRECTORY|.
+
+my $maybe_backslash = $on_windows ? '' : '\\';
+my $query = 'kpsewhich -expand-var=' .
+ "openin_any=$maybe_backslash\$openin_any," .
+ "openout_any=$maybe_backslash\$openout_any," .
+ "TEXMFOUTPUT=$maybe_backslash\$TEXMFOUTPUT";
+my $kpsewhich_output = `$query`;
+if (! $kpsewhich_output) {
+ # No TeX? (Note that |kpsewhich| should exist in MiKTeX as well.) In
+ # absence of |kpathsea| information, we get very paranoid.
+ ($openin_any, $openout_any) = ('p', 'p');
+ ($texmfoutput, $texmf_output_directory) = ('', '');
+ # Unfortunately, this warning can't make it into the log. But then again,
+ # the chances of a missing |kpsewhich| are very slim, and its absence would
+ # show all over the place anyway.
+ warning('I failed to execute "kpsewhich", is there no TeX system installed? ' .
+ 'Assuming openin_any = openout_any = "p" ' .
+ '(i.e. restricting all file operations to non-hidden files ' .
+ 'in the current directory of its subdirectories).');
+} else {
+ $kpsewhich_output =~ /^openin_any=(.*),openout_any=(.*),TEXMFOUTPUT=(.*)/;
+ ($openin_any, $openout_any, $texmfoutput) = @{^CAPTURE};
+ $texmf_output_directory = $ENV{'TEXMF_OUTPUT_DIRECTORY'};
+ if ($openin_any =~ '^\$openin_any') {
+ # When the |open*_any| variables are not expanded, we assume we're
+ # running MiKTeX. The two config settings below correspond to TeXLive's
+ # |openin_any| and |openout_any|; afaik, there is no analogue to
+ # |TEXMFOUTPUT|.
+ $query = 'initexmf --show-config-value=[Core]AllowUnsafeInputFiles ' .
+ '--show-config-value=[Core]AllowUnsafeOutputFiles';
+ my $initexmf_output = `$query`;
+ $initexmf_output =~ /^(.*)\n(.*)\n/m;
+ $openin_any = $1 eq 'true' ? 'a' : 'p';
+ $openout_any = $2 eq 'true' ? 'a' : 'p';
+ $texmfoutput = '';
+ $texmf_output_directory = '';
+ }
+}
+
+# An output directory should exist, and may not point to the root on Linux. On
+# Windows, it may point to the root, because being absolute also implies
+# containing the drive; see |sanitize_filename|.
+sub sanitize_output_dir {
+ return unless my $d = shift;
+ sanitize_path($d);
+ # On Windows, |rootdir| returns |\|, so it cannot possibly match |$d|.
+ return $d if -d $d && $d ne rootdir();
+}
+
+$texmfoutput = sanitize_output_dir($texmfoutput);
+$texmf_output_directory = sanitize_output_dir($texmf_output_directory);
+
+# We don't delve into the real script when loaded from the testing code.
+return 1 if caller;
+
+# \paragraph{Arguments}
+
+my $usage = "usage: $PROG [-h] [-P PDF] [-p] [-k] [-F {latex,plain,context}] [-f] " .
+ "[-L {PDF::API2,PDF::Builder}] [-q] [-m] [-V] mmz\n";
my $Help = <<END;
-Extract extern pages out of the document PDF.
+Extract extern pages produced by package Memoize out of the document PDF.
positional arguments:
- mmz the record file produced by Memoize: doc.mmz when compiling doc.tex
+ mmz the record file produced by Memoize:
+ doc.mmz when compiling doc.tex
+ (doc and doc.tex are accepted as well)
options:
- --help, -h show this help message and exit
- --version, -V show version and exit
- --pdf PDF, -P PDF extract from file PDF
- --prune, -p remove the extern pages after extraction
- --keep, -k do not mark externs as extracted
- --force, -f extract even if the size-check fails
- --log LOG, -l LOG the log file
- --warning-template WARNING_TEMPLATE, -w WARNING_TEMPLATE
- \warningtext in the template will be replaced by the warning message
- --quiet, -q don't describe what's happening
- --embedded, -e prefix all messages to the standard output with the script name
- --mkdir, -m create a directory (and exit)
+ -h, --help show this help message and exit
+ -P PDF, --pdf PDF extract from file PDF
+ -p, --prune remove the extern pages after extraction
+ -k, --keep do not mark externs as extracted
+ -F, --format {latex,plain,context}
+ the format of the TeX document invoking extraction
+ -f, --force extract even if the size-check fails
+ -q, --quiet describe what's happening
+ -L, --library {PDF::API2, PDF::Builder}
+ which PDF library to use for extraction (default: PDF::API2)
+ -m, --mkdir create a directory (and exit);
+ mmz argument is interpreted as directory name
+ -V, --version show program's version number and exit
For details, see the man page or the Memoize documentation.
END
-my ($pdf_arg, $prune, $keep, $log, $quiet, $embedded, $force, $mkdir, $help, $print_version);
-my $warning_template = '\warningtext';
+my @valid_libraries = ('PDF::API2', 'PDF::Builder');
Getopt::Long::Configure ("bundling");
GetOptions(
- "pdf|P=s" => \$pdf_arg,
+ "pdf|P=s" => \$pdf_file,
+ "prune|p" => \$prune,
"keep|k" => \$keep,
- "prune|p" => \$prune,
- "log|l=s" => \$log,
+ "format|F=s" => \$format,
+ "force|f" => \$force,
"quiet|q" => \$quiet,
- "embedded|e" => \$embedded,
- "force|f" => \$force,
- "warning-template|w=s" => \$warning_template,
+ "library|L=s" => \$pdf_library,
"mkdir|m" => \$mkdir,
+ "version|V" => \$print_version,
"help|h|?" => \$help,
- "version|V" => \$print_version,
) or die $usage;
+
if ($help) {print("$usage\n$Help"); exit 0}
-if ($print_version) { print("memoize-extract.pl of Memoize $VERSION\n"); exit 0 }
-die $usage unless @ARGV == 1;
-my $message_prefix = $embedded ? basename($0) . ': ' : '';
-print("\n") if ($embedded);
+if ($print_version) { print("$PROG of Memoize $VERSION\n"); exit 0 }
-my @output_paths = (dir()->absolute->resolve);
-my $texmfoutput = `kpsewhich --var-value=TEXMFOUTPUT`;
-$texmfoutput =~ s/^\s+|\s+$//g;
-if ($texmfoutput) {
- my $texmfoutput_dir = dir($texmfoutput)->absolute->resolve;
- push(@output_paths, $texmfoutput_dir) unless $texmfoutput_dir->dir_list == 1 && ! $texmfoutput_dir->volume;
-}
+die "${usage}$PROG: error: the following arguments are required: mmz\n"
+ unless @ARGV == 1;
-sub paranoia {
- my $file = $_[0];
- die "${message_prefix}Cannot create a hidden file or dir: $file"
- if $file->basename =~ /^\./;
- my $parent = $file->parent->absolute->resolve;
- die "${message_prefix}Cannot write outside the current working or output directory tree: $file"
- unless grep($_->contains($parent), @output_paths);
-}
+die "${usage}$PROG: error: argument -F/--format: invalid choice: '$format' " .
+ "(choose from 'latex', 'plain', 'context')\n"
+ unless grep $_ eq $format, ('', 'latex', 'plain', 'context');
-my $mmz_arg = $ARGV[0];
-my $mmz_file = file($mmz_arg);
-my $mmz_dir = $mmz_file->parent;
+die "${usage}$PROG: error: argument -L/--library: invalid choice: '$pdf_library' " .
+ "(choose from " . join(", ", @valid_libraries) . ")\n"
+ if $pdf_library && ! grep $_ eq $pdf_library, @valid_libraries;
+$header = $format ? basename($0) . ': ' : '';
+
+# start a new line in the TeX terminal output
+print("\n") if $format;
+
+# \paragraph{Initialization}
+
+# With |--mkdir|, argument |mmz| is interpreted as the directory to create.
if ($mkdir) {
- my $dir = dir($mmz_arg)->cleanup;
- my $current = dir(File::Spec->catpath($dir->volume,
- $dir->is_absolute ? File::Spec->rootdir : File::Spec->curdir,
- ''))->absolute;
- for my $c ($dir->components) {
- $current = $current->subdir($c);
- if (-d $current) {
- $current = $current->resolve;
- } else {
- paranoia($current);
- mkdir($current);
- print("${message_prefix}Created directory $current\n") unless $quiet;
- }
- }
- exit;
-} else {
- die "${message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '$mmz_file'\n" unless $mmz_file->basename =~ /\.mmz$/;
+ make_directory($ARGV[0]);
+ exit 0;
}
-# Enable in-place editing (of the .mmz file).
-paranoia($mmz_file) unless $keep;
-$^I = "" unless $keep;
+# Normalize the |mmz| argument into a |.mmz| filename.
+my $mmz_file = $ARGV[0];
+$mmz_file = with_suffix($mmz_file, '.mmz')
+ if suffix($mmz_file) eq '.tex';
+$mmz_file = with_name($mmz_file, name($mmz_file) . '.mmz')
+ if suffix($mmz_file) ne '.mmz';
-my $pdf_file = $pdf_arg ? file($pdf_arg) : $mmz_dir->file($mmz_file->basename =~ s/\.mmz$/\.pdf/r)->cleanup;
-paranoia($pdf_file) if $prune;
+# Once we have the |.mmz| filename, we can open the log.
+if ($format) {
+ my $_log = find_out(with_suffix($mmz_file, '.mmz.log'));
+ paranoia_out($_log);
+ info("Logging to '$_log'");
+ $log = $_log;
+ open LOG, ">$log";
+}
-if ($log) {
- paranoia(file($log));
- open LOG, ">$log";
+# Now that we have opened the log file, we can try loading the PDF processing
+# library.
+if ($pdf_library) {
+ eval "use $pdf_library";
+ error("Perl module '$pdf_library' was not found",
+ 'Have you followed the instructions is section 1.1 of the manual?')
+ if ($@);
} else {
- *LOG = *STDERR;
+ for (@valid_libraries) {
+ eval "use $_";
+ if (!$@) {
+ $pdf_library = $_;
+ last;
+ }
+ }
+ if (!$pdf_library) {
+ error("No suitable Perl module for PDF processing was found, options are " .
+ join(", ", @valid_libraries),
+ 'Have you followed the instructions is section 1.1 of the manual?');
+ }
}
-my ($pdf, %extern_pages);
-print "${message_prefix}Extracting externs from $pdf_file:\n" unless $quiet;
+# Catch any errors in the script and output them to the log.
+$SIG{__DIE__} = \&die_handler;
+$SIG{__WARN__} = \&warn_handler;
+close(STDERR);
+open(STDERR, ">", \$stderr);
+# Find the |.mmz| file we will read, but retain the original filename in
+# |$given_mmz_file|, as we will still need it.
+my $given_mmz_file = $mmz_file;
+$mmz_file = find_in($mmz_file, 1);
+if (! -e $mmz_file) {
+ info("File '$given_mmz_file' does not exist, assuming there's nothing to do");
+ endinput();
+}
+paranoia_in($mmz_file);
+paranoia_out($mmz_file,
+ 'I would have to rewrite this file unless option --keep is given.')
+ unless $keep;
+
+# Determine the PDF filename: it is either given via |--pdf|, or constructed
+# from the |.mmz| filename.
+$pdf_file = with_suffix($given_mmz_file, '.pdf') if !$pdf_file;
+$pdf_file = find_in($pdf_file);
+paranoia_in($pdf_file);
+paranoia_out($pdf_file,
+ 'I would have to rewrite this file because option --prune was given.')
+ if $prune;
+
+# Various initializations.
+my $pdf;
+my %extern_pages;
+my $new_mmz;
my $tolerance = 0.01;
-my $done_message = "${message_prefix}Done (there was nothing to extract).\n";
-
-while (<>) {
- if (/^\\mmzNewExtern *{(?P<extern_path>(?P<prefix>.*?)(?P<code_md5sum>[0-9A-F]{32})-(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P<page_n>[0-9]+)}{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}/) {
- my $extern_file = file($+{extern_path});
- if (! $extern_file->is_absolute) {
- $extern_file = $mmz_dir->file($+{extern_path});
- }
- paranoia($extern_file);
- my $page = $+{page_n};
- my $expected_width_pt = $+{expected_width};
- my $expected_height_pt = $+{expected_height};
- my $c_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '.memo');
- my $cc_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '-' . $+{context_md5sum} . '.memo');
- if (!(-e $c_memo_file && -e $cc_memo_file)) {
- print LOG ($warning_template =~ s/\\warningtext/Not extracting page $page into extern $extern_file, because the associated (c)c-memo does not exist/gr), "\n\\endinput\n";
- last;
- }
- eval { $pdf = PDF::API2->open($pdf_file->stringify) unless $pdf; };
- if ($@) {
- print LOG ($warning_template =~ s/\\warningtext/Cannot read file "$pdf_file". Perhaps you have to load Memoize earlier in the preamble?/gr), "\n\\endinput\n";
- close LOG;
- die "${message_prefix}File '$pdf_file' cannot be read, bailing out.\n";
- }
- my $extern = PDF::API2->new();
- $extern->version($pdf->version);
- $extern->import_page($pdf, $page);
- my $extern_page = $extern->open_page(1);
- my ($x0, $y0, $x1, $y1) = $extern_page->get_mediabox();
- my $width_pt = ($x1 - $x0) / 72 * 72.27;
- my $height_pt = ($y1 - $y0) / 72 * 72.27;
- my $warning = '';
- if (abs($width_pt - $expected_width_pt) > $tolerance
- || abs($height_pt - $expected_height_pt) > $tolerance)
- {
- $warning = "I refuse to extract page $page from $pdf_file, because its size (${width_pt}pt x ${height_pt}pt) is not what I expected (${expected_width_pt}pt x ${expected_height_pt}pt)";
- print LOG ($warning_template =~ s/\\warningtext/$warning/gr), "\n";
- }
- if ($warning && !$force) {
- unlink $extern_file;
- } else {
- $extern->saveas($extern_file->stringify);
- $done_message = "${message_prefix}Done.\n";
- print STDOUT "${message_prefix} Page $page --> $extern_file\n" unless $quiet;
- $extern_pages{$page} = 1 if $prune;
- print("%") unless $keep;
- }
+info("Extracting new externs listed in '$mmz_file' " .
+ "from '$pdf_file' using Perl module $pdf_library");
+my $done_message = "Done (there was nothing to extract)";
+$indent = ' ';
+$texindent = '\space\space ';
+my $dir_to_make;
+
+# \paragraph{Process \texttt{.mmz}}
+#
+# We cannot process the .mmz file using in-place editing. It would fail when
+# the file is writable but its parent directory is not.
+
+open (MMZ, $mmz_file);
+while (<MMZ>) {
+ my $mmz_line = $_;
+ if (/^\\mmzPrefix *{(?P<prefix>)}/) {
+ # Found |\mmzPrefix|: create the extern directory, but only later, if an
+ # extern file is actually produced. We parse the prefix in two steps
+ # because we have to unquote the entire prefix.
+ my $prefix = unquote($+{prefix});
+ warning("Cannot parse line '$mmz_line'") unless
+ $prefix =~ /(?P<dir_prefix>.*\/)?(?P<name_prefix>.*?)/;
+ $dir_to_make = $+{dir_prefix};
+ } elsif (/^\\mmzNewExtern\ *{(?P<extern_path>.*?)}{(?P<page_n>[0-9]+)}#
+ {(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}/x) {
+ # Found |\mmzNewExtern|: extract the extern page into an extern file.
+ $done_message = "Done";
+ my $ok = 1;
+ my %m_ne = %+;
+ # The extern filename, as specified in |.mmz|:
+ my $extern_file = unquote($m_ne{extern_path});
+ # We parse the extern filename in a separate step because we have to
+ # unquote the entire path.
+ warning("Cannot parse line '$mmz_line'") unless
+ $extern_file =~ /(?P<dir_prefix>.*\/)?(?P<name_prefix>.*?)#
+ (?P<code_md5sum>[0-9A-F]{32})-#
+ (?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf/x;
+ # The actual extern filename:
+ my $extern_file_out = find_out($extern_file);
+ paranoia_out($extern_file_out);
+ my $page = $m_ne{page_n};
+ # Check whether c-memo and cc-memo exist (in any input directory).
+ my $c_memo = with_name($extern_file,
+ $+{name_prefix} . $+{code_md5sum} . '.memo');
+ my $cc_memo = with_name($extern_file,
+ $+{name_prefix} . $+{code_md5sum} .
+ '-' . $+{context_md5sum} . '.memo');
+ my $c_memo_in = find_in($c_memo);
+ my $cc_memo_in = find_in($cc_memo);
+ if ((! access_in($c_memo_in) || ! access_in($cc_memo_in)) && !$force) {
+ warning("I refuse to extract page $page into extern '$extern_file', " .
+ "because the associated c-memo '$c_memo' and/or " .
+ "cc-memo '$cc_memo' does not exist");
+ $ok = '';
+ }
+ # Load the PDF. We only do this now so that we don't load it if there
+ # is nothing to extract.
+ if ($ok && ! $pdf) {
+ if (!access_in($pdf_file)) {
+ warning("Cannot open '$pdf_file'", '');
+ endinput();
+ }
+ # Temporarily disable error handling, so that we can catch the error
+ # ourselves.
+ $SIG{__DIE__} = undef; $SIG{__WARN__} = undef;
+ # All safe, |paranoia_in| was already called above.
+ eval { $pdf = $pdf_library->open($pdf_file, msgver => 0) };
+ $SIG{__DIE__} = \&die_handler; $SIG{__WARN__} = \&warn_handler;
+ error("File '$pdf_file' seems corrupted. " .
+ "Perhaps you have to load Memoize earlier in the preamble",
+ "In particular, Memoize must be loaded before TikZ library " .
+ "'fadings' and any package deploying it, and in Beamer, " .
+ "load Memoize by writing \\RequirePackage{memoize} before " .
+ "\\documentclass{beamer}. " .
+ "This was the error thrown by Perl:" . "\n$@") if $@;
+ }
+ # Does the page exist?
+ if ($ok && $page > (my $n_pages = $pdf->page_count())) {
+ error("I cannot extract page $page from '$pdf_file', " .
+ "as it contains only $n_pages page" .
+ ($n_pages > 1 ? 's' : ''), '');
+ }
+ if ($ok) {
+ # Import the page into the extern PDF (no disk access yet).
+ my $extern = $pdf_library->new(outver => $pdf->version);
+ $extern->import_page($pdf, $page);
+ my $extern_page = $extern->open_page(1);
+ # Check whether the page size matches the |.mmz| expectations.
+ my ($x0, $y0, $x1, $y1) = $extern_page->get_mediabox();
+ my $width_pt = ($x1 - $x0) / 72 * 72.27;
+ my $height_pt = ($y1 - $y0) / 72 * 72.27;
+ my $expected_width_pt = $m_ne{expected_width};
+ my $expected_height_pt = $m_ne{expected_height};
+ if ((abs($width_pt - $expected_width_pt) > $tolerance
+ || abs($height_pt - $expected_height_pt) > $tolerance) && !$force) {
+ warning("I refuse to extract page $page from $pdf_file, " .
+ "because its size (${width_pt}pt x ${height_pt}pt) " .
+ "is not what I expected " .
+ "(${expected_width_pt}pt x ${expected_height_pt}pt)");
+ } else {
+ # All tests were successful, let's create the extern file.
+ # First, the containing directory, if necessary.
+ if ($dir_to_make) {
+ make_directory($dir_to_make);
+ $dir_to_make = undef;
+ }
+ # Now the extern file. Note that |paranoia_out| was already
+ # called above.
+ info("Page $page --> $extern_file_out");
+ $extern->saveas($extern_file_out);
+ # This page will get pruned.
+ $extern_pages{$page} = 1 if $prune;
+ # Comment out this |\mmzNewExtern|.
+ $new_mmz .= '%' unless $keep;
+ }
+ }
}
- print unless $keep;
+ $new_mmz .= $mmz_line unless $keep;
+ stderr_to_warning();
}
+close(MMZ);
+$indent = '';
+$texindent = '';
+info($done_message);
-print $done_message unless $quiet;
+# Write out the |.mmz| file with |\mmzNewExtern| lines commented out. (All safe,
+# |paranoia_out| was already called above.)
+if (!$keep) {
+ open(MMZ, ">", $mmz_file);
+ print MMZ $new_mmz;
+ close(MMZ);
+}
-if ($pdf and $prune) {
- paranoia($pdf_file);
- my $pruned_pdf = PDF::API2->new();
+# Remove the extracted pages from the original PDF. (All safe, |paranoia_out|
+# was already called above.)
+if ($prune and keys(%extern_pages) != 0) {
+ my $pruned_pdf = $pdf_library->new();
for (my $n = 1; $n <= $pdf->page_count(); $n++) {
- if (! $extern_pages{$n}) {
- $pruned_pdf->import_page($pdf, $n);
- }
+ if (! $extern_pages{$n}) {
+ $pruned_pdf->import_page($pdf, $n);
+ }
}
- $pruned_pdf->save($pdf_file->stringify);
- print("${message_prefix}The following extern pages were pruned out of the PDF: ",
- join(",", sort(keys(%extern_pages))) . "\n") unless $quiet;
+ $pruned_pdf->save($pdf_file);
+ info("The following extern pages were pruned out of the PDF: " .
+ join(",", sort(keys(%extern_pages))));
}
-if ($log) {
- print LOG "\\endinput\n";
- close LOG;
-}
+endinput();
+
+# Local Variables:
+# fill-column: 79
+# after-save-hook: pl2dtx
+# End:
Modified: trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.py
===================================================================
--- trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.py 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Build/source/texk/texlive/linked_scripts/memoize/memoize-extract.py 2024-01-03 21:18:51 UTC (rev 69285)
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-
+#
# This file is a part of Memoize, a TeX package for externalization of
# graphics and memoization of compilation results in general, available at
# https://ctan.org/pkg/memoize and https://github.com/sasozivanovic/memoize.
@@ -18,166 +18,552 @@
# The files belonging to this work and covered by LPPL are listed in
# <texmf>/doc/generic/memoize/FILES.
-__version__ = '2023/10/10 v1.0.0'
+__version__ = '2024/01/02 v1.1.0'
-import argparse, re, sys, os, pdfrw, subprocess, itertools
-from pathlib import Path
+import argparse, re, sys, os, subprocess, itertools, traceback, platform
+from pathlib import Path, PurePath
+# We will only try to import the PDF processing library once we set up the
+# error log.
-parser = argparse.ArgumentParser(
- description = "Extract extern pages out of the document PDF.",
- epilog = "For details, see the man page or the Memoize documentation.",
- prog = 'memoize-extract.py',
-)
-parser.add_argument('--pdf', '-P', help = 'extract from file PDF')
-parser.add_argument('--prune', '-p', action = 'store_true',
- help = 'remove the extern pages after extraction')
-parser.add_argument('--keep', '-k', action = 'store_true',
- help = 'do not mark externs as extracted')
-parser.add_argument('--force', '-f', action = 'store_true',
- help = 'extract even if the size-check fails')
-parser.add_argument('--log', '-l', default = os.devnull, help = 'the log file')
-parser.add_argument('--warning-template', '-w', default = '\warningtext',
- help = '\warningtext in the template will be replaced by the warning message')
-parser.add_argument('--quiet', '-q', action = 'store_true',
- help = "describe what's happening")
-parser.add_argument('--embedded', '-e', action = 'store_true',
- help = "prefix all messages to the standard output with the script name")
-parser.add_argument('--mkdir', '-m', action = 'store_true',
- help = 'create a directory (and exit)')
-parser.add_argument('mmz',
- help = 'the record file produced by Memoize: doc.mmz when compiling doc.tex')
-parser.add_argument('--version', '-V', action = 'version',
- version = f"%(prog)s of Memoize " + __version__)
+# \paragraph{Messages}
-args = parser.parse_args()
+# The messages are written both to the extraction log and the terminal (we
+# output to stdout rather than stderr so that messages on the TeX terminal and
+# document |.log| appear in chronological order). Messages are automatically
+# adapted to the TeX |--format|.
-message_prefix = parser.prog + ': ' if args.embedded else ''
-if args.embedded:
- print()
+# The format of the messages. It depends on the given |--format|; the last
+# entry is for t the terminal output.
-# Even a bit more paranoid than required:
-# (a) disallowing TEXMFOUTPUT=/ (while allowing C:\ on Windows)
-# (b) waiting for access to '-output-directory'.
-output_paths = [Path.cwd()]
-if texmfoutput := subprocess.run(
- ['kpsewhich', '--var-value=TEXMFOUTPUT'],
- capture_output = True).stdout.decode().strip():
- texmfoutput_dir = Path(texmfoutput).resolve()
- if len(texmfoutput_dir.parts) != 1 or texmfoutput_dir.drive:
- output_paths.append(texmfoutput_dir)
+ERROR = {
+ 'latex': r'\PackageError{{{package_name}}}{{{short}}}{{{long}}}',
+ 'plain': r'\errhelp{{{long}}}\errmessage{{{package_name}: {short}}}',
+ 'context': r'\errhelp{{{long}}}\errmessage{{{package_name}: {short}}}',
+ None: '{header}{short}.\n{long}',
+}
-def paranoia(f):
- p = Path(f).resolve()
- if p.stem.startswith('.'):
- raise RuntimeError(f"{message_prefix}Cannot create a hidden file or dir: {f}")
- if not any(p.is_relative_to(op) for op in output_paths):
- raise RuntimeError(f"{message_prefix}Cannot write outside the current working or output directory tree: {f}")
+WARNING = {
+ 'latex': r'\PackageWarning{{{package_name}}}{{{texindent}{text}}}',
+ 'plain': r'\message{{{package_name}: {texindent}{text}}}',
+ 'context': r'\message{{{package_name}: {texindent}{text}}}',
+ None: r'{header}{indent}{text}.',
+}
-mmz_file = Path(args.mmz)
+INFO = {
+ 'latex': r'\PackageInfo{{{package_name}}}{{{texindent}{text}}}',
+ 'plain': r'\message{{{package_name}: {texindent}{text}}}',
+ 'context': r'\message{{{package_name}: {texindent}{text}}}',
+ None: r'{header}{indent}{text}.',
+}
+
+# Some variables used in the message routines; note that |header| will be
+# redefined once we parse the arguments.
+
+package_name = 'memoize (python-based extraction)'
+exit_code = 0
+log = None
+header = ''
+indent = ''
+texindent = ''
+
+# The message routines.
+
+def error(short, long):
+ if not args.quiet:
+ print(ERROR[None].format(short = short, long = long, header = header))
+ if log:
+ long = long.replace('\\', '\\string\\')
+ print(
+ ERROR[args.format].format(
+ short = short, long = long, package_name = package_name),
+ file = log)
+ global exit_code
+ exit_code = 11
+ endinput()
-if args.mkdir: # Here, argument "mmz" is interpreted as the directory to create.
- # We cannot simply say
- # paranoia(mmz_file)
- # mmz_file.mkdir(parents = True, exist_ok = True)
- # because have be paranoid about the intermediate directories as well:
- # memoize-extract.py -m a/../../BAD/../<cwd-name>/b
- # Note that paranoia might kick in only after a part of the given path was
- # already created. This is in accord to how "mkdir" behaves wrt existing
- # files.
- for folder in itertools.chain(reversed(mmz_file.parents), (mmz_file,)):
- if not folder.is_dir():
- paranoia(folder)
- folder.mkdir(exist_ok = True)
- if not args.quiet:
- print(f"{message_prefix}Created directory {folder}")
- sys.exit()
-elif mmz_file.suffix != '.mmz':
- raise RuntimeError(f"{message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '{mmz_file}'")
+def warning(text):
+ if log:
+ print(
+ WARNING[args.format].format(
+ text = text, texindent = texindent, package_name = package_name),
+ file = log)
+ if not args.quiet:
+ print(WARNING[None].format(text = text, header = header, indent = indent))
+ global exit_code
+ exit_code = 10
+
+def info(text):
+ if text and not args.quiet:
+ print(INFO[None].format(text = text, header = header, indent = indent))
+ if log:
+ print(
+ INFO[args.format].format(
+ text = text, texindent = texindent, package_name = package_name),
+ file = log)
+
+# Mark the log as complete and exit.
+def endinput():
+ if log:
+ print(r'\endinput', file = log)
+ log.close()
+ sys.exit(exit_code)
-mmz_dir = mmz_file.parent
-pdf_file = Path(args.pdf) if args.pdf else mmz_file.with_suffix('.pdf')
-paranoia(pdf_file)
-pdf = None
-extern_pages = []
-new_mmz = []
-args.log is os.devnull or paranoia(Path(args.log))
-re_newextern = re.compile(r'\\mmzNewExtern *{(?P<extern_fn>(?P<prefix>.*?)(?P<code_md5sum>[0-9A-F]{32})-(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P<page_n>[0-9]+)}{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}')
-tolerance = 0.01
-done_message = f"{message_prefix}Done (there was nothing to extract)."
+# \paragraph{Permission-related functions}
-# Complication: I want to use 'with', but don't have to open stderr.
-with open(args.log, 'w') as log:
- log = sys.stderr if args.log is os.devnull else log
+# |paranoia_in|/|out| should work exactly as
+# |kpsewhich -safe-in-name|/|-safe-out-name|.
+def paranoia_in(f, remark = ''):
+ if f and not _paranoia(f, openin_any):
+ error(f"I'm not allowed to read from '{f}' (openin_any = {openin_any})",
+ remark)
+
+def paranoia_out(f, remark = ''):
+ if f and not _paranoia(f, openout_any):
+ error(f"I'm not allowed to write to '{f}' (openout_any = {openout_any})",
+ remark)
+
+def _paranoia(f, mode):
+ # |mode| is the value of |openin_any| or |openout_any|. |f| is a
+ # |pathlib.Path| object.
+ return (
+ # In mode `any' (|a|, |y| or |1|), we may access any file.
+ mode in 'ay1'
+ or (
+ # Otherwise, we are at least in the restricted mode, so we should not
+ # open dot files on Unix-like systems (except file called |.tex|).
+ not (os.name == 'posix' and f.stem.startswith('.') and f.stem != '.tex')
+ and (
+ # If we are precisely in the restricted mode (|r|, |n|, |0|),
+ # then there are no further restrictions.
+ mode in 'rn0'
+ # Otherwise, we are in the paranoid mode (officially |p|, but
+ # any other value is interpreted as |p| as well). There are
+ # two further restrictions in the paranoid mode.
+ or (
+ # We're not allowed to go to a parent directory.
+ '..' not in f.parts
+ and
+ # If the given path is absolute, is should be a descendant
+ # of either |TEXMF_OUTPUT_DIRECTORY| or |TEXMFOUTPUT|.
+ (not f.is_absolute()
+ or
+ is_ancestor(texmf_output_directory, f)
+ or
+ is_ancestor(texmfoutput, f)
+ )))))
+
+# On Windows, we disallow ``semi-absolute'' paths, i.e.\ paths starting with
+# the |\| but lacking the drive. On Windows, |pathlib|'s |is_absolute| returns
+# |True| only for paths starting with |\| and containing the drive.
+def sanitize_filename(f):
+ if f and platform.system() == 'Windows' and not (f.is_absolute() or not f.drive):
+ error(f"\"Semi-absolute\" paths are disallowed: '{f}'", r"The path must "
+ r"either contain both the drive letter and start with '\', "
+ r"or none of these; paths like 'C:foo' and '\foo' are disallowed")
+
+def access_in(f):
+ return os.access(f, os.R_OK)
+
+# This function can fail on Windows, reporting a non-writable file or dir as
+# writable, because |os.access| does not work with Windows' |icacls|
+# permissions. Consequence: we might try to write to a read-only current or
+# output directory instead of switching to the temporary directory. Paranoia
+# is unaffected, as it doesn't use |access_*| functions.
+def access_out(f):
try:
- mmz = mmz_file.open()
- except FileNotFoundError:
- print(f'''{message_prefix}File "{mmz_file}" does not exist, assuming there's nothing to do.''',
- file = sys.stderr)
+ exists = f.exists()
+ # Presumably, we get this error when the parent directory is not
+ # executable.
+ except PermissionError:
+ return
+ if exists:
+ # An existing file should be writable, and if it's a directory, it
+ # should also be executable.
+ return os.access(f, os.W_OK) and (not f.is_dir() or os.access(f, os.X_OK))
else:
- if not args.quiet:
- print(f"{message_prefix}Extracting externs from {pdf_file}")
+ # For a non-existing file, the parent directory should be writable.
+ # (This is the only place where function |pathlib.parent| is used, so
+ # it's ok that it returns the logical parent.)
+ return os.access(f.parent, os.W_OK)
+
+# This function finds the location for an input file, respecting
+# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the
+# filesystem. It returns an absolute file as-is. For a relative file, it
+# tries |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (always),
+# and |TEXMFOUTPUT| directory (if defined), in this order. The first readable
+# file found is returned; if no readable file is found, the file in the current
+# directory is returned.
+def find_in(f):
+ sanitize_filename(f)
+ if f.is_absolute():
+ return f
+ for df in (texmf_output_directory / f if texmf_output_directory else None,
+ f,
+ texmfoutput / f if texmfoutput else None):
+ if df and access_in(df):
+ return df
+ return f
+
+# This function finds the location for an output file, respecting
+# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the
+# filesystem. It returns an absolute file as-is. For a relative file, it
+# tries |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (unless
+# |TEXMF_OUTPUT_DIRECTORY| is defined), and |TEXMFOUTPUT| directory (if
+# defined), in this order. The first writable file found is returned; if no
+# writable file is found, the file in either the current or the output
+# directory is returned.
+def find_out(f):
+ sanitize_filename(f)
+ if f.is_absolute():
+ return f
+ for df in (texmf_output_directory / f if texmf_output_directory else None,
+ f if not texmf_output_directory else None,
+ texmfoutput / f if texmfoutput else None):
+ if df and access_out(df):
+ return df
+ return texmf_output_directory / f if texmf_output_directory else f
+
+# This function assumes that both paths are absolute; ancestor may be |None|,
+# signaling a non-path.
+def is_ancestor(ancestor, descendant):
+ if not ancestor:
+ return
+ a = ancestor.parts
+ d = descendant.parts
+ return len(a) < len(d) and a == d[0:len(a)]
+
+# A paranoid |Path.mkdir|. The given folder is preprocessed by |find_out|.
+def mkdir(folder):
+ folder = find_out(Path(folder))
+ if not folder.exists():
+ paranoia_out(folder)
+ # Using |folder.mkdir| is fine because we know that
+ # |TEXMF_OUTPUT_DIRECTORY|/|TEXMFOUTPUT|, if given, exists, and that
+ # ``folder'' contains no |..|.
+ folder.mkdir(parents = True, exist_ok = True)
+ # This does not get logged when the function is invoked via |--mkdir|,
+ # as it is not clear what the log name should be.
+ info(f"Created directory {folder}")
+
+_re_unquote = re.compile(r'"(.*?)"')
+def unquote(fn):
+ return _re_unquote.sub(r'\1', fn)
+
+# \paragraph{Kpathsea}
+
+# Get the values of |openin_any|, |openout_any|, |TEXMFOUTPUT| and
+# |TEXMF_OUTPUT_DIRECTORY|.
+
+kpsewhich_output = subprocess.run(['kpsewhich',
+ f'-expand-var='
+ f'openin_any=$openin_any,'
+ f'openout_any=$openout_any,'
+ f'TEXMFOUTPUT=$TEXMFOUTPUT'],
+ capture_output = True
+ ).stdout.decode().strip()
+if not kpsewhich_output:
+ # No TeX? (Note that |kpsewhich| should exist in MiKTeX as well.) In
+ # absence of |kpathsea| information, we get very paranoid, but still try to
+ # get |TEXMFOUTPUT| from an environment variable.
+ openin_any, openout_any = 'p', 'p'
+ texmfoutput, texmf_output_directory = None, None
+ # Unfortunately, this warning can't make it into the log. But then again,
+ # the chances of a missing |kpsewhich| are very slim, and its absence would
+ # show all over the place anyway.
+ warning('I failed to execute "kpsewhich"; , is there no TeX system installed? '
+ 'Assuming openin_any = openout_any = "p" '
+ '(i.e. restricting all file operations to non-hidden files '
+ 'in the current directory of its subdirectories).')
+else:
+ m = re.fullmatch(r'openin_any=(.*),openout_any=(.*),TEXMFOUTPUT=(.*)',
+ kpsewhich_output)
+ openin_any, openout_any, texmfoutput = m.groups()
+ texmf_output_directory = os.environ.get('TEXMF_OUTPUT_DIRECTORY', None)
+ if openin_any == '$openin_any':
+ # When the |open*_any| variables are not expanded, we assume we're
+ # running MiKTeX. The two config settings below correspond to TeXLive's
+ # |openin_any| and |openout_any|; afaik, there is no analogue to
+ # |TEXMFOUTPUT|.
+ initexmf_output = subprocess.run(
+ ['initexmf', '--show-config-value=[Core]AllowUnsafeInputFiles',
+ '--show-config-value=[Core]AllowUnsafeOutputFiles'],
+ capture_output = True).stdout.decode().strip()
+ openin_any, openout_any = initexmf_output.split()
+ openin_any = 'a' if openin_any == 'true' else 'p'
+ openout_any = 'a' if openout_any == 'true' else 'p'
+ texmfoutput = None
+ texmf_output_directory = None
+
+# An output directory should exist, and may not point to the root on Linux. On
+# Windows, it may point to the root, because we only allow absolute filenames
+# containing the drive, e.g.\ |F:\|; see |is_absolute|.
+def sanitize_output_dir(d_str):
+ d = Path(d_str) if d_str else None
+ sanitize_filename(d)
+ return d if d and d.is_dir() and \
+ (not d.is_absolute() or len(d.parts) != 1 or d.drive) else None
+
+texmfoutput = sanitize_output_dir(texmfoutput)
+texmf_output_directory = sanitize_output_dir(texmf_output_directory)
+
+class NotExtracted(UserWarning):
+ pass
+
+# We don't delve into the real script when loaded from the testing code.
+if __name__ == '__main__':
+
+ # \paragraph{Arguments}
+
+ parser = argparse.ArgumentParser(
+ description = "Extract extern pages produced by package Memoize "
+ "out of the document PDF.",
+ epilog = "For details, see the man page or the Memoize documentation.",
+ prog = 'memoize-extract.py',
+ )
+ parser.add_argument('-P', '--pdf', help = 'extract from file PDF')
+ parser.add_argument('-p', '--prune', action = 'store_true',
+ help = 'remove the extern pages after extraction')
+ parser.add_argument('-k', '--keep', action = 'store_true',
+ help = 'do not mark externs as extracted')
+ parser.add_argument('-F', '--format', choices = ['latex', 'plain', 'context'],
+ help = 'the format of the TeX document invoking extraction')
+ parser.add_argument('-f', '--force', action = 'store_true',
+ help = 'extract even if the size-check fails')
+ parser.add_argument('-q', '--quiet', action = 'store_true',
+ help = "describe what's happening")
+ parser.add_argument('-m', '--mkdir', action = 'store_true',
+ help = 'create a directory (and exit); '
+ 'mmz argument is interpreted as directory name')
+ parser.add_argument('-V', '--version', action = 'version',
+ version = f"%(prog)s of Memoize " + __version__)
+ parser.add_argument('mmz', help = 'the record file produced by Memoize: '
+ 'doc.mmz when compiling doc.tex '
+ '(doc and doc.tex are accepted as well)')
+
+ args = parser.parse_args()
+
+ header = parser.prog + ': ' if args.format else ''
+
+ # Start a new line in the TeX terminal output.
+ if args.format:
+ print()
+
+ # \paragraph{Initialization}
+
+ # With |--mkdir|, argument |mmz| is interpreted as the directory to create.
+ if args.mkdir:
+ mkdir(args.mmz)
+ sys.exit()
+
+ # Normalize the |mmz| argument into a |.mmz| filename.
+ mmz_file = Path(args.mmz)
+ if mmz_file.suffix == '.tex':
+ mmz_file = mmz_file.with_suffix('.mmz')
+ elif mmz_file.suffix != '.mmz':
+ mmz_file = mmz_file.with_name(mmz_file.name + '.mmz')
+
+ # Once we have the |.mmz| filename, we can open the log.
+ if args.format:
+ log_file = find_out(mmz_file.with_suffix('.mmz.log'))
+ paranoia_out(log_file)
+ info(f"Logging to '{log_file}'");
+ log = open(log_file, 'w')
+
+ # Now that we have opened the log file, we can try loading the PDF
+ # processing library.
+ try:
+ import pdfrw
+ except ModuleNotFoundError:
+ error("Python module 'pdfrw' was not found",
+ 'Have you followed the instructions is section 1.1 of the manual?')
+
+ # Catch any errors in the script and output them to the log.
+ try:
+
+ # Find the |.mmz| file we will read, but retain the original filename
+ # in |given_mmz_file|, as we will still need it.
+ given_mmz_file = mmz_file
+ mmz_file = find_in(mmz_file)
+ paranoia_in(mmz_file)
+ if not args.keep:
+ paranoia_out(mmz_file,
+ remark = 'This file is rewritten unless option --keep is given.')
+ try:
+ mmz = open(mmz_file)
+ except FileNotFoundError:
+ info(f"File '{given_mmz_file}' does not exist, "
+ f"assuming there's nothing to do")
+ endinput()
+
+ # Determine the PDF filename: it is either given via |--pdf|, or
+ # constructed from the |.mmz| filename.
+ pdf_file = find_in(Path(args.pdf)
+ if args.pdf else given_mmz_file.with_suffix('.pdf'))
+ paranoia_in(pdf_file)
+ if args.prune:
+ paranoia_out(pdf_file,
+ remark = 'I would have to rewrite this file '
+ 'because option --prune was given.')
+
+ # Various initializations.
+
+ re_prefix = re.compile(r'\\mmzPrefix *{(?P<prefix>.*?)}')
+ re_split_prefix = re.compile(r'(?P<dir_prefix>.*/)?(?P<name_prefix>.*?)')
+ re_newextern = re.compile(
+ r'\\mmzNewExtern *{(?P<extern_path>.*?)}{(?P<page_n>[0-9]+)}'
+ r'{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}')
+ re_extern_path = re.compile(
+ r'(?P<dir_prefix>.*/)?(?P<name_prefix>.*?)'
+ r'(?P<code_md5sum>[0-9A-F]{32})-'
+ r'(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf')
+ pdf = None
+ extern_pages = []
+ new_mmz = []
+ tolerance = 0.01
+ dir_to_make = None
+ info(f"Extracting new externs listed in '{mmz_file}' from '{pdf_file}'")
+ done_message = "Done (there was nothing to extract)"
+ indent = ' '
+ texindent = '\space\space '
+
+ # \paragraph{Process \texttt{.mmz}}
+
for line in mmz:
- if m := re_newextern.match(line):
- extern_file = mmz_dir / m['extern_fn']
- paranoia(extern_file)
- page_n = int(m['page_n'])-1
- c_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '.memo')
- cc_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '-' + m['context_md5sum'] + '.memo')
- if not (c_memo.exists() and cc_memo.exists()):
- print(args.warning_template.replace('\warningtext', f'Not extracting page {page_n} into extern {extern_file}, because the associated (c)c-memo does not exist'), file = log)
- continue
- if not pdf:
- try:
- pdf = pdfrw.PdfReader(pdf_file)
- except pdfrw.errors.PdfParseError:
- print(f'{message_prefix}File "{pdf_file}" cannot be read, bailing out.', file = sys.stderr)
- print(args.warning_template.replace('\warningtext', f'Cannot read file "{pdf_file}". Perhaps you have to load Memoize earlier in the preamble?'), file = log)
- args.keep = True
- break
- extern = pdfrw.PdfWriter(extern_file)
- page = pdf.pages[page_n]
- expected_width_pt, expected_height_pt = float(m['expected_width']), float(m['expected_height'])
- mb = page['/MediaBox']
- width_bp, height_bp = float(mb[2]) - float(mb[0]), float(mb[3]) - float(mb[1])
- width_pt = width_bp / 72 * 72.27
- height_pt = height_bp / 72 * 72.27
- warning = None
- if abs(width_pt - expected_width_pt) > tolerance \
- or abs(height_pt - expected_height_pt) > tolerance:
- warning = (
- f'I refuse to extract page {page_n+1} from "{pdf_file}", '
- f'because its size ({width_pt}pt x {height_pt}pt) is not '
- f'what I expected ({expected_width_pt}pt x {expected_height_pt}pt)')
- print(args.warning_template.replace('\warningtext', warning), file = log)
- if warning and not args.force:
- extern_file.unlink(missing_ok = True)
- else:
+ try:
+ if m_p := re_prefix.match(line):
+ # Found |\mmzPrefix|: create the extern directory, but only
+ # later, if an extern file is actually produced. We parse
+ # the prefix in two steps because we have to unquote the
+ # entire prefix.
+ prefix = unquote(m_p['prefix'])
+ if not (m_sp := re_split_prefix.match(prefix)):
+ warning(f"Cannot parse line {line.strip()}")
+ dir_to_make = m_sp['dir_prefix']
+ elif m_ne := re_newextern.match(line):
+ # Found |\mmzNewExtern|: extract the extern page into an
+ # extern file.
+ done_message = "Done"
+ # The extern filename, as specified in |.mmz|:
+ unquoted_extern_path = unquote(m_ne['extern_path'])
+ extern_file = Path(unquoted_extern_path)
+ # We parse the extern filename in a separate step because
+ # we have to unquote the entire path.
+ if not (m_ep := re_extern_path.match(unquoted_extern_path)):
+ warning(f"Cannot parse line {line.strip()}")
+ # The actual extern filename:
+ extern_file_out = find_out(extern_file)
+ paranoia_out(extern_file_out)
+ page_n = int(m_ne['page_n'])-1
+ # Check whether c-memo and cc-memo exist (in any input
+ # directory).
+ c_memo = extern_file.with_name(
+ m_ep['name_prefix'] + m_ep['code_md5sum'] + '.memo')
+ cc_memo = extern_file.with_name(
+ m_ep['name_prefix'] + m_ep['code_md5sum']
+ + '-' + m_ep['context_md5sum'] + '.memo')
+ c_memo_in = find_in(c_memo)
+ cc_memo_in = find_in(cc_memo)
+ if not (access_in(c_memo_in) and access_in(cc_memo_in)) \
+ and not args.force:
+ warning(f"I refuse to extract page {page_n+1} into extern "
+ f"'{extern_file}', because the associated c-memo "
+ f"'{c_memo}' and/or cc-memo '{cc_memo}' "
+ f"does not exist")
+ raise NotExtracted()
+ # Load the PDF. We only do this now so that we don't load
+ # it if there is nothing to extract.
+ if not pdf:
+ if not access_in(pdf_file):
+ warning(f"Cannot open '{pdf_file}'")
+ endinput()
+ try:
+ # All safe, |paranoia_in| was already called above.
+ pdf = pdfrw.PdfReader(pdf_file)
+ except pdfrw.errors.PdfParseError as err:
+ error(rf"File '{pdf_file}' seems corrupted. Perhaps you "
+ rf"have to load Memoize earlier in the preamble",
+ f"In particular, Memoize must be loaded before "
+ f"TikZ library 'fadings' and any package "
+ f"deploying it, and in Beamer, load Memoize "
+ f"by writing \RequirePackage{{memoize}} before "
+ f"\documentclass{{beamer}}. "
+ f"This was the error thrown by Python: \n{err}")
+ # Does the page exist?
+ if page_n >= len(pdf.pages):
+ error(rf"I cannot extract page {page_n} from '{pdf_file}', "
+ rf"as it contains only {len(pdf.pages)} page" +
+ ('s' if len(pdf.pages) > 1 else ''), '')
+ # Check whether the page size matches the |.mmz|
+ # expectations.
+ page = pdf.pages[page_n]
+ expected_width_pt = float(m_ne['expected_width'])
+ expected_height_pt = float(m_ne['expected_height'])
+ mb = page['/MediaBox']
+ width_bp = float(mb[2]) - float(mb[0])
+ height_bp = float(mb[3]) - float(mb[1])
+ width_pt = width_bp / 72 * 72.27
+ height_pt = height_bp / 72 * 72.27
+ if (abs(width_pt - expected_width_pt) > tolerance
+ or abs(height_pt - expected_height_pt) > tolerance) \
+ and not args.force:
+ warning(
+ f"I refuse to extract page {page_n+1} from '{pdf_file}' "
+ f"because its size ({width_pt}pt x {height_pt}pt) "
+ f"is not what I expected "
+ f"({expected_width_pt}pt x {expected_height_pt}pt)")
+ raise NotExtracted()
+ # All tests were successful, let's create the extern file.
+ # First, the containing directory, if necessary.
+ if dir_to_make:
+ mkdir(dir_to_make)
+ dir_to_make = None
+ # Now the extern file. Note that |paranoia_out| was
+ # already called above.
+ info(f"Page {page_n+1} --> {extern_file_out}")
+ extern = pdfrw.PdfWriter(extern_file_out)
extern.addpage(page)
- if not args.quiet:
- print(f"{message_prefix} Page {page_n+1} --> {extern_file}", file = sys.__stdout__)
extern.write()
- done_message = f"{message_prefix}Done."
+ # This page will get pruned.
if args.prune:
extern_pages.append(page_n)
+ # Comment out this |\mmzNewExtern|.
if not args.keep:
line = '%' + line
- if not args.keep:
- new_mmz.append(line)
+ except NotExtracted:
+ pass
+ finally:
+ if not args.keep:
+ new_mmz.append(line)
mmz.close()
- if not args.quiet:
- print(done_message)
+ indent = ''
+ texindent = ''
+ info(done_message)
+
+ # Write out the |.mmz| file with |\mmzNewExtern| lines commented
+ # out. (All safe, |paranoia_out| was already called above.)
if not args.keep:
- paranoia(mmz_file)
with open(mmz_file, 'w') as mmz:
for line in new_mmz:
print(line, file = mmz, end = '')
+
+ # Remove the extracted pages from the original PDF. (All safe,
+ # |paranoia_out| was already called above.)
if args.prune and extern_pages:
pruned_pdf = pdfrw.PdfWriter(pdf_file)
pruned_pdf.addpages(
page for n, page in enumerate(pdf.pages) if n not in extern_pages)
pruned_pdf.write()
- if not args.quiet:
- print(f"{message_prefix}The following extern pages were pruned out of the PDF:",
- ",".join(str(page+1) for page in extern_pages))
- if args.log is not os.devnull:
- print(r'\endinput', file = log)
+ info(f"The following extern pages were pruned out of the PDF: " +
+ ",".join(str(page+1) for page in extern_pages))
+
+ # Report that extraction was successful.
+ endinput()
+
+ # Catch any errors in the script and output them to the log.
+ except Exception as err:
+ error(f'Python error: {err}', traceback.format_exc())
+
+# Local Variables:
+# fill-column: 79
+# after-save-hook: py2dtx
+# End:
Added: trunk/Master/texmf-dist/doc/generic/memoize/CHANGELOG.md
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/CHANGELOG.md (rev 0)
+++ trunk/Master/texmf-dist/doc/generic/memoize/CHANGELOG.md 2024-01-03 21:18:51 UTC (rev 69285)
@@ -0,0 +1,59 @@
+# Memoize changelog
+
+For the development history, see [Memoize's GitHub
+repository](https://github.com/sasozivanovic/memoize).
+
+## 2024/1/02 v1.1.0
+
+* Improve the extraction scripts:
+ * respect `TEXMF_OUTPUT_DIRECTORY`;
+ * respect `openin_any` and `openout_any`;
+ * implement `--format`;
+ * improve error reporting;
+ * drop the `Path::Class` dependency for the Perl script;
+ * allow for `PDF::Builder` in the Perl script;
+ * implement `--library` in the Perl script;
+ * set an appropriate exit code on exit;
+ * and several further minor changes.
+
+* Remove key `path` in favour of `prefix`.
+
+* `mkdir` is now initially `true`, but the directory is only created if `mkdir
+ command` is non-empty (and it is empty initially). The definition of `(no)
+ memo dir` is accordingly simpler.
+
+* The directory name is now appended to the value `mkdir command` when
+ constructing the system call.
+
+* A workaround for compatibility with package `morewrites`.
+
+* Process package options using the new LaTeX mechanism to avoids the issue of
+ spaces in package options. The remaining issue of `/` is addressed by
+ implementing option `options`.
+
+* Add the missing commands to `nomemoize` and `memoizable`, and implement a
+ generic variant of the latter (`memoizable.code.tex`).
+
+* Implement auto-key `to context`.
+
+* Write a c-memo even upon abortion.
+
+* Demote warning messages "memoization aborted" & "marked as unmemoizable" to
+ info messages.
+
+* Implement biblatex support.
+
+* Support `\DiscardShipoutBox`.
+
+* Advance the counter underlying `\pgfpictureid` when utilizing a tikzpicture
+ (`memoize tikz`).
+
+* Remove the `\pgfsys at getposition` hack for `tikzpicture`s.
+
+## 2023/10/10 v1.0.0
+
+* A complete, fully documented reimplementation.
+
+## 2020/07/17 v0.1
+
+* The proof of concept, available only at GitHub.
Property changes on: trunk/Master/texmf-dist/doc/generic/memoize/CHANGELOG.md
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Modified: trunk/Master/texmf-dist/doc/generic/memoize/FILES
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/FILES 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/generic/memoize/FILES 2024-01-03 21:18:51 UTC (rev 69285)
@@ -7,19 +7,20 @@
doc/generic/memoize/memoize-doc.sty
doc/generic/memoize/LICENCE
doc/generic/memoize/INSTALL.md
+doc/generic/memoize/memoize-doc.pdf
doc/generic/memoize/memoize-doc-common.sty
doc/generic/memoize/memoize-code.sty
doc/generic/memoize/yadoc.sty
+doc/generic/memoize/memoize-doc.mst
doc/generic/memoize/memoize-clean.1.md
-doc/generic/memoize/memoize.tex
doc/generic/memoize/memoize-code.pdf
doc/generic/memoize/memoize-code.tex
doc/generic/memoize/memoize-extract.1.md
-doc/generic/memoize/memoize.mst
+doc/generic/memoize/CHANGELOG.md
+doc/generic/memoize/memoize-doc.tex
doc/generic/memoize/examples.zip
doc/generic/memoize/examples-src.zip
doc/generic/memoize/FILES
-doc/generic/memoize/memoize.pdf
doc/generic/memoize/README.md
scripts/memoize/memoize-clean.pl
scripts/memoize/memoize-extract.py
@@ -28,13 +29,14 @@
tex/latex/memoize/memoize.sty
tex/latex/memoize/memoizable.sty
tex/latex/memoize/nomemoize.sty
-tex/context/memoize/t-nomemoize.tex
-tex/context/memoize/t-memoizable.tex
-tex/context/memoize/t-memoize.tex
+tex/context/third/memoize/t-nomemoize.tex
+tex/context/third/memoize/t-memoizable.tex
+tex/context/third/memoize/t-memoize.tex
tex/plain/memoize/nomemoize.tex
tex/plain/memoize/memoize-extract-one.tex
tex/plain/memoize/memoizable.tex
tex/plain/memoize/memoize.tex
+tex/generic/memoize/memoizable.code.tex
source/generic/memoize/memoize.edtx
source/generic/memoize/memoize.ins
source/generic/memoize/Makefile
Modified: trunk/Master/texmf-dist/doc/generic/memoize/examples-src.zip
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/doc/generic/memoize/examples.zip
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/doc/generic/memoize/memoize-clean.1.md
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize-clean.1.md 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize-clean.1.md 2024-01-03 21:18:51 UTC (rev 69285)
@@ -2,8 +2,8 @@
title: memoize-clean
section: 1
header: User Manual
-footer: memoize-clean of Memoize v1.0.0
-date: October 10, 2023
+footer: memoize-clean of Memoize v1.1.0
+date: January 02, 2024
hyphenate: false
---
Modified: trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.pdf
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.sty
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.sty 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.sty 2024-01-03 21:18:51 UTC (rev 69285)
@@ -23,7 +23,8 @@
\RequirePackage{memoize-doc-common}
% Typesetting the code & indentation
-\MacroIndent=0pt
+%\MacroIndent=0pt
+\AddToHook{begindocument}{\MacroIndent=0pt }
\MacroTopsep = 0pt
\MacrocodeTopsep = 7pt plus 2pt minus 2pt
\AddToHook{env/macrocode/before}{\macrocodepar}%
@@ -137,22 +138,33 @@
\newcommand\hang[2]{%
\setbox0=\hbox{\MacroFont \string#2\relax}% "\relax" is short enough to never trigger a hanging indent
\ifdim\wd0>\dimexpr\marginparwidth\relax
- \hangindent\dimexpr\wd0-\marginparwidth\relax
+ \hangindent\dimexpr\wd0-\marginparwidth+\hang at next@extra\relax
\hangafter-#1\relax
+ \gdef\hang at next@extra{0pt}%
\AddToHookNext{env/macrocode/before}{\par\nohang}%
\AddToHookNext{para/after}{\aftergroup\nohang}%
\fi
\ignorespaces
}
-\newcommand\nohang{\hangindent 0pt\relax}
-\newcommand\indentmacrocode{%
- \settowidth\MacroIndent{\theCodelineNo\ }%
- \addtolength\MacroIndent{\hangindent}%
+\def\AdjustNextHang#1{\gdef\hang at next@extra{#1}\ignorespaces}
+\gdef\hang at next@extra{0pt}
+\newcommand\nohang{%
+ \hangindent 0pt\relax
+ \gdef\hang at next@extra{0pt}%
}
+\newcommand\indentmacrocode[1][0pt]{%
+ \settowidth\@tempdima{\reset at font\scriptsize\arabic{CodelineNo}\ }%
+ \setlength\ExtraMacroIndent{#1}%
+ \addtolength\MacroIndent{\dimexpr\hangindent+\@tempdima+\ExtraMacroIndent}%
+}
+\newdimen\ExtraMacroIndent
\newcommand\noindentmacrocode{%
\setlength\MacroIndent{0em}%
\AddToHookNext{env/macrocode/begin}{\MacrocodeTopsep 0pt\relax}%
}
+\def\theCodelineNo{%
+ \reset at font\scriptsize\textcolor{gray}{\arabic{CodelineNo}}%
+ \hspace*{\dimexpr\leftskip-\MacroIndent+\ExtraMacroIndent}}%
% and now ... autohang!
% (the macros should be listed without spaces after commas)
@@ -165,6 +177,7 @@
advice=\macro{hang},
advice=\autokey{hang},
advice=\mmzautokey{hang},
+ advice=\advicekey{hang},
advice=\collargskey{hang},
advice=\handlerskey{hang},
advice=\adviceinstallkey{hang},
@@ -204,7 +217,7 @@
\def\doline{%
\expandafter\futurelet\expandafter\firsttoken\expandafter\dolinei\line\fi}
\def\dolinei{\ifx\firsttoken\newlabel}%
-\openin0=memoize.aux
+\openin0=memoize-doc.aux
\loop
\unless\ifeof0
\read0 to \line
Modified: trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.tex
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.tex 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize-code.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -61,47 +61,23 @@
\DocInput{\docdir/collargs.dtx}
-\clearpage
-\restoregeometry
-
\section{The scripts}
\label{sec:code:scripts}
-\def\continuation{\llap{\textcolor{purple}{$\rightarrow$}\,}}
-
\subsection{The Perl extraction script \texttt{memoize-extract.pl}}
+\DocInput{\docdir/memoize-extract.pl.dtx}
-\lstinputlisting[firstline=20, % manual (exclude the license information)
- language=Perl,
- breaklines, breakindent=4em, postbreak=\continuation,
- commentstyle=,
-]{../memoize-extract.pl}
-
\subsection{The Python extraction script \texttt{memoize-extract.py}}
+\DocInput{\docdir/memoize-extract.py.dtx}
-\lstinputlisting[firstline=20, % manual (exclude the license information)
- language=Python,
- breaklines, breakindent=4em, postbreak=\continuation,
- commentstyle=,
-]{../memoize-extract.py}
-
\subsection{The Perl clean-up script \texttt{memoize-clean.pl}}
+\DocInput{\docdir/memoize-clean.pl.dtx}
-\lstinputlisting[firstline=20, % manual (exclude the license information)
- language=Perl,
- breaklines, breakindent=4em, postbreak=\continuation,
- commentstyle=,
-]{../memoize-clean.pl}
-
\subsection{The Python clean-up script \texttt{memoize-clean.py}}
+\DocInput{\docdir/memoize-clean.py.dtx}
-\lstinputlisting[firstline=20, % manual (exclude the license information)
- language=Python,
- breaklines, breakindent=4em, postbreak=\continuation,
- commentstyle=,
-]{../memoize-clean.py}
-
\clearpage
+\restoregeometry
\IndexPrologue{\section*{Index}Numbers written in red refer to the code line
where the corresponding entry is defined; numbers in blue refer to the code
Added: trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.mst
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.mst (rev 0)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.mst 2024-01-03 21:18:51 UTC (rev 69285)
@@ -0,0 +1 @@
+encap '`'
Added: trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.pdf
===================================================================
(Binary files differ)
Index: trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.pdf
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.pdf 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.pdf 2024-01-03 21:18:51 UTC (rev 69285)
Property changes on: trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.pdf
___________________________________________________________________
Added: svn:mime-type
## -0,0 +1 ##
+application/pdf
\ No newline at end of property
Modified: trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.sty
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.sty 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.sty 2024-01-03 21:18:51 UTC (rev 69285)
@@ -406,7 +406,7 @@
\let\mmzPrefix\gobble
\let\mmzNewCMemo\gobble
\let\mmzNewCCMemo\gobble
- \def\mmzNewExtern##1##2##3##4##5{}%
+ \def\mmzNewExtern##1##2##3##4{}%
\let\mmzUsedCMemo\gobble
\let\mmzUsedCCMemo\gobble
\let\mmzUsedExtern\gobble
@@ -683,7 +683,7 @@
\ifregion{%
\AddToHook{begindocument/before}{%
\makeatletter
- \InputIfFileExists{memoize.aux}{}{}%
+ \InputIfFileExists{memoize-doc.aux}{}{}%
\makeatother
}%
\AddToHook{enddocument}{%
@@ -706,6 +706,6 @@
% \captionsetup{labelformat=empty}
% Local Variables:
-% TeX-master: "memoize.tex"
+% TeX-master: "memoize-doc.tex"
% TeX-engine: luatex
% End:
Added: trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.tex
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.tex (rev 0)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -0,0 +1,9314 @@
+\documentclass[a4paper,11pt]{article}
+\usepackage[margin=2cm,top=1cm,bottom=1cm,nohead,includefoot]{geometry}
+
+\usepackage{nomemoize}
+\usepackage{memoize-doc}
+\mmzset{
+ % recompile,
+ % readonly,
+ % disable,
+ % inverse search=false,
+ % include context in ccmemo,
+ % direct ccmemo input,
+ % trace,
+}
+\def\exampledir{examples/}
+
+\title{\pkg[white]{Memoize}}
+\author{Sašo Živanović\\[2mm]
+ \emailsymbol~\url(mailto:){saso.zivanovic at guest.arnes.si}\\
+ \homepagesymbol~\url(http://){spj.ff.uni-lj.si/zivanovic}\\
+ \faGithub~\url(https://){github.com/sasozivanovic}}
+\datefrompackageversion{memoize}
+\preto\packagever{v}
+
+\hypersetup{
+ pdftitle={Memoize},
+ pdfauthor={Sašo Živanović},
+ pdfsubject={externalization},
+ pdfkeywords={LaTeX, externalization, memoization}
+}
+
+\usepackage{makeidx}
+\pdfsystem{makeindex -s memoize-doc.mst \jobname.idx}
+\makeindex
+\usepackage[hangindent=1em,justific=raggedright,font=small]{idxlayout}
+\appto\indexfont{\def\pkgcolor{gray}}
+
+\AddToHook{begindocument}{% so that the region compilation takes these in
+ \docForeign{
+ scope={
+ after/.style={index annotation/.prefix={\hologo{LaTeX}\space}},
+ cmd={name=begin}, cmd={name=end},
+ cmd={name=label}, cmd={name=@currentlabel},
+ cmd={name=ref}, cmd={name=pageref},
+ cmd={name=index},
+ cmd={name=NewDocumentCommand}, cmd={name=GetDocumentCommandArgSpec},
+ env={name=verbatim},
+ cmd={name=ReadonlyShipoutCounter},
+ hook={name=begindocument},
+ cmd={name=usepackage},
+ },
+ scope={
+ before/.style={index annotation=\hologo{TeX} primitive},
+ cmd={name=shipout},
+ cmd={name=hbox}, cmd={name=vbox},
+ cmd={name=ignorespaces}, cmd={name=unskip},
+ cmd={name=errmessage},
+ cmd={name=catcode},
+ cmd={name=mag},
+ },
+ scope={
+ before/.style={index annotation=\hologo{eTeX} primitive},
+ cmd={name=scantokens},
+ cmd={name=currentgrouplevel},
+ },
+ scope={
+ before/.style={index annotation=\hologo{pdfTeX} primitive},
+ cmd={name=quitvmode},
+ },
+ scope={
+ before/.style={index annotation=\hologo{LuaTeX} primitive},
+ pdfcmd={name=savepos},
+ cmd={name=pdfvariable},
+ },
+ pdfvariable={name=compresslevel},
+ pdfvariable={name=horigin},
+ pdfvariable={name=vorigin},
+ scope={
+ after/.style={index annotation/.prefix={\hologo{ConTeXt}\space}},
+ cmd={name=realpageno},
+ },
+ pdfcmd={name=primitive, index annotation=\hologo{LuaTeX}/\hologo{XeTeX} primitive},
+ register={name=pdfmajorversion},
+ register={name=pdfminorversion},
+ scope={
+ after/.style={index annotation/.prefix={\hologo{plainTeX}\space}},
+ cmd={name=llap}, cmd={name=rlap},
+ },
+ scope={
+ pkg=forest,
+ env={name=forest},
+ cmd={name=Forest},
+ },
+ scope={
+ pkg=tikz, keypath=/tikz,
+ cmd={name=tikz},
+ env={name=tikzpicture},
+ key={name=remember picture},
+ key={name=overlay},
+ cmd={name=tikzexternalize},
+ cmd={name=pgfsys at getposition},
+ },
+ file={name=texmf.cnf},
+ env={name=tcblisting, pkg=tcolorbox},
+ cmd={name=vref, pkg=varioref},
+ cmd={name=cref, pkg=cleveref},
+ cmd={name=crefrange, pkg=cleveref},
+ xparse type={name=l, index annotation/.append={\ (up to begin-group)}},
+ xparse type={name=u, index annotation/.append={\ (until tokens)}},
+ xparse type={name=r, index annotation/.append={\ (required delimited)}},
+ xparse type={name=R, index annotation/.append={\ (required delimited with default)}},
+ xparse type={name=d, index annotation/.append={\ (optional delimited)}},
+ xparse type={name=D, index annotation/.append={\ (optional delimited with default)}},
+ xparse type={name=o, index annotation/.append={\ (standard optional)}},
+ xparse type={name=O, index annotation/.append={\ (standard optional with default)}},
+ xparse type={name=t, index annotation/.append={\ (optional token)}},
+ xparse type={name=s, index annotation/.append={\ (optional star)}},
+ xparse type={name=m, index annotation/.append={\ (mandatory)}},
+ xparse type={name=g, index annotation/.append={\ (optional group)}},
+ xparse type={name=G, index annotation/.append={\ (optional group with default)}},
+ xparse type={name=v, index annotation/.append={\ (verbatim)}},
+ xparse type={name=e, index annotation/.append={\ (embellishments)}},
+ xparse type={name=E, index annotation/.append={\ (embellishments with defaults)}},
+ xparse modifier={name=+, index annotation/.append={\ (long)}},
+ xparse modifier={name=!, index annotation/.append={\ (don't ignore spaces)}},
+ xparse modifier={name=>, index annotation/.append={\ (argument processor)}},
+ generic={name=shell-escape, name prefix=-, label prefix:=-,
+ index annotation=option of \hologo{TeX} binaries},
+ generic={name=enable-write18, name prefix=--, label prefix:=--,
+ index annotation=option of \hologo{TeX} binaries},
+ generic={name=TEXMFOUTPUT, index annotation={environment variable,
+ or variable in \reffile[into index=false]{texmf.cnf}}},
+ generic={name=TEXMF\_OUTPUT\_DIRECTORY, label=TEXMF_OUTPUT_DIRECTORY,
+ index annotation={environment variable, or variable in
+ \reffile[into index=false]{texmf.cnf}}},
+ generic={name=openout\_any, label=openout_any,
+ index annotation={variable in \reffile[into index=false]{texmf.cnf}}},
+ generic={name=openin\_any, label=openin_any,
+ index annotation={variable in \reffile[into index=false]{texmf.cnf}}},
+ }%
+ \docindex{%
+ cmd={name=pdfprimitive, see={\refcmd[into index=false,short]{primitive}}},
+ cmd={name=pdfsavepos, see={\refcmd[into index=false,short]{savepos}}},
+ cmd={name=pdfhorigin, see={\docref[into index=false]{pdfvar:horigin}}},
+ cmd={name=pdfvorigin, see={\docref[into index=false]{pdfvar:vorigin}}},
+ cmd={name=pdfcompresslevel, see={\docref[into index=false]{pdfvar:compresslevel}}},
+ pdfvariable={name=majorversion, see={\docref[into index=false,long]{reg:pdfmajorversion}}},
+ pdfvariable={name=minorversion, see={\docref[into index=false,long]{reg:pdfminorversion}}},
+ }%
+}
+
+\begin{document}
+
+
+\maketitle
+\thispagestyle{empty}
+
+\begin{abstract}
+ Memoize is a package for externalization of graphics and memoization of
+ compilation results in general, allowing the author to reuse the results of
+ compilation-intensive code.
+ Memoize
+ \begin{enumerate*}[(i)]
+ \item induces very little overhead, as all externalized graphics is produced
+ in a single compilation. It features
+ \item automatic recompilation upon the change of code or user-adjustable
+ context, and
+ \item automatic externalization of \TikZ pictures and Forest trees, easily
+ extensible to other commands and environments. Furthermore, Memoize
+ \item supports cross-referencing, \TikZ overlays and Beamer,
+ \item works with all major engines and formats, and
+ \item is adaptable to any workflow.
+ \end{enumerate*}
+\end{abstract}
+
+\vfill
+
+\ExampleName{titlepage}
+\makeexample{\examplename.pdf N=2} % N=2, because we show the .mmz file later
+\begingroup
+\tcbset{page box/.style={
+ box align=center,boxsep=2mm,top=0pt,bottom=0pt,left=0pt,right=0pt,
+ enhanced, remember as=#1,
+ }}%
+\begin{tcboxedraster}
+ [raster columns=1]
+ {title=The two steps of externalization of graphics in Memoize}
+ \tcbinputexample{
+ title=\tt doc.tex,
+ listing only,
+ remember as=tex,
+ }
+ \begin{tcolorbox}[example title=doc.pdf, no attachment, remember as=pdf1]
+ \includeexamplepdf[page box=extern1]{page=1,trim=1in 1in 1in 1in}\hfill
+ \includeexamplepdf[page box=extern2]{page=2,trim=1in 1in 1in 1in}\hfill
+ \includeexamplepdf[page box=extern3]{page=3,trim=1in 1in 1in 1in}\hfill
+ \includeexamplepdf[page box=docpage]{page=4}%
+ \end{tcolorbox}
+ \vspace{10mm}
+ \begin{tcboxedraster}[raster columns=3, raster column skip=1cm]{blankest}
+ \begin{tcolorbox}[example title=.pdf]\centering
+ \includeexamplepdf[page box=externfile1]{page=1,trim=1in 1in 1in 1in}\hfill
+ \end{tcolorbox}
+ \begin{tcolorbox}[example title=.pdf]\centering
+ \includeexamplepdf[page box=externfile2]{page=2,trim=1in 1in 1in 1in}\hfill
+ \end{tcolorbox}
+ \begin{tcolorbox}[example title=.pdf]\centering
+ \includeexamplepdf[page box=externfile3]{page=3,trim=1in 1in 1in 1in}\hfill
+ \end{tcolorbox}
+ \end{tcboxedraster}
+\end{tcboxedraster}
+
+\begin{tcolorbox}[title=Using the externalized graphics]
+ \begin{tcolorbox}[example title=doc.pdf,
+ halign=center, remember as=pdf2]
+ \tcbox[remember, enhanced]{\footnotesize
+ We all love Ti\emph{k}Zlings!
+ \def\mytikzling#1#2{%
+ \tcbox[blankest, baseline=1mm, remember as=used extern #1]
+ {\tikz[x=0.3cm,y=0.3cm]{#2;}}}%
+ \mytikzling1{\penguin} is a penguin,
+ \mytikzling2{\koala} is a koala, and
+ \mytikzling3{\owl} is an owl.
+ }
+ \end{tcolorbox}
+\end{tcolorbox}
+
+\vfill
+
+\begin{tcolorbox}[colback=emphcolor, fontupper=\footnotesize]
+ This manual also documents packages Advice
+ ({\datefrompackageversion{advice}v\packagever}) and CollArgs
+ ({\datefrompackageversion{collargs}v\packagever}). These are auxiliary
+ packages which were developed alongside Memoize, but are distributed as
+ independent packages as they might be useful outside the context of Memoize,
+ as well. See sections~\ref{sec:advice} and \ref{sec:ref:advice} for Advice, and
+ sections~\ref{sec:collargs} and \ref{sec:ref:collargs} for CollArgs.
+ \begin{tikzpicture}[remember picture, overlay]
+ \draw[->, snake arrow, thick, red] (tex.west) to[out=-90-60, in=90+60] (pdf1.west);
+ \draw[->, snake arrow, thick, red] (tex.east) to[out=-90+19, in=90-19] (pdf2.east);
+ \draw[->, thick, red] (extern1.south) to[out=south, in=north] (externfile1);
+ \draw[->, thick, red] (extern2.south) to[out=south, out looseness=1.1, in=north, in looseness=0.9] (externfile2);
+ \draw[->, thick, red] (extern3.south) to[out=south, out looseness=0.7, in=north, in looseness=0.9] (externfile3);
+ \draw[->, thick,blue] (externfile1) to[out=south, in=north] ([yshift=0.8mm]used extern 1.north);
+ \draw[->, thick,blue] (externfile2) to[out=south, in=north] (used extern 2.north);
+ \draw[->, thick,blue] (externfile3) to[out=south, in=north] ([yshift=0.3mm]used extern 3.north);
+ \end{tikzpicture}
+\end{tcolorbox}
+\endgroup
+
+
+
+\section*{Introduction}
+\label{sec:intro}
+
+\begin{tcolorbox}[
+ title={\introboxtitle{What} is externalization and why you might want it?},
+ ]
+ If you have ever worked on a long document full of \TikZ pictures and maybe
+ Forest trees, you have probably had some compilation-enforced coffee breaks
+ --- even on modern computers, compiling pictures, trees and such takes a lot
+ of time. And you might have wondered, why do I need to compile these
+ pictures over and over again? --- after all, I'm not changing them anymore!
+ Enter \emph{externalization}, a mechanism designed to deal precisely with
+ your issue, by saving the produced pictures into separate PDFs and including
+ those PDFs in subsequent compilations --- in no time at all!
+\end{tcolorbox}
+
+\begin{tcolorbox}[
+ title={\introboxtitle{Why} yet another externalization package?},
+ ]
+ \TikZ, the popular and all-powerful graphics language for \TeX, ships
+ including an externalization library (described in \PGFmanual{53}). \TikZ's
+ library does an excellent job, but with one caveat. Assume you're using it
+ for the first time (or after a clean-up) on that long document full of \TikZ
+ pictures and Forest trees. It will take ages to produce all the externalized
+ graphics. Why? Because to get you up to speed, your document has to be
+ compiled many many times --- once for each and every externalized picture.
+ Even with \TikZ's advanced mechanism for skipping the parts of the document
+ irrelevant for the picture at hand, the first externalization can be a
+ daunting task.
+\end{tcolorbox}
+
+\begin{tcolorbox}[
+ title={\introboxtitle{How} does Memoize save your time?},
+ ]
+ There is a reason why \TikZ uses an entire compilation cycle to produce a
+ single externalized picture: \TeX\ itself can only produce a single PDF per
+ compilation (at least at the moment). Memoize evades this limitation by
+ dumping the externalized pictures right in the middle of the document
+ (ouch!). More precisely, an externalized picture occurs in the PDF twice,
+ first on a special page of its own and then on a regular page, where you
+ intended it to be. The daring dump obviously necessitates a second step of
+ the procedure, when those special pages are \emph{extracted} into separate
+ PDFs, called \emph{externs}, which are then included into the document in
+ subsequent compilations.
+
+ This two-step procedure, illustrated on the cover page of this manual, is
+ very fast. The first step, which externalizes \emph{all} the pictures into
+ the document itself (the squiggly red arrow), takes virtually no more time
+ than a regular compilation. The time needed for the second step, extraction
+ (the normal red arrows), depends on the system setup, but it ranges from
+ little to almost none.
+\end{tcolorbox}
+
+\begin{tcolorbox}[
+ title={\introboxtitle{When} should I use Memoize?},
+ ]
+ In short, whenever you are writing a document containing lots of \TikZ
+ pictures, Forest trees, or other time-consuming constructs, and you are bored
+ waiting for the compilation to finish.
+
+ Using Memoize on a paper containing a single picture does not make much of a
+ difference. But with more complex documents, the speed-up can be immense.
+ For example, the compilation time of my 400-page book containing about 160
+ Forest trees was reduced by more than half, and the compilation time of a
+ 260-page Beamer presentation with a hundred complex dynamic trees went from
+ four minutes to a mere half minute!
+\end{tcolorbox}
+
+
+\begin{tcolorbox}[
+ title={\introboxtitle{How} much extra work does Memoize require?},
+ ]
+ In principle, none.
+
+ For one, while allowing for manual memoization of selected document chunks
+ (\S\ref{sec:tut:memoize}), Memoize features a system which automatically
+ triggers memoization at each invocation of selected commands and
+ environments. Out of the box, Memoize \emph{automemoizes} \TikZ
+ pictures\noprint{\refenv{tikzpicture}} and Forest
+ trees\noprint{\refenv{forest}}, but the author can easily submit (almost) any
+ command or environment to automemoization (\S\ref{sec:tut:automemoization}).
+ Memoize also does its best to automatically prevent memoization of code that
+ cannot be externalized, like \TikZ pictures with \refkey{/tikz/remember
+ picture}, and to abort memoization in case the memoized code yields any
+ errors.
+\end{tcolorbox}
+
+
+\begin{tcolorbox}[
+ title={\introboxtitle{Why} is Memoize not called Externalize?},
+ ]
+ Fundamentally, Memoize is about producing and utilizing \emph{memos} ---
+ pieces of \hologo{TeX} code replicating the effect of the compilation of a
+ document chunk in a computationally less intensive manner. Typically, each
+ memo has an associated extern, which is where the effect of
+ \emph{typesetting} is stored, but conceptually, memos come first. For
+ example, the extern is included back into the document by the memo, and a
+ memo may be associated with any number of externs, including zero.
+
+ Memos solve several externalization-related problems in a generic fashion,
+ allowing for a multitude of applications. For example, they store the
+ information about the associated externs, so that an extern can be integrated
+ back into the document as a box with the original orientation, width, height
+ and depth. They solve the problem of cross-referencing from and into the
+ memoized code by storing its \emph{context} (\S\ref{sec:cross-referencing})
+ and replicating any \cs{label}s which occur in it. They are also crucial for
+ externalizing pictures in Beamer frames overlay by overlay.
+
+ Incidentally, the term ``memoization'' is used with some programming
+ languages to refer to the process of remembering the result of the function,
+ along with the given arguments, so that on subsequent invocations of the
+ function with the same arguments, the result can be returned from memory
+ rather than recomputing it. I would say that, give or take the functions,
+ what Memoize does fits the bill.
+\end{tcolorbox}
+
+
+\begin{tcolorbox}[
+ title={\introboxtitle{How} can I make my command (auto)memoizable?},
+ ]
+ Any command which interacts with the rest of the document, like a command
+ which produces a float, must receive special treatment. Some issues can be
+ resolved from within Memoize. Other issues require a memoization-compatible
+ (re)implementation of the command. It is in the hope that package writers
+ will adapt their ``difficult'' commands to Memoize that this package offers a
+ documented interface to the memoization process, fully described in
+ sections~\ref{sec:tut:package}, \ref{sec:memoization-drivers}
+ and~\ref{sec:tut:automemoization-details}.
+
+ An advanced user might also want to know that Memoize ships with two
+ auxiliary packages, which form the base of Memoize's automemoization feature.
+ Package Advice implements a generic framework for extending the functionality
+ of selected commands and environments, while package CollArgs provides a
+ command which can determine the argument scope of any command whose argument
+ structure conforms to \pkg{xparse}'s argument specification.
+\end{tcolorbox}
+
+\begin{tcolorbox}[
+ title={\introboxtitle{What else} is out there?},
+ ]
+ Not long before submitting Memoize to CTAN, I became aware of another new
+ externalization package, \pkg{robust-externalize}, and it seems that the same
+ happened to the author of that package \Smiley, who found the
+ proof-of-concept version of Memoize, which was available
+ at \href{https://github.com/sasozivanovic/memoize}{GitHub} for a while.
+
+ The key idea behind \pkg{robust-externalize} seems to be to extract the code
+ submitted to externalization into separate files, and add the necessary
+ preamble. While a compilation from scratch takes more time than with Memoize
+ (but less than with \TikZ library), the approach allows for parallel
+ compilation of externs --- nice!
+\end{tcolorbox}
+
+\newpage
+\tableofcontents
+
+
+\section{Before you start}
+\label{sec:setup}
+
+\subsection{Installing the extern extraction software}
+\label{sec:install}
+
+Good news: using Memoize can be as easy as writing
+\EmphVerbatim{\usepackage{memoize}} in the preamble.
+
+Bad news: Memoize won't work out of the box. The culprit is the extern
+extraction --- the process which ships the externalized graphics from the main
+document into separate extern files; for details, see the title page
+illustration and the ``How'' box in the Introduction. For the extraction to
+work, you will probably have to install some additional software, and you might
+also have to allow your \hologo{TeX} to execute the extraction script. But
+there's a silver lining: once Memoize is set up, it is set up for good.
+
+\begin{tcolorbox}[title=What do I have to do?]
+
+ In principle, all you have to do for Memoize to work \emph{under the default
+ configuration} is install Perl library
+ \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2}; see the Perl section
+ below for the installation guideline.
+
+\end{tcolorbox}
+
+Consult section~\ref{sec:extraction} if you want to use an extraction method
+other than the default \refmmz{extract=perl}-based method or adapt Memoize to
+your particular workflow (for example, if you're compiling via a Makefile).
+
+If you installed Memoize through the package manager of your \hologo{TeX}
+distribution, your system should be already set up to allow the execution of
+Memoize's extraction scripts. If this is not the case, please contact either
+me or the maintainer of your distribution; until the issue is resolved, you
+have to either
+\begin{enumerate}[(a)]
+\item compile documents loading Memoize with command-line option
+ \docref{-shell-escape} (on \hologo{TeX}Live) or
+ \docref{--enable-write18} (on MiK\hologo{TeX}), or
+\item set up the restricted shell escape mode to allow for the execution of
+ \refscript{memoize-extract.pl}.\footnote{\label{fn:shell-escape-commands}On
+ \hologo{TeX}Live, |tlmgr conf texmf shell\_escape| will tell you which shell
+ escape mode is currently in effect (|p| = partial = restricted, |f| = disabled
+ and |t| = enabled); if necessary, use |tlmgr conf texmf shell\_escape p| to
+ set it to restricted. Then, execute \code{tlmgr conf texmf
+ shell\_escape\_commands} to get the list of currently allowed commands
+ \meta{current}, then add the script by executing \code{tlmgr conf texmf
+ shell\_escape\_commands \meta{current},memoize-extract.pl}.
+
+ On MiK\hologo{TeX}, the shell escape mode can be reported by |initexmf
+ --show-config-value=[Core]ShellCommandMode|, and restricted mode set by
+ |initexmf --set-config-value=[Core]ShellCommandMode=Restricted|, and you
+ get the \meta{current} list by \code{initexmf
+ --show-config-value=[Core]AllowedShellCommands[]}, and add to it by
+ \code{initexmf
+ --set-config-value=[Core]AllowedShellCommands[]=\meta{current};memoize-extract.pl}.}
+\end{enumerate}
+
+Once you have set up your system, I advise you to follow the instructions in
+section~\ref{sec:tut:test} to test if the setup was successful.
+
+
+\begingroup
+\setlength\intextsep{0pt}
+
+\vspace{-1ex}% manual
+
+\paragraph{Perl} If you're running GNU/Linux or macOS, Perl is most likely
+already installed on your system. On Windows, you might have to install it. I
+tested Memoize with Strawberry Perl, available
+at \url(https://){strawberryperl.com}; see \url(https://){www.perl.org} for
+other options.
+
+\begin{wrapfigure}[2]{r}{0.25\linewidth}
+ \raisebox{0.5mm}[\dimexpr\height-0.5mm][\depth]{%
+ \mmznext{no verbatim}
+ \begin{tcolorbox}[
+ % example title=Installing the required library in Perl,
+ % attach shifted boxed title to top right,
+ % top=2.5mm,
+ fontupper=\small,
+ left=2mm,
+ force nobeforeafter,
+ ]
+ |cpan PDF::API2|
+ \end{tcolorbox}%
+ }%
+\end{wrapfigure}
+Once Perl is installed on your system, you will also need to install the PDF
+processing library \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} (or
+its fork, \hreftt{https://metacpan.org/pod/PDF::Builder}{PDF::Builder}). On
+some GNU/Linux distributions, this library is included as a package --- just
+use your package manager to install it. Otherwise, install it from
+\hreftt{https://www.cpan.org}{CPAN} using
+\hreftt{https://metacpan.org/pod/CPAN}{cpan} tool, as shown in the box.
+
+\vspace{-1ex}% manual
+
+\paragraph{Python} Installing (Python and) the required Python library is only
+necessary if you decide to use the Python-based extraction script; see
+section~\ref{sec:extraction}.
+
+If you're running GNU/Linux, install Python using your package manager.
+Otherwise, download the installer from \url(https://){www.python.org}. You can
+install Python even if you don't have administrator privileges: simply uncheck
+the ``Install launcher for all users'' when running the installer.
+
+\begin{wrapfigure}[2]{r}{0.25\linewidth}
+ \raisebox{0.2mm}[\dimexpr\height-0.2mm][\depth]{%
+ \mmznext{no verbatim}
+ \begin{tcolorbox}[
+ % example title=Installing the required library in Python,
+ % attach shifted boxed title to top right,
+ % top=2.5mm,
+ fontupper=\small,
+ left=2mm,
+ nobeforeafter,
+ ]
+ |pip install pdfrw2|
+ \end{tcolorbox}%
+ }%
+\end{wrapfigure}
+
+Once Python is installed on your system, you will also need to install the PDF
+processing library |pdfrw2| library (or its predecessor, |pdfrw|, which will
+work just as well). Some GNU/Linux distributions offer this library as a
+package; if this is not the case, and on other operating systems, install it
+from \href{https://pypi.org}{The Python Package Index}
+using \href{https://pip.pypa.io}{pip} tool (if you
+run \href{https://pip.pypa.io}{pip} as a superuser, it will install the library
+system-wide, otherwise locally), as shown in the box.
+
+\endgroup % \intextsep
+
+
+\subsection{The configuration commands}
+\label{sec:mmzset}
+
+Memoize can be configured using command \Emph{\refcmd{mmzset}\marg{keylist}}
+(and friends). The \meta{keylist} argument is a comma-separated list of
+configuration settings, each setting having the form \meta{key}|=|\meta{value},
+or sometimes just \meta{key}. The \meta{keylist} argument is processed by package
+\pkg{pgfkeys} (see \PGFmanual{88}), so you can use all the bells and whistles
+of that fantastic PGF utility (like easily defining your own styles).
+
+Here's some examples of \refcmd{mmzset}. For one, to whet your appetite to learn
+about the various keys in the \refkeypath{/mmz} path, but more importantly now, to show
+you that white-space is irrelevant in the \meta{keylist} argument, so you can
+format the keylist as you wish --- as long as it does not contain an empty
+line.\footnote{I like to add a comma after the final key as well, as shown in
+ the bottom left example, because if I don't, I often forget to insert it when
+ I add more keys.}
+
+\begin{tcbraster}[raster columns=4, raster valign=bottom]
+ \begin{tcboxedraster}[raster columns=1]{blankest, raster multicolumn=3, valign=bottom}
+ \begin{tcblisting}{listing only}
+~\mmzset~{memo dir,recompile,memoize=circuitikz}
+ \end{tcblisting}
+ \begin{tcblisting}{listing only}
+~\mmzset~{memo dir, recompile, memoize=circuitikz}
+ \end{tcblisting}
+ \end{tcboxedraster}
+ \begin{tcblisting}{listing only}
+~\mmzset~{
+ memo dir,
+ recompile,
+ padding=2in}
+ \end{tcblisting}
+\end{tcbraster}\par
+\begin{tcbraster}[raster columns=4, raster valign=top]
+ \begin{tcblisting}{listing only, raster multicolumn=2}
+~\mmzset~{
+ memo dir, readonly,
+ memoize=\qrcode{om},
+ deactivate=\label,
+ disable,
+}
+ \end{tcblisting}
+ \begin{tcblisting}{listing only}
+~\mmzset~{
+ memo dir,
+ recompile,
+ % comment
+ padding=2in
+}
+ \end{tcblisting}
+ \begin{tcblisting}{listing only, bad}
+~\mmzset~{
+ memo dir,
+ recompile,
+
+ padding=2in
+}
+ \end{tcblisting}
+\end{tcbraster}
+
+Command \refcmd{mmzset} can be used any time after loading the package. It is
+very common in the preamble, but also useful in the document body, where its
+effect is local to the \hologo{TeX} group.\footnote{Any keys with a non-local
+ effect are explicitly marked as such in the reference section.} For example,
+you can use the idiom on the left below to force recompilation of a single
+Forest tree somewhere in the middle of the document (in the code listings
+below, highlighting marks the resulting scope of the \refmmz{recompile}
+directive). However, as applying some setting to a single piece of
+automatically memoized code is common, Memoize provides a special command for
+the occasion: the keys given as the argument of \Emph{\refcmd{mmznext}} will be
+applied only at the next instance of automemoization, overriding any keys set
+by \refcmd{mmzset} in case of a conflict. If the command is given more than
+once, only the final invocation takes effect.
+
+\begin{tcbraster}[raster columns=2, raster equal height=rows, raster force size=false]
+\begin{tcblisting}{listing only, mark region={3}{6}}
+~{~
+ ~\mmzset~{recompile}
+ \begin{forest}
+ [VP[V][DP]]
+ \end{forest}
+ % ...
+~}~
+\end{tcblisting}
+\begin{tcblisting}{listing only, add to width=-1em, mark region={3}{5}}
+% ...
+~\mmznext~{recompile}
+\begin{forest}
+ [VP[V][DP]]
+\end{forest}
+% ...
+
+\end{tcblisting}
+\end{tcbraster}
+
+I like to follow |\usepackage{memoize}| by \refcmd{mmzset}, but if you prefer,
+you can also provide the document-wide configuration as \Emph{package options.}
+For example, the following are equivalent --- both will force
+re-externalization of all the externs in the document.
+
+\begin{tcbraster}[raster columns=2, raster equal height=rows]
+ \begin{tcblisting}{listing only}
+\usepackage{memoize}
+\mmzset{recompile}
+ \end{tcblisting}
+ \begin{tcblisting}{listing only}
+\usepackage[recompile]{memoize}
+ \end{tcblisting}
+\end{tcbraster}
+
+
+\subsection{The configuration file}
+\label{sec:memoize.cfg}
+
+Memoize allows you to configure settings which apply to more than just one
+document. It does that by attempting to load file \Emph{\reffile{memoize.cfg}}
+just before executing package options. Given how \hologo{TeX} searches for
+files, the location of this file determines whether it applies system-wide,
+user-wide or directory-wide. The directory-wide location is clearly the
+directory itself. The user-wide and system-wide location depend on the
+\hologo{TeX} distribution and which format(s) you want to use
+\reffile{memoize.cfg} with:\footnote{The |memoize| subfolder is not
+ obligatory.}
+
+\begin{center}
+ \meta{the relevant texmf tree root}|/tex/|\meta{format}|/memoize/memoize.cfg|
+\end{center}
+
+You can say |generic| for \meta{format} if you want the configuration file to
+be accessible by all formats, otherwise \meta{format} should be one of the
+formats supported by Memoize: |plain|, |latex| or |context|. The texmf root
+directory depends on the distribution, here's how to figure out what it is:
+
+\begin{tcboxedraster}
+ [raster columns=1, raster valign=top, raster force size=false]
+ {title=The roots of TEXMF trees}
+\begin{tcolorbox}[title={\hologo{TeX} Live\hfill\url[white](https://){tug.org/texlive}}]
+ \begin{description}[itemsep=0pt]
+ \item[user-wide]
+ |tlmgr conf texmf TEXMFHOME|
+ or |kpsewhich -var-value TEXMFHOME|\\
+ the default (on Linux): |/home/|\meta{username}|/texmf|
+ \item[system-wide]
+ |tlmgr conf texmf TEXMFLOCAL|
+ or |kpsewhich -var-value TEXMFLOCAL|\\
+ the default (on Linux): |/usr/local/texlive/texmf-local|
+ \end{description}
+
+ \smallskip
+ Don't forget to run |texhash| or |mktexlsr| after creating \reffile{memoize.cfg}.
+\end{tcolorbox}
+\begin{tcolorbox}[title={MiK\hologo{TeX}\hfill\url[white](https://){miktex.org}}, before upper={\parskip0.3\baselineskip plus 2pt\relax},]
+ Open the MiK\hologo{TeX} Console, select the ``Settings'' page and then the
+ ``Directories'' tab.
+
+ If there is a folder marked with the ``Generic'' purpose with attribute
+ ``User'' (for a user-wide \reffile{memoize.cfg}) or ``Common'' (for a system-wide
+ \reffile{memoize.cfg}), that's the folder you are looking for. Otherwise, create a
+ folder following MiK\hologo{TeX}'s instructions and add it to the list.
+
+ Don't forget to click ``Refresh file name database'' (in the ``Tools'' menu
+ of the MiK\hologo{TeX} Console) after creating \reffile{memoize.cfg}.
+\end{tcolorbox}
+\end{tcboxedraster}
+
+Note that a directory config file will override the user config, and the user
+config will in turn override the system-wide one. This should not concern you
+too much, because you will probably only want to use the user-wide config
+anyway,\cprotect\footnote{If you want to have a directory-wide configuration
+ based on (rather than overriding) the user-wide configuration, your could
+ write down the real user-wide config in, say, |memoize.user.cfg| (located
+ user-wide), and then |\input| this file by both the user-wide and the
+ directory-wide \reffile{memoize.cfg}. Of course, the same logic can be used to base
+ a user-wide config on the system-wide one.} which might look something like
+this:
+
+\ExampleName{memoize}
+\makeexample{\examplename.cfg.c1}
+\tcbinputexample[.cfg]{listing only}
+
+I recommend including \refmmz{memo dir} in your \reffile{memoize.cfg},
+as shown in the first line. It reduces the clutter in the document
+directories; see section~\ref{sec:tut:memodir} for details.
+
+The \refmmz{extract} line concludes the story on ``permanently'' selecting the
+extraction method, started in chapter~\ref{sec:setup}. As the final note,
+observe that \refmmz{extract} and other extraction-related keys like
+\refmmz{perl extraction options} only make sense as a package option or as a
+\refcmd{mmzset} key in \reffile{memoize.cfg}. They will have no effect as a
+\refcmd{mmzset} key in the document, because the extraction happens while the
+package is loaded.
+
+
+
+\section{Your first memoized documents}
+\label{sec:tut}
+
+\subsection{Let's see if it works!}
+\label{sec:tut:test}
+
+\ExampleName{test}
+\noprint{\refcmd{tikz}\refenv{tikzpicture}\refenv{forest}}%
+
+Take example file \texttt{\examplename.tex},\footnote{Where can you find
+ the example files? For one, they are integrated into this manual, so if your
+ PDF viewer supports attachments, you can simply click on the paperclip icon
+ on the top right of the example box (even if you're offline). Otherwise,
+ visit the |examples| subdirectory of wherever you found this document
+ \Smiley. Online, the Memoize documentation can be found at CTAN:
+ \url{https://ctan.org/pkg/memoize}; and if your \hologo{TeX} installation
+ includes the documentation, you should also find it in directory \meta{the
+ root of your \hologo{TeX} installation}|/doc/generic/memoize|.
+
+ Incidentally, while we present a full example document in this section, many
+ code listings will only present the parts of the file relevant for the
+ discussion, for brevity. The example \emph{files}, however, will remain
+ full, compilable documents, including the document preamble etc.} or some
+simple document containing a \TikZ picture or a Forest tree, add
+\EmphVerbatim{\usepackage{memoize}} to the preamble,\footnote{Memoize likes to
+ be loaded early. If you get the warning \texttt{Cannot read file
+ \meta{jobname}.pdf}, move \cs{usepackage}\bracestt{memoize} up the
+ preamble; see section~\ref{sec:loading-order} for details.} and compile it
+twice.
+
+\begin{itemize}
+\item The first compilation of the example should produce a three-page PDF.
+ The first two pages are the \emph{extern pages} holding the externalized
+ graphics, while the final page is the (sole page of the) real document. Note
+ that you can see each extern twice: first on a page of its own, and then
+ wherever it belongs to in the real document.
+\item At the second compilation, the extern pages should have disappeared from
+ the PDF, meaning they were successfully extracted into extern files, which
+ are now embedded into the main document.
+\end{itemize}
+
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ sidebyside,
+ comment={\centering
+ \includeexamplepdf[extern page,left=2mm,right=2mm]{page=1,trim=1in 1in 1in 1in}\quad
+ \includeexamplepdf[extern page,left=2mm,right=2mm]{page=2,trim=1in 1in 1in 1in}\\[2ex]
+ \includeexamplepdf[document page]{page=3}
+ },
+}
+
+You might want to play with this example a bit now.
+For example, if you reverse the order of the \TikZ picture and the Forest
+tree, you should notice that the externs don't get recompiled. You won't see
+any extern pages again until you change the actual code of the picture or the
+tree --- or until you add some other picture or tree, of course.
+
+If you \emph{don't} want to automatically memoize \TikZ pictures and/or Forest
+trees, you can switch off their \emph{automemoization} using key
+\Emph{\refmmz{deactivate}}. This key takes a list of command and environment
+names. As you can see below, the command and the environment must be
+deactivated separately.
+
+\begin{tcblisting}{listing only}
+\mmzset{
+ deactivate={\tikz, tikzpicture}, % deactivate automemoization of all TikZ pictures
+ deactivate=\tikz, % deactivate only the command
+ deactivate=tikzpicture, % deactivate only the environment
+}
+\end{tcblisting}
+
+
+\subsection{Memoizing by hand}
+\label{sec:tut:memoize}
+
+In the previous section, we have compiled our very first document which used
+Memoize. In that document, we only had to load the package, as Memoize knows
+how to externalize \TikZ pictures and Forest trees without any help from the
+author. But what if you want to externalize some other code? The manual way of
+doing this is by surrounding the code by a \Emph{\refenv{memoize}} environment,
+or by making it the argument of the \Emph{\refcmd{mmz}} command. The only
+difference between the two is that the environment, but not the command,
+ignores any spaces surrounding the given code.
+
+\ExampleName{manual}
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ sidebyside, lefthand ratio=0.528,
+ comment={\centering
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\hfill
+ \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in}
+ },
+}
+
+Both the \refenv{memoize} environment and the \refcmd{mmz} command take a
+configuration keylist as the optional argument, so their full syntax is
+\cs{begin}\marg{\refenv{memoize}}\oarg{keylist}\meta{code to be
+ externalized}\cs{end}\marg{\refenv{memoize}} and and
+\refcmd{mmz}\oarg{keylist}\marg{code to be externalized}. The keys given in
+this optional argument take precedence over the keys set by \refcmd{mmzset}.
+Note that \refcmd{mmznext} does not apply to manual memoization.
+
+Manual memoization is great for one-shot memoizations, but you can use it
+within your own macros as well. For example, assume that you don't want to
+externalize \TikZ pictures in general (so you have \refmmz{deactivate}d
+automemoization of the \cs{tikz} command, as explained at the end of
+section~\ref{sec:tut:test}), but that you want to easily memoize selected
+pictures. You could define a memoized variant of the \cs{tikz} command, as shown
+below (and similary for the environment). (Note the |%| comment characters in
+the definition of |\mmztikz|. The definition was intentionally broken into
+several lines to remind you that the spaces around the argument of \refcmd{mmz}
+matter.)
+
+\ExampleName{mmztikz}
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ comment={\centering
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
+ \quad
+ \includeexamplepdf[document page]{page=2}
+ },
+}
+
+\clearpage
+\subsection{Memoizing automatically}
+\label{sec:tut:automemoization}
+
+Out of the box, Memoize automatically externalizes \TikZ pictures and Forest
+trees. Let us see how other commands and environments can be submitted to this
+\emph{automemoization} process.
+
+We start with the simpler case of environments (fortunately, externalizing
+environments also makes sense more often than externalizing commands). You can
+submit an environment to automemoization by writing
+\EmphTT{\refmmz{auto}=\marg{environment name}\braces{\refmmzauto{memoize}}} (as
+a key in \refcmd{mmzset}). The natural (but not the only possible) location
+for this instruction is the preamble. Below, we automemoize environment
+\env{circuitikz} of package \pkg{circuitikz}, used for drawing electronic
+circuits.\footnote{In section~\ref{sec:tut:test}, we learned that
+ automemoization can be switched off using key \refmmz{deactivate}. Memoize
+ also offers key \refmmz{activate}, but you probably won't have to use it, as
+ an \refmmz{auto} declaration automatically activates the submitted command.}
+
+\ExampleName{automemoize-environment}
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ sidebyside, lefthand ratio=0.6,
+ comment={\centering
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in,scale=0.8}
+ },
+}
+
+Commands are a bit harder to automemoize, because Memoize cannot possibly know
+how far the arguments of a command extend (in contrast, the end of an
+environment is clearly marked). With commands, we must inform Memoize about
+their argument structure, which we achieve using key \refmmzauto{args} in the
+second argument of key \refmmz{auto}:
+\EmphTT{\refmmz{auto}=\meta{command}\braces{\refmmzauto{memoize},
+ \refmmzauto{args}=\marg{argument specification}}}. We can only leave out
+\refmmzauto{args} if the command was defined by \refcmd{NewDocumentCommand} or
+similar; in this case, Memoize can retrieve the argument specification on its
+own.
+
+And how does the \meta{argument specification} look like? It is a sequence of
+letters, each letter determining an argument type. Memoize recognizes the same
+argument types and their letters as package \pkg{xparse} (which defines
+\refcmd{NewDocumentCommand} and friends), so you should look at (section 1 of)
+the documentation of that package for details, if and when you need them.
+Here, we focus on the two most commonly used types, \docref{xparse:m} and \docref{xparse:o}, and add the
+optional star for good measure:
+\begin{center}
+ \begin{tabular}{lll}
+ \toprule
+ letter & argument type & example\\
+ \midrule
+ \docref{xparse:m} & mandatory argument \\
+ & --- either surrounded with braces & |\foo{arg}|\\
+ & --- or a single token & |\foo a|\\
+ \docref{xparse:o} & optional argument, surrounded with brackets & |\foo[arg]|\\
+ \docref{xparse:s} & optional star & |\foo*|\\
+ \bottomrule
+ \end{tabular}
+\end{center}
+
+Below, we write |args=om| because command \cs{qrcode} (of package \pkg{qrcode})
+takes two arguments: a bracketed optional argument, followed by a mandatory
+argument (in braces).
+
+\ExampleName{automemoize-command}
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ sidebyside, lefthand ratio=0.603, top=0mm,
+ comment={\centering
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad
+ \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in}
+ },
+}
+
+This should get you started with automemoization. We'll provide some further
+basic information in sections~\ref{sec:tut:working-on-a-picture}
+and~\ref{sec:tut:verbatim} of the tutorial, but only package writers will
+probably ever need the gory details from
+section~\ref{sec:tut:automemoization-details}. It is my sincere hope that they
+will support Memoize in their packages, where necessary, so that you don't even
+have to write the \refmmz{auto} declarations, but there is one thing you should
+know if you encounter a package supporting Memoize: you should load it
+\emph{after} Memoize!
+
+
+
+\subsection{Working on a picture}
+\label{sec:tut:working-on-a-picture}
+
+
+Memoize automatically recompiles a picture when the code producing the picture
+changes. However, sometimes we can modify a picture without changing its code,
+like when we modify the definition of a command used in the code. In the
+example below, a predefined style |emph| is applied to the node, producing a
+node with a red background. Let's say we compile the document (with
+memoization) and then change this style to set the yellow background.
+Curiously, the node will remain red.
+
+\ExampleName{recompile}
+\makeexample{\examplename.pdf N=3}
+\begin{tcbraster}[raster columns=2, raster valign=top, raster column skip=4mm]
+ \tcbinputexample{
+ after title pre={\ (version~1)},
+ comment={\centering
+ \includeexamplepdf[document page]{page=2}
+ },
+ }
+ \tcbinputexample[.tex][.c2]{
+ no attachment,
+ after title pre={\ (version~2)},
+ comment={\centering
+ \includeexamplepdf[document page]{page=1}
+ },
+ }
+\end{tcbraster}
+
+The curious thing happens (or rather, doesn't happen) because Memoize doesn't
+keep track of how commands and styles are defined; it just uses the extern file
+it created when the old style was in effect. To get a yellow node, we must ask
+Memoize to reexternalize the picture. The simplest way to do that is by using
+the \Emph{\refmmz{recompile}} key; below, we write
+\refcmd{mmznext}\bracestt{\refmmz{recompile}} just before the picture and
+compile the document again (remember from section~\ref{sec:mmzset} that
+whatever keys we provide through \refcmd{mmznext} only apply to the instance of
+automemoization). After the compilation, we may (and should) remove the
+\refmmz{recompile} directive (otherwise, Memoize will produce the extern page
+again and again).
+
+\tcbinputexample[.tex][.c3]{
+ no attachment,
+ after title pre={\ (version~3)},
+ sidebyside, lefthand ratio=0.6,
+ comment={\centering
+ \includeexamplepdf[document page]{page=2}
+ },
+}
+
+It is also common to put (again, for the space of a single compilation)
+\refcmd{mmzset}\braces{\refmmz{recompile}} in the preamble, or to use
+\refmmz{recompile} as the package option. Either will remake all the
+externalized graphics in the document, so you can be sure all of them use the
+latest version of your macros and styles.
+
+We'll revisit the issue of memoized code depending on macros and styles defined
+elsewhere in section~\ref{sec:tut:redefinitions}. In this section, we will
+learn how the issue can be \emph{avoided}, at least to some extent. One idea
+is to turn off memoization for the picture(s) we are currently working on;
+another idea is to let Memoize know which definitions the picture relies on.
+The simplest way to achieve the former is by using key \Emph{\refmmz{disable}};
+by putting it into a \hologo{TeX} group, we can localize its effect to the
+selected pictures.
+
+\ExampleName{disable}
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ sidebyside,
+ lefthand ratio=0.6,
+ top=0mm, % manual top, bottoms and \\[...] to fit the page
+ comment={\centering
+ \includeexamplepdf[extern page,bottom=0.8mm]{page=1,trim=1in 1in 1in 1in}\\[0.5mm]
+ \includeexamplepdf[extern page,bottom=0.8mm]{page=2,trim=1in 1in 1in 1in}\\[0.5mm]
+ \includeexamplepdf[document page,bottom=0.8mm]{page=3}
+ },
+}
+
+\begin{tcolorbox}[warning]
+ As you can imagine, key \refmmz{disable} is complemented by \refmmz{enable},
+ but it is perhaps worth mentioning a problem that can arise if you disable
+ memoization for a part of your document by enclosing it in a pair of
+ \refcmd{mmzset}\braces{\refmmz{disable}} and
+ \refcmd{mmzset}\braces{\refmmz{enable}}. Yes, it might work at the moment,
+ but say you later (e.g.\ when you are preparing the final version of the
+ document) decide to disable memoization for the entire document, and say you
+ try to do this by writing \refcmd{mmzset}\braces{\refmmz{disable}} in the
+ preamble. As shown below on the left, you're in for a surprise: memoization
+ will still be enabled in the part of the document following
+ \refcmd{mmzset}\braces{\refmmz{enable}}! The solution is to always disable
+ memoization for a part of the document by using
+ \refcmd{mmzset}\braces{\refmmz{disable}} in a \hologo{TeX} group (i.e.\ the
+ braces), as shown on the right. (In the examples below, the shaded areas
+ mark the parts of the document where memoization is \emph{disabled}.)
+\end{tcolorbox}
+
+\makeexample{disable-bad.tex.c1}
+\makeexample{disable-good.tex.c1}
+\makeexample{disable-nomemoize.tex.c1}
+\makeexample{disable-nommz.tex.c1}
+\makeexample{disable-auto-cmd.tex.c1}
+\makeexample{disable-auto-env.tex.c1}
+
+\begin{tcbraster}[raster columns=2, raster equal height=rows]
+ \tcbinputexample(disable-bad){
+ listing and comment, bad, mark region={4}{7},
+ comment={\raggedright The upper \refcmd{mmzset}\braces{\refmmz{disable}}
+ does not have the intended effect,
+ i.e.\ it doesn't apply to the whole document!}}
+ \tcbinputexample(disable-good){
+ listing and comment, mark region={4}{10},
+ comment={\raggedright The upper \refcmd{mmzset}\braces{\refmmz{disable}} applies
+ to the entire document, as expected.}}
+\end{tcbraster}
+
+In fact, it might be better to disable memoization using environment
+\Emph{\refenv{nomemoize}} or macro \Emph{\refcmd{nommz}}. I also like these
+commands because it is easy to add and remove prefix |no| to switch manual
+memoization (triggered using environment \refenv{memoize} or macro
+\refcmd{mmz}) off and on.
+
+\begin{tcbraster}[raster columns=2, raster equal height=rows]
+ \tcbinputexample(disable-nomemoize){listing and comment, mark region={2}{2},
+ comment=\raggedright Disable using the dedicated environment.}
+ \tcbinputexample(disable-nommz){listing and comment, mark region={2}{2},
+ comment=\raggedright Disable using the dedicated command.}
+\end{tcbraster}
+
+It is also possible to disable memoization for all occurrences of a selected
+command or environment. In fact, we're already familiar with the procedure
+from section \ref{sec:tut:automemoization}, where we used key
+\refmmzauto{memoize} inside the second argument of \refmmz{auto} to
+automatically memoize all instances of the command or environment given as the
+first argument. All we have to do to auto-disable rather than auto-memoize, is
+substitute \refmmzauto{nomemoize} for \refmmzauto{memoize}. Note that this
+prevents memoization of not only the given command or environment, but also of
+any (manual or automatic) memoization which would otherwise occur during its
+execution; for example, if |\foo| executes \cs{tikz} under the hood,
+autodisabling |\foo| prevents memoization of the inner \cs{tikz}, even though
+that command is normally automemoized.
+
+\begin{tcbraster}[raster columns=2, raster equal height=rows]
+ \tcbinputexample(disable-auto-cmd){
+ listing and comment,
+ mark region={3}{3}, mark region={5}{5},
+ comment=\raggedright Autodisable within a command.,
+ }
+ \tcbinputexample(disable-auto-env){
+ listing and comment,
+ mark region={3}{5},
+ comment=\raggedright Autodisable within an environment.,
+ }
+\end{tcbraster}
+
+All that said, Memoize actually offers a neater way to switch off the
+externalization for the picture I'm currently working on. The
+\Emph{\refmmz{readonly}} key instructs Memoize to use whatever externs it had
+already produced (thereby reducing the document compilation time), but to
+abstain from producing any new externs. In effect, the stuff we are currently
+working on does not undergo memoization and therefore does not produce the
+clutter which can potentially lead to trouble described in the \refmmz{recompile}
+examples above.
+
+What I like to do is load the package using |\usepackage[readonly]{memoize}|,
+work on stuff, and once I'm happy with the most recent pictures, remove
+\refmmz{readonly} from the package options for one compilation.
+
+\ExampleName{readonly}
+\makeexample{\examplename.pdf N=2}
+
+\begin{tcbraster}[raster columns=5, raster valign=top, raster column skip=4mm]
+ \tcbinputexample{
+ raster multicolumn=2,
+ after title pre={\ (work in progress)},
+ comment={\centering
+ \includeexamplepdf[document page]{page=1}
+ },
+ }
+ \tcbinputexample[.tex][.c2]{
+ raster multicolumn=3,
+ no attachment,
+ title={\tt readonly.tex} (the final version),
+ comment={\centering
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad
+ \includeexamplepdf[document page]{page=2}
+ },
+ }
+\end{tcbraster}
+
+We're now ready to tackle a completely different way of avoiding the issue, by
+informing Memoize which definitions the externalized picture depends on. We do
+this by appending these definitions to \emph{context} --- when the context of a
+picture changes, Memoize recompiles the picture, same as if the code of the
+picture itself was changed. (We will talk about context in more detail in
+section~\ref{sec:cross-referencing}.)
+
+A command can be added as a dependency using key \Emph{\refmmz{meaning to
+ context}}. Below, we make the following externalized picture depend on the
+definition of macro \cs{answer}; changing this definition will result in the
+recompilation of the extern. In general, \refmmz{meaning to context} accepts a
+comma-separated list of command and environment names, e.g.\
+\code{\refmmz{meaning to context}=\bracestt{\cs{foo},bar}} (note the braces).
+
+Memoize offers several variants of \refmmz{meaning to context}, applicable to
+various types of commands. For example, the easiest way of making the picture
+depend on the definition of a \pkg{pgfkeys} style is to use handler
+\Emph{\refkey{/handlers/.meaning to context}} --- note the dot in the name, and
+observe that \code{emph/.meaning to context} below is executed within
+\cs{tikzset}, not \refcmd{mmzset}; see \PGFmanual{87} to learn about key
+handlers.
+
+\ExampleName{meaning-to-context}
+\makeexample{\examplename.pdf N=2}
+\tcbinputexample{
+ after title pre={\ (version~1)},
+ sidebyside,
+ comment={\centering
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
+ },
+}
+
+\tcbinputexample[.tex][.c2]{
+ after title pre={\ (version~2)},
+ sidebyside,
+ comment={\centering
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
+ },
+ no attachment,
+}
+
+All variants of \refmmz{meaning to context} (see reference
+section~\ref{sec:ref:memoization:basic} for the full list) may be used within
+the externalized picture itself, e.g.\ \code{\cs{node}[emph,
+ emph/\refkey{/handlers/.meaning to context}]\bracestt{...}} is perfectly
+valid --- and also handy when you want to limit the effect of the handler to a
+single picture, as \refkey{/handlers/.meaning to context} cannot be used within
+\refcmd{mmznext}.
+
+
+
+\subsection{Keeping a clean house}
+\label{sec:tut:memodir}
+
+Memoize produces lots of auxiliary files. For each piece of memoized code, it
+produces two \emph{memo} files (we will learn more about these in
+section~\ref{sec:memos}), which will be joined by the extern PDF upon the next
+compilation.\footnote{If you're using the \hologo{TeX}-based extraction, each
+ extern (|.pdf|) is also accompanied by a log file (|.log|) produced during
+ the compilation that extracted the extern.} You can recognize these files
+easily: their names start with the name of your document and include one or two
+long hexadecimal numbers.
+
+\ExampleName{dirty-house}
+\makeexample{\examplename.pdf}
+{
+ \relaxmmzcommands
+ \edef\marshal{%
+ \noexpand\def\noexpand\mynewextern\examplename
+ }\marshal.#1-#2.pdf\relax{%
+ \forestset{
+ append={[\examplename.#1.memo\vphantom{p},pointcolor]},
+ append={[\examplename.#1-#2.memo\vphantom{p},pointcolor]},
+ append={[\examplename.#1-#2.pdf,pointcolor]},
+ }%
+ }
+ \tcbinputexample{
+ comment={%
+ \begin{tcolorbox}[reset, left=2mm, bottom=0mm,
+ my boxed title=the folder contents (after two compilations),
+ attach shifted boxed title to top right,
+ ]
+ \begin{forest}
+ before typesetting nodes={for tree={
+ font=\scriptsize\tt, grow'=0, folder, s sep=0}},
+ [\meta{the document folder}, inner xsep=0,
+ TeX={%
+ \def\mmzNewExtern##1##2##3##4{\mynewextern##1\relax}%
+ \input{\examplepath.mmz}%
+ },
+ [\examplename.tex]
+ [\examplename.pdf]
+ [\normalsize\dots, no edge]
+ [\examplename.mmz, tikz={\draw[dotted](.east)--+(5em,0ex) node [anchor=south west, yshift=-1ex, align=justify, text width=14em, font=\footnotesize]{This is another auxiliary file produced by Memoize. We will mention it at the end of the section.};}]
+ ]
+ \end{forest}
+ \end{tcolorbox}
+ },
+ }
+}
+
+To top it off, changing the memoized code will produce new memo (and extern)
+files, with the old files staying in place. This is all by design --- the
+first hexadecimal number in these filenames is the MD5 sum of the memoized code
+and that's how Memoize knows which memo belongs to which piece of
+code\footnote{The second hexadecimal number in the memo and extern filenames is
+ the MD5 sum of the \emph{context}. The context is crucial for properly
+ externalizing code containing cross-references, see
+ section~\ref{sec:cross-referencing} for details.} --- but it has the downside
+that the folder containing your document can get quite cluttered (imagine the
+directory listing as above, but for a document with a hundred externalized
+pictures which you have been working on for a month).
+
+\ExampleName{clean-house}
+
+I like to keep a clean house by instructing Memoize to put memos and friends
+into their own directory. This can be achieved by writing
+\refcmd{mmzset}\braces{\Emph{\refmmz{memo dir}}} into the preamble (anytime
+after loading the package).\footnote{The \refmmz{memo dir} key is in fact
+ merely an abbreviation for \code{\refmmz{prefix}=\string\jobname.memo.dir};
+ use this key if you need more control over the name and location of the
+ auxiliary files. Furthermore, there is also the \refmmz{no memo dir} key,
+ which reverts the configuration back to the dirty default.} This will put
+the memo and the extern files into folder \meta{document name}|.memo.dir| (and
+it will also omit the \meta{document name} prefix in their filenames, because
+it makes no sense to repeat it there).
+
+\makeexample{\examplename.pdf}
+{%
+ \relaxmmzcommands
+ \edef\marshal{%
+ \noexpand\def\noexpand\mynewextern\examplename
+ }\marshal.memo.dir/#1-#2.pdf\relax{%
+ \forestset{
+ append={[#1.memo\vphantom{p},pointcolor]},
+ append={[#1-#2.memo\vphantom{p},pointcolor]},
+ append={[#1-#2.pdf,pointcolor]},
+ }%
+ }
+ \tcbinputexample{
+ comment={%
+ \begin{tcolorbox}[reset, left=2mm, bottom=0mm,
+ my boxed title=the folder contents (after two compilations),
+ attach shifted boxed title to top right,
+ ]
+ \begin{forest}
+ before typesetting nodes={for tree={
+ font=\scriptsize\tt, grow'=0, folder, s sep=0}},
+ [\meta{the document folder}, inner xsep=0
+ [\examplename.tex]
+ [\examplename.pdf]
+ [\normalsize\dots, no edge]
+ [\examplename.mmz]
+ [\examplename.memo.dir,pointcolor,
+ % tikz={\draw[dotted](.east)--+(9em,0ex) node [anchor=south west, yshift=-1ex, font=\footnotesize, emphshade=black!5!white, text width=15em, align=justify]{You need to create this directory by hand, or allow Memoize to create it by adding |mkdir| to the list of restricted shell escape commands.};},
+ tikz={\draw[dotted](.east)--+(9em,0ex) node [anchor=west, font=\footnotesize, emphshade=black!5!white]{This directory must be created, somehow.};},
+ TeX={%
+ \def\mmzNewExtern##1##2##3##4{\mynewextern##1\relax}%
+ \input{\examplepath.mmz}%
+ },
+ ]
+ ]
+ \end{forest}
+ \end{tcolorbox}
+ },
+ }
+}
+
+\begin{tcolorbox}[title={Why is \refmmz[link color=white]{memo dir} not the default?}]
+ The \code{clean-house} example most likely compiled just fine, and you are
+ wondering why \refmmz{memo dir} is not in effect by default. Well, out of
+ the box, \hologo{TeX} cannot create directories, so it is the fact that
+ Memoize \emph{can} create them (at least under the default settings) which
+ requires explanation. By default, Memoize triggers extraction by executing
+ the Perl extraction script \refscript{memoize-extract.pl}, and it is this
+ script which actually creates the memo directory. However, not everybody
+ will necessarily use this script \dots\ so \refmmz{memo dir} should not be
+ the default.
+
+ The Python extraction script \refscript{memoize-extract.pl}, used when
+ \refmmz{extract}|=|\refmmz{extract=python}, works the same as the Perl
+ variant. With \hologo{TeX}-based extraction
+ \refmmz{extract}|=|\refmmz{extract=tex}, things are different. If you are
+ compiling the document with a full shell escape mode
+ (\docref{-shell-escape}), Memoize successfully creates the directory with the
+ system command |mkdir|.\footnote{|mkdir| is the default value of key
+ \refmmz{mkdir command}, but executing the extraction method
+ \refmmz{extract=perl} or \refmmz{extract=python} overrides this default.}
+ However, if you're using the restricted shell escape mode, the attempt to
+ create the directory won't succeed unless you include |mkdir| among the
+ restricted shell escape commands (see footnote~\ref{fn:shell-escape-commands}
+ on page~\pageref{fn:shell-escape-commands} for how to do this, but note that
+ it is not recommended).
+
+ If you are using external extraction, you have to create directory
+ \texttt{\examplename.memo.dir} by hand, prior to the first compilation of the
+ document (with Memoize). This is the case even if you are performing the
+ extraction using one of the shipped extraction methods, and it is due to the
+ fact that Memoize needs the memo directory to be present even before extern
+ extraction, because it writes the |.memo| files into the same directory.
+ (When Memoize uses the extraction script to create the memo directory, it
+ does so completely independently of extraction, and prior to creating any
+ |.memo| files.)
+\end{tcolorbox}
+
+I actually suggest adding \refcmd{mmzset}\braces{\refmmz{memo dir}} into your
+user-wide \reffile{memoize.cfg} (see section~\ref{sec:memoize.cfg} for details
+on this file). This will keep all your houses clean --- without work! --- as
+Memoize will automatically use the memo directory for any document your create.
+
+It is always safe to delete memos (|.memo|) and externs (|.pdf|s residing next
+to memos), in the sense that you cannot lose data this way.\footnote{The same
+ goes for the extern |.log| files produced by the \hologo{TeX}-based
+ extraction.}
+
+\begin{itemize}
+
+\item Many memos and externs are typically stale anyway, i.e.\ they reflect
+ some previous state of your document and are not needed anymore. These files
+ can be deleted without any repercussions whatsoever (unless you later revert
+ to a previous version of the document, of course). In fact, you might
+ \emph{want} to delete them periodically, or at least once you finish writing
+ the document. As it is hard to figure out which memos/externs are stale,
+ Memoize ships with a clean-up script: writing \refscript{memoize-clean.pl}
+ \meta{document name}\dmmz (replace |.pl| with |.py| if you use Python rather
+ than Perl) into the command line will delete all the \emph{stale} auxiliary
+ files belonging to the document.
+
+\item If you delete a memo or an extern currently in use, you will trigger
+ recompilation of their code --- so deleting a memo or an extern is actually a
+ perfectly legal alternative to using the \refmmz{recompile} key!\footnote{For the
+ users of the \hologo{TeX}-based extraction: deleting the |.log| file does
+ \emph{not} trigger recompilation.}
+
+\end{itemize}
+
+It is also safe to delete the \dmmz file (or any other kind of record file, see
+section~\ref{sec:record-files}) residing next to your document's |.pdf|. The
+\dmmz file contains the information about which externs should be extracted
+from the |.pdf|. Deleting it before this is done (by default, before compiling
+the document again) will prevent the extraction (same as if providing the
+package option \code{\refmmz{extract}=\refmmz{extract=no}}) and ultimately
+result in the recompilation of the externs produced in the previous run.
+Deleting it after the extraction will have almost no effect: it will only only
+prevent the clean-up script from working (the \dmmz file also lists the
+currently active memos and externs, and thereby indirectly informs the clean-up
+script which files are stale). For further information on the \dmmz file, see
+section~\ref{sec:.mmz}.
+
+
+
+\subsection{Writing a book?}
+\label{sec:tut:multifile}
+
+Books and other long documents are usually produced from sources which reside
+in more than a single file, and to speed up the editing process, authors
+usually use some system which allows them to compile each chapter separately.
+Can Memoize --- designed for virtually the same task of speeding up the editing
+process --- work sensibly in this kind of situation? More precisely, can the
+book and the individual chapters share the memos and the externs? Yes they
+can! If we instruct Memoize to use the same memo directory for both the book
+job and the chapter jobs, then we can externalize graphics when compiling a
+chapter and have the externs included when compiling the book (and vice
+versa).\footnote{Package \pkg{docmute} makes \hologo{LaTeX} ignore the preamble of
+ the chapter file when including this file into the main document.} All we
+need to do is use our old friend \refmmz{memo dir} from section~\ref{sec:tut:memodir}
+--- we see now that this setting is good for more than just keeping a clean
+house!
+
+\ExampleName{book}
+\makeexample{\examplename.tex.c1}
+\makeexample{chapters/chapter1.tex.c1}
+
+\begin{tcbraster}[raster columns=2, raster valign=top]
+ \tcbinputexample{listing only}
+ \tcbinputexample(chapters/chapter1){listing only}
+\end{tcbraster}
+
+In the above example, the individual chapters reside in files stored in the
+|chapters| subdirectory, and that's why the |book.tex| preamble uses
+\code{\refmmz{memo dir}=chapters/book} (rather than \code{\refmmz{memo
+ dir}=book} or just \refmmz{memo dir}). However, Memoize has no trouble
+with a situation where the main file and the chapters reside in the same
+folder; the setup is even simpler, as we then say \code{\refmmz{memo dir}=book}
+in both the book and the chapter preamble. The more complicated situation was
+chosen to point out the following potential problem with the setup where the
+chapters reside in a subdirectory.
+
+If you're anything like me, you would first go for having a memo directory
+immediately contained in the project directory (so |examples/book.memo.dir|
+above) and set up \refmmz{memo dir} as shown below. Well, this won't work, or
+at least it won't work with the vanilla \hologo{TeX} Live, because \hologo{TeX}
+will refuse to \emph{write} into (memo) files outside the directory where it
+was executed,\cprotect\footnote{In \hologo{TeX} Live, the \reffile{texmf.cnf}
+ option controlling this behaviour is called \docref{openout_any}. By default, it is
+ set to |p| (paranoid), which ``disallow[s] opening dot files [and]
+ \emph{going to parent directories}, and restrict[s] absolute paths to be
+ under \docref{TEXMFOUTPUT}'' (emphasis mine).} and this is precisely what
+the chapter compilation is asked to do below.
+
+\begin{tcboxedraster}[raster columns=2, raster valign=top]{blankest, bad}
+ \begin{tcblisting}{example title, title=the main file, listing only}
+\mmzset{~memo dir=book~}
+% ...
+\input{chapters/chapter1.tex}
+ \end{tcblisting}
+ \begin{tcblisting}{example title, title=a chapter file, listing only}
+\mmzset{~memo dir=../book~}
+ \end{tcblisting}
+\end{tcboxedraster}
+
+\pagebreak % manual
+
+Section~\ref{sec:tut:working-on-a-picture} presented some ideas on how to work
+on a single picture. Those ideas can be all easily applied to the multi-file
+situation. For example, you could use \refmmz{readonly} on the chapter that you're
+working on (and that chapter only). This way, the preview of the chapter will
+not be tarnished by the extern pages, and if you periodically compile it
+without \refmmz{readonly}, or compile the book (which does not have the \refmmz{readonly}
+set), you will have a reasonably up-to-date set of externs.
+
+\begin{tcboxedraster}[raster columns=2, raster valign=top]{blankest}
+ \begin{tcblisting}{example title, title=the main file, listing only}
+\mmzset{memo dir=chapters/book}
+% ...
+\input{chapters/chapter1.tex}
+ \end{tcblisting}
+ \begin{tcblisting}{example title, title=the current chapter file, listing only}
+\mmzset{memo dir=book, ~readonly~}
+ \end{tcblisting}
+\end{tcboxedraster}
+
+\ExampleName{memoize.cfg._region_}
+\makeexample{\examplename.tex.c1}
+\begin{tcolorbox}[title=For \Emacs users]
+ I often use this \refmmz{readonly} trick myself, but with a twist. As an Emacs user,
+ I don't use a \hologo{TeX}-based mechanism (such as the \pkg{docmute} package) to
+ compile a chapter, but rely on the region compilation feature of Emacs'
+ AUC\hologo{TeX} package. AUC\hologo{TeX} offers a way to compile the current
+ buffer (if you don't know what an Emacs buffer is, read ``file'') or region
+ (roughly speaking, the selected text). It does that by putting the buffer or
+ the region into a file called |_region_.tex| while dressing it up in the
+ preamble of the original document (when I'm working on a multi-file document,
+ it correctly pulls the preamble from the main document). This results in a
+ compilable region file. My trick is to detect whether I'm compiling a region
+ (this is the job of |\ifregion|), and if so, put Memoize into the \refmmz{readonly}
+ mode (an alternative trick would be to \refmmz{disable} it).
+
+ This is the trick in a nutshell, but to make it really work we have to
+ address one further issue: the original document and the region have to share
+ memos and externs. This happens automatically if the original document sets
+ \refmmz{memo dir} explicitly (e.g.\ if a document called |doc.tex| contains
+ \code{\refmmz{memo dir}=doc} in the preamble), but I'm lazy and don't want to
+ write this in every document --- if I have to do that, what's the point of
+ \refmmz{memo dir} I put into my \reffile{memoize.cfg} in
+ section~\ref{sec:tut:memodir}? Fortunately, the region file starts with
+ |\message{ !name(|\meta{original document name}|.tex)}| to indicate the
+ origin. The complicated part of the code below (everything following
+ \refcmd{mmzset}\braces{\refmmz{readonly}} parses this header to extract the
+ \meta{original document name}, which is then fed to \refmmz{memo dir}. Now,
+ the trick works automatically for any document.\footnote{The assumption here
+ is that \refmmz{memo dir} is in effect for the original document. If not,
+ the trick can be adapted to use \refmmz{prefix}.}
+
+ \tcbinputexample{
+ title=\tt memoize.cfg,
+ attachment name=memoize-region.cfg,
+ listing only,
+ }
+\end{tcolorbox}
+
+
+\subsection{Writing a presentation?}
+\label{sec:tut:beamer}
+
+Memoize ships with built-in support for the most widespread \hologo{LaTeX}
+presentation class, Beamer, in the sense that it can externalize a picture
+which changes from overlay to overlay. Before we learn how to use that
+functionality, however, there's a peculiarity about loading Memoize in Beamer
+to address.
+
+\ExampleName{beamer}
+\makeexample{\examplename.pdf}
+
+\tcbinputexample{
+ comment and listing,
+ listing options app={lastline=2},
+ warning, no attachment, title=,
+ %
+ comment={Beamer opens the document PDF while loading the class, while
+ Memoize requires the PDF from the previous compilation intact in order to
+ extract the externs (when extraction is triggered internally, which is the
+ default setting). The solution is to load Memoize (a package) before
+ Beamer (a class), which can be done by using \cs{RequirePackage} instead of
+ the usual \cs{usepackage}. Easy, if hacky.},
+ %
+}
+
+To memoize a piece of code which produces different results on different
+overlays --- by virtue of containing |\pause|, |\only|, and\slash or related
+commands --- apply key \Emph{\refmmz{per overlay}}. Without this key,
+externalization of the picture will end badly, with a single extern (the final
+one) appearing on all overlays. The key may be invoked either from a prior
+\refcmd{mmznext} command,\footnote{Of course, \refmmz{per overlay} may also be
+ invoked from \refcmd{mmzset}, but I guess this won't make sense often. For
+ example, if you set it for the entire presentation, and the presentation
+ contains static memoized pictures as well, you will compile those pictures
+ more times than necessary: once for each overlay, whereas once per frame
+ would suffice. It might occasionally make sense, however, to use \refmmz{per
+ overlay} as an \refmmz{auto} option --- consult
+ section~\ref{sec:tut:verbatim} to learn what that is.} or executed in the
+memoized code itself. The example below illustrates the latter option, and
+also shows that we may invoke it via its full path,
+\refkeypath{/mmz}|/|\refmmz{per overlay}, when listed among options processed by
+\pkg{pgfkeys}.\footnote{Read section~\ref{sec:per-overlay} to learn how the
+ Beamer support is implemented. The implementation only uses Memoize's public
+ interface, so understanding it should help if you need to support some other
+ presentation package.}
+
+\tcbinputexample{
+ listing options app={firstline=3},
+ middle=1mm,
+ comment={\def\dpwidth{0.27\linewidth}\def\epwidth{0.15\linewidth}%
+ \raisebox{-\height}{\includeexamplepdf[extern page,left=0mm,right=0mm]{width=\epwidth,page=1,trim=1in 1in 0.4in 0.6in}}\hfill
+ \raisebox{-\height}{\includeexamplepdf[document page,left=1mm,right=1mm]{page=2,width=\dpwidth}}\hfill
+ \raisebox{-\height}{\includeexamplepdf[extern page,left=0mm,right=0mm]{width=\epwidth,page=3,trim=1in 1in 0.4in 0.6in}}\hfill
+ \raisebox{-\height}{\includeexamplepdf[document page,left=1mm,right=1mm]{page=4,width=\dpwidth}}%
+ },
+}
+
+\begin{tcolorbox}[warning]
+ If the memoized code changes the value of Beamer's pause counter
+ |beamerpauses|, e.g.\ by issuing a |\pause|, take care that (i) \refmmz{per
+ overlay} is executed prior to any changes of |beamerpauses|, and that (ii)
+ the final value of this counter in the memoized code is the same for all
+ overlays.
+\end{tcolorbox}
+
+
+\subsection{When stuff sticks out}
+\label{sec:tut:padding}
+
+Some constructs --- like plain \hologo{TeX}'s \refcmd{llap} and \refcmd{rlap},
+and, notably, \TikZ overlays --- fool \hologo{TeX} into thinking that the
+``size'' of the typeset material is different than what it actually is. This
+can cause trouble for externalization: a piece of your picture might disappear!
+In a sentence, the solution is to manually set the \emph{padding} of the
+externs, but let's slow down a bit.
+
+The \TikZ picture in the following example consists of node with a pin on the
+right, but let's say we want to horizontally center this picture so that only
+the node rather than the entire picture (including the pin) will be centered.
+This can be achieved by adding key \refkey{/tikz/overlay} to the pin (actually,
+we need to add it to both the pin and its edge). \TikZ normally updates the
+extents (called the bounding box) of the picture every time it puts something
+in it; when \refkey{/tikz/overlay} is in effect, however, these updates are
+temporarily disabled. In effect, the \refkey{/tikz/overlay} key on the pin
+below will fool \hologo{TeX} into thinking that the node is all there is to the
+picture, so centering will work as desired.
+
+\ExampleName{overlay}
+\makeexample{\examplename.pdf N=4}
+\tcbinputexample{
+ after title pre={\ (no memoization)},
+ comment={\centering
+ % \includeexamplepdf[document page]{page=1,trim=1.8in 7.4in 1.8in 1.7in}
+ \includeexamplepdf[document page]{page=1}
+ },
+}
+
+What happens when we try to externalize this picture? The example below shows
+what would happen if Memoize had no concept of \emph{padding} --- which we
+simulate by setting \code{\refmmz{padding}=0pt}.\footnote{Unlike in the rest of
+ the manual, the extern pages in this section are shown without trimming the
+ whitespace.} Along with the rest of \hologo{TeX}, Memoize would be fooled
+into thinking that the picture comprises of the node only, so the pin would
+never make it into the extern. You would end up with a document missing the
+pin!\footnote{On the first compilation, the document page containing the figure
+ without padding looks fine, as it uses the result of the compilation rather
+ than the extern file. But on the second compilation, when Memoize actually
+ uses the extern, the pin disappears.}
+
+\tcbinputexample[.tex][.c2]{
+ after title pre={\ (memoization without padding)},
+ attachment name=\examplename-no-padding.tex,
+ sidebyside, lefthand ratio=0.3,
+ comment={\centering
+ \includeexamplepdf[size=minimal,boxrule=0.5mm,extern page,
+ attach boxed title to top right={xshift=1.5mm,yshift=-1mm},
+ ]{page=1}
+ },
+}
+
+
+By default, Memoize puts an inch of space around (what it thinks is) the
+externalized picture, and if the overlayed parts of the picture fit into this
+inch of space, you will find them in the extern and therefore also in the
+document. In our example, however, the default padding is not enough --- the
+pin is only partially visible.\footnote{You might wonder why I didn't make the
+ default padding much bigger, like 10 inches. \hologo{TeX} wouldn't be
+ bothered (unless the resulting extern size exceeded its maximum dimension),
+ but you might be, because with such a large default padding, all the externs
+ would be huge, most often bigger than the document pages, and remember that
+ the externs are first dumped into the document, where they can bother you.}
+
+\tcbinputexample[.tex][.c3]{
+ after title pre={\ (memoization with default padding)},
+ attachment name=\examplename-default-padding.tex,
+ listing and comment,
+ middle=0.9mm,
+ comment={\centering
+ \includeexamplepdf[extern page,size=minimal,boxrule=0.5mm]{page=1}
+ },
+}
+
+The solution is to set the padding manually. Below, I used \refmmz{padding
+ right} to only increase the padding on the right side (clearly, we also have
+\refmmz{padding left}, \refmmz{padding top} and \refmmz{padding bottom}), but
+if you're not bothered by a large extern, you can just use
+\Emph{\refmmz{padding}}, which sets all four sides at once. By the way, having
+too much padding (almost) never hurts, and as you see, you can use (simple)
+arithmetics in the value of these keys.
+
+\tcbinputexample[.tex][.c4]{
+ after title pre={\ (memoization with extra padding)},
+ attachment name=\examplename-extra-padding.tex,
+ listing and comment,
+ middle=0.9mm,
+ comment={\centering
+ \includeexamplepdf[extern page,size=minimal,boxrule=0.5mm]{page=1}
+ },
+}
+
+\enlargethispage{1ex} % manual
+Incidentally, the \refmmz{padding} keys only change how
+the externalized picture is \emph{stored}. Memoize remembers the size of the
+extern as seen by \hologo{TeX} (e.g.\ the bounding box of the picture as
+reported by \TikZ, with overlayed parts of the picture protruding out of it),
+and it uses that size when integrating the extern into the document --- so
+everything works as it should!
+
+
+
+\subsection{The verbatim mode}
+\label{sec:tut:verbatim}
+
+Not all code will peacefully submit to memoization. In particular, this is the
+case for environments which process the environment body verbatim (or perform
+some other kind of \refcmd{catcode} magic). A simple environment of this kind is the
+standard \hologo{LaTeX} \refenv{verbatim}, but let us illustrate the issue with
+\refenv{tcblisting}, which typesets a code listing alongside its compiled effect.
+(This environment is defined by the |listings| library of package \pkg{tcolorbox}
+and was used extensively during the production of this manual.) To manually
+memoize a \refenv{tcblisting} environment, we enclose it in a \refenv{memoize}
+environment with a \Emph{\refmmz{verbatim}} key in the optional argument ---
+without this key, the example below would produce nothing but
+errors.\footnote{Memoize also offers a \emph{partial} verbatim mode, triggered
+ by key \refmmz{verb}; in this mode, the braces retain their usual category
+ codes. Also note that the effect of \refmmz{verbatim} can be ``undone'' by
+ key \refmmz{no verbatim}.}
+
+\ExampleName{verbatim-manual}
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ sidebyside,
+ comment={\centering
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\\[1ex]
+ \footnotesize
+ (The document page is the same as for the \texttt{verbatim-auto} example below.)
+ },
+}
+
+Using \refmmz{verbatim} from \refcmd{mmzset} or \refcmd{mmznext} works just as
+well, and the latter can be very useful with automemoization, when some
+environment (say, \env{tcolorbox}) generally does not require the verbatim mode,
+but a specific occurrence does (say, because it contains some verbatim
+construction such as \verb!|!\meta{verbatim text}\verb!|! of the \pkg{ltxdoc}
+class).
+
+However, for an environment such as |tcblisting|, it makes the most sense to
+declare it verbatim in general, so that all instances of the environment will
+be processed in the verbatim mode. This is simple to do: add \refmmz{verbatim}
+to the \refmmz{auto} keylist.
+
+\ExampleName{verbatim-auto}
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ sidebyside, % lefthand ratio=0.62,
+ comment={\centering
+ \footnotesize
+ (The extern page is the same as for the \texttt{verbatim-manual} example above.)\\[1ex]
+ \includeexamplepdf[document page]{page=2}\\
+ },
+}
+
+In fact, you can add any \refkeypath{/mmz} key to the \refmmz{auto} keylist,
+and the key will be applied to all occurrences of the command or the
+environment. For example, adding \refmmz{recompile} to the declaration of
+|tcblisting| above would recompile all and only the |tcblisting| environments;
+and as an \refmmz{auto} declaration only updates (rather than completely
+replaces) a previous declaration, you can also say things like
+\code{\refmmz{auto}=\refcmd{tikz}\braces{\refmmz{recompile}}} to recompile all
+\TikZ pictures produced by the \cs{tikz} command (handy, as you don't know how
+automemoization for \cs{tikz} was declared unless you've read
+section~\ref{sec:tut:automemoization-details} or looked at the Memoize's source
+code).
+
+
+\subsection{The final version of your document}
+\label{sec:tut:final}
+
+Bluntly put, you might want to disable Memoize when compiling the final version
+of your document, at least if you intend to distribute it in electronic form,
+for two reasons:
+\begin{itemize}
+\item An externalized picture cannot contain hyperlinks. Any hyperlinks (or
+ hyperlink anchors) contained in the original picture will silently disappear
+ during the production of the extern.
+\item When the document contains many externs, the size of the resulting PDF
+ can be several times the size of the PDF compiled without externalization.
+\end{itemize}
+
+Below, we list several ways of fully disabling Memoize. You're of course
+already familar with the first two ways, but what's this \refpkg{nomemoize}
+package? The rationale behind this package is that if you want to be
+absolutely sure that there is no trace of memoization in your document (for
+example, see the \refmmz{disable} -- \refmmz{enable} pitfall in
+section~\ref{sec:tut:working-on-a-picture}), the best thing to do is to not
+load the package at all. However, you have all those \refcmd{mmzset}s etc.\ in
+your source, so the document won't compile without
+\cs{usepackage}\braces{\refpkg{memoize}}, right? Right, but wrong. Enter
+\Emph{\refpkg{nomemoize}}, a dummy package which accepts all the commands that
+Memoize does, but does nothing. In effect, your document will compile, but you
+can be sure that not a single memo or extern was loaded or produced.
+
+\begin{tcbraster}[raster columns=3, raster force size=false, raster valign=top]
+ \begin{tcblisting}{listing only, add to width=2.4em, enhanced}
+\usepackage[~disable~]{memoize}
+ \end{tcblisting}
+ \begin{tcblisting}{listing only, add to width=-1.6em, enhanced}
+\usepackage{memoize}
+\mmzset{~disable~}
+ \end{tcblisting}
+ \begin{tcblisting}{listing only, add to width=-0.8em, enhanced}
+\usepackage{~no~memoize}
+ \end{tcblisting}
+\end{tcbraster}
+
+There is one issue you might need to resolve manually before package
+\refpkg{nomemoize} works as intended, though. If you have used any
+\refkeypath{/mmz} keys outside \refcmd{mmzset}, you need to list them in
+\refcmd{nommzkeys}. For example, if you used \refmmz{per overlay} in the manner
+illustrated in section~\ref{sec:tut:beamer}, i.e.\ as
+\refkeypath{/mmz}|/|\refmmz{per overlay} among the \TikZ keys, you need to
+write \refcmd{nommzkeys}\braces{\refmmz{per overlay}} into the document
+preamble.
+
+Another thing you might want to do once you have produced the final version of
+the document (in fact, just before you disable Memoize for good) is clean up.
+As we saw in sections~\ref{sec:tut:memodir} and~\ref{sec:tut:redefinitions},
+Memoize produces a lot of auxiliary files (memos and externs) and it keeps the
+old versions around! Once your document is prepared, you can reduce the
+clutter (and save some disk space) by deleting memos and externs belonging to
+the work-in-progress versions of your document, and keep only those used in the
+final version.
+
+You could achieve this by deleting all the memos and externs (if you're using
+the \refmmz{memo dir} directive, this amounts to the entire contents of the
+memo directory) and compiling your document for the final couple of times.
+However, there is an easier (but \hologo{TeX}-external) way: on the command
+line, change into the directory containing your (main) document and write
+\EmphTT{\refscript{memoize-clean.pl}} \meta{document name}\dmmz (substitute
+|.py| for |.pl| to use Python rather than Perl). The script will inspect the
+contents of the \dmmz record file to see which memos and externs were used in
+the final compilation, and delete all other memos and externs belonging to the
+given document.
+
+\begin{tcolorbox}[warning]
+ Deleting memos and externs is never an irreversible operation, as you can
+ always recreate them, but it is still wise to be cautious when cleaning up.
+ For one, avoid cleaning up after a compilation which produced errors; a
+ failed compilation can lead to an incomplete \dmmz file, which can in turn
+ lead to over-deletion. Another bad idea is cleaning up after disabling
+ Memoize for a part of a document, for the same reason.
+
+ All that said, Memoize takes some precautions itself. It will cowardly
+ refuse to perform the clean-up when the \dmmz file is missing the
+ end-of-file marker (|\endinput|), assuming that this indicates a fatal error
+ in the previous compilation. It will do the same in case the \dmmz file is
+ absent or empty. The latter is assumed to be a result of a globally
+ \refmmz{disable}d memoization, but note that clean-up will be performed if
+ memoization was disabled using package \refpkg{nomemoize}: that package does not
+ touch the \dmmz file, so cleaning up should work as intended.
+\end{tcolorbox}
+
+As the final note, memos and externs (cleaned-up or not) may be copied (along
+the document source) to another directory or machine, where they should be
+picked up by Memoize. There is no need to copy the \dmmz file (assuming that
+the document PDF contains no extern pages waiting for extraction).
+
+
+\section{Digging deeper}
+\label{sec:potential-pitfalls}
+
+
+
+\subsection{Good to know}
+\label{sec:tut:good-to-know}
+
+
+\paragraph{Line- and page-breaking} An extern can't be broken across lines or
+pages.
+
+Externalization of a chunk of code produces a PDF, which is included into the
+document at subsequent compilations as a picture --- an unbreakable object (a
+horizontal box) with fixed width and height. Therefore, the original code
+should produce an unbreakable object as well. For example, this means that you
+cannot externalize some text and expect \hologo{TeX} to break it across lines
+or pages on subsequent compilations. If you try, the compilation will succeed
+--- without an error! --- but your externalized text will end up in a single
+line, as shown below.
+
+\ExampleName{no-linebreaking}
+\makeexample{\examplename.pdf N=2}
+\tcbinputexample{
+ bad,
+ comment={\centering
+ \includeexamplepdf[extern page,to be continued on right]
+ {page=1,trim=1in 1in 14in 1in}\\[1ex]
+ \includeexamplepdf[document page]{page=2}\\[1ex]
+ \includeexamplepdf[document page,title=the expected document page]
+ [\examplepath.c2.pdf]{page=1}
+ },
+}
+
+That said, you \emph{can} externalize a paragraph or some other vertical mode
+material using \refmmz{capture}|=|\refmmz{capture=vbox}, but beware that the
+vertical spacing between the memoized material and its surroundings might
+change.
+
+
+\paragraph{\env{remember picture}}
+\TikZ pictures using this key cannot be externalized.
+
+Memoize will silently refuse to externalize any \TikZ picture using
+\refkey{/tikz/remember picture} (see \PGFmanual{17.13}). Such pictures
+interact with the outside world --- they either reference or are referenced by
+other pictures --- and are as such unsuitable for externalization. For
+example, while the colored boxes in this manual are generally externalized ---
+out of principle \Smiley\ --- the title page illustration is not, and it cannot
+be, because of the arrows connecting the various \TikZ pictures composing that
+illustration. Some packages use the \refkey{/tikz/remember picture} mechanism
+under the hood, and are thus subject to this limitation; one example is package
+\pkg{todonotes}, but in general, any package dealing with absolute positions on the
+page will be limited in this way.
+
+How does Memoize deal with this situation? Well, by cowardly refusing to
+externalize any code which uses \refkey{/tikz/remember picture} or a similar
+mechanism for dealing with absolute positions. Luckily, any such mechanism
+eventually boils down to the \hologo{TeX} primitive \refcmd{savepos}, so
+Memoize hacks --- or as we will say in this manual, \emph{advises} --- this
+primitive to abort any ongoing memoization. Initializing and then aborting the
+memoization takes some time, to be sure, but the overhead is negligible,
+especially in the light of the fact that \emph{not} aborting wreaks real havoc.
+
+Memoize actually provides a user interface for aborting memoization.
+Memoization can be aborted either manually, by executing \refcmd{mmzAbort}, or
+automatically. The latter is a generalization of the automemoization idea: a
+command such as \refcmd{savepos} can be advised to abort memoization by
+\refmmz{auto}|=|\refcmd{savepos}\bracestt{\refmmzauto{abort}}. In Memoize,
+commands take your advice seriously, so memoization will be aborted whenever
+the advised command or environment is encountered.
+
+
+\paragraph{Indirectly embedded environments} Such environments cannot be
+memoized.
+
+Read this if you got an error message such as |Environment "tikzpicture" ended
+as "foo"|.
+
+Some environments are defined so that they embed another environment using the
+idiom shown on the left: the begin-code of the outer environment opens the
+inner environment, and the end-code of the outer environment closes the inner
+environment. While this is a fine, and common, idiom, it messes up the
+memoization of the inner environment. In the example on the right, trying to
+\emph{auto}memoize a \env{minipage} environment (not recommended at all!) causes
+trouble with package \pkg{sectionbox}.\footnote{This is a \texttt{Package
+ collargs Error} because Memoize outsources the actual work of collecting
+ the environment body to the auxiliary package CollArgs, described in
+ section~\ref{sec:collargs}.}\fncomma\cprotect\footnote{Why does this happen?
+ As mentioned in section~\ref{sec:tut:memodir}, Memoize keeps track of memos
+ and externs by the MD5 sum of the memoized code. But to compute that sum for
+ an environment, Memoize has to \emph{grab} the environment body, meaning it
+ has to collect the body in advance. This presents no problem when
+ \refcmd{end}\meta{environment name} is already present in the input stream at the
+time \refcmd{begin}\meta{environment name} is executed, like when you use the
+ environment normally in your document, or when some macro expands so that it
+ produces both \cs{begin}\meta{environment name} and \cs{end}\meta{environment
+ name} simultaneously --- so there would be no problem above if
+ |\end{minipage}| occured in the beginning code of \env{sectionbox}. The idiom
+presented above is problematic for memoization because at the time \hologo{TeX}
+executes |\begin{sectionbox}|, putting |\begin{minipage}| into the input
+ stream, |\end{sectionbox}| is not yet executed and remains as it is. The
+ input stream therefore contains a pair of |\begin{minipage}| and
+ |\end{sectionbox}|. In the normal, non-memoizing course of events this
+ would not be a problem, because |\end{sectionbox}| would eventually expand to
+|\end{minipage}|. During memoization, however, this \emph{is} a problem,
+because, as we said, Memoize needs to grab the environment body: upon
+encountering |\begin{minipage}|, it looks in the input stream for
+ |\end{minipage}| --- but there is no |\end{minipage}| in the input stream,
+there is only |\end{sectionbox}|, and this results in the |Environment
+"minipage" ended as "sectionbox"| error.}
+
+\begingroup
+\ExampleName{sectionbox}
+\makeexample{\examplename.tex.c1}
+\mmzset{disable}
+\begin{tcbraster}[raster equal height]
+\begin{tcblisting}{listing only, bad, valign=center}
+\newenvironment{foo}% ... args
+{% the begin-code of foo
+ % ...
+ \begin{tikzpicture}
+ % ...
+}
+{% the end-code of foo
+ % ...
+ \end{tikzpicture}
+ % ...
+}
+\end{tcblisting}
+\tcbinputexample{
+ bad,
+ comment={\small\tt
+ ! Package collargs Error: Environment\\"minipage" ended as "sectionbox".
+ },
+ }
+\end{tcbraster}
+\endgroup
+
+What are your options in this kind of a situation?
+\begin{enumerate}
+\item The only way to perform any memoization here is to memoize the
+ \emph{outer} environment --- if that makes sense.\footnote{This avoids the
+ error because Memoize grabs and memoizes the outer environment, and while
+ it is memoizing it, further memoization is switched off.} You can do this
+ either on a case-by-case basis, by enclosing it in the \refenv{memoize}
+ environment, or automemoize it: \code{\refmmz{auto}=\marg{outer
+ environment}\braces{\refmmzauto{memoize}}}.
+\item But what if memoizing the outer environment is out of the question?
+ Then, the only way to avoid the error is to prevent the automemoization of
+ the inner environment.
+ \begin{enumerate}
+ \item If you are facing a single occurrence of the problem, it is perhaps
+ easiest to use key \refmmz{disable} just before the start of the outer
+ environment.
+ \item Otherwise, you can automatically disable memoization for the span of
+ each occurrence of the outer environment: \code{\refmmz{auto}=\marg{outer
+ environment}%
+ \braces{\refmmzauto{nomemoize}}}.
+ \item To deactivate the automemoization of the inner environment for the span
+ of the outer environment, but otherwise allow for memoization inside the
+ outer environment: \code{\refmmz{auto}=\marg{outer environment}%
+ \braces{\refmmzauto{noop}, \refmmz{deactivate}=\meta{inner
+ environment}}}. Key \refmmzauto{noop} does nothing but apply the
+ given options (in this case, \refmmz{deactivate}=\meta{inner environment})
+ to the advised command or environment.
+ \end{enumerate}
+\end{enumerate}
+
+\subsection{Extraction methods and modes}
+\label{sec:extraction}
+
+Remember that in Memoize, externalization is a two-step process. First,
+externs are typeset on separate pages of the main document, called extern
+pages. Then, these extern pages are extracted out of the main document PDF
+into extern files. The process is illustrated on the title page.
+
+Memoize is flexible in terms of which piece of software is used to perform
+extern extraction. It ships with three \emph{extraction methods}:
+
+\begin{description}
+\item[\refmmz{extract=perl}] A Perl script, \refscript{memoize-extract.pl}.
+ This method is the default because it is fast and because Perl is usually
+ already installed on a system running \hologo{TeX}. However, you will most
+ likely still need to install the PDF processing library
+ \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2}, or its fork
+ \hreftt{https://metacpan.org/pod/PDF::Builder}{PDF::Builder}; the
+ installation guidelines can be found in section~\ref{sec:install}.
+\item[\refmmz{extract=python}] A Python script, \refscript{memoize-extract.py}.
+ This method is even faster than the Perl script, though not by much. Try it
+ out if you have problems installing Perl or the required libraries, or if the
+ Perl script chokes on your document (see section~\ref{sec:troubleshooting}
+ for the list of known issues). Besides Python ($\geq 3.8$), you will also
+ need the Python library \hreftt{https://pypi.org/project/pdfrw/}{pdfrw} or
+ \hreftt{https://pypi.org/project/pdfrw2/}{pdfrw2}. For the installation
+ guidelines, see section~\ref{sec:install}.
+\item[\refmmz{extract=tex}] \hologo{TeX}-based extraction requires no
+ additional software, but it is much slower than the scripts. As \hologo{TeX}
+ can only produce a single PDF per compilation, an instance of \hologo{TeX}
+ (loading the entire document PDF) has to be invoked for each extern, and this
+ takes time (although the entire process is still much faster than the
+ venerable \TikZ externalization library).
+\end{description}
+
+Memoize is also flexible in terms of how extern extraction is triggered,
+providing two \emph{extraction modes}:
+
+\begin{description}
+\item[internal]\label{item:setup:who} By default, extern extraction is
+ triggered internally, i.e.\ by Memoize during the compilation of the
+ document; more precisely, any externs produced in a compilation are extracted
+ during the next compilation. To choose an extraction method other than the
+ default Perl script, load Memoize with the package option
+ \EmphTT{\refmmz{extract}=\meta{extraction method}}.
+
+\item[external] Loading Memoize with with package option
+ \EmphTT{\refmmz{extract}=\refmmz{extract=no}} instructs Memoize to \emph{not}
+ trigger the (internal) extraction. When instructed to use extraction
+ ``method'' \refmmz{extract=no}, Memoize expects you to trigger the extraction
+ yourself, in any way that is convenient to you: manually from the command
+ line, or automatically through your editor, a Makefile, etc. --- all Memoize
+ cares about is that the extraction takes place before the next compilation of
+ the document.
+\end{description}
+
+Summing up, the extraction mode and method are selected by providing the
+appropriate value to package option key \refmmz{extract}; the possible values
+are listed in the table below. Note that this key can only be used as a
+package option, or in \refcmd{mmzset} within \reffile{memoize.cfg}. In
+particular, it is disabled in the document preamble, because Memoize performs
+extraction while it is loaded.
+
+\medskip
+\begin{center}
+ \begin{tabular}{lll}
+ \toprule
+ extraction method&external program&Memoize invocation\\
+ \midrule
+ \refmmz{extract=perl}&\refscript{memoize-extract.pl}&\refcmd{usepackage}\braces{\refpkg{memoize}}\footnotemark\\
+ \refmmz{extract=python}&\refscript{memoize-extract.py}&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=python}}\braces{\refpkg{memoize}}\\
+ \refmmz{extract=tex}&|pdftex|&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=tex}}\braces{\refpkg{memoize}}\\
+ \refmmz{extract=no}&none (external extraction)&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=no}}\braces{\refpkg{memoize}}\\
+ \bottomrule
+ \end{tabular}%
+ \footnotetext{Or
+ \refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=perl}}%
+ \braces{\refpkg{memoize}}. This is useful if you have changed the default
+ using \reffile{memoize.cfg}.}%
+\end{center}
+\smallskip
+
+For internal extraction, \hologo{TeX} must be allowed to execute the external
+program implementing the chosen extraction method. Both |memoize-extract|
+scripts should be listed among restricted shell escape mode commands in your
+\hologo{TeX} distribution; their execution should therefore be allowed under
+the default, restricted shell escape mode. However, the |pdftex| program,
+executed by extraction method \refmmz{extract=tex}, is not listed there, nor
+should it be. If you are forced to use this fallback method, I suggest you
+compile documents loading Memoize under the full shell escape mode, by adding
+command-line option \docref{-shell-escape} (on \hologo{TeX}Live) or
+\docref{--enable-write18} (on MiK\hologo{TeX}) to the invocation of the
+\hologo{TeX} program. The answers linked from question
+``\href{https://tex.stackexchange.com/q/598818/16819}{How can I enable
+ shell-escape?}'' on \href{https://tex.stackexchange.com}{\hologo{TeX}
+ StackExchange} will tell you how you can ask your editor to do this for you.
+
+You may use any extraction method to perform external extraction. The simplest
+option is to use the Perl or the Python script. Supposing you are doing this
+manually from the command line, change into the directory containing your
+document, which should contain the auxiliary \dmmz file produced by Memoize,
+and execute:
+
+\begin{enumerate}[(a)]
+\item \refscript{memoize-extract.pl} \meta{document name}\dmmz \hfill (for
+ the Perl script)
+\item \refscript{memoize-extract.py} \meta{document name}\dmmz \hfill (for
+ the Python script)
+\end{enumerate}
+
+See sections~\ref{sec:.mmz} and \ref{sec:ref:extraction-perl-python} for
+further details on the \dmmz file and the extraction scripts.
+
+Things are a bit more complicated if you want to use the \hologo{TeX}-based
+extraction method externally, because an instance of |pdftex| needs to be
+invoked for each extern (and these have unwieldy names and can be many in
+number), but Memoize can help you here by producing a shell script or a
+makefile, executing which will extract all the externs at once. To have Memoize
+produce a shell script, use package option
+\code{\refmmz{record}=\refmmz{record=sh}} (or
+\code{\refmmz{record}=\refmmz{record=bat}} on Windows); package option
+\code{\refmmz{record}=\refmmz{record=makefile}} will make a makefile. By
+default, these files are named |memoize-extract.|\meta{document name} plus the
+|.sh|, |.bat| or |.makefile| suffix. If neither a shell script nor a makefile
+works for you, you can also define your own kind of \emph{record file}, to be
+processed by the external tool of your choice (and implementation) in order to
+extract the externs; see section~\ref{sec:new-record-file} to learn how to do
+this.
+
+
+\subsection{From cross-references to the context}
+\label{sec:cross-referencing}
+
+Cross-referencing presents a challenge to externalization, because without
+special provisions, the ``communication channel'' between the \cs{label} and the
+\cs{ref} is broken once we start utilizing the extern.
+
+One direction of the issue occurs when a \refcmd{label} within the memoized
+code is referenced by a \cs{ref} on the outside. Without the (built-in)
+workaround, the \cs{label} command would only be executed when the extern is
+being produced, but not on subsequent compilations of the document, when it is
+merely included. Memoize addresses this problem by generalizing
+externalization (which can only produce a picture, the extern) to memoization
+(which can additionally produce arbitrary code). When Memoize is externalizing
+code which contains a \cs{label}, it automatically replicates it into the
+\emph{memo}, which is input into the document on subsequent compilations. In
+effect, the memo--extern team will continue to produce the label even when it
+is utilized rather than compiled. As far as the author is concerned, \cs{label}s
+in memoized code ``just work,'' without any observable differences to the
+situation without memoization. This is why we will not discuss this direction
+of the issue here; a reader interested in how precisely the system works is
+invited to read section~\ref{sec:memos}.
+
+The other direction of the issue occurs when a \refcmd{ref} within the memoized
+code references \cs{label} on the outside. In this situation, the extern
+should be recompiled when the value of the label it refers to changes. Again,
+Memoize addresses this problem in full generality, by associating with each
+extern a \emph{context}, and recompiling the extern whenever the value of the
+context changes.\footnote{The dependency of an extern upon prior definitions
+ and such can also be addressed in a more \emph{ad hoc} manner, by recompiling
+ manually; we have already touched upon this subject in
+ section~\ref{sec:tut:working-on-a-picture}, and will revisit it in
+ section~\ref{sec:tut:redefinitions}.} All that needs to be done for \cs{ref}
+and friends, specifically, is to advise them to add their reference keys to the
+context.
+
+As we shall see presently, for the author, the only difference between a
+non-memoized and a memoized \cs{ref} is that the latter will take one more
+compilation cycle to ``stabilize'' the resulting document. (More precisely,
+the memoized situation will take one more cycle if the reference is undefined
+on the first compilation.) Then, we will show how we can teach Memoize about
+cross-referencing commands other than \cs{ref} and \cs{pageref}. Finally, we will
+learn about key \refmmz{context}, the backbone of the cross-referencing support
+in Memoize. (The inner workings of the context are further explained in
+section~\ref{sec:c-memos}.)
+
+\ExampleName{ref}
+\makeexample{\examplename.pdf N=7}
+
+When the memoized code contains a \cs{ref} referring to a label given in another
+part of the document, the code is recompiled when (and only when) the reference
+changes. Let us look at the following example, jumping in at the point where
+it was already compiled enough times that the resulting PDF had stabilized into
+a single (document) page with correct references. (Environment
+\refenv{nomemoize} disables memoization
+of \href{https://ctan.org/pkg/tikzlings}{Ti\emph{k}Zlings}, so that their
+externs don't disturb us, and we can focus on the \cs{tikz} command, which does
+get externalized and contains a \cs{ref}.)
+
+\tcbinputexample[.tex][.c3]{
+ after title pre={\ (with stable output after three compilations)},
+ middle=1mm,
+ comment={\centering
+ \includeexamplepdf[document page]{page=1}
+ }
+}
+
+Let us add an owl in front of the penguin. In the next compilation, neither
+the ``normal'' nor the memoized reference is yet updated, as expected --- in
+this compilation, the new value of the penguin label only makes it into the
+|.aux| file.
+
+\tcbinputexample[.tex][.c4]{
+ after title pre={\ (after the first compilation with the added owl)},
+ no attachment,
+ comment={\centering
+ \includeexamplepdf[document page]{page=1}
+ }
+}
+
+During the following compilation, the \refcmd{ref}s pick up the new value of the
+penguin label, and the \cs{ref} inside the automemoized \cs{tikz} command
+forces recompilation of the extern (\emph{how} this is done will be explained
+later).
+
+\tcbinputexample[.tex][.c5]{
+ after title pre={\ (after the second compilation with the added owl)},
+ comment only, no attachment,
+ comment={\centering
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad
+ \includeexamplepdf[document page]{page=2}
+ }
+}
+
+In the next compilation, the resulting PDF is finally stabilized, as the
+updated extern is (extracted and) included into the document.
+
+\tcbinputexample[.tex][.c6]{
+ after title pre={\ (after the third compilation with the added owl)},
+ comment only, no attachment,
+ comment={\centering
+ \includeexamplepdf[document page]{page=1}
+ }
+}
+
+The message to take home? When some memoized code contains a reference and
+that reference changes, it will take three compilation cycles (so, one more
+cycle than without memoization) for the resulting document to ``stabilize.''
+
+Out of the box, Memoize supports the standard \hologo{LaTeX} cross-referencing
+commands \cs{ref} and \refcmd{pageref}. To automatically recompile code
+containing some other cross-referencing command, like \refcmd{vref} of package
+\pkg{varioref}, we use the advising framework implemented by package Advice.
+This framework is a generalization of automemoization: we use the familiar
+\refmmz{auto}, but with advice offered by \Emph{\refmmzauto{ref}} rather than
+\refmmzauto{memoize}.
+
+{\ExampleName{vref}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{listing only}}
+
+Key \refmmzauto{ref} only works for commands which operate on a single
+reference key. However, that single key (which must be enclosed in braces) may
+be preceded by optional argumen(s) of any kind. Extensions to \refcmd{ref},
+e.g.\ the \pkg{hyperref}'s variant, which accepts an optional |*|, work out of
+the box. Furthermore, Memoize offers support for cross-referencing commands
+which work on multireferences and reference ranges, such as \pkg{cleveref}'s
+\refcmd{cref} and \refcmd{crefrange}. Those commands should be advised by
+\refmmz{auto} keys \refmmzauto{multiref} and \refmmzauto{refrange},
+respectively.
+
+We have jumped into first example of this section with the assumption that it
+had already been compiled several times, allowing the resulting PDF to
+stabilize. Let us now take a look at what happens at the very first, fresh
+compilation of our original example (the one without the owl). (Removing the
+|.aux| file before compiling the example again will start afresh.) The curious
+thing is that we don't get the extern page containing
+\tikz[baseline]\node[draw=red,thick,fill=yellow,anchor=base,font=\bf]{??};.
+This is so because by default, Memoize aborts a memoization containing an
+undefined reference.
+
+\tcbinputexample[.tex][.c1]{
+ after title pre={\ (after the fresh compilation of the original example)},
+ comment only, no attachment,
+ comment={\centering
+ \includeexamplepdf[document page]{page=2}
+ }
+}
+
+Now sometimes you might want to produce an extern even if it contains an
+undefined reference --- for example, you might intend to write the code
+containing the \cs{label} much later but enjoy the speed-up offered by Memoize
+until then. In that case, apply the \refmmz{auto} key \Emph{\refmmzauto{force
+ ref}} to \cs{ref}.
+
+\tcbinputexample[.tex][.c7]{
+ after title pre={\ (after the fresh compilation with \refmmzauto{force ref})},
+ attachment name=\examplename-force.tex,
+ comment={\centering
+ \includeexamplepdf[extern page][\examplepath.c1.pdf]{page=1,trim=1in 1in 1in 1in}\quad
+ \includeexamplepdf[document page][\examplepath.c1.pdf]{page=2}
+ }
+}
+
+However, note that when you use \refmmzauto{force ref}, \hologo{LaTeX} will
+\emph{not} complain about the undefined reference once the extern containing it
+is included (unless that reference also occurs in some non-memoized piece of
+code). Using \refmmzauto{force ref} is therefore a tiny bit dangerous, and
+this is why \refmmzauto{ref}, with the abortion mechanism, is the default
+handler for \cs{ref} and \refcmd{pageref}.
+
+As already noted in the previous section, \refcmd{ref} works by appending the
+cross-reference to the \emph{context}, the change of which triggers
+recompilation. Memoize initializes the context to contain the four
+\refmmz{padding}s --- as a result, an extern recompiles if we change the
+padding --- but we can append stuff to the context by ourselves, as well.
+Below, we use key \Emph{\refmmz{context}} to append the font size (we'll talk
+about the value given to this key a bit later); as a result, the picture is
+recompiled whenever the font size changes. Below, we change the font size
+using command |\small|; changing the default size with a class option such as
+|12pt| works as well.
+
+\ExampleName{fontsize}
+\makeexample{\examplename.pdf N=2}
+\tcbinputexample{
+ sidebyside, lefthand ratio=0.7, after title pre={\ (the first version)},
+ comment={
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
+ }
+}
+
+\tcbinputexample[.tex][.c2]{
+ sidebyside, lefthand ratio=0.7, after title pre={\ (the second version)},
+ no attachment,
+ comment={
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
+ }
+}
+
+How does this work? Key \refmmz{context} appends the given tokens to the
+\emph{context expression}. When creating an extern or trying to use it,
+Memoize (fully) expands this expression and computes the MD5 sum of the
+expansion. This \emph{context MD5 sum} then serves as a part of the extern's
+filename (see sections~\ref{sec:tut:memodir} and~\ref{sec:memos}). In effect,
+Memoize will only find (and utilize) the extern if \emph{the context MD5 sum
+ computed during (attempted) utilization matches the one computed during
+ memoization}.
+
+As revealed by looking at the \hologo{LaTeX} source code, \hologo{LaTeX} holds
+the current font size in macro |\f at size|, and above, we have effectively added
+the contents of this macro to the context. Now, why didn't we simply write
+\refmmz{context}|=\f at size|? First, we used |\csname ... \endcsname| because we
+were under the normal \hologo{LaTeX} catcode regime, where |@| cannot be a part
+of the command name. Of course, we could have temporarily changed the catcode
+of |@| using |\makeatletter| and |\makeatother|, but I would advise against
+that, because the approach does not work in general: it fails when key
+\refmmz{context} is used \emph{within} memoized code (we will explain why in
+section~\ref{sec:memos}). Another reason why I recommend the |\csname
+... \endcsname| approach is that it does not result in an error when the
+control sequence is not defined (|\csname ... \endcsname| will expand to
+|\relax| then); this behaviour is handy for undefined cross-references, for
+example. Second, why did I write |fsize={...}| around the control sequence?
+Well, because I'm being paranoid, really. Writing \refmmz{context}|={\csname
+ f at size\endcsname}| would work just as well, but I like to explicitly
+``announce'' the value to prevent any possibility of a conflict with an
+alternative context. Imagine that we don't use the ``announcements'' and we
+decide to add some other dimension instead of the font size to the context.
+Now if that dimension happened to have the same value as the font size, Memoize
+would incorrectly pick up the ``font size extern'' instead of producing a new
+one.
+
+It bears emphasizing that whatever you add to the context expression must be
+fully expandable, and also not merely declared as robust. So writing
+\code{\refmmz{context}=\cs{ref}\marg{key}}, for example, would be unwise, since
+it would not work as intended when package \pkg{hyperref} is loaded. (This
+package declares \refcmd{ref} as robust, so it won't expand to the
+cross-reference value.) You have to look up where the cross-references are
+stored internally; the cross-reference for \meta{key} turns out to be stored in
+the internal control sequence |\r@|\meta{key}, so it is |\csname
+r@|\meta{key}|\endcsname| that the \refmmzauto{ref} handler actually appends to
+the context.
+
+The padding and font-size contexts are useful quite generally. However, the
+context can be pretty command-specific, as well. Consider the \pkg{skak}
+package used for typesetting chess games. The board is drawn using command
+|\showboard|, but this command has no arguments, because it draws the state of
+the board that is reached by the moves given by command |\mainline|. Memoizing
+|\showboard| as such will therefore yield the wrong result --- all the boards
+will be one and the same board! The solution is to provide the correct
+context: we dig into the \pkg{skak} sources and realize that the current
+board is stored in macro |\csname chessgame.skak.mainline\endcsname|.
+
+\ExampleName{skak}
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ sidebyside, lefthand ratio=0.48,
+ comment={
+ \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in,scale=0.45}\quad
+ \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in,scale=0.45}
+ }
+}
+
+If you remove \code{\refmmz{context}=\bracestt{...}} from the code above, you
+will end up with a document where the final board drawn takes place of all the
+boards. This is so because in that case, all externs are written into the same
+file, so the final extern overwrites the previous ones, but note that you will
+only observe this after the second compilation, when the externs are actually
+used.
+
+
+
+\subsection{More on redefinitions and stale externs}
+\label{sec:tut:redefinitions}
+
+In this subsection, we elaborate on an issue touched upon at the beginning of
+section~\ref{sec:tut:working-on-a-picture}: what happens if the memoized code
+depends on some macro or style which gets redefined? The answer was
+``nothing,'' and one solution was to \refmmz{recompile} the code. Let us take
+the example from that section a bit further. We will propose no new solution
+or workaround, but deepen our understanding of the issue.
+
+\ExampleName{redefinitions}
+\makeexample{\examplename.pdf N=6}
+\tcbset{
+ recompile step/.style={
+ sidebyside, lefthand ratio=0.74, title=,
+ comment={\includeexamplepdf[blankest][\examplepath.c\theenumi.pdf]{#1}},
+ /utils/exec={\addtocounter{enumi}{1}},
+ enhanced, overlay app={\node[at=(frame.north west), font=\scriptsize, circle, fill=black!80!white, text=white, inner sep=1pt]{\theenumi};},
+ },
+}
+\begin{tcboxedraster}[raster columns=1]{%
+ title=Working on \texttt{\examplename.tex}, enhanced, breakable}
+ \setcounter{enumi}{0}%
+ \tcbox[blankest]{I like red. My emphasized nodes will have red background.}
+ \tcbinputexample[.tex][.c1]{recompile step={page=2}}
+ \tcbox[blankest]{Hmm, this particular node is really important,
+ let me put the text in italics as well!}
+ \tcbinputexample[.tex][.c2]{recompile step={page=2}, no attachment}
+ \tcbox[blankest]{You know what? Perhaps yellow background would work better
+ --- in general.}
+ \tcbinputexample[.tex][.c3]{recompile step={page=1}, bad, no attachment}
+ \tcbox[blankest]{How come my node is still red?! Oh yes, I changed the
+ style, so I have to recompile the extern!}
+ \tcbinputexample[.tex][.c4]{recompile step={page=2}, no attachment}
+ \tcbox[blankest]{Ahh, yellow background, that's much better. But you know
+ what, this double emphasis won't do after all, let me go back to the
+ upright shape.}
+ \tcbinputexample[.tex][.c5]{recompile step={page=1}, bad, no attachment}
+ \tcbox[blankest]{Red???!???!? Ok, I know that recompiling will help, but what
+ happened here?}
+ \tcbinputexample[.tex][.c6]{recompile step={page=2}, no attachment}
+\end{tcboxedraster}
+
+What happened is that the externs from steps~1 and~5 share the very same code.
+In step~1, this code was compiled when the red |emph| style was in effect, and
+that extern lingered and was eventually picked up again in step~5, Memoize
+having no idea that it is including an extern produced with the obsolete
+definition of the style.
+
+There are two points to this story. First (and forgetting for a moment about
+the context, which we started discussing in
+section~\ref{sec:cross-referencing}), Memoize identifies externs (and memos) by
+the code that produced them --- or more precisely, by the MD5 sum of the code,
+as each piece of code has a unique (well, unique enough) MD5 sum. Each extern
+is saved in a file whose name contains this MD5 sum; see
+section~\ref{sec:tut:memodir} for illustration. Generally, this is a very
+useful feature. You can move your picture to another location in the document,
+insert some other (externalized) picture in front of it, and so on, all without
+triggering recompilation of the extern(s). (None of this is possible with
+\TikZ's externalization library, which identifies the externs by the order in
+which they appear in the document.)
+
+The downside of the MD5 sum approach is the potential pitfall illustrated
+above, and the downside comes about because of the second point of the story:
+Memoize does not attempt to delete the ``old'' externs. (However, as described
+in section~\ref{sec:tut:memodir}, stale memos and externs can be easily removed
+using the \refscript{memoize-clean.pl} script.) That would be not only
+dangerous (as any deletion inherently is) but also potentially wasteful: what
+if you have only temporarily removed some code, or compiled only a portion of
+the document --- you surely wouldn't want your hard-won externs to disappear in
+such a situation!
+
+The pitfall described above applies to any command which depends on parameters
+which can be set prior to the invocation of the command, like \TikZ pictures,
+which depend on the settings given in \cs{tikzset}. After customizing the
+settings, you will have to recompile the externs:
+\refcmd{mmznext}\braces{\refmmz{recompile}} is useful when you only have to
+recompile a single extern; use \refcmd{mmzset}\braces{\refmmz{recompile}}, or
+the package option \refmmz{recompile}, to recompile all externs in the
+document; and there is also the middle road: if you have changed only Forest's
+settings, you can write
+\code{\refmmz{auto}=\braces{forest}\braces{\refmmz{recompile}}} to recompile
+all and only the Forest trees.
+
+Above, we have seen the ``same code, same extern'' issue manifested ``through
+time,'' i.e.\ Memoize was (incorrectly) reusing externs produced in previous
+compilations, but the issue can also manifest ``through space.'' This can
+happen if the same code appears twice in the same document --- but, crucially,
+with some parameters which it depends on changed from one occurrence to the
+next. Observe what happens in the following example, where the settings for
+|\progressbar| are changed by |\progressbarchange|. After the first
+compilation, everything looks fine. But as both extern pages were produced by
+the same code, they will be stored into the same file, the second one
+overwriting the first one. The second compilation pulls in the externs --- or
+rather, the single extern --- resulting in the document containing the second
+progressbar in the place of the first one as well.
+
+\ExampleName{progressbar}
+\makeexample{\examplename.pdf N=2}
+\tcbinputexample{
+ bad, float,
+ comment={
+ \begin{tcbitemize}[raster columns=3,raster equal height,halign=center,valign=center]
+ \tcbitem[title=After 1\textsuperscript{st} compilation]
+ \includeexamplepdf[extern page,remember as=extern1]{page=1,trim=1in 1in 1in 1in}\\
+ \includeexamplepdf[extern page,remember as=extern2]{page=2,trim=1in 1in 1in 1in}\\
+ \includeexamplepdf[document page]{page=3}
+ \tcbitem[title=After extern extraction\vphantom{p\textsuperscript{nd}}]
+ \includeexamplepdf[my boxed title=the extern file,attach shifted boxed title to top right,remember as=externfile]{page=2,trim=1in 1in 1in 1in}
+ \tcbitem[title=After 2\textsuperscript{nd} compilation]
+ \includeexamplepdf[document page,remember as=doc][\examplepath.c2.pdf]{page=1}
+ \end{tcbitemize}
+ \begin{tikzpicture}[remember picture,overlay]
+ \draw[->, thick, red] (extern1.east) to[out=east, in=west] ([yshift=0.5ex]externfile.west);
+ \draw[->, thick, red] (extern2.east) to[out=east, in=west] ([yshift=-0.5ex]externfile.west);
+ \draw[->, thick, blue] (externfile.east) to[out=east, in=west] (doc.west);
+ \end{tikzpicture}
+ },
+}
+
+The same ``extern duplication'' can arise due to how a particular command is
+implemented. Say we deactivate automemoization of Forest trees
+(\code{\refmmz{deactivate}=forest}), but keep on automemoizing \TikZ pictures.
+Forest uses \refenv{tikzpicture}\noprint{\refenv{forest}} under the hood (a
+lot); in particular, the tree itself is typeset as a \env{tikzpicture}
+environment. But the code that typesets it is the same for all trees,
+regardless of their content (the actual content of the tree is hidden in
+various macros and boxes, rather than ``pasted'' into the \env{tikzpicture}).
+Consequently, the final tree of the document will overwrite all other trees in
+the document, just as the second (and thus final) progress bar overwrote the
+first one above.\footnote{That is assuming that \hologo{TeX} doesn't simply
+ spew a bunch of errors. This can happen as well. In the interest of full
+ disclosure, compiling a Forest tree in the situation described above would
+ actually also produce --- but only in the first compilation --- a number of
+ small empty extern pages, one for each node of the tree. A promise: Forest
+ will soon fully support Memoize and (among other things) avoid this pitfall.
+ But the principle will remain.} Ouch!
+
+Generally speaking, this final sort of extern duplication issue can arise
+whenever we have an ``outer'' command that we don't want to (auto)memoize which
+uses an ``inner'' command that we \emph{do} want to automemoize. The solution
+is to use the \refmmz{auto} key \refmmzauto{nomemoize} on the outer command;
+remember that this key disables memoization for the space of the command or
+environment. For example, the correct way to ``deactivate'' automemoization of
+\env{forest} environments (but keep automemoizing \TikZ pictures) is
+\code{\refmmz{auto}=\bracestt{forest}\braces{\refmmzauto{nomemoize}}}.
+
+
+
+
+\subsection{Supporting Memoize in your package}
+\label{sec:tut:package}
+
+\subsubsection{Loading Memoize?}
+\label{sec:loading-memoize}
+
+So you want to support Memoize in your package? That's great!
+
+What form precisely this support will take of course depends on your package.
+For some commands, a simple \refmmz{auto} declaration will suffice; for other
+commands, you might need to write a dedicated \emph{memoization driver}, as
+explained in section~\ref{sec:memoization-drivers}. However, one thing is
+clear: you \emph{don't} want to require Memoize's presence by
+\cs{RequirePackage}\braces{\refpkg{memoize}} in your package. That would
+trigger memoization, but triggering memoization should be left at the sole
+discretion of the author. The question is, if you're not allowed to load
+Memoize, how can you even issue the \refmmz{auto} declaration?
+
+Well, it's not that you really want to \emph{memoize} anything; you want to
+make the commands of your package \emph{memoizable}. So:
+\cs{RequirePackage}\braces{\Emph{\refpkg{memoizable}}} --- and note that in
+`memoizable', the final `e' of `memoize' is dropped, apparently this is the
+correct way to spell it.
+
+Loading \refpkg{memoizable} does nothing if Memoize is already loaded, and
+behaves like package \refpkg{nomemoize} otherwise --- remember from
+section~\ref{sec:tut:final} that \refpkg{nomemoize} is a dummy package which
+accepts all the commands that Memoize does, but does nothing.
+
+I have decided to require that Memoize must be loaded before any package that
+supports it. Allowing for an arbitrary loading order would complicate the
+implementation (and possibly even turn out to be problematic), and furthermore,
+Memoize likes to be loaded early anyaway, because it needs to be loaded before
+the document PDF is opened if it is to perform the embedded extern extraction.
+I don't think the ordering requirement will cause any problems --- let me know
+if it does! --- but perhaps it is wise to inform the author about it in the
+documentation of your package (I did so at the end of
+section~\ref{sec:tut:automemoization}). Anyway, I have enforced the
+requirement by raising an error and refusing to load the package in case
+Memoize detects \refpkg{memoizable} to be loaded.
+
+Note that the loading order requirement implies that you can use
+|\@ifpackageloaded{memoize}| to specifically react to the presence Memoize, if
+necessary.
+
+
+\subsubsection{Memoizable design}
+\label{sec:memoizable-design}
+
+Many commands and environments can be submitted to externalization with a
+single-line \refmmz{auto} declaration, as illustrated in
+section~\ref{sec:tut:automemoization}, perhaps requiring an addition to the
+context (section~\ref{sec:cross-referencing}), or some pre- or post-memoization
+code (section~\ref{sec:tut:beamer}). In some situations, however, these simple
+approaches won't work. Most often, this will happen when the extern must be
+integrated into the document in some special way. For example, a command might
+internally create floating material, or surround the core typeset material with
+some stretchable space.\footnote{Commands and environments of package
+ \pkg{tcolorbox} exhibit both these issues (see \pkg{tcolorbox} options |float|,
+ |before| and |after|), and were in fact the inspiration for several technical
+ details of Memoize.} None of these behaviours can be replicated by merely
+including the extern; with respect to the stretchable space, remember that an
+extern, being a picture, has fixed size, so if our extra space ended up in the
+extern, it would lose the stretchability.
+
+The key to successful memoization of problematic commands is their design. In
+a nutshell, the idea is to \emph{break up the command's definition into two
+ parts, the outer command and the inner command, and only submit the inner
+ command to automemoization}. We will illustrate this with a simple
+environment --- |poormansbox| --- which produces a potentially framed box of
+the given width, and surrounds this box with some configurable material --- by
+default, this material will be stretchable vertical space, and this will be the
+source of our memoization problem. (In terms of user experience, the solution
+in this section will leave something to be desired, but we will revisit the
+example in section~\ref{sec:memoization-complex-single-driver} and make things
+right.)
+
+Let us first take a look at a document using our to-be-developed box
+environment. The |poormansbox| environment takes one optional argument, a
+keylist of options, which can also be set with the |\pmbset| command. This
+being a \textbf{p}oor \textbf{m}an's \textbf{b}ox, it doesn't recognize many
+options. One can set the |width| of the box, or request that it occurs in a
+|frame|, and the surrounding material can be configured using keys |before| and
+|after|. As we will see later in the listing of the package, the box is
+|\linewidth| wide by default, has no frame, and is surrounded by vertical glue
+|\vskip 2ex plus 1ex minus 1ex| (|2ex| of natural vertical space which may both
+stretch and shrink for |1ex|); furthermore, the default value of |before|
+contains |\centering| to center the box horizontally (centering is of course
+only observable when we change the width of the box).
+
+\ExampleName{poormansbox}
+\makeexample{\examplename.sty}
+\makeexample{\examplename.pdf}
+\tcbinputexample{
+ sidebyside, lefthand ratio=0.482,
+ comment={\centering
+ \includeexamplepdf[document page,left=0pt,right=0pt]
+ {page=1,scale=0.5,trim=0mm 3mm 0mm 3mm}
+ },
+}
+
+You might want to play with the example to see that the surrounding vertical
+space is indeed stretchable. The example is set up so that the surrounding
+space is shrunk a bit to fit all the material onto one page. But if you remove
+the final |\lipsum[144]|, the natural amount of all vertical space can be
+accommodated on the page, so you should observe an increase of vertical
+spacing.
+
+You might have noticed that the example contains nested |poormansbox|es: the
+second box (the one which contains |\lipsum[66]|) is nested within the first
+one (between |\lipsum[101]| and |\lipsum[75]|). This is intentional: when we
+will revisit the |poormansbox| example in
+section~\ref{sec:memoization-complex-single-driver}, the implementation will
+have to pay special attention to nesting (which presents no problem to the
+implementation in this section).
+
+As you can see in the package listing below (\texttt{\examplename.sty}), the
+implementation of our environment is straightforward. We first define the
+configuration command |\pmbset| and the option keys (we're using
+\pkg{pgfkeys}), and set the option defaults. Then, we move on to the
+environment itself: we apply the given options, execute the pre-code, typeset
+the box (which is a |minipage| of the given width, potentially wrapped in a
+|\fbox|), and execute the post-code.
+
+\tcbinputexample[.sty]{listing only, float}
+
+\ExampleName{poormansbox-memoizable}
+Now let's make our |poormansbox| externalizable (\texttt{\examplename.sty}). As
+announced above, the idea is to split the definition of the environment into
+the outer part (below, the user-level environment |poormansbox|), which
+(applies the options and) executes the pre- and the post-code, and the inner
+part (below, the macro |\@poormansbox|), which typesets the actual box. If we
+then then submit |\@poormansbox|, rather than |poormansbox|, to
+automemoization, the outer part will be executed at every compilation (giving
+us stretchable space if we request it), while the inner command will be
+executed (and memoized) at the first compilation, and substituted for the
+extern (the fixed-size box) at subsequent compilations.\cprotect\footnote{You
+ might have wondered why our definition of the |poormansbox| environment grabs
+ the body into an argument (|+b|, yielding |#2|), necessitating the use of
+ |\NewDocumentEnvironment| over the venerable |\newenvironment|. One reason
+ was that having the environment body as an argument simplifies wrapping the
+ |\fbox| around the |minipage|, but there is a more important reason. If we
+ did not grab the environment body, we would have to implement the internal
+ part of the definition as an environment (|@poormansbox|) as well, and embed
+ it into the user-level environment using the following idiom:
+ |\newenvironment{poormansbox}[2][]{...\begin{@poormansbox}}{\end{@poormansbox}...}|.
+ However, as illustrated in section~\ref{sec:tut:good-to-know}, automemoizing
+ an environment indirectly embedded in such a way produces an error, because
+ Memoize is prevented from collecting the environment body.}
+
+\makeexample{\examplename.sty}
+\makeexample{\examplename.pdf}% just to know if it compiles fine
+\tcbinputexample[.sty]{listing only, float}
+
+Looking at the definition of the internal |\@poormansbox| command, it might
+strike you as weird that we have equipped this command with an optional
+argument (|#1|) it never uses. However, this optional argument is crucial for
+memoization. It will become a part of the memoized code (note
+\code{\refmmzauto{args}=om} in the \refmmz{auto} declaration) and thereby
+ensure that Memoize will produce separate externs for invocations of
+|\@poormansbox| with the same environment body but different options; or in
+other words, it will ensure that changing the options recompiles the
+extern.\cprotect\footnote{Of course, this only holds for options given in the
+ optional arguments; if the user changes an option value using a prior
+ \cs{pmbset} (and that option does not occur in the optional argument),
+ Memoize won't detect the change. But the end-user knows about this issue, as
+ it was addressed in sections~\ref{sec:tut:working-on-a-picture}
+ and~\ref{sec:tut:redefinitions}, and she is also aware of two workarounds:
+ manual recompilation, or setting the context
+ (section~\ref{sec:cross-referencing}).
+
+ While we're on the subject of the context, note that it is also possible to
+ deploy context to trigger recompilation of the inner command upon change of
+ parameters it depends on. We could simply omit the optional argument of
+ \cs{@poormansbox} and add
+ \refmmz{context}|={width=\pmb at width,frame=\ifpmb at frame true\else false\fi}|,
+ to the \refmmz{auto} declaration. The advantage of such an approach is that
+ Memoize reacts to the change of parameters regardless of whether they are set
+ using the optional argument or \cs{pmbset}. However, the approach is
+ unfeasible for commands depending on many parameters: can you imagine listing
+ all the \TikZ options in the context? Not to mention that a particular
+ picture usually only depends on a small subset of these options --- by and
+ large, \TikZ externs would get recompiled too often if the context contained
+ all \TikZ options.}\footnote{I have toyed with the idea of
+ splitting (using \pkg{pgfkeys} key filtering) the given options into outer
+ options, relevant for the outer command, and inner options, relevant for the
+ inner command, and only passing the inner options to the inner command. The
+ thought was that would (i) avoid recompiling the extern when only outer
+ options change, as these options don't affect the inner command, and (ii)
+ avoid applying the inner options when utilizing the extern, as these options
+ don't affect the outer command. However, it then hit me that the end-user
+ might define a style which incorporated both inner and outer options --- I
+ know I do this with my |tcolorbox|es.}
+
+The downside to automemoizing an internal command is that this might be
+counter-intuitive for the author. For example, to deactivate automemoization
+of |poormansbox|, the author will have to write
+\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{@poormansbox}} (note the
+|\@|), but they will have no clue they have to do this unless they have
+carefully read |poormansbox|'s documentation. Even worse, the above
+\refcmd{mmzset} command will not work unless surrounded by |\makeatletter| and
+|\makeatother|, as it refers to an internal control sequence containing |@|.
+Well, Memoize offers \refmmz{auto csname}, \refmmz{activate csname} and
+\refmmz{deactivate csname}, so that |@| catagory code manipulations can be
+omitted by writing \refcmd{mmzset}\bracestt{\refmmz{deactivate
+ csname}|=@poormansbox|}, but still.
+
+Another downside could occur when you use the same (automemoized) internal
+command in service of several user interface commands. For the sake of
+illustration, assume we have also defined an UI-macro |\pmb| which again relies
+on |\@poormansbox|. How is the author to deactivate automemoization of |\pmb|
+but leave the |poormansbox| environment intact? This is how:
+\refcmd{mmzset}\bracestt{\refmmz{auto}=\cs{pmb}\bracestt{\refmmzauto{args}=m,
+ \refmmzauto{nomemoize}}}. Again, counter-intuitive; the author expects
+\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{pmb}} to work.
+
+One other consequence of this approach is that the code included in the c-memo
+(if \refmmz{include source in cmemo} is in effect) will not faithfully reflect
+the source: as shown in the c-memo listing below, it will contain
+|\@poormansbox{...}| instead of |\begin{poormansbox}...\end{poormansbox}| ---
+even if this might actually count as an advantage, as the discrepancy will at
+least inform the author who refuses to read the fine material accompanying our
+|poormansbox| that something funky is going on.
+
+\begingroup
+\relaxmmzcommands
+\def\mmzNewCMemo#1{% fetch the last c-memo filename
+ \def\mycmemo{#1}%
+}
+\input{\examplepath.mmz.c1}
+\sed{%
+ s/[]]\lbrace/]\n\space\space\space\space\space\space\space
+ \space\space\space\space\space\space\space\lbrace/;
+}{\exampledir\mycmemo}
+\tcbinputlisting{
+ listing only,
+ listing file=\exampledir\mycmemo,
+ example title,
+ title=the c-memo of the last \texttt{poormansbox} environment,
+ left=0.5em, right=0em,
+}
+\endgroup
+
+In a nutshell, automemoizing an internal command might be counter-intuitive for
+the author. But the core idea --- to support memoization of a resistant
+command by splitting its definition into the outer and the inner command --- is
+sound, and we will elaborate on this idea in
+section~\ref{sec:memoization-complex-single-driver}, where we will revisit our
+|poormansbox| example and develop a variant of this environment which is both
+memoizable and user-friendly.
+
+
+\section{Under the hood}
+\label{sec:under-the-hood}
+
+This chapter is written for three audiences: a curious user who wants to
+know how Memoize does what it does; a package writer who wants to support
+Memoize in a tricky situation; and myself, lest I forget why I made the design
+decisions that I made.
+
+\subsection{The entry point}
+\label{sec:Memoize}
+
+
+From the author's perspective, the functionality of this package is entered
+either through the manual memoization commands (macro \refcmd{mmz} and
+environment \refenv{memoize}), or via automemoization. And while that is
+correct, those user interface entry points merely determine what code is
+submitted to memoization, and set any options specific to the upcoming
+memoization. The real fun starts with command \Emph{\refcmd{Memoize}}, which
+is eventually executed by both manual and automatic memoization.
+
+Not every call to \refcmd{Memoize} results in memoization. Calling this macro
+has three possible outcomes. It can result in \emph{memoization}, which
+produces the memos and externs; in \emph{utilization} of the result of an
+earlier memoization (which boils down to inputting the memos); or in
+\emph{regular compilation}, whereby the code is compiled
+(almost)\footnote{\label{fn:regular-compilation-almost}This is absolutely true
+ for memoized code which is ``contained'' in the sense of not peeking into the
+ input stream following the memoized code. In general, code which fails to
+ satisfy this containment requirement is most likely simply not memoizable;
+ but there are borderline cases. For example, \refcmd{ignorespaces} at the
+ end of some code will have the expected effect in the absence of Memoize, but
+ no effect when executed either during memoization or regular compilation
+ under Memoize, simply because it will hit some code belonging to Memoize
+ rather than the continuation of the document. Memoize offers the
+ \refmmz{ignore spaces} provision to work around this specific problem.} as if
+Memoize was not there. Which outcome obtains depends on several factors. The
+decision logic is depicted below, and note that you can \refmmz{trace} the
+action on the terminal.
+
+\begin{center}
+ \begin{forest}
+ yes/.style={edge=->,edge label={node[font=\scriptsize,pos=0.4,left,#1]{yes}}},
+ no/.style={edge=->,edge label={node[font=\scriptsize,pos=0.4,right,#1]{no}}},
+ use/.style={content=utilization\\(of the cc-memo)\vphantom{dj},fill=orange,align=center},
+ memoize/.style={content=memoization\vphantom{dj},fill=green},
+ compile/.style={content=regular compilation\vphantom{dj},fill=emphcolor},
+ for tree={
+ l sep*=1.5,
+ if n children={0}{}{align=center},
+ child anchor=north,
+ },
+ [Is Memoize enabled?\\(\refcmd{ifmemoize}),
+ [Are we currently undergoing memoization?\\(\refcmd{ifmemoizing}), yes
+ [,compile, yes]
+ [Is the\\\refmmz{recompile}\\mode in effect?, no
+ [,memoize, yes]
+ [Do the relevant\\memos and externs exist?, no
+ [,use, yes]
+ [Is the\\\refmmz{readonly}\\mode in effect?,no
+ [,compile, yes]
+ [,memoize, no]
+ ]
+ ]
+ ]
+ ]
+ [,compile, no]
+ ]
+ \end{forest}
+\end{center}
+
+As the memoization options were already set by the user interface entry points,
+you might expect, quite reasonably, that \refcmd{Memoize} takes a single
+argument, the code submitted to memoization. After all, what more does it
+need? Clearly, executing this code is what produces the typeset material, and
+to detect whether the code has ``changed'' (in order to recompile the memos and
+externs), we compute the MD5 sum of this very code, don't we? Well, the
+reality is a bit more complicated. When it comes to automemoized commands, the
+code which the MD5 sum is computed off of (and which is displayed in the c-memo
+if \refmmz{include source in cmemo} is in effect) is not exactly the same as
+the code we compile (during either memoization or regular compilation). We'll
+see what the difference is in section~\ref{sec:tut:automemoization-details};
+what matters here is that we must provide \refcmd{Memoize} with both and that
+this macro therefore takes two arguments: the \emph{identification code}, which
+the MD5 sum is computed off of, and the \emph{executable code}, which, well, is
+the code that gets executed during memoization (or regular compilation).
+
+Let's illustrate this with an example which is probably entirely useless (but
+don't worry, we'll get to a realistic example in
+section~\ref{sec:tut:automemoization-details}). We first memoize some text
+manually, using command \refcmd{mmz}, and then do something very stupid: we use
+this very text as the identification code for the following \refcmd{Memoize},
+even if the executable code of that command is completely different. The
+second line of the typeset output should convince you that the first argument
+to that command was really used to produce the extern; and one further
+compilation should convince you that the first argument was indeed used to
+identify the extern: the extern produced by \refcmd{mmz} was overwritten by the
+extern produced by \refcmd{Memoize}, in the fashion of the |progressbar|
+example from section~\ref{sec:tut:redefinitions}.
+
+\ExampleName{memoize-internal}
+\makeexample{\examplename.pdf N=2}
+\tcbinputexample{
+ comment={%
+ \includeexamplepdf[document page,
+ after title pre={\ (after the first compilation)}
+ ]{page=3}\hfill
+ \includeexamplepdf[document page,
+ after title pre={\ (after the second compilation)}
+ ][\examplepath.c2.pdf]{page=1}
+ },
+}
+
+The example above also illustrates a(nother) peculiar feature of
+\refcmd{Memoize}. \refcmd{Memoize} does not open a new \hologo{TeX} group, but
+it \emph{expects a group to be opened prior to calling it}, as it will issue an
+|\endgroup| at some point. Specifically, the memoization group will be closed
+before regular compilation or utilization, but after memoization. If you want
+to know why, read the boxed text below.
+
+\begin{tcolorbox}[title=\string\Memoize\ and grouping, enhanced, breakable]
+ One important desideratum behind the design of Memoize was that using the
+ package should disrupt the original, Memoize-less compilation as little as
+ possible. In particular, if the memoized code contains local assignments
+ whose effect (in the original compilation) persists into the rest of the
+ document (until the end of the surrounding \hologo{TeX} group, of course),
+ wouldn't one want these local effects to persist when Memoize is around, as
+ well? Fortunately, most memoized code does not have persistent local effects
+ (at least for me, it is usually environments, like \env{tikzpicture}s, that I
+ want to memoize, and environments introduce a group anyway) --- fortunately,
+ because there are design reasons for enclosing memoization in a \hologo{TeX}
+ group (or two), and this enclosure will of course cancel the effect of local
+ assignments in the memoized code.
+
+ For one, the user interface memoization commands, such as \refcmd{mmz} and
+ automemoized commands, allow for options specific to a particular piece of
+ memoized code (the options given as the optional argument to manual
+ memoization, the next-options and the auto-options), and to delimit their
+ effect, it makes most sense to apply them in a group. I have toyed with the
+ idea of working around the introduction of a group by manually saving and
+ restoring all the options, but I quickly gave up on this line of thought.
+ For one, manually saving and restoring the options would be cumbersome and
+ error-prone, and probably also slower than using the group. But even worse,
+ all that work would not really solve the problem of the persistence of local
+ effects, because memoization itself introduces a group, as well: during
+ memoization, the typeset material is collected into a box, and opening a box
+ introduces a group. In some particular situations, this could be avoided by
+ typesetting the memoized code as-is and collecting the resulting material
+ using |\lastbox|, but this approach cannot work in general. In general,
+ memoization will take place in a group, so the issue of local effects must be
+ addressed in some other way. Memoize offers the following workaround: during
+ memoization, the memoized code can (globally) add code to the \refmmz{after
+ memoization} hook, which gets executed immediately after closing the
+ memoization group.
+
+ Does this mean it would be best if the user interface memoization commands
+ straightforwardly surrounded \refcmd{Memoize} by |\begingroup| and
+ |\endgroup|? For example, \refcmd{mmz} would open the memoization group, let
+ \refcmd{Memoize} do its work, and then close the group. Not really.
+ Remember that memoization is not the only possible outcome of calling
+ \refcmd{Memoize}. Perhaps we can at least retain the local effects of a
+ regular compilation, and of utilization?
+
+ We can, by finely tuning the timing of the memoization group closure within
+ \refcmd{Memoize}. This command is designed to close the memoization group
+ after memoization, but before regular compilation and utilization. Closing
+ the group after memoization makes sure that the given options are in effect
+ during this process. By closing the group prior to regular compilation,
+ regular compilation of the memoized code (which takes place when Memoize is
+ disabled, for example) is guaranteed to have (almost, see
+ footnote~\ref{fn:regular-compilation-almost}) exactly the same effect as the
+ compilation of that code in absence of Memoize; in particular, the effect of
+ any local assignments will persist into the rest of the document. Finally,
+ closing the group before utilization simplifies the construction of the memo
+ in the cases where we need to replicate local effects of the memoized code
+ --- the group closed, there is no need to smuggle local assignments out of a
+ memo.
+\end{tcolorbox}
+
+\subsection{Memos}
+\label{sec:memos}
+
+Up until now, we have pretended that there is a single kind of a memo file. In
+truth, there's two kinds: \emph{code memos}, or \emph{c-memos} for short; and
+\emph{code--context memos}, or \emph{cc-memos} for short. In this section, we
+will learn what they are for, and how they look like --- and also a bit on how
+they are produced, even if the details on that will have to wait until
+section~\ref{sec:memoization-drivers}.
+
+We will see that when Memoize utilizes memos, c-memos are processed first. But
+conceptually, cc-memos are more important, so we will start the discussion with
+these.
+
+\subsubsection{Cc-memos (and extern inclusion)}
+\label{sec:cc-memos}
+
+When it is input, a cc-memo replicates the effect of the memoized code. This
+includes the reproduction of its visual output, which takes the form of
+inclusion of any externs produced by memoization. And yes, you got the
+implication right: a cc-memo can have any number of associated externs,
+including zero, even if the most common case is that of exactly one extern per
+cc-memo. The number of externs mostly depends on the memoization driver (see
+section~\ref{sec:memoization-drivers}); the default driver always produces
+exactly one extern.
+
+A cc-memo is located in the directory determined by key \refmmz{prefix}
+(everything up to the final |/| in the value of this key, or the current
+directory if this value contains no |/|). You can recognize it by its
+filename, which has the following form (\meta{prefix name} is everything
+following the final |/| in the value of \refmmz{prefix}):
+\begin{center}
+ \code{\meta{prefix name}\meta{code md5sum}-\meta{context md5sum}.memo}
+\end{center}
+
+In fact, this is how Memoize recognizes --- or rather, searches for --- a
+cc-memo as well: Memoize will utilize a cc-memo when the code and the context
+MD5 sum computed during an attempted utilization match the code and the context
+MD5 sum computed during some previous memoization (for details on the context
+MD5 sum, see section~\ref{sec:c-memos}). In detail, a cc-memo is created at
+the end of memoization, at which point Memoize computes the MD5 sum of the
+memoized code and the MD5 sum of the context, and writes the results of
+memoization into the cc-memo identified by (the prefix and) these two MD5 sums.
+And when Memoize, on a subsequent compilation, encounters a piece of memoized
+code, it again computes the MD5 sum of that code and the MD5 sum of the
+context, and tries to input the cc-memo identified by (the prefix and) these
+two MD5 sums. If the inputting is successful, we have utilized the cc-memo
+(which in the typical case amounts to including the one associated extern); if
+the cc-memo cannot be found, Memoize starts the memoization process, which
+creates the memos and the externs.
+
+Let us take a look at the contents of a cc-memo in detail. Here's a typical
+cc-memo (it belongs to the titlepage penguin):
+
+\begingroup
+\relaxmmzcommands
+\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
+ \def\myccmemo{#1}%
+ \endinput
+}
+\input{\exampledir titlepage.mmz.c1}
+\sed{%
+ s/\cmd{quitvmode} \cmd{mmzIncludeExtern}/\cmd{quitvmode}\n\cmd{mmzIncludeExtern}/;
+ s/\(\marg\)\cmd{global}/\1\n\cmd{global}/;
+}{\exampledir\myccmemo}
+\tcbinputlisting{
+ listing only,
+ listing file=\exampledir\myccmemo,
+ example title=\myccmemo,
+}
+\endgroup
+
+A cc-memo begins by listing the externs which the memo will (actually, might)
+attempt to include into the document. When the cc-memo is input, each
+\refcmd{mmzResource} command checks if the given extern exists. If some
+existence check fails, Memoize enters the memoization mode, same as if the
+cc-memo itself did not exist. If all the resources pass the existence check,
+Memoize inputs the core of the cc-memo, i.e.\ everything following the
+\refcmd{mmzMemo} marker.
+
+The core might contain arbitrary code, but most often, it will consist of only
+two commands. The first one is \refcmd{quitvmode} and it is included if the
+extern was \refmmz{capture}d into a horizontal box (which is the usual
+situation). The second one is \refcmd{mmzIncludeExtern}, and it is this
+command which actually includes the extern into the document upon inputting the
+cc-memo. The core code is executed without introducing any groups, i.e.\ the
+effect of any local assignments in the cc-memo will persist into the code
+following the memoized code.
+
+Command \refcmd{mmzIncludeExtern} takes nine parameters. The first is the
+sequential number of the extern associated with the cc-memo, starting with 0;
+usually, this is simply 0 as most memos are associated with a single extern.
+The second one is a \refcmd{hbox} or \refcmd{vbox}, noting the type of the box
+the memoized code was externalized into. The next three numbers are the
+expected width, height and the depth of the extern. Finally, we have the four
+\refmmz{padding} amounts (left, bottom, right and top). We should arrive at
+the expected size after trimming the extern PDF by the
+padding amounts; Memoize will complain if we don't.
+
+Let's look at a more interesting cc-memo. Using the advising framework,
+described in section~\ref{sec:tut:automemoization-details}, Memoize hacks
+\refcmd{label} to support \cs{label}s inside memoized code --- the following
+code ``just works.''
+
+\ExampleName{label}
+\makeexample{\examplename.pdf N=3}
+\tcbinputexample{
+ comment={\centering
+ \includeexamplepdf[document page, after title pre={\ (compilation 1)},
+ left=1.5mm, right=1mm]{page=2}\quad
+ \includeexamplepdf[document page, after title pre={\ (compilation 2)},
+ left=1.5mm, right=1mm][\examplepath.c2.pdf]{page=1}\quad
+ \includeexamplepdf[document page, after title pre={\ (compilation 3)},
+ left=1.5mm, right=1mm][\examplepath.c3.pdf]{page=1}
+ },
+}
+
+Everything seems normal --- after the first compilation, we get ``\textbf{??}''
+because the label has not made it into the |.aux| file yet, but in subsequent
+compilations, we learn where the penguin lives --- but it is far from normal
+under the hood. If we de-hacked \cs{label} by writing
+\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{label}}, the third compilation
+(and subsequent compilations) would revert to ``\textbf{??}''. Why would that
+happen? The memoized code containing the \cs{label}s is only executed in the
+first compilation; in the subsequent compilations, we're simply inputting the
+cc-memo, so the memoized code, including any \cs{label}s in contains, is not
+compiled, and the labels don't get into the |.aux| file anymore.
+
+The \cs{label} hack deploys Memoize's ability to put arbitrary code into the
+cc-memo. During memoization, the memoized code may add arbitrary code to
+register \refcmd{mmzCCMemo}, and the contents of this register at the end of
+the memoization form the free-form part of the cc-memo.\footnote{This is also
+ how the above-described code containing \refcmd{mmzIncludeExtern} gets into
+ the cc-memo. The code is produced by \refcmd{mmzExternalizeBox} and appended
+ to \refcmd{mmzCCMemo} by the default memoization driver
+ \refcmd{mmzSingleExternDriver}; see section~\ref{sec:memoization-drivers} for
+ details.} When the hacked \cs{label} is encountered during memoization, it
+appends \refcmd{mmzLabel}\marg{label name}\marg{current label value} to
+\refcmd{mmzCCMemo}, so this command winds up in the cc-memo. It is then a
+simple job for \refcmd{mmzLabel}, executed when the cc-memo is input at
+subsequent compilations, to temporarily store \meta{current label value} (i.e.\
+the contents of \refcmd{@currentlabel} at the time the \cs{label} was invoked) back
+into \cs{@currentlabel} and to execute \cs{label}\marg{label name}. In effect, any
+\cs{label} command contained within the memoized code is executed at every
+compilation, even if the memoized code itself is not compiled.
+
+\begingroup
+\relaxmmzcommands
+\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
+ \def\myccmemo{#1}%
+ \endinput
+}
+\input{\examplepath.mmz.c1}
+\sed{%
+ s/~//g;
+ s/\cmd{quitvmode} \cmd{mmzLabel}/\cmd{quitvmode}\n\cmd{mmzLabel}/;
+ s/\(\marg\)\cmd{mmzIncludeExtern}/\1\n\cmd{mmzIncludeExtern}/;
+ s/\(\cmd{mmzLabel}\) *\(\marg\marg\) */~\1\2~\space/g;
+ s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/;
+}{\exampledir\myccmemo}
+\tcbinputlisting{float,
+ listing only, right=1.5mm,
+ listing file=\exampledir\myccmemo,
+ example title=\myccmemo,
+}
+\endgroup
+
+We will continue the discussion of \refcmd{label} in section~\ref{sec:label+} using
+a funkier example.
+
+
+\subsubsection{C-memos (and context)}
+\label{sec:c-memos}
+
+As explained in the previous section, a cc-memo belonging to a piece of
+memoized code is identified by two MD5 sums: the MD5 sum of the memoized code,
+and the MD5 sum of the associated context. However, when Memoize encounters
+some code submitted to memoization, the context expression is not yet fully
+known, as it may be adjusted by the memoized code itself during memoization ---
+and this potential adjustment is crucial for \cs{ref} and friends to work as
+advertised (see section~\ref{sec:cross-referencing}). Upon being invoked,
+Memoize therefore cannot immediately attempt to input the cc-memo; it needs to
+first learn about the context adjustments. Here's where c-memos enter the
+picture: \emph{the primary job of a c-memo is to store the context adjustments
+ made by the memoized code}. Let's see how this works in detail.
+
+Same as cc-memos, c-memos are located in the directory determined by key
+\refmmz{prefix}, and their filenames start by \meta{prefix name} determined by
+the same key. However, a c-memo belonging to some memoized code is identified
+by the MD5 sum of that code alone:
+\begin{center}
+ \code{\meta{prefix name}\meta{code md5sum}.memo}
+\end{center}
+
+The c-memo is created at the end of the memoization process. At that time, the
+context expression is fully known, as the memoized code was already processed.
+Even more, Memoize keeps track of both the state of the context expression
+prior to memoization, stored in token register \refcmd{mmzContext}, and of the
+\emph{additions} to the context expression made by the memoized code, which are
+stored in token register \refcmd{mmzContextExtra}. (Incidentally, key
+\refmmz{context} automatically adapts to the situation by appending to
+\refcmd{mmzContext} outside memoization and to \refcmd{mmzContextExtra} during
+memoization.) The complete context expression is the concatenation of the
+contents of these two registers, but it is only the context expression
+additions, i.e.\ the contents of \refcmd{mmzContextExtra}, which Memoize stores
+into the c-memo, with the idea that during subsequent compilations, the initial
+context (\refcmd{mmzContext}) will be set up again via ``normal'' compilation,
+while inputting the c-memo will restore the additions, jointly reconstructing
+the complete context expression associated with a piece of memoized code to
+what it was at the end of memoization.
+
+We can now complete the picture of a utilization attempt started in
+section~\ref{sec:cc-memos}. Memoize begins by trying to input the c-memo; this
+can be done as the c-memo can be identified based solely on (the MD5 sum of)
+the memoized code. If the c-memo does not exist, Memoize starts the
+memoization process, which will produce the memos and the externs. But if it
+does exist, inputting it reconstructs the context expression to the state at
+the end of memoization. Therefore, as the MD5 sum of the \emph{expansion} of
+the context expression \emph{at the end of memoization} is baked into the
+cc-memo filename, trying to load the cc-memo identified by (the prefix, the
+code MD5 sum and) the MD5 sum of the \emph{expansion} of the context expression
+\emph{at attempted utilization} will succeed precisely when the context
+remained unchanged from memoization to attempted utilization.
+
+All this might have sounded very complicated, but in the end, most c-memos are
+quite boring, the titlepage penguin's c-memo shown below being no exception. A
+c-memo starts with the \refcmd{mmzMemo} marker, which is always followed by a
+(global) assignment to token register \refcmd{mmzContextExtra}, holding the
+context expression additions. As promised, the c-memo below is boring: it
+assigns an empty token list to this register, leaving the context expression
+as-is. Next comes the free-form part of the memo. Below, it is boringly empty
+as well (just the percent sign), but in principle, it will contain any code
+gathered in register \refcmd{mmzCMemo} during memoization; see
+\ref{sec:per-overlay} for an example. A c-memo is concluded by an optional
+part consisting of the \refcmd{mmzSource} marker, followed by the memoized
+code. The source code section is not used by Memoize in any way and can be
+switched off by \code{\refmmz{include source in cmemo}=false}; it is included
+by default so that an interested user can know which code produced which memo,
+which can be useful if one wants to trigger recompilation of an extern by
+deleting the corresponding memo. Incidentally, any newlines in the source code
+are lost in the c-memo replica (unless \refmmz{verbatim} is in effect), but we
+will only see this once we arrive at the \pkg{beamer} example below.
+
+\begingroup
+\relaxmmzcommands
+\def\mmzNewCMemo#1{% fetch the first c-memo filename
+ \def\mycmemo{#1}%
+ \endinput
+}
+\input{\exampledir titlepage.mmz.c1}
+\tcbinputlisting{
+ listing only,
+ listing file=\exampledir\mycmemo,
+ example title=\mycmemo,
+}
+\endgroup
+
+A c-memo of code containing a cross-reference will prove more interesting. The
+following c-memo was produced by the |ref| example from section
+\ref{sec:cross-referencing}. As we know from that section, when a \refcmd{ref} (as
+hacked by Memoize's \refmmzauto{ref} key) occurs in some memoized code, it
+appends the cross-reference to the context. In the c-memo below, this is
+reflected by the (global) assignment of an expression containing the
+cross-reference macro to token register \refcmd{mmzContextExtra}, holding
+the context expression additions.
+
+\begingroup
+\relaxmmzcommands
+\def\mmzNewCMemo#1{% fetch the first c-memo filename
+ \def\mycmemo{#1}%
+ \endinput
+}
+\input{\exampledir ref.mmz.c1}
+\sed{%
+ s/~//g;
+ s/\cmd{global}.*\marg/~\0~/;
+}{\exampledir\mycmemo}
+\tcbinputlisting{
+ listing only,
+ listing file=\exampledir\mycmemo,
+ example title=\mycmemo,
+}
+\endgroup
+
+
+\subsubsection{More on \texorpdfstring{\cs{label}}{\textbackslash label}}
+\label{sec:label+}
+
+A \refcmd{label} inside memoized code works out of the box in the usual situation
+when label value is fully determined by the memoized code, as in the example in
+section~\ref{sec:cc-memos}, where the memoized code contained the outermost
+(and only) \env{enumerate} environment. However, the out of the box approach does
+not work if the label value is (fully or partially) determined outside the
+memoized code. To illustrate the problem, and some potential solutions, we
+define two very simple enumeration environments, |listi| and |listii|, which
+use counters |counti| and |countii|, and which are intended as the outer and
+the inner environment, respectively. Our interest here is in the inner
+environment, |listii|. While it prefixes each item by an indented
+|\thecountii)|, the label is a composite of both counters:
+|\thecounti\thecountii|. The label is stored into \refcmd{@currentlabel}, so
+referencing works as usual. However, problems arise when we automemoize the
+inner environment.
+
+\ExampleName{label+}
+\makeexample{\examplename.tex.c1 N=6}
+\makeexample{\examplename.pdf N=2}
+\tcbinputexample{
+ sidebyside, lefthand ratio=0.53,
+ comment={\centering\includeexamplepdf[document page][\examplepath.c2.pdf]{page=1}},
+}
+
+While the result looks fine at first, changing the order of |listii|
+environments, for example by moving ``pets'' below ``domestic'', will result in a
+problem: the reference at the bottom will remain unchanged. This is so because
+the reference text is baked into the cc-memo, as shown below.
+
+\begingroup
+\relaxmmzcommands
+\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
+ \def\myccmemo{#1}%
+ \endinput
+}
+\input{\examplepath.mmz.c1}
+\sed{%
+ s/~//g;
+ s/\cmd{mmzLabel} \marg\marg/~\0~/;
+ s/\(\marg\)\(\cmd{mmzIncludeExtern}\)/\1 \2/;
+ s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/;
+}{\exampledir\myccmemo}
+\tcbinputlisting{
+ listing only,
+ listing file=\exampledir\myccmemo,
+ example title=\myccmemo,
+}
+\endgroup
+
+How can we remedy this? The manual option is to force the recompilation of the
+extern by putting an (invisible) reference to the outer item into the inner
+item: add |\label{item:pets}| to item ``pets'' and refer to it at ``dog'' by
+\refcmd{mmzNoRef}|{item:pets}|.%
+\attachexample[\examplename mmzNoRef.tex][\examplepath.tex.c3.attachment]
+
+An automatic variant of the recompilation solution is to add \cs{@currentlabel}
+to the context upon memoizing |listii|. This can be achieved by adding
+\refmmz{context}|={@currentlabel={\csuse{@currentlabel}}}| to the \refmmz{auto}
+declaration for |listii|. The downside of this approach is that every |listii|
+will get reexternalized upon movement, whether it actually contains a label or
+not.\attachexample[\examplename context.tex][\examplepath.tex.c4.attachment]
+
+In fact, given that the externs produced by the inner environment do not
+contain the value of the outer counter, it seems wasteful to recompile any
+extern just to change the reference. And indeed, it is possible to avoid this,
+but the approach unfortunately requires adapting the inner environment code
+(and this is why I have not illustrated the problem using an environment of an
+elaborate package like \pkg{enumitem}). The idea is to ``unbake'' the
+reference to the outer item in the cc-memo. We can achieve this by changing
+|listii| to define \cs{@currentlabel} to be |\unexpanded{\thecounti}\thecountii|.
+Under this definition, the cc-memo will contain |\mmzLabel
+{item:dog}{\thecounti a}|, and rearranging |listii| environments will produce
+(upon two compilations, of course) the correct reference without recompiling
+the extern. Note again, however, that this solution can only work when the
+value of the outer counter does not appear in the extern, i.e.\ it would not
+work the ``dog'' item was prefixed by |1a)| rather than simply |a)|. In those
+cases, one should deploy one of the other solutions.%
+\attachexample[\examplename listii.tex][\examplepath.tex.c5.attachment]
+
+The final solution, presented below, is an elaboration on the second one.
+Rather than append \cs{@currentlabel} to the context immediately upon beginning
+to memoize environment |listii|, we will at that point redefine \cs{label} to do
+that. In effect, changing the location of |listii| will only recompile it if
+it contains a \cs{label}.
+
+At announced, we redefine \cs{label} once the memoization of |listii| begins,
+so within \refmmz{at begin memoization}.\footnote{Another generally good
+ location for such redefinitions is among the auto-options of |listii|. We
+ could include an \refmmz{auto}\cs{label}\marg{...} there, or a |/utils/exec|
+ with \refcmd{AdviceSetup}. However, in this particular case this would be
+ wasteful, as it would be applied regardless of whether memoization will take
+ place or not, whereas we only need the redefined \cs{label} when memoizing.}
+However, we do not redefine \cs{label} directly, as Memoize advises this
+control sequence out of the box (see
+section~\ref{sec:tut:automemoization-details} for details). What we redefine
+is the command which the advising framework executes instead of \cs{label} ---
+its so-called \refmmzauto{outer handler} --- and we do this by calling
+\refcmd{AdviceSetup}, the low-level variant of the familiar key
+\refmmz{auto}.\footnote{We could have also used \refmmz{auto}, but we don't,
+ because (a) \refcmd{AdviceSetup} is faster, (b) it is easier to prepend
+ material to a handler using the low-level interface, and (c) I wanted to
+ showcase \refcmd{AdviceSetup}.} The first argument of \refcmd{AdviceSetup}
+is the installation path (|/mmz|), the second one the command or environment we
+are submitting to the framework (\cs{label}), and the third one the setup
+\emph{code} --- here lies the biggest difference between \refmmz{auto} and
+\refcmd{AdviceSetup}: the former expects a keylist, and the latter \hologo{TeX}
+code which directly manipulates settings macros like
+\refcmd{AdviceOuterHandler} (for the full list, see
+section~\ref{sec:tut:automemoization-details} or~\ref{sec:ref:advice}).
+
+\tcbinputexample[.tex][.c6]{listing only, float,
+ attachment name=\examplename auto.tex,
+}
+
+Within \refcmd{AdviceSetup}, we prefix (using macro \cs{preto} of package
+\pkg{etoolbox}) the original outer handler \refcmd{AdviceOuterHandler} by code
+which causes \cs{outerlabeltocontext} to be executed at the end of memoization
+(by globally appending this macro to \refcmd{mmzAtEndMemoizationExtra};
+``Extra'' because we're appending during memoization). It is
+\cs{outerlabeltocontext} which then appends \refcmd{@currentlabel} to the
+context (\refcmd{mmzContextExtra}; again, ``Extra'' because we're appending
+during memoization),\footnote{As a courtesy, we clear out macro
+ \cs{outerlabeltocontext} once it did it's job, so that multiple \cs{label}s
+ do not include multiple \cs{@currentlabel}s into the context. But the code
+ would work even without this addendum, can you see why?} and it is crucial
+that this happens at the end of memoization rather than when \cs{label} is
+executed. When \cs{label} is executed, we're inside an inner item, and
+\cs{@currentlabel} refers to that item, while at the end of memoization, the
+value of this macro equals the value at the beginning of memoization, namely
+the label of the outer item. While the outer list remains unshuffled, the
+value of \refcmd{@currentlabel} that contributes to the context MD5 sum during
+the utilization attempt will therefore match the value which contributed to the
+context MD5 sum during memoization, resulting in matching MD5 sums and
+therefore in actual utilization of the extern; once the outer list is shuffled,
+this will cease to be the case and the extern will be recompiled.
+
+
+\subsubsection{The Beamer support explained}
+\label{sec:per-overlay}
+
+The implementation of \refmmz{per overlay}, which makes memoization sensitive
+to Beamer overlays, provides an example of a complex interaction between
+various components of memoization. At the core, the Beamer support works by
+adjusting the context, but we will also have the occasion to observe the
+free-form part of the c-memo, add a bit of extra code to the cc-memo, and
+deploy several memoization hooks. We will show the complete Beamer support
+code later on; let us build our understanding of that code step by step.
+(Before you read on, you might want to refresh your memory about the \pkg{beamer}
+example from section~\ref{sec:tut:beamer}, as we will refer to it in the
+present section.)
+
+The core idea behind \refmmz{per overlay} is to append the current beamer
+overlay number to the context:\cprotect\footnote{As we saw in
+ section~\ref{sec:tut:beamer}, it is convenient to execute \refmmz{per
+ overlay} inside memoized code. But remember, from
+ section~\ref{sec:cross-referencing}, that when \refmmz{context} is executed
+ from within the memoized code, its argument winds up in the c-memo. As the
+ c-memo is processed under the normal category code regime, where |@| is not a
+ letter, we have to access |\beamer at overlaynumber| using the |\csname
+ ... \endcsname| construct.} \refmmz{context}|={overlay=\csname
+ beamer at overlaynumber\endcsname}|. This makes Memoize produce a separate
+extern for each overlay. However, only the first of these externs will get
+utilized on subsequent compilations, in general at least. Even worse, we will
+lose each frame whose creation is driven solely by the memoized code. We will
+lose the second overlay in our example (i.e.\ in the \pkg{beamer} example from
+section~\ref{sec:tut:beamer}) as the second overlay was only created because
+Beamer encountered |only={2}{...}| (resolving to |\only<2>{...}| under the
+hood) inside the picture code; once we utilize the extern instead of compiling
+the picture on the first overlay, the |\only| command is not executed anymore,
+so Beamer thinks it is done with the frame.
+
+As the compilation of our picture is substituted by utilization of its
+cc-memos, we have to somehow drive the creation of the necessary overlays from
+these files. An easy way to achieve this is to furnish them with a dummy
+|\only<|\meta{final overlay number}|>{}|,\footnote{`The final overlay' here
+ should be understood as relative to our memoized picture, i.e.\ as the final
+ overlay containing the memoized picture.}\footnote{Actually, putting this
+ \cs{only} command only into the first cc-memo would suffice, but would be
+ harder to implement.} but there is a problem: the final overlay number is
+unknown when we're memoizing our picture --- it is unknown even when we're
+memoizing the picture on final overlay itself (we simply don't know yet that
+this overlay will end up being the final one), let alone during the memoization
+on the first overlay.
+
+The solution exploits the fact that \emph{the c-memo is rewritten at each
+ memoization}: at each memoization of our picture, we store the the
+\emph{current} overlay number to the c-memo; after all memoizations, the c-memo
+will thus contain the number of the \emph{final} overlay containing our
+memoized picture. To access this number from the cc-memo, we store it as a
+macro definition, and then use the defined macro, |\mmzBeamerOverlays|, in the
+overlay specification of the dummy |\only|. Below, you can see all this in
+code, as the argument to \refmmz{at begin memoization}.
+
+\ExampleName{per-overlay-v1}
+\makeexample{\examplename.sty}
+\tcbinputexample[.sty]{
+ listing only,
+ title={The implementation of \refmmz{per overlay} (first attempt)},
+}
+
+A couple of remarks are in order here. First, the definition of
+|\mmzBeamerOverlays| in the c-memo is global, because it will be accessed from
+the cc-memo, but the cc-memo is input after closing the memoize group (which
+the c-memo \emph{is} processed in). Second, using \refmmz{at begin
+ memoization} makes it possible to use \refmmz{per overlay} both outside and
+during memoization: if \refmmz{at begin memoization} is executed outside
+memoization, its argument is (locally) stored into hook
+\refcmd{mmzAtBeginMemoization}, to be called at the beginning of each
+memoization; if the key executed outside memoization, the argument is executed
+immediately. Third, the Memoize keys in the definition of \refmmz{per overlay}
+are prefixed with \code{\refkeypath{/mmz}/}, so that this key can be called from
+\pkg{pgfkeys} option lists of other packages, for example the option list of
+the \env{tikzpicture} environment, as shown in the example in
+section~\ref{sec:tut:beamer}.
+
+I used the above version of \refmmz{per overlay} for quite a while. In
+general, it worked as I expected, but there were glitches. Occasionally, the
+picture would appear on the wrong overlay, or I would get an extra overlay, or
+perhaps lose an overlay. Eventually, I figured out this happens when I play
+with the overlay structure of the frame: when I add or remove a \cs{pause} or
+similar. In hindsight, it is easy to see what was happening. Once the picture
+is memoized, it is fixed, forever, which extern will appear on which overlay.
+I cannot expect the extern--overlay correlation to change just because I added
+a \cs{pause} in front of the picture. Furthermore, the number of overlays the
+memos will drive to be created is fixed as well. If I memoize the picture
+while it follows a \cs{pause}, and the picture creates 10 overlays, the c-memo
+will define |\mmzBeamerOverlays| to 11. So what, if I then remove that
+\cs{pause}! The c-memo will still define |\mmzBeamerOverlays| to 11, and drive
+the creation of 11 overlays --- one too many.
+
+By now, the road ahead is probably clear --- we put the |beamerpauses| counter
+into the context --- but we will see there are still obstacles on the way. The
+issue is that the context is evaluated at the \emph{end} of memoization (so
+that those cross-references from section~\ref{sec:cross-referencing} actually
+get into it). However, the memoized code might contain a \cs{pause} or similar
+itself, and change the value of |beamerpauses|. For one, this means that we
+have to write down the changed value of |beamerpauses| into the cc-memo; below,
+we do this using key \refmmz{at end memoization} (the code given code to this
+key is executed after the driver but before Memoize writes down the memos and
+ships out the extern pages; the key itself may be executed either before or
+during memoization). Furthermore, if the memoized code changes the value of
+|beamerpauses|, the value of |beamerpauses| at the attempted utilization, which
+would nicely match the value from the start of memoization, will never match
+the changed, final value from the end of memoization, in effect preventing our
+hard-won externs from ever getting utilized.
+
+We therefore have to invent a way to get the memoization-\emph{initial} value
+of |beamerpauses| into the context.\footnote{This imposes a requirement on the
+ in-code usage of \refmmz{per overlay}, namely, that it should be executed
+ prior to any changes of |beamerpauses|.} Fine, we store it in a
+macro,\footnote{We must define \cs{mmzBeamerPauses} globally, because
+ \refmmz{per overlay} can be arbitrarily deeply embedded in the memoized
+ code.} |\mmzBeamerPauses|, when we start the memoization (\refmmz{at begin
+ memoization}), and put |\mmzBeamerPauses| rather than |beamerpauses| into the
+context. Will this work? Not yet, because |\mmzBeamerPauses| is undefined at
+utilization. We need to set up the context expression so that it will expand
+to the value of |\mmzBeamerPauses| at (the end of) memoization, and to the
+value of |beamerpauses| at utilization. This leads to the
+|pauses=\ifmemoizing...| part of the context expression in (the final version
+ of) the Beamer support code below.
+
+\medskip % manual, to avoid an orphan
+
+\ExampleName{per-overlay}
+\makeexcerpt{per-overlay}
+\tcbinputexample[.tex][.excerpt]{%
+ listing only, one file,
+ left=0pt, % manual indentation gobble
+ title={The implementation of \refmmz{per overlay}},
+}
+
+Are we done? Almost. The final issue is that once we have introduced support
+for pauses, we have to relativize |\mmzBeamerOverlays| (the final overlay
+number) to |beamerpause|. So instead of a simple |\gdef\mmzBeamerOverlays| in
+the first version, we define |\mmzSetBeamerOverlays|\nohyphen\marg{beamer
+ pauses}\marg{final overlay number}, which sets |\mmzBeamerOverlays| only if
+\meta{beamer pauses} argument matches the value of |beamerpauses| (at its
+invocation in the c-memo). Well, the macro has some other housekeeping to do
+as well: it is self-replicating, so that during potential memoization, the
+|\mmzBeamerOverlays| values belonging to non-current |beamerpauses| values get
+rewritten into the c-memo.\footnote{As replication should only occur during
+ memoization (actually, it \emph{can} only occur then, anyway), the
+ instruction to append to the c-memo is appended to the
+ \cs{mmzAtBeginMemoization} hook (the low-level interface to \refmmz{at begin
+ memoization}). Note that the assignment to this hook must be local (once
+ local, always local), and it \emph{can} be local because of a little
+ implementation detail: while the c-memo is processed in the memoize
+ \hologo{TeX} group, we don't open an additional group to process it; so the
+ local effects from c-memo will persist into memoization (but not into
+ utilization, because remember that the memoize group is closed before
+ inputting the cc-memo).} (Fine, there is another fine detail, regarding
+anti-pollution: the macro also ensures that, relative to \marg{beamer pauses},
+only the instance with the greatest \meta{final overlay number} is replicated.)
+
+We are now truly done, and we can look at the final result, the c-memo and the
+cc-memo belonging to the extern on the first overlay of the example from
+section~\ref{sec:tut:beamer}. Specifically, look at the |\mmzSetBeamerOverlays
+{1}{2}|, which says that the extern chain started when |beamerpauses| equals 1
+should continue up to overlay 2, and at the (expanded) context included at the
+end of the cc-memo, courtesy of \refmmz{include context in ccmemo}, where you
+can see that the cc-memo will be used when on the first overlay (|overlay=1|)
+when preceded by no \cs{pause} command (|pauses=1|).
+
+\begingroup
+\relaxmmzcommands
+\def\mmzNewCMemo#1{% fetch the first c-memo filename
+ \def\mycmemo{#1}%
+}
+\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
+ \def\myccmemo{#1}%
+ \endinput
+}
+\input{\exampledir beamer.mmz.c1}
+\sed{%
+ s/~//g;
+ s/\cmd{mmzSetBeamerOverlays} \marg\marg/~\0~/;
+}{\exampledir\mycmemo}
+\sed{%
+ s/~//g;
+ s/overlay=[0-9]*/~\0~/;
+ s/pauses=[0-9]*/~\0~/;
+ s/\(\cmd{quitvmode}\) \(\cmd{only}\)/\1\n\2/;
+ s/\rbrace\(\cmd{mmzIncludeExtern}\)/\rbrace\percentchar\n\1/;
+ s/\rbrace\(\cmd{setcounter}\)/\rbrace\percentchar\n\1/;
+ s/\(\marg\)\cmd{global}/\1\n\cmd{global}/;
+}{\exampledir\myccmemo}
+
+\tcbinputlisting{
+ listing only,
+ listing file=\exampledir\mycmemo,
+ example title=\mycmemo,
+}
+
+\tcbinputlisting{
+ listing only,
+ listing file=\exampledir\myccmemo,
+ example title=\myccmemo,
+}
+\endgroup
+
+
+\subsection{Record files}
+\label{sec:record-files}
+
+We have seen that externalization is a two-step process in Memoize: as it is
+impossible for \hologo{TeX} to create multiple PDFs during a single
+compilation, the externs are first dumped into the document PDF as special
+extern pages, and only later extracted from the main document into separate PDF
+files. But extraction requires a complete PDF, which is unavailable even at
+the very end of the compilation which produces the externs. The externs can
+therefore only be extracted \emph{after} that compilation (either before or at
+the beginning of the next one), and this necessitates some form of
+communication whereby the memoization step informs the extraction step which
+pages should be extracted from the document PDF and into which (PDF) files they
+should be stored. This communication is implemented through auxiliary files
+called \emph{record files}.
+
+\subsubsection{The \texttt{.mmz} file}
+\label{sec:.mmz}
+
+By default, Memoize records the information needed for the extraction in a file
+named \meta{document name}\dmmz,\footnote{For \hologo{TeX}perts: the
+ \meta{document name} is of course the expansion of \cs{jobname}.} henceforth
+a \dmmz file. In fact, this file contains more than information on externs
+created during the last compilation: it records which memos and externs were
+either used or created during the compilation. The full information contained
+in the \dmmz file is used by the clean-up script \refscript{memoize-clean.pl}
+to safely remove stale memos and externs. Let us take a look at the \dmmz file
+produced by the titlepage illustration. In fact, we have two versions of this
+file, as it changes upon the second compilation.
+
+\ExampleName{titlepage}
+\makeexample{\examplename.pdf N=2}
+\sed{%
+ s/~//g;
+ s/\cmd{mmzNewExtern}/~\0~/;
+}{\examplepath.mmz.c1}
+\tcbinputexample[.mmz][.c1]{
+ listing only, one file, no attachment, float,
+ after title pre={\ (after the first compilation)},
+ listing options app={breakatwhitespace=false,prebreak=\coloredpercentchar},
+}
+\tcbinputexample[.mmz][.c2]{
+ listing only, one file, no attachment, float,
+ after title pre={\ (after subsequent compilations)},
+ listing options app={breakatwhitespace=false,prebreak=\coloredpercentchar},
+}
+
+As you can see, the \dmmz file takes the form of a \hologo{TeX} script (the
+format was chosen because it facilitated the implementation of the internally
+triggered \hologo{TeX}-based extraction). The crucial lines in this file, and
+the only lines used by the extraction script, occur in the first version of the
+file: they contain command \refcmd{mmzNewExtern}, which informs the extraction
+script that it should extract the document page given by the second argument
+into the extern file given by the first argument.\footnote{If you look at the
+ \dmmz file after extracting the externs using \refscript{memoize-extract.pl}
+ without the \refscript{memoize-extract.pl--keep} option, you will find that
+ the \refcmd{mmzNewExtern} commands are commented out; this is to prevent
+ multiple extractions (even if they are harmless).} (The following two
+arguments provide the expected width and height of the extern; the
+extraction script may check whether the extern size conforms to these
+expectations, but this is not crucial, as the extern size is checked every time
+it is included anyway.)
+
+A \dmmz file also contains a record of the memos (both c-memos and cc-memos)
+created in the last compilation; this information is provided by the sole
+argument of commands \refcmd{mmzNewCMemo} and \refcmd{mmzNewCCMemo}. And once
+memos and externs get used in subsequent compilations, the \dmmz file will
+reflect this with \refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo} and
+\refcmd{mmzUsedExtern}, as shown in the second version of the file above.
+
+Finally, both versions illustrate that a \dmmz file always begins with command
+\refcmd{mmzPrefix} and ends with the |\endinput| marker. The argument of
+\refcmd{mmzPrefix} is the prefix to the memo and extern files, as
+determined by the invocation of key \refmmz{prefix}. The initial
+\refcmd{mmzPrefix} line is written to the \dmmz file at the beginning of the
+document, but an additional \refcmd{mmzPrefix} line will occur for every
+invocation of \refmmz{prefix} in the document body. Finally, the |\endinput|
+marker signals that the \dmmz file is complete.
+
+As mentioned above, the full contingent of \dmmz file commands is only used by
+the clean-up script \refscript{memoize-clean.pl}. By default, this script
+removes all memos and externs with the prefix given by \refcmd{mmzPrefix}
+(relative to the directory hosting the \dmmz file) but those listed by any of
+\refcmd{mmzNewCMemo}, \refcmd{mmzNewCCMemo}, \refcmd{mmzNewExtern},
+\refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo} and \refcmd{mmzUsedExtern}.
+Furthermore, the clean-up script will cowardly refuse to delete anything if the
+\dmmz file does not end with |\endinput|, as this means that the compilation
+ended prematurely and that the \dmmz file might not mention all memos and
+externs actually used in the document. If given option
+\refscript{memoize-clean.pl--all}, the clean-up script removes even the memos
+and externs mentioned in the \dmmz file, and as this option is intended to
+bring Memoize to a clean slate after any ``disasters,'' the
+\refscript{memoize-clean.pl--all} mode also ignores the potential absence of
+the |\endinput| marker. Incidentally, the \refscript{memoize-clean.pl--all}
+mode is also the raison d'être for \refcmd{mmzPrefix}: while the prefix is
+usually recognizable from \refcmd{mmzNewCMemo} and friends, these commands
+might not make it into the \dmmz file in a fatally failed compilation, but it
+is precisely such compilations that could occasionally require the full
+clean-up.
+
+\subsubsection{Defining a new record type}
+\label{sec:new-record-file}
+
+The \dmmz file is not the only kind of a record file that can be produced by
+Memoize. Out of the box, it can also write down the extraction instructions
+into a makefile or a shell script. These are useful on systems which have to
+employ the \hologo{TeX}-based extraction but cannot trigger it internally.
+Running the \hologo{TeX}-based extraction manually would be painful, as it must
+be done on extern-by-extern basis, so Memoize offers to automate the extraction
+by a makefile or a shell script; here, the record file is named
+\code{memoize-extract.\meta{document name}.\meta{record type}} by default,
+where \meta{record type} is either \refmmz{record=makefile},
+\refmmz{record=sh} (for shell scripts on Linux), or \refmmz{record=bat} (for
+shell scripts on Windows).
+
+To turn on recording of an alternate record type, use key
+\refmmz{record}|=|\meta{record type}. Memoize can record any number of files
+simultaneously, so saying \code{\refmmz{record}=\refmmz{record=sh}} will
+produce the shell script alongside \dmmz (Memoize internally executes
+\code{\refmmz{record}=\refmmz{record=mmz}} to start recording the \dmmz file);
+this should not be a problem, but if you really want to disable the \dmmz file
+production, you can say \refmmz{no record}.
+
+The predefined record types are defined through a generic system open to the
+user. To define an additional record type, one needs to define, using
+\pkg{pgfkeys}, the relevant hooks of the form \refkeypath{/mmz/record/record
+ type}|/|\meta{hook}. The following \meta{hook}s can be defined (the hooks
+not needed for the record file type may be left undefined):
+\begin{itemize}
+\item Key \refmmz{record/record type/begin} will be executed at the beginning
+ of the document; it will receive no argument. Use it to open the record
+ file.
+\item Key \refmmz{record/record type/end} will be executed at the end of the
+ document; it will receive no argument. Use it to close the record file.
+\item Key \refmmz{record/record type/prefix} will be executed at the end of the
+ document and at every invocation of key \refmmz{prefix} in the document body;
+ it will receive a single argument, the prefix determined by key
+ \refmmz{prefix}.
+\item Keys \refmmz{record/record type/new cmemo}, \refmmz{record/record
+ type/used cmemo}, \refmmz{record/record type/new ccmemo} and
+ \refmmz{record/record type/used ccmemo} will be executed after creating or
+ inputting a memo; they will receive a single argument, the full path to the
+ memo.
+\item Key \refmmz{record/record type/used extern} will be executed after an
+ extern was included into the document; it will receive a single argument, the
+ full path to the extern.
+\item Key \refmmz{record/record type/new extern} will be executed after
+ creating creating an extern, more precisely at the end of memoization, right
+ after shipping out the extern page. It will receive a single argument, the
+ full path to the extern, but additionally, Memoize prepares the following
+ macros:
+ \begin{itemize}
+ \item \refcmd{ne:externbasepath} holds the full path to the extern, but (unlike |#1|)
+ without the |.pdf| suffix;
+ \item \refcmd{ne:pagenumber} holds the ``physical'' page number of the extern page in
+ the document (the numbering starts by $1$);
+ \item \refcmd{ne:expectedwidth} and \refcmd{ne:expectedheight} hold the width and
+ the height (total height, i.e.\ the sum of \hologo{TeX}'s height and depth)
+ of the extern page.
+ \end{itemize}
+\end{itemize}
+
+Below, we present two simple examples of a record file. The first type simply
+records the names of all memos and externs used or created by Memoize; the
+resulting file could be included by |.gitignore| to have
+\hreftt{https://git-scm.com}{git} automatically ignore all files produced by
+Memoize. The second type lists the new externs, each preceded by its page
+number in the |.pdf|; this file could be fed to a custom extern extraction
+tool.
+
+\ExampleName{record-files}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{listing only}
+
+\ExampleName{record-extern-pages}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{listing only}
+
+Finally, note that (unlike memos and externs) record files are auxiliary files
+and may be deleted at any time after the extraction of the externs produced in
+the final compilation --- actually, even if these externs were not yet
+extracted, deleting the record file(s) will merely force their recompilation.
+
+
+\subsection{The memoization process}
+\label{sec:memoization-drivers}
+
+We now turn to the memoization process itself. The job of memoization is to,
+while compiling the given code in a regular fashion, prepare the cc-memo
+(which, when it is input, will replicate the effect of the given code),
+alongside any externs that the cc-memo will include (these hold the typeset
+material to be replicated). Clearly, merely compiling the code cannot have
+this effect (unless that code was written specifically to support memoization;
+more on this later), and this is why the memoized code is typically wrapped by
+a \emph{memoization driver}, which can be set using key \Emph{\refmmz{driver}}.
+We'll inspect the default memoization driver, \refcmd{mmzSingleExternDriver},
+in the first subsection, and we will learn how to write specialized drivers in
+the remaining subsections. But first, let us say some words about a
+grouping-related \hologo{TeX}nical detail we need to take care about during
+memoization.
+
+During memoization, we have to collect certain information, like build the
+contents of the cc-memo. Some of that information might be contributed by the
+memoized code itself. For example, a \cs{label} ``adds itself'' to the cc-memo
+(by appending to token register \refcmd{mmzCCMemo}); a
+\refkey{/tikz/remember picture} aborts memoization (by issuing
+\refcmd{mmzAbort}); etc. The issue is that the memoized code might open any
+number of \hologo{TeX} groups; we have no idea how deeply embedded the
+\cs{label} or \refkey{/tikz/remember picture} might be. Therefore, we have
+to collect all the information about the ongoing memoization \emph{globally}:
+all assignments to \refcmd{mmzCCMemo} must be global; \refcmd{mmzAbort} sets
+the underlying conditional globally; etc. (Clearly, all these global variables
+are initialized at the start of memoization.)
+
+This was the easy part. An additional complication arises with some options
+which may be set either outside memoization, or during this process. For
+example, you can append the font size to the context expression in the preamble
+(see section~\ref{sec:cross-referencing}), so that the externs will be
+automatically recompiled when the font size changes, and clearly, this context
+adjustment should respect \hologo{TeX} grouping; but a \cs{ref} or some other
+cross-referencing command in the memoized code needs to append to the context
+as well, and as this \cs{ref} occurs \emph{within} the memoized code, the
+assignment must be global, as explained above.
+
+Mixing the local and global assignments to the token register
+\refcmd{mmzContext}, which holds the (in the actual implementation, local)
+context expression, will not do. For one, we \emph{do} want to restore the
+pre-memoization context expression after we have memoized the code, and
+furthermore, mixing local and global assignments to the same variable is not
+recommended for save stack reasons anyway.
+
+Memoize addresses this issue by having \emph{two} context registers,
+\refcmd{mmzContext} and \refcmd{mmzContextExtra} --- when computing the context
+MD5 sum (which happens at the end of memoization), the two registers are
+concatenated (the local one comes first). A package writer should know when to
+use which register, and how. Outside memoization, one should assign to
+\refcmd{mmzContext} --- \emph{locally}. During memoization, one should assign
+to \refcmd{mmzContextExtra} --- \emph{globally}. The user interface key
+\refmmz{context} respects this requirement automatically: it locally appends to
+\refcmd{mmzContext} outside memoization, and it globally appends to
+\refcmd{mmzContextExtra} during memoization. (The same idea is applied to the
+post-memoization hooks \refmmz{at end memoization} and \refmmz{after
+ memoization}.)
+
+
+\subsubsection{The default memoization driver}
+\label{sec:memoization-driver-default}
+
+The default memoization driver, \refcmd{mmzSingleExternDriver}, produces
+exactly one extern, which contains whatever is typeset by the code submitted to
+memoization. The driver compiles the code into a horizontal or vertical box
+depending on the value of key \refmmz{capture}. Let us look at the definition
+of the driver line by line:
+
+\makeexcerpt{single-extern-driver}
+\tcbinputexample[.tex][.excerpt]{%
+ listing only, one file,
+ listing options app={numbers=left, numberstyle=\tiny, numbersep=0.5em},
+ title=The default memoization driver,
+ float,
+}
+
+\begin{enumerate}
+\item Macro \refcmd{mmzSingleExternDriver} (and in fact any memoization
+ driver) takes a single argument, the code to compile. Memoize will call the
+ driver with the code given as the second argument to \refcmd{Memoize}, but
+ wrapped in a macro which re-reads it using \refcmd{scantokens} when
+ \refmmz{verbatim} is in effect.
+\item If we're capturing into a horizontal box
+ (\refmmz{capture}|=|\refmmz{capture=hbox}), we put \refcmd{quitvmode} into
+ the cc-memo --- putting it to the very beginning should make sure that any
+ replicated \cs{label} and \cs{index} commands refer to the correct page.
+\item We compile the given code, storing the typeset material into a box
+ (above, a temporary box called |\mmz at box|). |\mmz at capture| resolves into a
+ box construction command, depending on the value \refmmz{capture}.
+\item Macro \refcmd{mmzExternalizeBox} instructs Memoize to externalize the box
+ given as its first argument. However, this macro does not directly produce
+ an extern page or write any instructions into the cc-memo; the road to this
+ final destination is indirect. \refcmd{mmzExternalizeBox} has two effects.
+ First, it adds the contents of the given box (above, |\mmz at box|) to an
+ internal box dedicated to holding all the externs produced in this
+ memoization (the contents of |\mmz at box| remain as they are) --- it is only at
+ the end of memoization that the contents of this internal box are shipped off
+ to extern pages. Second, \refcmd{mmzExternalizeBox} produces the code which
+ will include the extern into the document on subsequent compilations (this
+ will be a call to \refcmd{mmzIncludeExtern}, potentially prefixed by
+ \refcmd{quitvmode}; see section~\ref{sec:cc-memos} for details). This code
+ is stored into the token register gives as the second argument (above,
+ |\mmz at temptoks|), and it is the responsibility of the driver to include it
+ into the cc-memo. (In the interest of full disclosure,
+ \refcmd{mmzExternalizeBox} also updates the list of externs produced in this
+ memoization. At the end of memoization, this list is written to the
+ beginning of the cc-memo, resulting in the \refcmd{mmzResource} lines
+ preceding the \refcmd{mmzMemo} marker.)
+\item The construction of the cc-memo is indirect as well. In the third line
+ of the definition, we globally append the extern-inclusion code residing in
+ |\mmz at temptoks| to token register \refcmd{mmzCCMemo}. At the end of
+ memoization, the contents of \refcmd{mmzCCMemo} are written into the cc-memo,
+ preceded by the \refcmd{mmzMemo} marker.
+\item We put the typeset material into the document, again preceded by
+ \refcmd{quitvmode} when capturing in a horizontal box.
+\end{enumerate}
+
+You might wonder why the construction of the extern pages and the cc-memo (and
+actually, of the c-memo as well) is indirect, as described above.
+\begin{itemize}
+\item For one, the indirect construction facilitates potential abortion of
+ memoization (see section~\ref{sec:tut:good-to-know}). With the indirect route,
+ aborting is easy --- as nothing was permanently written anywhere yet, Memoize
+ simply skips the final part of the process, where extern boxes are shipped
+ into extern pages and the memo registers written into memo files --- and also
+ clean: if \refcmd{mmzExternalizeBox} immediately shipped out the extern
+ pages, these pages would remain in the document even in the case of abortion.
+\item Even more importantly, the cc-memo filename contains the \meta{context
+ md5sum} (see section~\ref{sec:cc-memos}), but the context expression is not
+ yet fully known when memoization starts --- remember (from
+ section~\ref{sec:cross-referencing}) that a \cs{ref} in the memoized code will
+ update the context! The cc-memo can therefore only be opened at the end of
+ memoization, which necessitates a buffer (i.e.\ the \refcmd{mmzCCMemo}
+ register) for storing its contents during memoization.
+\end{itemize}
+
+
+\subsubsection{Pure memoization}
+\label{sec:pure-memoization}
+
+The default memoization driver discussed above is really an externalization
+driver: it produces a single extern. We now move to examples of drivers with
+other functions, starting with a pure memoization driver, which does not
+externalize any typeset output --- simply because it does not call
+\refcmd{mmzExternalizeBox} at any point --- but rather remembers the result
+of a (pgfmath) computation (let's pretend that the computation is
+time-consuming).
+
+\ExampleName{pgfmathparse}
+\makeexample{\examplename.pdf}
+\begingroup
+\relaxmmzcommands
+\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
+ \def\myccmemo{#1}%
+ \endinput
+}
+\input{\examplepath.mmz.c1}
+\tcbinputexample{%
+ comment={%
+ \tcbinputlisting{
+ listing only, width=.4\linewidth, nobeforeafter,
+ listing file=\exampledir\myccmemo,
+ example title, title=the cc-memo,
+ }\quad
+ \includeexamplepdf[document page]{page=1}
+ },
+}
+\endgroup
+
+Command \refcmd{mmz} above memoizes its mandatory argument with the memoization
+\refmmz{driver} set to the previously defined macro \cs{mmzPgfmathDriver}.
+Just as the default driver above, \cs{mmzPgfmathDriver} first executes the
+given code. However, there is no need to do this in the context of a
+|\setbox|, as the memoized code, which is obviously expected to consist of a
+single \cs{pgfmathparse} call, does not typeset anything: \cs{pgfmathparse}
+evaluates the given expression and stores the result into macro
+\cs{pgfmathresult}. The driver has two jobs: first, it must store this result
+into the cc-memo, to be utilized in subsequent compilations; second, because
+the assignment to \cs{pgfmathresult} (within \cs{pgfmathparse}) is local, the
+driver also needs to somehow smuggle the result out of the \cs{endgroup} issued
+by \refcmd{Memoize} and thereby make it into the following code (the final line
+of the example, which typesets the equation). Both jobs are easy enough: the
+expansion of |\def\noexpand\pgfmathresult{\pgfmathresult}| (in this case,
+|\def\pgfmathresult{42.0}|) is (globally) appended both to the token register
+\refcmd{mmzCCMemo}, which Memoize later writes into the cc-memo, and to the
+macro underlying the \refmmz{after memoization} hook, whose contents are
+executed \emph{after} closing the memoization group.
+
+Let us consider an alternative implementation of the same goal of memoizing the
+result of a pgfmath computation, showcasing a couple of useful tricks.
+
+\ExampleName{pgfmathparse-embellished}
+\makeexample{\examplename.pdf}
+\tcbinputexample{listing only}
+
+For one, this ``embellished'' example reminds us that we can list the
+\refmmz{driver} key among the auto-options (even if I don't really recommend
+automemoizing \cs{pgfmathparse}). But even more importantly, the example shows
+that the driver consist of more than a single control sequence; the only
+requirement is that the given driver code will consume the memoized code. In
+this example, we have developed a generic smuggling driver and applied it to
+\cs{pgfmathresult} in particular --- \cs{pgfmathresult} will become the first
+argument of \cs{mmzSmuggleOneDriver}, and the memoized code will become its
+second argument.
+
+In the first version of the example, we have appended the same code to macro
+\refcmd{mmzCCMemo} and to macro \refcmd{mmzAfterMemoizationExtra} --- no
+surprise here, as we want the effect of memoization and utilization to be the
+same. In the embellished version, we advertise another way to achieve the same
+effect, a way which might be useful for complicated drivers: we simply smuggle
+out the entire cc-memo. The idea works even when memoization procudes externs;
+in that case, however, the driver also has to say \cs{mmzkeepexternstrue} ---
+conditional \refcmd{ifmmzkeepexterns} decides whether Memoize keeps the externs
+around, in memory, even after shipping them out (but they are always gone at
+the start of the next memoization).
+
+Finally, remember that the default context expression contains the
+\refmmz{padding} values. However, these really have no place in the context
+expression of some purely memoized code. We have therefore emptied out the
+context expression using \refmmz{clear context}.
+
+
+\subsubsection{Multiple externs per memo}
+\label{sec:memoization-multiple-externs}
+
+In the next example, we show how to produce multiple externs for a single piece
+of memoized code. The usage case I find most appealing is breaking the typeset
+material, like a table, across pages --- but of course, table-breaking is too
+complicated an example, so we illustrate the idea by defining command
+|\countdown|, which counts down from the given number, typesetting each number
+into its own line. Clearly, if we were to externalize a call to this command
+using the default memoization driver, page breaking would stop working, as the
+entire countdown would be seen as a single, unbreakable box. To externalize it
+properly, the chunks of the countdown that should appear on separate pages must
+be externalized into separate externs, as shown below.
+
+\ExampleName{countdown}
+\makeexample{\examplename.sty N=2}
+\makeexample{\examplename.pdf}
+\tcbinputexample{%
+ listing options app={
+ basicstyle=\ttfamily\footnotesize,
+ },
+ middle=1mm, bottom=0mm,
+ comment={\centering
+ \relaxmmzcommands
+ \def\mmzUsedExtern##1{%
+ \saveexpandmode\fullexpandarg
+ \StrGobbleRight{##1}{4}[\mytemp]%
+ \StrRight{\mytemp}{5}[\mytemp]%
+ \restoreexpandmode
+ \relaxmmzcommands
+ }%
+ \def\mmzNewExtern##1##2##3##4{\mmzUsedExtern{##1}}
+ \def\myexternname##1{extern \dots\texttt{\mytemp##1.pdf}}%
+ \input{\examplepath.mmz}%
+ \begin{tabular}{ccc}
+ \includeexamplepdf[extern page,title/.expand once=\myexternname{}]
+ {page=2,trim=1in 1in 1in 1in,scale=0.333}&
+ \includeexamplepdf[extern page,title/.expand once=\myexternname{-1}]
+ {page=3,trim=1in 1in 1in 1in,scale=0.333}&
+ \includeexamplepdf[extern page,title/.expand once=\myexternname{-2}]
+ {page=4,trim=1in 1in 1in 1in,scale=0.333}\\
+ \includeexamplepdf[document page]{page=1,scale=0.333}&
+ \includeexamplepdf[document page]{page=5,scale=0.333}&
+ \includeexamplepdf[document page]{page=6,scale=0.333}
+ \end{tabular}
+ },
+}
+
+To achieve this, we will have to integrate the memoization driver into the very
+code of |\countdown|. This approach contrasts sharply with the standard
+memoization driver, which is simply wrapped around the memoized code. Let's
+say we have implemented a non-memoization-aware variant of |\countdown| as a
+loop which gathers the countdown lines into a vertical box, and periodically,
+when this box holds all the material that will fit onto the page, places it
+into the main vertical list (i.e.\ on the page).\footnote{Of course,
+ implementing \cs{countdown} this way would be idiotic; a sane implementation
+ would simply spit out the countdown lines one by one, and let \hologo{TeX}
+ deal with page-breaking. However, remember that we are pretending that we
+ are typesetting (and page-breaking) some complex material, like a table; in
+ such a case, the loop outlined in the main text would make perfect sense.}
+To have this command support memoization, we have to externalize our box every
+time we're placing it into the main vertical list. This is precisely what we
+do in the definition of |\countdowntypeset| below:\footnote{We omit the
+ definition of the core algorithm of \cs{countdown} in the listing, because it
+ is mostly irrelevant for our discussion, and only show the
+ memoization-related code.} the final line of this macro adds the material to
+the main vertical list, and the preceding lines externalize it (the two lines
+inside \cs{ifmemoizingcountdown} should be familiar from the definition of the
+standard memoization driver; we'll explain about the conditional below); note
+that Memoize automatically deals with the fact that our box is vertical. As a
+result of having our memoization driver integrated into the loop of the core
+command, we can create as many externs as necessary, complete with the code in
+the cc-memo for including each and every one of them on subsequent
+compilations. Each extern eventually makes it into its own extern file, and
+note that the filenames of the non-first externs have their sequential number
+(we start counting at $0$) appended to the basename, as shown in the
+example.\cprotect\footnote{The \refmmz{auto} declaration of |\countdown| adds
+ some relevant parameters to the context (see
+ section~\ref{sec:cross-referencing}). The countdown will be recompiled upon
+ change of either the font size (|f at size|), the text height (|\textheight|),
+ or the height of the material in the main vertical list collected so far
+ (|\pagetotal|). The |\pagetotal| parameter is especially important;
+ including it makes sure that the countdown will be recompiled when it is
+ pushed up or down the page. Also note that we want the context to record the
+ value of |\pagetotal| when the (automemoized) |\countdown| is encountered
+ (rather than at the end of memoization), so we expand it when applying the
+ auto-options.}
+
+\tcbinputexample[.sty]{listing only, after title pre={\ (version 1)}}
+
+Of course, the chunks of the countdown should only be externalized when the
+code is actually being memoized, and not, say, when Memoize is disabled or
+performing regular compilation. (Note that this is a problem that only affects
+integrated drivers and not wrapped drivers such as the default driver.) The
+first thought is to detect whether we're undergoing memoization using
+conditional \refcmd{ifmemoizing}, which Memoize sets to true at the start of
+every memoization. This conditional is used in the run conditions of advice
+for \cs{ref} and \cs{label}, the idea being that they should add stuff to the
+context (\cs{ref}) and the cc-memo (\cs{label}) only when undergoing
+memoization. However, deploying \refcmd{ifmemoizing} in the current example
+would not be exactly right. It would work well with the main document as it
+is, but it would fail if |\countdown| was called from a piece of code that was
+independently submitted to memoization, e.g.\
+\refcmd{mmz}|{\countdown{30}}|.\footnote{Such embedding occurs more often than
+ you might think. For example, \env{forest} calls \env{tikzpicture} under the
+ hood, and both environments are automemoized.} In that case, both the
+\refcmd{mmz} driver and the |\countdown| integrated driver would get executed,
+resulting in the creation (and in subsequent compilations, utilization) of four
+externs: first, the |\countdown| driver would externalize each countdown chunk
+separately, and then, the \refcmd{mmz} driver would externalize them, all
+together, yet again. You can try this out by replacing
+\cs{ifmemoizingcountdown} in \cs{countdowntypeset} by \refcmd{ifmemoizing} (and
+wrapping the \cs{countdown} call in \cs{mmz}).
+
+The solution to the \refcmd{ifmemoizing} problem deployed in the example is to
+declare a new, \cs{countdown}-specific memoization conditional, and set it to
+true in \cs{countdown}'s formal driver, i.e.\ the macro set as the
+\refmmz{driver} in the \refmmz{auto} declaration for \cs{countdown}. In fact,
+Memoize can do most of this for you: when we write \refmmzauto{integrated
+ driver}|=countdown|, Memoize creates the countdown-specific memoization
+conditional and declares the formal driver which sets this conditional to true;
+you only have to access this conditional in your code, and you should do this
+using the \hologo{LaTeX}-style conditional \refcmd{IfMemoizing}, as shown
+below.\footnote{You shouldn't directly use the plain \hologo{TeX}
+ countdown-specific conditional created by \refmmzauto{integrated driver} ---
+ to prevent accidental access, Memoize doesn't actually name it
+ \cs{ifmemoizingcountdown} --- because this conditional is undefined when
+ Memoize is not loaded, i.e.\ when only package \refpkg{memoizable} is in
+ effect. Furthermore, \refcmd{IfMemoizing} addresses a problem faced by
+ integrated drivers of potentially recursive commands; we will talk about this
+ in section~\ref{sec:memoization-complex-single-driver}.}
+
+\tcbinputexample[.sty][.c2]{listing only, after title pre={\ (version 2)},
+ attachment name=\examplename-integrated-driver.sty,
+}
+
+
+\subsubsection{Driver-based memoizable design}
+\label{sec:memoization-complex-single-driver}
+
+In the previous section, we used the integrated driver approach to produce
+memos including multiple externs, but the approach can be useful for one-extern
+memos as well, when the extern must be integrated into the document in some
+special way. We already discussed such situations in
+section~\ref{sec:memoizable-design}, where we suggested to split a
+``difficult'' command into the outer command and the inner command, and only
+submit the inner command to automemoization. However, the vanilla flavour of
+this approach had a negative impact on the user interface to automemoization.
+In this section, we will deploy the memoization driver to overcome the issue.
+
+\ExampleName{poormansbox-driver}
+\makeexample{\examplename.sty}
+\makeexample{\examplename.pdf N=2}
+
+Let us revisit the |poormansbox| example from
+section~\ref{sec:memoizable-design}. Remember that that environment produced
+a potentially framed box of a certain width, surrounded by some pre- and
+post-code, and that the issue was that the pre- and the post-code should not be
+memoized, but rather executed at every invocation of the command, as it was
+primarily intended to put some stretchable vertical space around the box.
+
+The document source\attachexample\ and the resulting PDF of the example are the
+same as in section~\ref{sec:memoizable-design}, so we will not repeat them
+here, but jump directly into a revised definition of the environment. We will
+retain the core idea from the original implementation: the outer command will
+execute the pre- and the post-code, and the inner command will typeset the box.
+But unlike in the original implementation, we will not automemoize the inner,
+internal command (this was the source of the author's discomfort) but the outer,
+user-level command --- and we will equip it with a custom memoization driver.
+The major idea here is to have the driver compose a cc-memo which not only
+includes the extern, but also executes the outer command.
+
+\tcbinputexample[.sty]{listing only, float}
+
+In detail, the implementation (partially shown in the |.sty| listing) is as
+follows. The outer command (|\poormansbox at outer|) first applies the options
+(|#1|) and then wraps the pre-code (|\pmb at before|) and the post-code
+(|\pmb at after|) around some arbitrary code (|#2|). During memoization or
+regular compilation, the outer command is invoked through the |poormansbox|
+environment, and you can see that in the definition of that environment, the
+second argument to |\poormansbox at outer| is a call to the inner command
+(|\poormansbox at inner|; this command takes two arguments, the options and the
+environment body). During utilization, the outer command is invoked from the
+cc-memo,\cprotect\footnote{As you can see, in the cc-memo the outer command is invoked
+ by |\csuse{poormansbox at outer}|. A straight |\poormansbox at outer| would not
+ work because we're in the middle of the document where |@| is not a letter,
+ and including a |\makeatletter| in front of it (and in a group) only works if
+ \refmmz{direct ccmemo input} was in effect. Under the default, indirect
+ cc-memo input regime, the core cc-memo is tokenized before |\makeatletter| can
+ take effect.} and as you can see in the cc-memo listing below, the second
+argument to |\poormansbox at outer| there is a call to \refcmd{mmzIncludeExtern}.
+
+\begingroup
+\relaxmmzcommands
+\def\mmzNewCCMemo#1{%
+ \def\mmzNewCCMemo##1{% fetch the second cc-memo filename
+ \def\myccmemo{##1}%
+ \endinput
+ }%
+}
+\input{\examplepath.mmz.c1}
+\sed{%
+ s/~//g;
+ s/\(\cmd{csuse} *{poormansbox at outer}\) *\({\nobrace*\marg\nobrace*}\)\({.*}\rbrace\)/~\1~\2~\3~/;
+ s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/;
+}{\exampledir\myccmemo}
+\tcbinputlisting{
+ listing only, left=1.5mm, right=1mm,
+ listing options app={basicstyle=\tt\small},
+ listing file=\exampledir\myccmemo,
+ example title, title=the second poor man's box's cc-memo,
+}%
+\endgroup
+
+And how does |\poormansbox at outer| get into the cc-memo, which normally only
+includes a call to \refcmd{mmzIncludeExtern}, you ask? This is the job of the
+memoization driver, which is in this case integrated into the inner command.
+The overall shape of the driver is the same as the shape of the standard
+driver, discussed in section~\ref{sec:memoization-driver-default}: typeset the
+extern material into a box, externalize this box, append the extern-inclusion
+code to the cc-memo, and put the extern box into the document. It is the
+cc-memo part which interests us right now: unlike the standard driver, we don't
+simply append the contents of |\mmz at temptoks|, i.e.\ a
+\refcmd{mmzIncludeExtern} call; we rather append a call to
+|\poormansbox at outer|, which gets the \refcmd{mmzIncludeExtern} call as its
+second argument (and the options as its first argument).
+
+The core part of the driver, which externalizes the box and appends to the
+cc-memo, is embedded inside the true branch of conditional
+\refcmd{IfMemoizing}|[1]{pmb}|. We already used this conditional in
+section~\ref{sec:memoization-multiple-externs}, but without the optional
+argument. Such usage will not work here, because it is not recursion-safe.
+Unlike in the \cs{countdown} situation, one |poormansbox| environment can be
+embedded in another one (and, in our example, it is). If we deployed
+\refcmd{IfMemoizing}|{pmb}| in the inner command, the driver would be executed for
+\emph{both} the outer and the inner instance of the environment, whereas it
+should really be executed only for the outer instance.
+
+When used in a recursion-safe way, i.e.\ with the optional argument,
+\refcmd{IfMemoizing} first tests whether the auxiliary command-specific
+conditional from the previous section is true, and then proceeds to compare the
+current group level (\hologo{eTeX}'s \refcmd{currentgrouplevel}) to the group
+level at the start of memoization (which Memoize stored in
+\refcmd{memoizinggrouplevel}). Only if these group levels match do we know
+that we're working on the outer instance of the environment, and that we should
+therefore execute the memoization driver. Importantly, though, the two group
+levels are compared modulo the offset, given as the optional parameter to
+\refcmd{IfMemoizing}: in our example, the offset is 1, because the driver is
+located inside the |poormansbox| environment, which opens a group --- note that
+0 zero (no offset) is not the default optional parameter; the absence of the
+optional parameter indicates that the non-recursion safe method should be
+used.\footnote{Even this approach is not completely bullet-proof. It will only
+ work when the inner instance of the command is guaranteed to occur in an
+ additional group, i.e.\ when our command opens up a group for any free-form
+ code. I will assume that situations which require externalization of a
+ potentially recursive command which, for some reason, cannot open the group
+ before processing a free-form argument, are rare enough to not warrant a
+ generic solution here.}
+
+
+\FloatBarrier
+
+
+\subsubsection{Shipout}
+
+Memoize is a hypocrite: when it is creating extern pages, it uses
+\refcmd{primitive}\refcmd{shipout} to bypass the regular shipout routine of the
+format, but it is offended if anyone else does that.
+
+Memoize bypasses the regular shipout because the extern pages should really not
+be modified or discarded by a foreign package. But using the primitive
+\refcmd{shipout} means that extern shipouts can't be detected by another
+package, at all. To facilitate peaceful coexistence with a potential package
+which needs to know about our extern pages, we offer public counter
+\refcmd{mmzExternPages} holding the number of externs shipped out so far. And
+if anyone really needs to \emph{do} something at every extern shipout, they can
+always (ab)use \refmmz[show keypath]{record/record type/new extern} as a
+post-extern-shipout hook.
+
+The other side of the story is about Memoize needing to know the ``physical''
+page numbers of its externs in the document PDF --- how else are we to extract
+them? Memoize computes these page numbers by adding the values of several
+counters: \refcmd{mmzRegularPages}, which holds the number of regular shipouts;
+the above-mentioned \refcmd{mmzExternPages}, which holds the number of extern
+shipouts; and \refcmd{mmzExtraPages}, which holds the number of other shipouts.
+The latter counter should be advanced by a package which, like Memoize,
+bypasses the regular shipout routine.
+
+\hologo{LaTeX} and \hologo{ConTeXt} kindly provide the number of regular
+shipouts as publicly a accessible counter, so we define
+\refcmd{mmzRegularPages} as synonymous with their
+\refcmd{ReadonlyShipoutCounter} and \refcmd{realpageno}. In \hologo{plainTeX}, we
+have to hijack the \refcmd{shipout} control sequence and count regular shipouts
+ourselves; as we have to hijack it while it still refers to the
+\refcmd{shipout} primitive, this format provides another reason for preferring
+Memoize to be loaded early.
+
+
+\subsection{Automemoization}
+\label{sec:tut:automemoization-details}
+
+Automemoization is a mechanism that automatically memoizes the result of the
+compilation of certain commands and environments. Writing Memoize, I went to
+great lengths to make it flexible, yet easy to use. This resulted in
+automemoization deploying two specifically developed auxiliary packages:
+package Advice, which provides a generic framework for extending the
+functionality of selected commands and environments, and package CollArgs,
+which provides a command for collection of the arguments conforming to the
+given (slightly extended) \pkg{xparse} argument specification.
+
+This section lists the considerations which went into designing the system,
+followed by short tutorials on both auxiliary packages, which include several
+examples of how Memoize uses the underlying advising framework.
+
+\paragraph{(De)activation}
+Ideally, all commands and environments where memoization makes sense would
+support Memoize (or Memoize would support them) and nothing would ever go
+wrong. In this dream world, memoization would be completely transparent to the
+author. However, things will go wrong, so at the very least, we need to
+offer the author a \emph{simple} way to \emph{selectively} switch automemoization
+on and off. This is achieved by keys \refmmz{activate} and
+\refmmz{deactivate}.
+
+\paragraph{Submission}
+Of course, there will be commands without official support by either Memoize or
+the package which defines them; clearly, at the moment when I write this, all
+commands but \cs{tikz}, \env{tikzpicture} and \env{forest} are such. Or, the
+author might want to automemoize his or her own command. Ideally, submitting a
+new command to automemoization would be as simple as
+|memoize=|{\meta{command}}, and for environments, this is in fact achievable,
+although the actual interface is
+\refmmz{auto}|=|\braces{\meta{environment}}\braces{\refmmzauto{memoize}}. But
+simply submitting the name cannot work for commands, because commands are where
+we encounter the major \hologo{TeX}nical problem with automemoization: we need
+to somehow collect the arguments of the command --- without executing the
+command itself.
+
+\paragraph{Argument collection using CollArgs}
+\hologo{TeX} being \hologo{TeX}, automatically determining the scope of a
+command in general is just plain impossible. Note that inspecting the
+|\meaning| is not enough in general, because the ``real'' and the formal
+arguments of a command can, and quite often do, differ wildly. The author (or the
+package writer) will need to tell Memoize about the argument structure of the
+command. And as there is already a nice and general argument specification on
+the market --- I'm obviously referring to the argument specification of package
+\pkg{xparse}, which was recently even integrated into the core \hologo{LaTeX}
+--- why not use that? Memoize comes with an auxiliary package CollArgs, which
+(given the slightly extended \pkg{xparse}-style argument specification)
+collects the arguments of a command into a single entity. All the user needs
+to write to enable automemoization for a command is thus
+\refmmz{auto}|=|\meta{command}\braces{\refmmzauto{memoize},
+ \refmmzauto{args}|=|\braces{\meta{argument specification}}}. Even simpler,
+when it comes to commands defined by \pkg{xparse}'s \refcmd{NewDocumentCommand}
+or friends, writing \refmmz{auto}|=|\meta{command}\braces{\refmmzauto{memoize}}
+will suffice, as the argument specification of these commands can be retrieved
+by \refcmd{GetDocumentCommandArgSpec}.
+
+\paragraph{Weird commands}
+Not every argument structure can be described using \pkg{xparse}'s argument
+specification, a case in point being \refcmd{tikz} with its totally
+idiosyncratic syntax --- and if Memoize won't support \cs{tikz}, why have it at
+all? The interface to automemoization must be flexible enough to cover even
+the craziest commands, and this is why Memoize allows for arbitrary argument
+collectors. These are defined by the advanced user or package writer and then
+declared to be used for parsing the argument structure of a command by writing
+\code{\refmmz{auto}=\marg{command}\bracestt{...,\refmmzauto{collector}=\meta{argument
+ collector}}}.
+
+\paragraph{Over and above automemoization: handlers}
+The framework facilitating automemoization must cover more than just that. For
+one, it sometimes makes sense to automatically \emph{prevent} memoization
+during the execution of certain commands (as in the \refmmzauto{nomemoize}
+example in section~\ref{sec:tut:redefinitions}). It follows that the action
+performed to an invocation of a command should not be fixed. In the advising
+framework, implemented by the auxiliary package Advice, we assign each
+advised command a handler --- a command which does the real work of
+memoizing or whatever. Crucially, the handler and the collector are
+independent of each other, allowing a single memoization handler to handle
+commands with both standard and non-standard argument structure, and allowing a
+single collector to serve either the memoization or the no-memoization handler.
+
+\paragraph{Over and above automemoization: the outer and the inner handler}
+Second, unlike the memoization handler set by \refmmzauto{memoize}, not all
+handlers work with the entire argument list of the advised command. Some
+handlers don't care about the arguments of the advised command at all:
+\refmmzauto{abort} simply aborts memoization whenever the advised command is
+executed. Other handlers are only intended to advise a single command or a
+small family of commands, and need to inspect specific arguments of the advised
+command: for example, \refmmzauto{ref} needs to append the internal
+cross-reference macro to the context, and it of course constructs the name of
+this macro from the reference key. For all such handlers, it would be plain
+wasteful to first collect the arguments and then tear them apart to inspect
+them (or not). The advising framework therefore recognizes two kinds of
+handlers. The abortion and the cross-reference handler are examples of an
+\emph{outer} handler, which is simply placed in front of the arguments of the
+handled command as they are, without invoking the collector. The memoization
+handler, on the other hand, is an example of an \emph{inner} handler, which
+receives the entire argument list from the collector (as a single argument) ---
+or more precisely, even \refmmzauto{memoize} sets up an outer handler, but this
+outer handler doesn't do much more than invoke the collector, which in turn
+invokes the inner handler.
+
+\paragraph*{Run conditions} are another, if minor, lego piece of the
+advising framework. Using key \refmmzauto{run conditions}, the user can set the
+conditions under which the (outer) handler is executed; for example,
+cross-reference commands are only advised when memoization is underway. And
+the same goes for \refcmd{label}, and for the \refmmzauto{replicate}d
+\refcmd{index}, and for \refcmd{savepos}, upon which memoization must be
+aborted. The bottom-line is that run conditions repeat across handlers, so
+it makes sense to separate them out as an independent component of the
+framework, with the added bonus that the system can make sure that an
+invocation of a advised command which does not satisfy the run conditions
+will incur as litte overhead as possible.
+
+\paragraph{Bailout handler}
+An automemoized command applies the next-options (set by \refcmd{mmznext}), but
+what happens when the run conditions are not satisfied? If nothing happened,
+the existing next-options might apply to the next instance of
+(auto)memoization, which would not be what the author intended. This is why
+Advice introduces the bailout handler, a piece of code executed before the
+original command when the run conditions are not met. Obviously, the bailout
+handler for memoization clears out the next-options (and does not process
+them).
+
+\paragraph{The structure of advice}
+Together, the components mentioned above form a piece of \emph{advice}:
+
+\begin{center}
+ \begin{forest}
+ /tikz/edge node/.style={midway, font=\scriptsize, yshift=0.5ex},
+ for tree={l sep=5ex, % manual, to avoid a page break
+ child anchor=north, s sep+=2em},
+ [activated?
+ [\refmmzauto{run conditions}?,
+ edge=dashed, edge label={node[edge node, left]{yes}}
+ [\refmmzauto{outer handler}\meta{args},
+ edge=->, edge label={node[edge node, left]{yes}},
+ [default:,
+ edge=dashed,
+ [\refmmzauto{collector}\meta{args}, edge=->,
+ [\refmmzauto{inner handler}
+ \textcolor{red}{\bracestt{\textcolor{black}{\meta{args}}}},
+ edge=->
+ ]
+ ]
+ ]
+ [other:\\whatever\rlap\footnotemark,
+ align=center, edge=dashed]
+ ]
+ [\refmmzauto{bailout handler},
+ edge=->, edge label={node[edge node, right]{no}},
+ [\meta{original command}\meta{args}, edge=->]
+ ]
+ ]
+ [\meta{original command}\meta{args},
+ edge=dashed, edge label={node[edge node, right]{no}},
+ ]
+ ]
+ \end{forest}%
+ \footnotetext{The handler may do whatever as long as it consumes all and
+ only the arguments of the original command.}
+\end{center}
+
+
+
+\paragraph{Deferred activation}
+Memoize needs to be loaded early, but activation should take place late, so
+that it can surely override the submitted commands; a case in point,
+\pkg{hyperref} redefines \refcmd{ref} very late. To address the issue, the
+advising framework implements the deferred \refmmz{activation} regime, under
+which (de)activation commands are not executed but collected in a special
+style, \refmmz{activate deferred}. Memoize deploys the deferred activation
+regime throughout the preamble, and executes \refmmz{activate deferred} at the
+latest possible \docref{hook:begindocument} hook; as a bonus, it also offers
+the author a way to avoid automatic activation completely by invoking key
+\refmmz{manual}.
+
+
+\paragraph{Install anywhere}
+Once all this machinery is developed, why not offer it to others as well?
+
+Once I decided to offer Advice (at the time, still called Auto) as a standalone
+package (and I freely admit that the framework got much cleaner once I
+separated its code out of Memoize) it became immediately clear that if it is
+to serve as a generic framework, it should be possible for multiple packages to
+use it without interfering with each other. The package thus allows any number
+of installations into different namespaces, each namespace a \pkg{pgfkeys}
+keypath. The installation is a breeze:
+\cs{pgfkeys}\bracestt{\meta{namespace}/\refkey{/handlers/.install advice}}.
+
+
+\subsubsection{Using package Advice}
+\label{sec:advice}
+
+In this section, we will provide some examples of handler declarations, mainly
+based on how Memoize deploys the advising framework.
+
+\paragraph{\refmmzauto[show keypath]{memoize}}
+
+In section~\ref{sec:tut:automemoization}, the author was instructed to submit a
+command to automemoization by writing
+\code{\refmmz{auto}=\meta{command}\bracestt{\refmmzauto{memoize},...}}. The
+auto-key \refmmzauto{memoize} is a style (defined by Memoize rather than
+Advice) which sets the appropriate components of the automemoization advice.
+Residing in keypath \refkeypath{/mmz/auto}, it is effectively defined as
+follows:
+
+\begin{tcblisting}{listing only}
+\mmzset{
+ auto/memoize/.style={
+ run if memoization is possible,
+ bailout handler=\mmz at auto@bailout,
+ outer handler=\mmz at auto@outer,
+ inner handler=\mmz at auto@memoize
+ }
+}
+\end{tcblisting}
+
+The heart of this advice is its inner handler, which actually triggers
+memoization by executing \refcmd{Memoize}. Remember that the first argument of
+\refcmd{Memoize} is the code which the md5sum is computed off of. This
+argument must therefore be identical to the author's invocation of the
+automemoized command or environment. Given what Advice offers, this is easy to
+construct: \refcmd{AdviceReplaced} holds the code replaced by the advice, and
+the \meta{arguments} of the automemoized command are waiting for us in |#1|.
+The second argument of \refcmd{Memoize} will be similar, but as this is the
+code which will get compiled, we have to execute the original definition of the
+command, followed by the (unbraced!) \meta{arguments} as |#1|; this is a job
+for \refcmd{AdviceOriginal}. (Note that \emph{executing}
+\refcmd{AdviceReplaced} would run the auto-handler again, resulting in an
+infinite loop! Or at least a pile of errors.)
+
+However, the overly simplistic approach shown below won't necessarily work.
+The issue is that the arguments of \refcmd{Memoize} contain
+\refcmd{AdviceReplaced} and \refcmd{AdviceOriginal} themselves, instead of their
+contents, i.e.\ (first) expansions.
+
+\begin{tcblisting}{listing only, bad}
+\long\def\mmz at auto@memoize#1{%
+ \Memoize{\AdviceReplaced#1}{\AdviceOriginal#1}%
+}
+\end{tcblisting}
+
+Regarding the first argument, the problem is that the code md5sum will be
+computed off of the token list \refcmd{AdviceReplaced}\meta{arguments} ---
+exactly as you see it.\footnote{If you inspected a c-memo, you would find
+ \refcmd{AdviceReplaced}\meta{arguments} in the \refcmd{mmzSource} section.}
+This implies that two commands sharing exactly the same \meta{arguments} will
+receive the same (c-)memo. For example, if you automemoized first
+|\textit{foo}| and then |\textbf{foo}|, both would come out as a bold ``foo''
+upon utilization.
+
+The second argument illustrates a general issue about the lifespan of
+\refcmd{AdviceOriginal} and other |\Advice...| commands.\footnote{The list of
+ all commands available only within the handler can be found in the
+ documentation of key \refmmzauto{outer handler} in
+ section~\ref{sec:ref:advice}.} By executing \refcmd{Memoize}, we leave the
+advice and thereby cannot be sure that when expanded, \refcmd{AdviceOriginal}
+will mean what it means at the moment. In general, another piece of advice might be
+triggered until its expansion, or the group might be closed, etc. For example,
+the author may have issued \refcmd{mmznext}\bracestt{\refmmz{at begin
+ memoization}=\cs{label}\marg{key}}, and as the pre-memoization code is
+executed before the memoization driver, the \cs{label}, which is submitted to
+Advice so that any \cs{label}s inside the memoized code ``just work'', would
+execute another piece of advice, redefining \refcmd{AdviceOriginal} and friends.
+Effectively, you'd end up memoizing an invocation of the \cs{label} command.
+
+The bottom line is that while the code following the above template
+\emph{might} sometimes work, Advice offers no guarantees that it will, so I
+advise against using it. The actual definition of the memoization inner
+handler is shown below. In this definition, we expand \refcmd{AdviceReplaced}
+and \refcmd{AdviceOriginal} --- exactly once! --- into the respective arguments
+of \refcmd{Memoize}; of course, as the entire invocation of \refcmd{Memoize} is
+expanded, we have to guard against expanding the collected arguments (|#1|) and
+\refcmd{Memoize} itself.\footnote{As the final touch, the handler also contains
+ \refcmd{ignorespaces} after the invocation of \refcmd{Memoize}, if this was
+ requested using \refmmz{ignore spaces}. Note that this could not be done
+ without pre-expanding the \cs{ifmmz at ignorespaces} conditional, as
+ \refcmd{Memoize} closes the group in which the auto- and the next-options are
+ applied.} The result of the expansion is shown under the code: the first and
+the second line assume we are automemoizing command |\foo| and environment
+|bar|, respectively. Note that \refcmd{AdviceOriginal} expands into an
+invocation of \refcmd{AdviceGetOriginal}, a command which may be safely used
+outside the advice; the first argument of this command is the auto-namespace
+(in our case, \refkeypath{/mmz}), and the second argument is the advised
+command. For a \hologo{LaTeX} environment, the advised command is actually
+\refcmd{begin}, and this is why the call of \refcmd{AdviceGetOriginal} is of
+course followed by the environment name.
+
+\makeexcerpt{_auto-memoize-inner}
+\tcbinputexample[.tex][.excerpt]{%
+ listing and comment, one file, no attachment,
+ title={The implementation of the \refmmzauto{inner handler}
+ for automemoization},
+ comment={%
+ $\rightarrow$ \refcmd{Memoize}\braces{\cs{foo}\Arg1}%
+ \braces{\refcmd{AdviceGetOriginal}\bracestt{/mmz}\bracestt{\cs{foo}}\Arg1}\\
+ $\rightarrow$ \refcmd{Memoize}\braces{\cs{begin}\bracestt{bar}\Arg1}%
+ \braces{\refcmd{AdviceGetOriginal}%
+ \bracestt{/mmz}\bracestt{\cs{begin}}\bracestt{bar}\Arg1}
+ }
+}
+
+Let us now move backwards in time and look at the outer handler installed by
+\refmmzauto{memoize}. It is very simple, but performs an important function of
+applying the auto- and the next-options (in this order), which also
+necessitates opening a group (closed by \refcmd{Memoize}). The final line
+invokes the argument collector, which then calls the inner handler; remember
+that the invocation of \refcmd{AdviceCollector} is the sole function of the
+default outer handler.
+
+\makeexcerpt{_auto-memoize-outer}
+\tcbinputexample[.tex][.excerpt]{%
+ listing only, one file, no attachment,
+ title={The implementation of the \refmmzauto{outer handler}
+ for automemoization},
+}
+
+Moving even further back in time, we arrive at the run conditions. The
+\refmmzauto{memoize} style invokes \refmmzauto{run if memoization is possible},
+defined as \code{run conditions=\cs{mmz at auto@rc at if@memoization at possible}}, with
+the installed macro as shown below. Indeed, memoization only makes when
+Memoize is \refmmz{enable}d (which we test using \refcmd{ifmemoize}), but we're
+not already ``inside \refcmd{Memoize}'' (which we test using
+\refcmd{ifinmemoize}). The latter condition is true when we're either
+memoizing or regularly compiling some code submitted to memoization (see the
+diagram in section~\ref{sec:Memoize}). Note that it is not necessary to invoke
+\refcmd{AdviceRunfalse} in branches where the run conditions are not satisfied.
+
+\makeexcerpt{_auto-run-if-memoization-is-possible}
+\tcbinputexample[.tex][.excerpt]{%
+ listing only, one file, no attachment,
+ title={The implementation of
+ \refmmzauto{run if memoization is possible}},
+}
+
+While it is clear that double memoization is a no-no, why should we avoid
+memoizing inside a regular compilation? Imagine that Memoize decides not to
+memoize a Forest tree, perhaps because \refmmz{readonly} is in effect. Under
+the hood, Forest creates many \env{tikzpicture}s. Should all of them be
+(auto)memoized now? Certainly not.
+
+Finally, what happens when the run conditions are not met? Not much, but
+something important nevertheless: by consuming the next-options, the
+\refmmzauto{bailout handler} makes sure they will not erroneously apply to the
+next instance of (auto)memoization.
+
+\makeexcerpt{_auto-memoize-bailout}
+\tcbinputexample[.tex][.excerpt]{%
+ listing only, one file, no attachment,
+ title={The implementation of the \refmmzauto{bailout handler}
+ for automemoization},
+}
+
+The only component of the automemoization advice not determined by style
+\refmmzauto{memoize} is the argument collector, which allows the user to submit
+a command with a weird argument structure to automemoization simply by setting
+key \refmmzauto{collector} in addition to executing \refmmzauto{memoize}. For
+example, Memoize submits \refcmd{tikz} to automemoization by loading
+\reffile{advice-tikz.code.tex}, which contains Advice's definition of the
+\cs{tikz} collector \refcmd{AdviceCollectTikZArguments}, and issuing the
+following declaration.\footnote{This is a simplification, see
+ \refmmzauto{memoize tikz} for the full story.}
+
+\begin{tcblisting}{
+ example title={Declaring automemoization of command \cs{tikz}},
+ listing only,
+ }
+ auto=\tikz{memoize, collector=\AdviceCollectTikZArguments},
+\end{tcblisting}
+
+\paragraph{\refmmzauto[show keypath]{ref}}
+
+The cross-reference advice presents an example of an outer handler radically
+different from the default outer handler. This outer handler does not invoke
+the collector at all. As shown below, it grabs the argument of \refcmd{ref} (or
+whichever cross-referencing command) on its own --- remember that the outer
+handler receives the arguments of the handled command ``as they are,'' i.e.\
+uncollected. It then asks \refcmd{mmzNoRef} to do the real job of getting the
+reference key into the context, and finally executes the original \cs{ref}.
+
+\ExampleName{_auto-ref}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{%
+ listing only, no attachment,
+ title={\hypercolor{link}{white}A simplified\footnotemark{} definition
+ of \refmmzauto{ref}},
+ % This only works if there is a single footnote in the memoized code.
+ /mmz/context/.expanded={footnote=\numexpr\thefootnote-1},
+}\footnotetext{The real outer handler allows for arbitrary optional arguments of
+ the cross-referencing command, and shares code with \refmmzauto{force
+ ref}.}
+
+The run conditions of this style are agonizingly simple: \refmmzauto{run if
+ memoizing} sets \refmmzauto{run conditions} to a macro defined as
+\refcmd{ifmemoizing}\hyp\refcmd{AdviceRuntrue}\hyp\cs{fi}.
+
+\paragraph{\refmmzauto[show keypath]{abort}}
+
+The advice for aborting memoization is very simple --- it merely executes
+\refcmd{mmzAbort} --- but also very sneaky. Here, the \refmmzauto{run
+ conditions} do the real work of aborting memoization, while the ``real,''
+i.e.\ outer handler, never even gets executed; note the absence of
+\refcmd{AdviceRuntrue}, which implies \refcmd{AdviceRunfalse}, which triggers
+the execution of the original command after the run conditions are ``checked.''
+
+\makeexcerpt{_auto-abort}
+\tcbinputexample[.tex][.excerpt]{%
+ listing only, one file, no attachment,
+ title={The definition of \refmmzauto{abort}},
+}
+
+The point here is that executing \refcmd{mmzAbort} (itself a single-liner
+setting an internal conditional) is cheaper than testing for the real run
+conditions (\refmmzauto{run if memoizing}) and aborting only if they are
+satisfied. Of course, the trick only works because (i) the advice doesn't
+need to inspect any arguments of advised command, and because (ii) setting the
+internal abortion conditional outside memoization does no damage.
+
+\paragraph{Advice in chains}
+
+A command may be submitted to several instances of the advising framework,
+i.e.\ instances installed under different keypaths. In the example below, we
+submit |\foo| both to the instance of Advice installed in keypath |/one| and to
+the one installed in keypath |/two|. Under |/one|, the result of |\foo{...}|
+will be boxed (|\fboxWrap|); under |/two|, in will be parenthesized
+(|\parenWrap|). The order in which this happens depends on the order in which
+|\foo| was activated under different keypaths. If we first activate it under
+|/one| with the boxing effect and then under two with the parenthesizing
+effect, the box will appear within parenthesis; if we reverse the activation
+order, the parenthesis will appear inside the box.
+
+\ExampleName{chained-advice}
+\makeexample{\examplename.tex.c1 N=2}
+\tcbinputexample{comment=\input{\examplepath.tex.c2}}
+
+First of all, looking at the code above, you have probably noticed the absence
+of key \refmmz{auto}. This is because by default, \refkey{/handlers/.install
+ advice} defines the \emph{setup key} to be |advice| --- Memoize overrides
+this default by installing the framework with \refkey{/handlers/.install
+ advice}|=|\bracestt{\refkey{/advice/install/setup key}=auto, ...}.
+
+Next, |advice'| is a variant of |advice| which prevents automatic activation
+upon setup (and the same holds for |auto'| vs.\ |auto| in Memoize). We have
+used the bar variant above to make it clear that it is the order of activation,
+rather than declaration by |advice|\slash \refmmz{auto}, which matters in
+determining which handler is applied first.
+
+Finally, note that the deactivation order must be the reverse of the activation
+order. So if we activate |\foo| first in |/one| and then in |/two|, we should
+deactivate it in |/two| first and in |/one| next, otherwise Advice will
+complain.
+
+
+\paragraph{A simple collector}
+
+Let us implement a collector for a command which accepts one (standard
+\hologo{LaTeX}) optional argument and one mandatory argument; in \pkg{xparse}
+terms, a command with argument specification |om|.
+
+Using \cs{NewDocumentCommand}, such a collector is very easy to implement. We
+simply define a command with signature |om| and distinguish two possibilities
+regarding the presence of the optional argument, which we test using
+\cs{IfValueTF}. If the true branch, we pass a \emph{braced} |[#1]{#2}| to the
+inner handler, which we invoke by \refcmd{AdviceInnerHandler}; in the false
+branch, we omit the optional argument, passing it an (additionally) braced
+|{#2}|.
+
+\ExampleName{om-collector-NewDocumentCommand}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{listing only}
+
+Defining a functionally equivalent collector using \cs{newcommand} would be a
+bit more involved, as \hologo{LaTeX2e} does not offer a standardized way to
+test for the \emph{presence} of the optional argument. Consider the following
+collector, whose optional argument has the same default value as the advised
+command. Is it functionally equivalent to the one above?
+
+\ExampleName{om-collector-newcommand}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{listing only}
+
+While there will be no visual difference, there is a difference under the hood.
+If you compile both documents, you will see that the first one creates three
+memos\slash externs, while the second one only creates two: |\foo{...}| does
+not have its own memo anymore, but creates and uses the same memo as
+|\foo[green]{...}|.
+
+While the second version might sometimes be preferred, perhaps even in the
+context of memoization, the initial collector, which deploys command
+\refcmd{CollectArguments}, behaves like the \cs{NewDocumentCommand}-defined
+collector above,\footnote{In fact, there is a slight difference after all.
+ While the above-defined collector won't distinguish between the single-token
+ mandatory argument given with or without braces, \refcmd{CollectArguments}
+ will again faithfully replicate the original argument tokens.} as it attempts
+to perfectly replicate the command invocation. Furthermore, this behaviour
+makes it unnecessary for the author to provide the default values of optional
+arguments (and even allows them to replace |O{default}| in the argument
+specification by \docref{xparse:o}).
+
+We now turn to the package CollArgs, which implements the actual argument
+collection; we'll revisit the initial collector of Advice at the end of the
+following subsection.
+
+
+
+\subsubsection{Using package CollArgs}
+\label{sec:collargs}
+
+Automemoization is implemented on top of the framework offered by package
+Advice, and that package in turn couldn't really work as intended without
+package CollArgs. A regular user of Memoize shouldn't need to know anything
+about CollArgs, but a package writer wanting to support Memoize might have to.
+
+The package provides two public commands, \refcmd{CollectArguments} and
+\refcmd{CollectArgumentsRaw}; we'll focus on the former first.
+\refcmd{CollectArguments} takes three arguments: optional \meta{options} in the
+form of a \pkg{pgfkeys} keylist; a mandatory \meta{argument specification} in a
+(slightly extended) \pkg{xparse} format; and the \meta{next-code}:
+\begin{center}
+ \refcmd{CollectArguments}\oarg{options}\marg{argument specification}\marg{next-code}%
+ \textcolor{gray}{\meta{tokens}}
+\end{center}
+Following the three formal arguments of \refcmd{CollectArguments} are some
+\meta{tokens} --- the rest of the document, really --- and the job of
+\refcmd{CollectArguments} is to figure out the extent to which these
+\meta{tokens} conform to the given \meta{argument specification}. In other
+words, \refcmd{CollectArguments} will consume as many of the \meta{tokens} as a
+\meta{command} defined by \refcmd{NewDocumentCommand}\meta{command}\marg{argument
+ specification}\bracestt{...} would. Once these \meta{argument tokens} are
+collected, \refcmd{CollectArguments} executes the \meta{next-code} with the
+\meta{argument tokens} given as a single, braced argument (clearly, the
+\meta{rest} of the \meta{tokens}, i.e.\ the non-consumed tokens, will follow):
+\begin{center}
+ \meta{next-code}\marg{argument tokens}\textcolor{gray}{\meta{rest}}
+\end{center}
+
+In the example below, we define macro |\PrintAndDo|, which takes two arguments,
+a command and the collected arguments of that command, prints out which command
+we're about to execute and with what arguments, and then executes that command
+with those arguments --- |#1#2| at the end of the definition. Note that |#2|
+immediately following |#1| is not braced, so
+|\PrintAndDo\makebox{[5em][r]{text}}| executes |\makebox[5em][r]{text}|.
+
+{\tolerance 1000 Executing |\PrintAndDo\makebox{[5em][r]{text}}| directly would
+ thus yield the first line of the result below --- and in fact, this is
+ precisely what gets executed to yield that line, but in a roundabout fashion.
+ Given the argument specification |oom| (two optional arguments followed by a
+ mandatory argument), \refcmd{CollectArguments} figures out how many tokens
+ following its formal arguments conform to this argument specification ---
+ below, these would be |[5em][r]{text}| following
+ \refcmd{CollectArguments}\hyp|{oom}{\PrintAndDo\makebox}|--- and puts them,
+ braced, behind its \meta{next-code} argument, |\PrintAndDo\makebox|, yielding
+ |\PrintAndDo|\hyp|\makebox{[5em][r]{text}}|.\par}
+
+\ExampleName{collargs-makebox}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{listing and compile}
+
+Seeing the arguments of |\makebox| without the immediately preceding |\makebox|
+might seem strange, but remember that \refcmd{CollectArguments} is about the
+arguments of a command, not about the command's control sequence. It doesn't
+know or care which command the argument tokens ``belong'' to, as long as they
+conform to the given specification. In the example above, it is only in |#1#2|
+of |\PrintAndDo| that |\makebox| is ``reunited'' with its arguments, but note
+that the reunion is far from obligatory.
+
+CollArgs supports all the argument types (and modifiers) that \pkg{xparse}
+does, including the environment type \docref{xparse:b}, as exemplified below.
+Again, the code below might seem strange, as it features an |\end{minipage}|
+without the matching |\begin{minipage}|, but the logic is similar as for
+ commands: just as \refcmd{CollectArguments} occurs in front of the command
+ arguments, without the command itself, so it occurs in front of the
+ environment body, without the opening of that body. However, while
+ \refcmd{CollectArguments} never needs to know the command name, we need to
+ inform it of the environment name, so that it can find the end of the
+ environment. This can be achieved as shown below, using key
+ \refcollargs{environment} in the optional argument of the command, or by our
+ extension to the \pkg{xparse} argument specification, where the environment
+ argument type \docref{xparse:b} may be followed by a braced environment name.
+ In the example below, we could therefore also invoke argument collection by
+ \refcmd{CollectArguments}\braces
+ {\docref{xparse:+}\docref{xparse:b}\braces{\env{minipage}}} (we have preceded
+ \docref{xparse:b} with a \docref{xparse:+} to allow for an environment body
+ containing paragraph tokens).\footnote{CollArgs automatically adapts to the
+ format, i.e.\ it knows that environments are tagged by \cs{}\meta{name} and
+ \cs{end}\meta{name} in \hologo{plainTeX} and by \cs{start}\meta{name} and
+ \cs{stop}\meta{name} in \hologo{ConTeXt}.}
+
+\ExampleName{collargs-minipage}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{comment=\input{\examplepath.tex.c1}}
+
+You might wonder why didn't we provide \refcmd{CollectArguments} in the
+previous example with argument specification |omb| --- after all, the
+\env{minipage} environment takes an optional and a mandatory argument. While that
+would work, and produce the same result,\footnote{Unless argument processing
+ was in effect; see section~\ref{sec:ref:collargs} for details.} note that
+\refcmd{CollectArguments} is only interested in finding the scope of the
+arguments, and grabbing everything until |\end{minipage}| is the same as first
+grabbing the optional argument, maybe, then the mandatory argument, and finally
+the argument body.
+
+\refcmd{CollectArguments} not only supports \pkg{xparse}'s verbatim argument
+type \docref{xparse:v}, it can grab an argument of \emph{any} type in the
+verbatim mode, triggered by option \refcollargs{verbatim}.\footnote{We refer to
+ the verbatim mode triggered by \refcollargs{verbatim} as the full verbatim
+ mode, where all characters are of category ``other''. There is also the
+ partial verbatim mode, triggered by \refcollargs{verb}, where braces retain
+ their normal category codes.} We illustrate this key below, where we also use
+option \refcollargs{tags}, which makes CollArgs automatically surround the
+grabbed environment body with the begin tag \refcmd{begin}\marg{environment
+ name} and the end tag \cs{end}\marg{environment name}, and use
+\refcmd{scantokens} to execute the grabbed environment. Consult
+section~\ref{sec:ref:collargs} for the full reference on the verbatim mode and
+its limitations.
+
+% A harmless warning:
+% Missing character: There is no ^M (U+000D) in font [lmmono10-regular]:!
+% because \texttt in the example is fed a (verbatim) carriage return
+\ExampleName{collargs-verbatim}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{listing and compile}
+
+Finally, CollArgs extends the \pkg{xparse} specification by modifier
+\docref{xparse:amp}, which allows the user to specify options which apply only
+to the following argument, as opposed to the options given as the optional
+argument of \refcmd{CollectArguments}, which apply to all the arguments. A
+third way to invoke the environment body collection in the above example is
+thus \refcmd{CollectArguments}%
+\bracestt{\docref{xparse:amp}\bracestt{\refcollargs{environment}=minipage}+b}.
+
+Both the single-argument and the common options can be given not only as
+\pkg{pgfkeys} keys, but also in the raw, ``programmer's interface'' format.
+Every option key has a corresponding macro; for example, key
+\refcollargs{environment} is matched by macro \refcmd{collargsEnvironment}.
+The macros are listed alongside their corresponding keys in the reference
+section~\ref{sec:ref:collargs}; here, we merely learn how to use them.
+
+To use raw options for a single argument, double the ampersand in the argument
+specification. Therefore, the fourth way to specify the environment name is
+\code{\docref{xparse:amp}\docref{xparse:amp}\bracestt{%
+ \refcmd{collargsEnvironment}\bracestt{minipage}}\docref{xparse:+}\docref{xparse:b}}.
+
+To set the raw options for all arguments, use \refcmd{CollectArgumentsRaw}, the
+second public command of the package. This command is exactly like
+\refcmd{CollectArguments}, excepts that it expects the options in the raw
+format and as a \emph{mandatory} argument:
+
+\begin{center}
+ \refcmd{CollectArgumentsRaw}\marg{raw options}\marg{argument specification}%
+ \marg{next-code}\textcolor{gray}{\meta{tokens}}
+\end{center}
+
+This leads us to the fifth way to set the environment name (an overkill, I
+know): \refcmd{CollectArgumentsRaw}\hyp
+\bracestt{\refcmd{collargsEnvironment}\bracestt{minipage}}\bracestt{+b}\marg{next-code}.
+Furthermore, you can use a mixture of raw and key--value options: the raw
+option commands include \refcmd{collargsSet}, which applies the given option
+keylist. The idea here (incarnated by both Auto and Memoize) is that the
+package will provide CollArgs with the raw options, for speed, while the
+author can supplement them in the friendly keylist format --- and this leads
+us to the sixth, and thankfully final way to set the environment name:
+\refcmd{CollectArgumentsRaw}\bracestt{\refcmd{collargsSet}%
+ \bracestt{\refcollargs{environment}=minipage}}\bracestt{+b}\marg{next-code}.
+
+
+\paragraph{The initial collector}
+
+As the final example, let us study Advice's initial collector; this is a macro
+which is used as the collector when key \refmmzauto{collector} is not given.
+This macro is not really \refcmd{CollectArguments}, as we sometimes state to
+simplify matters, but a macro which acts as the ``bridge'' between Advice and
+CollArgs, by compiling an invocation of \refcmd{CollectArgumentsRaw} from the
+given advice setup, and executing it.
+
+The bridge macro is shown below in its full glory, but it is really less
+complicated than it might appear at first sight. In line~2, we use
+\refcmd{AdviceIfArgs} to see whether the argument structure of the handled
+command was given by the user. If it wasn't, we assume that the handled
+command was defined using \cs{NewDocumentCommand} or similar, and use
+\refcmd{GetDocumentCommandArgSpec} to retrieve it (line~3; note that
+\refcmd{AdviceName} holds the handled control sequence) and store it into
+\refcmd{AdviceArgs} (line~4), which also receives the argument specification
+when given by the user via key \refmmzauto{args}.
+
+\makeexcerpt{_advice-CollectArgumentsRaw}(../../advice.edtx)
+\tcbinputexample[.tex][.excerpt]{%
+ listing only, one file, no attachment,
+ listing options app={numbers=left, numberstyle=\tiny, numbersep=0.5em},
+ title={The definition of the initial collector},
+}
+
+Lines~6--17 are somewhat of an expansion mess, because we have to construct the
+invocation of the CollArgs' collector from the advice setup stored in various
+macros. But once we think away all the (non-)expansion commands, we're left
+with \refcmd{CollectArgumentsRaw} plus the following three arguments:
+\begin{enumerate}
+\item The raw options (lines 8--11):
+ \begin{enumerate}
+ \item In line 8, the advised command's control sequence is designated as the
+ \refcollargs{caller}. The effect is that if the given arguments don't
+ conform to the specification, the error thrown seems to come from the
+ advised command rather than some internal CollArgs macro. The author will
+ be grateful for this little detail.
+ \item In line 9, we add any \refmmzauto{raw collector options} set by Advice
+ (plus the package deploying Advice, like Memoize); user-given options are of
+ course possible, but not really expected here, because:
+ \item In lines 10--11, we add the user-given \refmmzauto{collector options},
+ if there are any, embedded under \refcmd{collargsSet}.
+ \end{enumerate}
+\item The argument specification (line 14).
+\item The \refmmzauto{inner handler} (line 15).
+\end{enumerate}
+
+
+
+\section{Reference}
+\label{sec:reference}
+
+In this section, the extra information about keys, offered at the right of a
+key in parenthesis, may contain the initial value of a key, and also a default
+value of a key. In this context, the terms ``initial'' and ``default'' have
+the meaning employed by the \pkg{pgfkeys} utility (\PGFmanual{87}). The term
+``initial value'' applies to the setting underlying a key (when there is no
+such setting, the key is marked as a ``style''), and refers to the value of
+this setting that is set by the package. In other contexts, we would call this
+the default or the package-default value, but in the \pkg{pgfkeys} parlance,
+the term ``default value'' applies to a key taking an argument, and refers to
+the value that is passed to the key in the absence of that argument.
+(Honestly, I only keep to this convention in the reference section; elsewhere,
+I often say ``default'' or ``package-default'' and mean the initial value.)
+
+Another convention I keep to in this section is the color-coding of the keys
+and commands. Green background indicates a basic key or command, which any
+user might want to know about. Red background indicates other, more or less
+advanced keys and commands.
+
+
+\subsection{Loading and initialization}
+\label{sec:ref:loading}
+\paragraph{\hologo{LaTeX}}
+
+Load Memoize by \cs{usepackage}\braces{\docaux{package}{memoize}} or
+\cs{usepackage}\oarg{options}\bracestt{memoize}. The latter form functions
+almost identically to the former followed by
+\refcmd{mmzset}\bracestt{\meta{options}}, with two exceptions.
+
+First, when used as package options, \meta{options} may not contain the slash
+character (|/|), which is necessary to invoke \pkg{pgfkeys} handlers, because
+is it misinterpreted by \hologo{LaTeX}.\footnote{The historic \hologo{LaTeX}
+ constraint prohibiting spaces in package options does not apply anymore.} In
+the rare situations requiring key handling in the package options, use option
+\refmmz{options}.
+
+Second, key \refmmz{extract} and other extraction-related keys such as
+\refmmz{perl extraction options} should normally be used as a package option,
+or within \reffile{memoize.cfg}. Because Memoize extracts the externs while it
+is being loaded, executing these keys after the package is loaded will have a
+different effect; see the documentation of \refmmz{extract} for details.
+
+Memoize extracts the externs while it is being loaded (and not, say, at the
+beginning of the document) because extern extraction can only be performed
+before the output PDF is opened, and some packages cause it to be opened when
+they are loaded. This also implies that Memoize must be loaded before any such
+package. In particular, it must be loaded before \PGF library |fadings| (see
+section~\ref{sec:fadings-problem}) and before the \pkg{beamer} class (see
+section~\ref{sec:tut:beamer}).
+
+If you're familiar with the \TikZ externalization library, you might wonder
+whether Memoize has an equivalent of the \refcmd{tikzexternalize} command. It
+doesn't. Memoize assumes that if you loaded it, you want to use it --- but you
+can always disable it using key \refmmz{disable}, or even by loading package
+\refpkg{nomemoize} in its stead.
+
+In \hologo{LaTeX}, initialization and finalization are completely automatic.
+Memoize defines several initialization and finalization styles ---
+\docaux{key}{begindocument/before}, \docaux{key}{begindocument},
+\docaux{key}{begindocument/end} and \docaux{key}{enddocument/afterlastpage}
+---and executes them at the cognominal \hologo{LaTeX} hooks.
+
+\paragraph{\hologo{plainTeX}}{}
+
+Load Memoize by |\input memoize|. As package options cannot be provided in
+\hologo{plainTeX}, the author must trigger extraction from \refcmd{mmzset}
+using key \refmmz{extract}; I recommend doing this immediately after loading
+the package. This key may be invoked with or without a value. In the latter
+case, Memoize will extract using the package default method
+\refmmz{extract=perl}, unless its has been overriden from
+\reffile{memoize.cfg}.
+
+Furthermore, as \hologo{plainTeX} has no concept of a document body, the text
+must be manually enclosed in \refcmd{mmzset}\bracestt{\docaux{key}{begin
+ document}} and \refcmd{mmzset}\bracestt{\docaux{key}{end document}}; this
+is where the initialization and finalization hooks described above will be
+executed. Note that \refmmz{extract}, when used, must precede this enclosure.
+
+Finally, \hologo{plainTeX} has another reason for preferring the early loading
+of the package. In this format, Memoize must redefine \refcmd{shipout}, at a
+time the meaning of this control sequence is still primitive. In particular,
+this means that Memoize must be loaded before \pkg{atbegshi}.
+
+\begin{tcboxedraster}[raster column skip=1cm]{blankest}
+ \ExampleName{plainTeX}%
+ \makeexample{\examplename.tex.c1}%
+ \tcbinputexample{listing only, title=A minimal \hologo{plainTeX} example}
+ %
+ \ExampleName{ConTeXt}%
+ \makeexample{\examplename.tex.c1}%
+ \tcbinputexample{listing only, title=A minimal \hologo{ConTeXt} example}
+\end{tcboxedraster}
+
+\paragraph{\hologo{ConTeXt}}{}
+
+Load Memoize by \cs{usemodule}\bracketstt{memoize} or
+\cs{usemodule}\bracketstt{memoize}\oarg{options}. Unlike in \hologo{LaTeX},
+there are no restrictions on characters allowed within \meta{options}; the
+remarks on key \refmmz{extract} and loading order are the same as for
+\hologo{LaTeX}.
+
+In \hologo{LaTeX}, Memoize automatically executes its initialization and
+finalization code when at the beginning and the end of the document body. Due
+to my very limited experience with \hologo{ConTeXt}, and its project structure
+in particular, I don't know what the appropriate place for initialization and
+finalization would be in \hologo{ConTeXt}. I therefore provisionally leave it
+to the author to execute \refcmd{mmzset}\bracestt{\refmmz{begin document}} and
+\refcmd{mmzset}\bracestt{\refmmz{end document}} manually, and hope for advice
+on how to handle this properly.
+
+
+\paragraph{Auxiliary packages}
+
+\begin{doc}{package={name=nomemoize}}
+ Loading this package instead of Memoize completely disables memoization, but
+ does not require removing any Memoize commands from the document (they all
+ become no-ops). This package accepts any package options (and ignores them).
+\end{doc}
+
+\begin{doc}{package={name=memoizable}}
+ This package is a programmer's stub: if Memoize is loaded, it does nothing;
+ otherwise, it provides the no-op variants Memoize commands. This package
+ accepts no package options. Generic packages, e.g.\ \TikZ libraries, can
+ also load this package via |\input memoizable.code.tex|. See
+ section~\ref{sec:loading-memoize} for details.
+\end{doc}
+
+
+\subsection{Configuration}
+\label{sec:ref:configuration}
+
+
+\begin{doc}{easy,cmd={name=mmzset, par=\marg{options}}}
+
+ Update the Memoize configuration.
+
+ The \meta{options} are a comma-separated list of \meta{key}|=|\meta{value}
+ pairs. They are processed using the \pkg{pgfkeys} utility of PGF/\TikZ (see
+ \PGFmanual{87}), with the default path set to \docaux{key path}{mmz}.
+
+ The changes are local to the current \hologo{TeX} group, except for keys
+ where explicitly noted otherwise.
+\end{doc}
+
+\begin{doc}{easy,
+ cmd={name=mmznext, par=\marg{options}},
+ }
+
+ This command accepts the same \meta{options} as \refcmd{mmzset}, but
+ interprets them as \emph{next-options} --- options which will be applied to
+ the next, and only the next, \emph{auto}memoized command or environment.
+ (Remember that a command or environment is submitted to automemoization by
+ \refmmz{auto}|=|\marg{command or
+ environment}\bracestt{\refmmzauto{memoize},...}; see
+ sections~\ref{sec:tut:automemoization}, \ref{sec:tut:working-on-a-picture}
+ and \ref{sec:ref:advicememoization} for details.)
+
+ Remarks for the author:
+
+ \begin{itemize}
+ \item Key \refmmz{enable} has no effect inside \refcmd{mmznext}.
+ \item If \refcmd{mmznext} is used more than once preceding an automemoized
+ command, only the final invocation takes effect.
+ \item The next-options also apply to commands and environments for which
+ memoization is autodisabled via \refmmz{auto}|=|\marg{command or
+ environment}\bracestt{\refmmzauto{nomemoize},...}.
+ \item It is safe to set the next-options in front of a command submitted to
+ automemoization which does not actually undergo memoization in this
+ particular instance. In other words, the absence of memoization will not
+ cause the next-options to ``leak'' to the next automemoized command.
+ \item Only the linear (execution) order of \refcmd{mmznext} and the
+ automemoized command matters. Key \refcmd{mmznext} will correctly apply to
+ a single following automemoized command even if it occurs outside the group
+ which that command is executed from; and it will apply to the following
+ automemoized command even if it is called within a group closed before that
+ command is executed.
+ \end{itemize}
+
+ Remarks for the programmer:
+ \begin{itemize}
+ \item The next-options are set globally.
+ \item The effect of \refcmd{mmznext} is \emph{not} cumulative. Consequently,
+ \refcmd{mmznext}\braces{} clears the next-options.
+ \item The next-options are applied by executing \refcmd{mmzAutoInit} within
+ the advice. Any piece of advice applying the next options should also clear them
+ when the \refmmzauto{run conditions} are not met. This is streamlined by
+ style \refmmzauto{apply options}, intended for use within \refmmz{auto}
+ declarations. Out of the box, this style is deployed by
+ \refmmzauto{memoize}, \refmmzauto{nomemoize} and \refmmzauto{noop}, but it
+ may be used by any piece of advice. Note that the \refmmzauto{outer handler}
+ declared by this style opens a group (to apply the options in) but leaves
+ it to the (undeclared) \refmmzauto{inner handler} to close that group.
+ \item Key \refmmz{enable} has no effect inside \refcmd{mmznext} because when
+ Memoize is disabled upon encountering an automemoized command, the advice
+ bails out without ever applying the next-options. More generally, this
+ applies to any advised command whose run conditions require
+ \refcmd{ifmemoize} to be true. Key \refmmz{disable}, on the other hand,
+ takes effect, because \refcmd{ifmemoize} is checked within \refcmd{Memoize}
+ as well.
+ \end{itemize}
+\end{doc}
+
+\begin{doc}{easy,file={name=memoize.cfg, desc=file}}
+ The configuration file, loaded just before processing the package options.
+ It will typically contain a \refcmd{mmzset} command, but it may contain any
+ \hologo{TeX} code.
+
+ As for any other file loaded by \hologo{TeX}, the location of the file
+ determines whether it applies system-wide, user-wide or directory-wide.
+
+ This file is also loaded by package \refpkg{nomemoize}, on the off chance it
+ defines some commands other than \refkeypath{/mmz} keys. (It is not loaded
+ by package \refpkg{memoizable}, though).
+\end{doc}
+
+\begin{doc}{key={name=options, par=\marg{options}}}
+ Execute \meta{options} as if they were given an an argument to \refcmd{mmzset}.
+
+ This option primarily exists to allow any key--value pair accepted by
+ \refcmd{mmzset} to be used as a package option. In particular, this applies
+ key handlers (see \PGFmanual{87}), because their invocation includes a slash
+ (|/|). For example, one cannot directly use
+ \begin{center}
+ \code{\cs{usepackage}[\refmmz{perl extraction
+ options}/.prefix=\braces{\braces{\refscript{memoize-extract.pl--quiet}}} ]\braces{memoize}}
+ \end{center}
+ to add (prepend) option \refscript{memoize-extract.pl--quiet} to the
+ invocation of the perl extraction script \refscript{memoize-extract.pl}, but
+ this will work:
+ \begin{center}
+ \code{\cs{usepackage}[\refmmz{options}=\braces{\refmmz{perl extraction
+ options}/.prefix=\braces{\braces{\refscript{memoize-extract.pl--quiet}}} ]\braces{memoize}}}
+ \end{center}
+\end{doc}
+
+\begin{doc}{cmd={name=nommzkeys, par=\marg{key}}}
+ In package \refpkg{memoize}, this command is a no-op; in packages
+ \refpkg{nomemoize} and \refpkg{memoizable}, it defines key
+ \refkeypath{/mmz}|/|\meta{key} as a no-op.
+
+ As explained in section~\ref{sec:tut:final}, use this command to declare any
+ \refkeypath{/mmz} keys you have used outside \refcmd{mmzset} when switching
+ to package \refpkg{nomemoize}.
+\end{doc}
+
+
+\subsection{Memoization}
+\label{sec:ref:memoization}
+
+\subsubsection{Manual memoization commands}
+\label{sec:ref:memoization:manual}
+
+\begin{doc}{easy,
+ cmd={name=mmz, par=\oarg{options}\marg{code}},
+ env={name=memoize, par=\oarg{options}},
+ }
+ Submit \meta{code} or \meta{environment body} to memoization.
+
+ Prior to memoization, the configuration is locally updated by executing
+ \meta{options} given as the optional argument to this command, i.e.\ the
+ given options take precedence to options previously set via \refcmd{mmzset}.
+ Note that next-options, set by \refcmd{mmznext}, are not applied.
+
+ The effect of the macro and of the environment version of this command is the
+ same, except that the command version memoizes \meta{code} exactly as-is,
+ while the environment version trims away any spaces at the beginning and the
+ end of the code.\footnote{What actually happens is that at the beginning of
+ the environment body, all space tokens will be discarded. At the end of
+ the body, no spaces are actually discarded; Memoize simply issues an
+ \refcmd{unskip}. This should not matter to a regular user who simply
+ writes down the environment.} The space-trimming feature of the environment
+ ensures that you can write |\begin{memoize}| and |\end{memoize}| in separate
+ lines (as shown above), but no extra space will creep into the extern.
+
+ The space-trimming feature of the environment, which trims spaces at the
+ beginning and at the end of the \meta{environment body}, should not be
+ confused with the effect of \refmmz{ignore spaces}, which ignores spaces
+ following the environment end-tag (in \hologo{LaTeX}, |\end{...}|) --- and
+ which does not apply to manual memoization at all!
+
+ The argument of \refcmd{mmz} \emph{must} be enclosed in braces.
+\end{doc}
+
+\begin{doc}{easy,
+ cmd={name=nommz, par=\oarg{options}\marg{code}},
+ env={name=nomemoize, par=\oarg{options}},
+ }
+ Disable Memoize for the span of the compilation of \meta{code} or
+ \meta{environment body}.
+
+ This command consumes the \meta{options} in the same way as
+ \refcmd{mmz}\slash \refenv{memoize} described above. The macro and the
+ environment version of the command exhibit the same space-trimming behaviour
+ as their \refcmd{mmz}\slash \refenv{memoize} counterparts, and the argument
+ of \refcmd{nommz} \emph{must} be enclosed in braces.
+\end{doc}
+
+
+\subsubsection{Basic configuration}
+\label{sec:ref:memoization:basic}
+
+\begin{doc}{easy,
+ key={name=enable},
+ key={name=disable},
+ }
+ Enable or disable the functionality of the package.
+
+ What happens when Memoize is enabled depends on the memoization mode
+ (\refmmz{normal}, \refmmz{readonly}, \refmmz{recompile}) and many other
+ factors. When the package is disabled, it neither creates new memos and
+ externs, nor uses the existing ones; this applies to both manual and
+ automatic memoization. The effect is close to not having Memoize loaded, or
+ to loading NoMemoize, but it is not completely the same; for example, the
+ record file (\dmmz) \emph{is} updated while Memoize is disabled, reflecting
+ the fact that nothing was memoized (or utilized) in the disabled
+ state.\footnote{Cleaning the folder (\S\ref{sec:cleanup-scripts}) after
+ disabling the package for the entire document is thus a bad idea.}
+
+ If these keys are used in the preamble, their effect is delayed until the
+ beginning of the document, to ensure that Memoize is never enabled in the
+ preamble. Other than that, all these keys do is set the \hologo{TeX}
+ conditional \refcmd{ifmemoize}, which you may use in your code to test
+ whether Memoize is enabled. You may also use \refcmd{memoizetrue} and
+ \refcmd{memoizefalse}, as long as you never enable the package in the
+ preamble.
+
+ Key \refmmz{enable} cannot be applied to automemoized commands via
+ \refcmd{mmznext}. It will take effect for manual memoization, though, and
+ key \refmmz{disable} will work for both, as expected.
+\end{doc}
+
+\begin{doc}{easy,
+ key={name=normal, desc=the initial mode},
+ key={name=readonly},
+ key={name=recompile},
+ }
+
+ Select the memoization mode.
+
+ Each piece of code submitted to (either manual or automatic) memoization is
+ associated to several files: one c-memo, one cc-memo, and some externs (zero
+ or more, typically one). When Memoize encounters a piece of code submitted
+ to memoization, it takes one of the following actions:
+
+ \begin{description}
+ \item[memoization] The code is compiled in a special way which produces the
+ associated memos and externs.
+ \item[utilization] The code is not compiled. Its effect is replicated by
+ processing the cc-memo; typically, this includes the single extern into the
+ document.
+ \item[regular compilation] The code is compiled as if Memoize was absent or
+ disabled (the memos and externs are neither utilized nor produced).
+ \end{description}
+
+ \begin{minipage}[t]{0.48\linewidth}
+ The action taken depends on the memoization mode and on whether all the
+ memos and externs associated with the code exist, as shown in the table on
+ the right. Note that a single missing memo or extern implies the ``no''
+ column of the table, and that memoization will create \emph{all} the
+ associated memos and externs, even those which already exist.
+ \end{minipage}
+ \hfill
+ \begin{tabular}[t]{@{}lll@{}}
+ \toprule
+ &\multicolumn{2}{c}{Do the memos and externs exist?}\\
+ \cmidrule{2-3}
+ mode&yes&no\\
+ \midrule
+ normal&utilization&memoization\\
+ readonly&utilization®ular compilation\\
+ recompile&memoization&memoization\\
+ \bottomrule
+ \end{tabular}
+
+ The memoization mode only has effect when Memoize is \refmmz{enable}d. Mode
+ selection is orthogonal to enabling\slash disabling the package; for example,
+ if you switch to a new mode while the package is disabled, the new mode will
+ be in effect once the package is enabled.
+\end{doc}
+
+
+\begin{doc}{easy,
+ desc={style},
+ key={name=verbatim},
+ key={name=verb},
+ key={name=no verbatim, desc={style, the initial mode}},
+ }
+ When \refmmz{verbatim} or \refmmz{verb} is in effect, the code submitted to
+ memoization is read verbatim; \refmmz{no verbatim} reverts to the normal,
+ non-verbatim collection of the code. This applies to both manual and
+ automatic memoization.
+
+ The long version, \refmmz{verbatim}, switches to the full verbatim mode,
+ where all characters are assigned category code 12 (other). With the short
+ version, \refmmz{verb}, the braces, |{| and |}|, retain category codes 1 and
+ 2, which can be useful for verbatim collection of optional arguments. For
+ details, see the documentation of CollArgs' \refcollargs{verbatim} in
+ section~\ref{sec:ref:collargs}.
+
+ Under the hood, these keys have two effects. First, they are passed on to the
+ argument collector (typically, \refcmd{CollectArguments} of the auxiliary
+ package CollArgs; for details, see section~\ref{sec:ref:advicememoization}),
+ instructing it to collect the code in the specified fashion, as described
+ above. Second, if the collected verbatim code is eventually compiled (either
+ regularly, or memoized), Memoize first rescans it using \refcmd{scantokens}.
+\end{doc}
+
+\begin{doc}{easy,
+ key={name=padding left, par=\meta{dimension},
+ desc={no default, initially |1| |in|}},
+ key={name=padding right, par=\meta{dimension},
+ desc={no default, initially |1| |in|}},
+ key={name=padding top, par=\meta{dimension},
+ desc={no default, initially |1| |in|}},
+ key={name=padding bottom, par=\meta{dimension},
+ desc={no default, initially |1| |in|}},
+ }
+ Set the left/right/top/bottom padding of the extern in the extern PDF.
+
+ Without padding, the (PDF) page holding the extern would tightly fit the
+ bounding box of the extern. These keys enlarge the extern page by the given
+ amounts, so that any parts of the extern lying outside the bounding box will
+ be correctly included when using the extern. See
+ section~\ref{sec:tut:padding} for details.
+
+ \meta{dimension} is evaluated with \hologo{eTeX}'s |\dimexpr|, and may
+ contain control sequences \docaux{cmd}{width}, \docaux{cmd}{height} and
+ \docaux{cmd}{depth}, which will refer to the dimensions of the extern.
+ \refcmd{width} and friends behave like dimension registers, so it is ok to write
+ e.g.\ \refmmz{padding right}|=0.5\width|.
+
+ The default padding is what \hologo{pdfTeX} puts around the page anyway, 1
+ inch, but we use |1 in| rather than |1 true in|, which is the true default
+ value of PDF registers \docref{pdfvar:horigin} and \docref{pdfvar:vorigin},
+ as we want the padding to adjust with \refcmd{mag}nification.
+\end{doc}
+
+\begin{doc}{easy,key={name=padding, par=\meta{dimension}, desc={style, no default}}}
+ Set all the above |padding| keys to the given value.
+\end{doc}
+
+\begin{doc}{
+ easy,
+ key={name=context, par=\meta{tokens}, desc={cumulative, no default,
+ initially set by \refmmz{padding to context}}},
+ key={name=clear context},
+ }
+ These keys append the given \meta{tokens} to the \emph{context expression},
+ or clear this expression. Memoized code gets recompiled whenever the
+ expansion of the context expression changes.
+
+ The \meta{tokens} must be fully expandable (modulo protection); in
+ \hologo{LaTeX}, they will be expanded by |\protected at edef| when calculating
+ the md5sum of the context.
+
+ The context expression is evaluated at the end of the memoization, and at
+ utilization attempts after inputting the c-memo (note that the c-memo
+ contains any additions to the context expression accumulated during
+ memoization). At evaluation, the given context expression is fully expanded,
+ yielding the \emph{context value}, whose md5 sum forms the \meta{context md5
+ sum} part of the filename of the cc-memo and the extern.
+
+ These keys may be used both prior to the memoization process, or during
+ memoization. In the former case, their effect is local to the current group;
+ in the latter case, the effect is global, so that the changes surely survive
+ until the end of memoization, when the c-memo, where the context expression
+ is stored, is written into a file.
+
+ Under the hood, these keys manipulate token registers
+ \docaux{cmd}{mmzContext} and \docaux{cmd}{mmzContextExtra}, changing the
+ contents of the former while not memoizing, and the contents of the latter
+ during memoization. These token registers may also be manipulated directly
+ by the user, as long as one keeps to the convention of adjusting
+ \refcmd{mmzContext} locally and only while not memoizing, and adjusting
+ \refcmd{mmzContextExtra} globally and only during memoization.
+\end{doc}
+
+\begin{doc}{
+ key={easy, name=meaning to context,
+ par=\marg{comma-separated list of commands and environment names}},
+ key={name=csname meaning to context, par=\marg{control sequence name}},
+ key={name=key meaning to context,
+ par=\marg{full path to a pgfkeys command key}},
+ key={name=key value to context,
+ par=\marg{full path to a pgfkeys value key}},
+ }
+ These keys append the definition of the given construct to the context.
+
+ Essentially, the ``meaning'' keys append \cs{meaning}\meta{control sequence}
+ to the context, for the appropriate \meta{control sequence}. For example,
+ \refmmz{meaning to context} appends \cs{meaning}\cs{foo} when given \cs{foo}
+ as in item in the list, and it appends the internal environment macros
+ appropriate to the format when given an environment name. Similarly,
+ \refmmz{key meaning to context} resolves \meta{control sequence} to the
+ internal macro holding the key command.
+
+ Key \refmmz{key value to context} should be used on keys which store values,
+ e.g.\ keys initialized by \pkg{pgfkeys} handler \code{.initial}; see
+ \PGFmanual{87.3.4 and \S87.4.5}.
+
+ All the keys prefix the meaning\slash value by the name of the command\slash
+ environment\slash key, in order to prevent ambiguous contexts, see
+ section~\ref{sec:cross-referencing} for details. Furthermore, they all
+ operate through \code{\cs{csname}...\cs{endcsname}} construct, allowing one
+ to safely add internal commands to the context.
+\end{doc}
+
+\begin{doc}{
+ keypath={/handlers}, desc=handler,
+ key={easy, name=.meaning to context},
+ key={name=.value to context},
+ }
+ These are the handler variants of \refmmz{key meaning to context} and
+ \refmmz{key value to context}.
+\end{doc}
+
+
+\begin{doc}{key={name=padding to context,desc=style}}
+ This key appends the values of the \refmmz{padding} keys to the context,
+ causing the memoized code to be recompiled whenever the padding values are
+ changed. This key is used to initialize \refmmz{context}, so the author
+ normally shouldn't have to use this key.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzNoRef, par=\marg{reference key}, desc=\hologo{LaTeX} only},
+ cmd={name=mmzForceNoRef, par=\marg{reference key}, desc=\hologo{LaTeX} only},
+ }
+ These macros append the current value of the \meta{reference key} to the
+ context,\footnote{More precisely, it is \meta{reference key}|=|\marg{current
+ value} which is appended.} causing the memoized code to be recompiled
+ when the reference changes.
+
+ If the reference key is undefined, \refcmd{mmzNoRef} aborts memoization,
+ while \refcmd{mmzForceNoRef} uses \cs{relax} as the reference string.
+
+ These commands are deployed in the implementation of \refmmzauto[show
+ keypath]{ref}, \refmmzauto{force ref} and friends. If the
+ cross-referencing commands you are using are advised by these keys, you most
+ likely have no need of these macros.
+\end{doc}
+
+\begin{doc}{easy, key={name=per overlay,desc=style}}
+ This style is only defined in the Beamer class. When applied, the memoized
+ code will produce a cc-memo (and extern) for each overlay of the frame. For
+ implementation, see section~\ref{sec:per-overlay}.
+\end{doc}
+
+\begin{doc}{
+ key={name=capture, par=\docAux{of=key:/mmz/capture, value={name=hbox}, text=\alt,
+ value={name=vbox,of=key:/mmz/capture}},
+ desc={no default, initially \refmmz{capture=hbox}}}
+ }
+ Select the capture mode. This setting only applies to the default
+ memoization \refmmz{driver}.
+
+ By default, it is assumed that the memoized code should be executed in the
+ horizontal mode, so the default memoization driver captures the output of the
+ memoized code in a \refcmd{hbox} --- and also issues a \refcmd{quitvmode}
+ (both in the document and in the cc-memo), just in case the memoized code
+ occurs at the start of the paragraph.
+
+ Use \refmmz{capture}|=|\refmmz{capture=vbox} to execute the memoized code in
+ the vertical mode: Memoize will capture the output of the memoized code in a
+ \refcmd{vbox}, and avoid issuing \refcmd{quitvmode}. For example, this
+ capture mode is necessary to memoize a \refenv{verbatim} environment.
+\end{doc}
+
+
+\subsubsection{Inside the memoization process}
+\label{sec:ref:memoization:process}
+
+\begin{doc}{easy, cmd={name=mmzAbort}}
+ This command aborts the ongoing memoization.
+
+ The memoization will proceed as usual (i.e.\ the extern boxes and the cc-memo
+ code will be produced), but at the end of this process, no memos will be
+ produced, no externs shipped out to extern pages, and no record files
+ updated.
+\end{doc}
+
+\begin{doc}{easy, cmd={name=mmzUnmemoizable}}
+ This command aborts the ongoing memoization, and marks the submitted code as
+ unmemoizable.
+
+ The ongoing memoization produces a c-memo setting conditional
+ \docaux{cmd}{ifmmzUnmemoizable} to true. Upon utilizing this c-memo, the
+ system switches to regular compilation.
+
+ For example, if you are automemoizing |tcolorbox|es of package
+ \pkg{tcolorbox}, you will want to refrain from memoizing boxes marked as
+ |breakable| or |float|ing. Simply aborting the memoization cannot do the
+ trick here, as memoization compiles the submitted code in a \hologo{TeX} box.
+ Marking a |breakable| or |float|ing |tcolorbox| as unmemoizable (either
+ manually using this macro, or automatically using \refmmz{auto key}) makes
+ sure that after the first compilation when memoization is attempted, the box
+ will be compiled regularly, and will have the intended ability to break
+ across pages, or float.
+\end{doc}
+
+
+\begin{doc}{
+ cmd={name=Memoize, par=\marg{key}\marg{code}},
+ }
+ Depending on various factors, this command either memoizes \meta{code} under
+ key \meta{key}, utilizes the results of a previous memoization, or performs a
+ regular compilation of \meta{code}.
+
+ The outcome of executing this command --- memoization, utilization or regular
+ compilation --- depends upon the Memoize's state (\refcmd{ifmemoize},
+ \refcmd{ifmemoizing}) and mode (\refmmz{normal}, \refmmz{readonly},
+ \refmmz{recompile}), and the existence of the relevant memos and externs.
+ The decision process is depicted in section~\ref{sec:Memoize}.
+
+ This command expects to be executed in a dedicated group, which it will close
+ itself.
+
+ Invoking memoization through \refcmd{Memoize} might be useful for packages
+ which want to save the results of intensive computations, regardless of
+ whether the author loads (and enables) memoization or not. However, this
+ usage is not yet officially allowed, because there is currently no way to
+ load the core memoization routines without loading the entire package,
+ thereby forcing the author to use Memoize.
+\end{doc}
+
+\begin{doc}{
+ key={
+ name=driver, par=\marg{code},
+ desc={no default, initially \refcmd{mmzSingleExternDriver}}
+ },
+ }
+ This key sets \meta{code} as the memoization driver.
+
+ Given some code submitted to memoization, the memoization driver should
+ produce the memos and externs which will replicate the effect of that code
+ (while retaining its regular effect). For details, see
+ section~\ref{sec:memoization-drivers}.
+
+ Typically, the \meta{code} argument of this key will consist of a single
+ control sequence (the driver control sequence), but any amount of tokens is
+ allowed. Memoize executes the driver followed by the code which it is
+ supposed to memoize, in braces, and only cares that the driver consumes that
+ code.
+\end{doc}
+
+
+\begin{doc}{
+ key={name=at begin memoization, par=\meta{code}, desc={cumulative, initially empty}},
+ key={name=at end memoization, par=\meta{code}, desc={cumulative, initially empty}},
+ key={name=after memoization, par=\meta{code}, desc={cumulative, initially empty}},
+ }
+ Use these keys to set up memoization hooks.
+
+ These keys may be used both prior to the memoization process, or during
+ memoization. In the former case, their effect is local to the current group;
+ in the latter case, the code given to \refmmz{at begin memoization} is
+ executed immediately, while the assignment performed by the other two keys is
+ global, so that the changes surely survive until the end of memoization.
+
+ The code given to hook \refmmz{at begin memoization} is kept in macro
+ \docaux{cmd}{mmzAtBeginMemoization}, while the content of the other two hooks
+ resides in two macros per hook: \docaux{cmd}{mmzAtEndMemoization} and
+ \docaux{cmd}{mmzAtEndMemoizationExtra}, and \docaux{cmd}{mmzAfterMemoization}
+ and \docaux{cmd}{mmzAfterMemoizationExtra}. All these macros may be
+ manipulated directly by the user,\footnote{Use \cs{appto}, \cs{eappto},
+ \cs{gappto} and \cs{xappto} of package \pkg{etoolbox} (loaded by Memoize)
+ to easily append code to these macros.} as long as one keeps to the
+ convention of adjusting the macros without ``\texttt{Extra}'' locally and
+ only while not memoizing, and adjusting the macros with ``\texttt{Extra}''
+ globally and only during memoization. The ``\texttt{Extra}'' macros require
+ global assignments as they might be manipulated by code residing within a
+ \hologo{TeX} group of any depth.
+
+ These complications explained, let us take a look at how memoization proceeds
+ to learn when the hooks are used:
+ \begin{enumerate}
+ \item Initialize various conditionals, macros and token registers. (Here is
+ where the ``\texttt{Extra}'' hooks are cleared.) Remember that at this
+ point, we're inside a group opened by \refcmd{Memoize}.
+ \item Execute \refmmz{at begin memoization} hook, i.e.\ the contents of
+ macro \refcmd{mmzAtBeginMemoization}.
+ \item Execute the memoization \refmmz{driver}.
+ \item Execute \refmmz{at end memoization} hook, i.e.\ the contents of macros
+ \refcmd{mmzAtEndMemoization} and \refcmd{mmzAtEndMemoizationExtra} (in this
+ order).
+ \item Write out the memos and ship out the externs to extern pages (unless
+ memoization was aborted).
+ \item Close the memoization group.
+ \item Execute \refmmz{after memoization} hook, i.e.\ the contents of
+ macros \refcmd{mmzAfterMemoization} and \refcmd{mmzAfterMemoizationExtra}
+ (in this order).
+ \end{enumerate}
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzCMemo, desc={token register, global, empty at the start of memoization}},
+ }
+ This token register mediates the construction of the c-memo. During
+ memoization (and only during memoization), arbitrary code may be added to
+ this register; at the end of memoization, Memoize writes out its contents to
+ the free-form part of the c-memo.
+
+ All assignments to this register should be global. Use \refcmd{gtoksapp} and
+ \refcmd{xtoksapp} to easily append tokens to the register.
+\end{doc}
+
+
+\begin{doc}{
+ cmd={name=mmzCCMemo, desc={token register, global, empty at the start of memoization}},
+ }
+ This token register mediates the construction of the cc-memo. During
+ memoization, the memoization driver should append cc-memo code to
+ \refcmd{mmzCCMemo}; at the end of memoization, Memoize writes out its
+ contents to the cc-memo (preceded by the list of produced externs).
+
+ All assignments to this register should be global. Local assignments would
+ not work, because the memoized code may contain commands, like \cs{label} and
+ \cs{ref}, which contribute content to cc-memo as well, but these commands may
+ appear within a \hologo{TeX} group of any depth.
+
+ Use \refcmd{gtoksapp} and \refcmd{xtoksapp} to easily append tokens to the
+ register.
+\end{doc}
+
+\begin{doc}{
+ par=\meta{token register}\marg{tokens},
+ cmd={name=toksapp},
+ cmd={name=gtoksapp},
+ cmd={name=etoksapp},
+ cmd={name=xtoksapp},
+ }
+ These commands append the given \meta{tokens} to the \meta{token register}.
+ \refcmd{etoksapp} and \refcmd{xtoksapp} expand the \meta{tokens} before
+ appending them; \refcmd{gtoksapp} and \refcmd{xtoksapp} perform a global
+ assignment.
+
+ These commands are actually provided by CollArgs, and they are defined only
+ if they don't already exist; in particular, note that \hologo{LuaTeX}
+ provides them as primitives.
+
+ Unlike the \hologo{LuaTeX} primitive variant, these commands require the
+ \meta{token register} to be given by a (|\toksdef|fed) control sequence; it
+ cannot be given as |\toks|\meta{number}.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=ifmemoize},
+ cmd={name=memoizetrue},
+ cmd={name=memoizefalse},
+ }
+ Use the \hologo{TeX}-style conditional \refcmd{ifmemoize} to test whether
+ Memoize is currently enabled. Within the document body, the conditional may
+ be set using \refcmd{memoizetrue} and \refcmd{memoizefalse}, which are then
+ functionally equivalent to \refmmz{enable} and \refmmz{disable}. Do not set
+ the conditional in the preamble of the document (unless you really know what
+ you are doing).
+\end{doc}
+
+\begin{doc}{
+ cmd={name=ifmemoizing,desc={readonly}},
+ }
+ Use this \hologo{TeX}-style conditional to test whether Memoize is currently
+ memoizing. It may be only inspected; you should \emph{never} set this
+ conditional yourself.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=ifinmemoize,desc={readonly}},
+ }
+ Use this \hologo{TeX}-style conditional to test whether Memoize is currently
+ active, in the sense of either memoizing or regularly compiling some code ---
+ so inside a call to \refcmd{Memoize}. The conditional may be only inspected;
+ you should \emph{never} set it yourself.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzSingleExternDriver, par=\marg{code}},
+ }
+ This is the default memoization \refmmz{driver}, producing exactly one extern
+ containing whatever is typeset by the submitted \meta{code}.
+
+ The \meta{code} is compiled either within a horizontal or vertical box,
+ depending on the value of key \refmmz{capture}. In the case of a horizontal
+ capture, the driver makes sure that the horizontal mode is entered prior to
+ both typesetting the resulting box in the document, or utilizing the extern.
+
+ For the implementational details, see
+ section~\ref{sec:memoization-driver-default}.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzExternalizeBox, par=\marg{box}\marg{token register}},
+ }
+ This macro is indended to be called by memoization drivers to produce an
+ extern page. The given \meta{box} is dumped into the document as a separate
+ extern page, while the \meta{token register} receives the cc-memo extern
+ inclusion code.
+
+ The \meta{box} may be given either as a control sequence (declared via
+ |\newbox|), or as box number. The resulting extern page will contain a
+ \emph{copy} of the given box, padded by the \refmmz{padding} values in effect
+ at the time of invocation of \refcmd{mmzExternalizeBox}.
+
+ An implementation detail is that \refcmd{mmzExternalizeBox} does not ship out
+ the extern page immediately. This action is delayed until the end of the
+ memoization process; more precisely, it is carried out (in tandem with
+ writing out the c-memo and the cc-memo) between execution of hooks \refmmz{at
+ end memoization} and \refmmz{after memoization}. This delay guarantees
+ that no extern pages are produced in the event of aborting memoization, even
+ if the abortion is triggered after executing \refcmd{mmzExternalizeBox}.
+
+ The \meta{token register} may be given either as a control sequence (declared
+ via |\newtoks|) or as control sequence |\toks| followed by the register
+ number. The register will receive the code, which, when executed from the
+ cc-memo, includes the extern file into the main document. This code consists
+ of a single invocation of \refcmd{mmzIncludeExtern}. It is the
+ responsibility of the driver to include the code received by \meta{token
+ register} in the register \refcmd{mmzMemo}, whose contents are, unless
+ memoization is aborted, written into the cc-memo. (See
+ \refcmd{ifmmzkeepexterns} and \refmmz{after memoization} to learn about
+ another way to use the code received by \meta{token register}.)
+
+ The invocation of \refcmd{mmzIncludeExtern} in the produced extern-inclusion
+ code is adapted to the type of the box (horizontal or vertical), which is
+ detected automatically --- the memoization driver does not need to inform
+ \refcmd{mmzExternalizeBox} about this type explicitly.\footnote{This does not
+ negate the need for key \refmmz{capture}, which applies to the default ---
+ and therefore generic --- memoization driver. This driver cannot know
+ whether the memoized code would prefer to be compiled in a horizontal or
+ vertical box. It is precisely key \refmmz{capture} which gives the user an
+ opportunity to inform Memoize about this preference. Only once the
+ memoized code is compiled into a box of the appropriate type, it is trivial
+ to detect the type of that box.}
+\end{doc}
+
+\begin{doc}{
+ cmd={name=ifmmzkeepexterns, desc={initially \cs{iffalse}}},
+ cmd={head prefix=\texttt{\string\global}, name=mmzkeepexternstrue},
+ cmd={head prefix=\texttt{\string\global}, name=mmzkeepexternsfalse},
+ }
+ Setting this conditional to true makes Memoize keep the extern boxes in the
+ global temporary storage even after shipping them out as extern pages. (The
+ temporary storage is emptied at the start of the next memoization.)
+
+ The extern inclusion code received by the \refcmd{mmzCCMemo} when executing
+ \refcmd{mmzExternalizeBox} is primarily meant to be executed by inputting the
+ cc-memo file; i.e.\ when the cc-memo is input, \refcmd{mmzIncludeExtern} is
+ defined to include the extern file into the document. However, it sometimes
+ makes sense to execute the cc-memo contents immediately after memoization;
+ for example, if memoization produces several externs, intricately integrated
+ into the surrouding environment, it might be cumbersome to replicate their
+ typesetting both in the memoizing compilation and in the cc-memo code ---
+ easier to build up the cc-memo code and execute it right after memoization.
+ This is why Memoize, just before executing the contents of \refmmz{after
+ memoization} hook, redefines \refcmd{mmzIncludeExtern} to include externs
+ from the temporary storage rather than from (at that point still
+ non-existing) extern files. However, as this mechanism requires Memoize to
+ keep the externs around even after memoization, it is not enabled by default:
+ it must be enabled by (globally) setting conditional
+ \refcmd{ifmmzkeepexterns} to true.
+\end{doc}
+
+\begin{doc}{
+ key={keypath=/mmz/auto, name=integrated driver, par=\marg{name}, desc=style}
+ }
+ Use this key to easily setup a memoization driver which is integrated into
+ the command itself.
+
+ \begin{tcolorbox}[warning]
+ This is an auto-key residing in keypath \refkeypath{/mmz/auto}.
+ \end{tcolorbox}
+
+ An integrated driver must have a way of telling whether it is memoizing or
+ regularly compiling the code. This key declares a driver-specific
+ conditional which may be inspected, using \refcmd{IfMemoizing}, to determine
+ this. The conditional is set to true by the formal \refmmz{driver} of the
+ command (set up by the invocation of this key), executed at the start of
+ memoization; it should never be set elsewhere. See
+ section~\ref{sec:memoization-multiple-externs} for details and an example.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=IfMemoizing, par=\oarg{offset}\marg{name}\marg{true code}\marg{false code}}
+ }
+ This \hologo{LaTeX}-style conditional is meant to be used by the
+ \refmmzauto{integrated driver} with the given \meta{name}. It tests whether
+ this particular driver is currently memoizing some code.
+
+ Potentially recursive commands are supported via the optional argument
+ \meta{offset}. If given, the conditional will only execute the \meta{true
+ code} when the current \hologo{TeX} group level matches the \hologo{TeX}
+ group level at the time of the invocation of the formal driver (held in
+ \refcmd{memoizinggrouplevel}), plus the \meta{offset}. In effect, the inner
+ invocation of the integrated driver will perform a regular compilation. For
+ details, see section~\ref{sec:memoization-complex-single-driver}.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=memoizinggrouplevel,desc={readonly}},
+ }
+ During memoization, this macro holds the \hologo{TeX} group level in effect
+ at the start of the memoization.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzRegularPages, desc=readonly counter},
+ }
+ This counter holds the number of pages shipped out (so far) by the format's
+ regular shipout routine. \emph{Do not change its value!}
+
+ In \hologo{LaTeX}, this counter is synonymous with
+ \refcmd{ReadonlyShipoutCounter}, and in \hologo{ConTeXt}, it is synonymous
+ with \refcmd{realpageno}. Memoize does not touch its value.
+
+ In \hologo{plainTeX}, Memoize hijacks the \refcmd{shipout} control sequence
+ to count (and only to count) regular shipouts. In order for its value to be
+ realistic, Memoize should be loaded before other packages which hack
+ \cs{shipout} --- in particular, before \pkg{atbegshi}.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzExternPages, desc=readonly counter},
+ }
+ This counter holds the number of extern pages Memoize has shipped out (so
+ far). \emph{Do not change its value!}
+
+ A third-party tool may inspect this counter to have a realistic count of
+ shipped-out pages.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzExtraPages, desc=counter},
+ }
+ This counter holds the number of pages shipped out (so far) in a way not
+ tracked by either \refcmd{mmzRegularPages} or \refcmd{mmzExternPages}. It
+ \emph{should} be advanced by any code which performs such shipouts, or
+ Memoize won't work correctly.
+\end{doc}
+
+
+\subsubsection{Tracing}
+\label{sec:ref:memoization:tracing}
+
+\begin{doc}{
+ key={name=trace, conditional=false},
+ }
+ When tracing is on, Memoize shows information about its decision processes on
+ the terminal. You can learn whether the memoized code is being memoized,
+ utilized or regularly compiled; find out the md5sum of the code and which
+ input line it comes from; etc.
+
+ This key has the syntax of a conditional, but there is no underlying
+ \hologo{TeX} conditional. The low-level interface for switching the tracing
+ on and off consists of macros \docaux{cmd}{mmzTracingOn} and
+ \docaux{cmd}{mmzTracingOff}.
+
+ To learn about tracing the ``auto'' part of the automemoization process, also
+ \refcmd{AdviceTracingOn}.
+\end{doc}
+
+\begin{doc}{
+ key={name=include source in cmemo, conditional=true},
+ }
+
+ As a courtesy towards a curious user and a debugging aid, Memoize can include
+ a copy of the memo source in the c-memo. This feature is switched on by
+ default, but as the package itself never uses that information, it can be
+ safely switched off at any time.
+\end{doc}
+
+\begin{doc}{
+ key={name=include context in ccmemo, conditional=false},
+ }
+ When this key is in effect, the expanded context expression is appended to
+ the cc-memo, behind the \docaux{cmd}{mmzThisContext} marker.
+
+ Memoize never uses the context information from the cc-memo; this information
+ is only for tracing purposes.
+\end{doc}
+
+\begin{doc}{
+ key={name=direct ccmemo input, conditional=false},
+ }
+ When this key is set to false, a cc-memo is processed indirectly: it is first
+ read into a token register, and it is the contents of this register which are
+ executed. When the key is set to true, the cc-memo is simply |\input|ted.
+
+ The indirect execution is implemented to facilitate inverse search. Under
+ the direct cc-memo input, inverse search pointed at an included extern will
+ visit the cc-memo, which is not practical; under the indirect regime, the
+ inverse search will work as expected, and this is why the indirect cc-memo
+ input is the default.
+
+ The overhead produced by the default indirect input method seems negligible,
+ but there are other factors which might make the user switch to the direct
+ input. For one, a cc-memo changing some category codes will require direct
+ input (no such cc-memos are ever produced out of the box). Less crucially,
+ sometimes one would like to use the inverse search to figure whether a part
+ of the document was produced by regular compilation or utilization, and which
+ memos\slash externs were utilized if the latter. Figuring this out under the
+ indirect input regime is harder: (i) reading the tracing information shown by
+ \refmmz{trace} is the surest way to learn what's going on, although (ii)
+ visual inspection of the externs and (iii) grepping through the |.memo.dir|
+ folder for particular code often help, as well.
+
+ Both input methods use the same cc-memos; there is no need to
+ \refmmz{recompile} the memos when switching the cc-memo input method. Note
+ that the default indirect input method crucially relies on cc-memos ending
+ with \refcmd{mmzEndMemo}; this macro should not appear in the cc-memo itself.
+\end{doc}
+
+
+
+\subsubsection{Internal memo commands}
+\label{sec:ref:memoization:internal}
+
+The end-user should never have to use these commands. They are not formally
+marked as internal by a |@| in their name only because doing so would
+complicate |\input|ting the memos due to the category code changes it would
+require.
+
+\begin{doc}{
+ cmd={name=mmzMemo},
+ }
+ This macro marks the beginning of a c-memo and a cc-memo core. Without it,
+ utilization of a memo will not work.
+\end{doc}
+
+\begin{doc}{cmd={name=mmzSource}}
+ This macro marks the beginning of the memoized source in the c-memo. That
+ source is not used by Memoize in any way. It's inclusion into the c-memo may
+ be switched off by \refmmz{include source in cmemo}|=false|.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzResource, par=\marg{filename}},
+ }
+ This is an internal command, which only occurs in a cc-memo. It checks
+ whether file \meta{filename} exists and is non-empty, and triggers
+ recompilation of the memoized code if the check fails.
+\end{doc}
+
+\begin{doc}{
+ cmd={
+ name=mmzIncludeExtern,
+ par=\marg{seq}\braces{\refcmd{hbox}\alt\refcmd{vbox}}%
+ \marg{expected width}\marg{expected height}\marg{expected depth}%
+ \marg{padding left}\marg{padding bottom}\marg{padding right}\marg{padding top},
+ }
+ }
+
+ This is an internal command, which only occurs in a cc-memo. It includes the
+ extern identified by the sequential number \meta{seq} into the document as a
+ box of the specified type (horizontal or vertical). The extern is trimmed by
+ the given padding values. After trimming, the command checks whether the
+ size of the resulting box matches the given expectations; if it doesn't, a
+ warning is yielded.
+
+ Before this command is executed, the externs should be listed by a sequence
+ of \refcmd{mmzResource} commands; \meta{seq} refers to the sequential number
+ of an extern in this sequence.
+
+ This command may also be executed by executing the entire contents of
+ \refcmd{mmzCCMemo} after memoization.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzLabel, par=\marg{label key}\marg{label value}}
+ }
+ This is an internal command written into the cc-memo by the auto-handler of
+ \refcmd{label}. It temporarily stores \meta{label value} into
+ \refcmd{@currentlabel} and then executes \cs{label}\marg{label name}.
+\end{doc}
+
+\begin{doc}{
+ cmd={name=mmzEndMemo},
+ }
+ This macro marks the end of a cc-memo. It is used to grab the cc-memo core
+ (everything between \refcmd{mmzMemo} and \refcmd{mmzEndMemo}) under the
+ indirect cc-memo input regime, i.e.\ when \refmmz{direct ccmemo input} is not
+ in effect.
+\end{doc}
+
+
+\subsection{Location of memos and externs}
+\label{sec:ref:dirs}
+
+\begin{doc}{easy,
+ key={name=memo dir,par=\meta{name}, desc={style, default \texttt{\string\jobname}}},
+ }
+ A convenient way to store memos and externs in a dedicated directory (see
+ sections~\ref{sec:tut:memodir} and~\ref{sec:tut:multifile} for the tutorial).
+
+ Without an argument, this key places these files in subdirectory
+ \meta{document name}|.memo.dir| of the current directory. With an argument,
+ these files are stored in subdirectory \meta{name}|.memo.dir|. In both
+ cases, memo and extern filenames contain no prefix, as \meta{name} already
+ occurs in the directory name.
+
+ The definition of this style is very simple: |memo
+ dir/.style={|\refmmz{prefix}|={#1.memo.dir/}}| (note the slash). Feel free
+ to redefine it, or to define another style to suit your needs.
+\end{doc}
+
+\begin{doc}{easy,key={name=no memo dir,desc=style}}
+ A convenient way to undo the effect of \refmmz{memo dir} and revert to the
+ initial \refmmz{prefix} setting which locates memos and externs in the
+ current directory, with filenames prefixed by \meta{document name}|.| (mind
+ the dot).
+\end{doc}
+
+\begin{doc}{
+ key={name=prefix, par=\meta{prefix},
+ desc={no default, initially \texttt{\string\jobname.}}},
+ }
+ This key determines the location of memo and extern files, and the initial
+ part of their filenames.
+
+ If \meta{prefix} contains no |/|, memos and externs are stored in the current
+ directory, alongside the output PDF, and their filenames begin with
+ \meta{prefix} (and continue with the identifier, see below). For example,
+ this is what happens with the initial value of this key, where memo and
+ extern filenames are prefixed by \meta{jobname} (i.e.\ the name of the
+ document) plus |.| (a dot). Such a situation is shown in the |dirty-house|
+ example in section~\ref{sec:tut:memodir}.
+
+ If \meta{prefix} contains slashes, everything up to the final slash
+ determines the directory which memos and externs will be stored into, and the
+ part after the final slash determines the starting part of their filenames; a
+ slash (|/|) as a directory separator should be used even on Windows, where
+ the system directory separator is a backslash (|\|). For example,
+ \refmmz{memo dir} sets \refmmz{prefix} to |\jobname.memo.dir/|. As shown in
+ the |clean-house| example in section~\ref{sec:tut:memodir}, this results in
+ memos and externs stored in directory \meta{jobname}|.memo.dir|, with their
+ filenames consisting solely of the identifier.
+
+ In detail, the paths to memos and externs are constructed as shown below,
+ where \meta{code md5 sum} and \meta{context md5 sum} identify the memoized
+ code and the \refmmz{context} in effect at its memoization, and $N$ is the
+ sequential number of the extern with respect to that code and context ($N$ is
+ usually $0$, as memoization normally produces a single
+ extern).\footnote{Normally, these paths are relative to the current directory
+ (i.e.\ the directory where \hologo{TeX} is executed from; usually, this
+ will be the directory where the compiled |.tex| file resides). However,
+ when \hologo{TeX} is invoked with option |-output-directory|, these paths
+ are relative to the specified output directory. Furthermore, when a memo
+ or extern cannot be written into the current\slash output directory, it
+ will be stored into the temporary directory \docref{TEXMFOUTPUT}, if
+ specified.}
+ \begin{center}
+ \begin{tabular}{r@{ }l}
+ c-memo:&\meta{prefix}\meta{code md5 sum}|.memo|\\
+ cc-memo:&\meta{prefix}\meta{code md5 sum}|-|\meta{context md5 sum}|.memo|\\
+ extern:&\meta{prefix}\meta{code md5 sum}|-|\meta{context md5 sum}-$N$|.pdf|\\
+ &(where ``-$N$'' is only present when $N\neq 0$, i.e.\ for non-first externs)\\
+ \end{tabular}
+ \end{center}
+
+ \begin{tcolorbox}[warning,title=The final slash matters]
+ To reiterate, the presence vs.\ absence of a slash (|/|) determines whether
+ memos and externs are stored in a dedicated directory or not. For example,
+ if you want to store memos and externs in directory |memos|, you should set
+ \code{\refmmz{prefix}=memos/}, with the final slash. Without the final
+ slash, these files would end up in the current directory.
+ \end{tcolorbox}
+
+ In principle, the directory specified by this key must already exist.
+ However, Memoize does it best to create it for you, and should succeed at
+ least when extraction method is set to \refmmz{extract=perl} or
+ \refmmz{extract=python}. See also \refmmz{mkdir} and \refmmz{mkdir command}.
+
+ When invoked from the document body, each invocation of this key records the
+ new prefix by invoking \refmmz{record/record type/prefix} (this typically
+ results in a \refcmd{mmzPrefix} entry in the \dmmz file) and attempts to
+ create memo directory if \refmmz{mkdir} is in effect. When \refmmz{prefix}
+ is executed in the document preamble, these actions are only carried out at
+ the beginning of the document, for the final value of the key.
+
+
+\end{doc}
+
+\begin{doc}{key={name=mkdir, conditional=true}}
+ When this key is set to true and the value of \refmmz{mkdir command} is
+ non-empty, Memoize will attempt to create the memo directory set by
+ \refmmz{prefix} if it does not yet exist.
+
+ The directory is created using the system command specified by \refmmz{mkdir
+ command}. The directory creation takes place at the beginning of the
+ document and at every subsequent invocation of key \refmmz{prefix}.
+\end{doc}
+
+\begin{doc}{
+ key={name=mkdir command, par=\meta{system command},
+ desc={no default, initially empty}},
+ }
+ This key sets the command used to create the memo directory specified
+ by \refmmz{prefix} when \refmmz{mkdir} is in effect.
+
+ Memoize attempts to create the directory by executing \meta{system command}
+ followed by (a space and) \meta{directory}, where \meta{directory} is the
+ directory part of the \meta{prefix} set by \refmmz{prefix}. The directory
+ creation takes place at the beginning of the document and at every subsequent
+ invocation of key \refmmz{prefix}.
+
+ This key could be set to |mkdir| (on Linux, |mkdir -p| would be an even
+ better choice), however, this is not advisable as |mkdir| does not respect
+ \hologo{TeX}'s output restrictions, set by \docref{openout_any} in
+ \hologo{TeX}Live and |[Core]AllowUnsafeInputFiles| in MiK\hologo{TeX}.
+ Further note that as the value of this key is a system command, an
+ appropriate shell escape mode must be in effect to execute it successfully;
+ again, not something to be taken lightly.
+
+ The extractions scripts shipped with Memoize accept option
+ \refscript{memoize-extract.pl--mkdir}, which makes them behave as a safe
+ variant of |mkdir|, i.e.\ a |mkdir| which respects \hologo{TeX}'s output
+ restrictions. Therefore, extraction methods \refmmz{extract=perl} and
+ \refmmz{extract=python} set \refmmz{mkdir command} to
+ \refscript{memoize-extract.pl} \refscript{memoize-extract.pl--mkdir} and
+ \refscript{memoize-extract.py} \refscript{memoize-extract.pl--mkdir},
+ respectively. In effect, as the initial extraction method is
+ \refmmz{extract=perl}, the memo directory (if set via \refmmz{prefix} or
+ \refmmz{memo dir}) should be created under the initial settings without any
+ user intervention.
+\end{doc}
+
+
+\subsection{Extern extraction}
+\label{sec:ref:externs}
+
+\begin{doc}{easy,
+ key={name=extract,par=\marg{extraction method},
+ desc={preamble-only, initially \refmmz{extract=perl}, default: see below}},
+ }
+ This key selects or executes the extern \meta{extraction method}, i.e. the
+ method which Memoize will use to extract the extern pages out of the document
+ PDF.
+
+ Out of the box, Memoize recognizes the following \meta{extraction method}
+ keywords: \docAux{easy, of=key:/mmz/extract, value={name=perl}, comma,
+ value={name=python}, comma, value={name=tex}, and, value={name=no}}. The
+ first three keywords trigger extern extraction using the methods documented
+ in sections~\ref{sec:ref:extraction-perl-python}
+ and~\ref{sec:ref:extraction-tex}. The final keyword (not available in
+ \hologo{plainTeX}) instructs Memoize to \emph{not} perform the extraction at
+ the end of loading the package; it should be used when extraction is
+ performed externally (for details, see section~\ref{sec:setup}). Additional
+ methods may be installed by defining key \docaux{key
+ path}{mmz/extract}|/|\meta{extraction method}.
+
+ When invoked from \reffile{memoize.cfg} or used as a package option, this key
+ \emph{selects} the extraction method. In this case, the key has no default
+ value, i.e.\ it is illegal to use it without an argument. The method
+ selected by the package option overrides the method selected in
+ \reffile{memoize.cfg}, which in turn overrides the package-initial value
+ \refmmz{extract=perl}.
+
+ In \hologo{LaTeX} and \hologo{ConTeXt}, the selected method is automatically
+ executed at the end of loading the package. This does not happen in
+ \hologo{plainTeX}, because we want to allow the author to override the
+ initial extraction method (\refmmz{extract=perl}) or the extraction method
+ specified in \reffile{memoize.cfg}, even if \hologo{plainTeX} has no package
+ options. In \hologo{plainTeX}, internal extraction must be triggered by an
+ explicit invocation of \refmmz{extract} in the ``document preamble'' ---
+ i.e.\ between |\input memoize| and \refcmd{mmzset}\braces{\refmmz{begin
+ document}}. If given an \meta{extraction method} argument, the key will
+ execute the given extraction method; otherwise, it will execute either the
+ initial method, \refmmz{extract=perl}, or the overwide specified in
+ \reffile{memoize.cfg}.
+
+ In general, Memoize guards against triggering the extraction more than once.
+ In formats other than \hologo{plainTeX}, invoking key \refmmz{extract} from
+ the document preamble is thus only allowed when
+ \refmmz{extract}|=|\refmmz{extract=no} was previously selected. In this
+ case, \refmmz{extract} in the document preamble behaves as in
+ \hologo{plainTeX}, i.e.\ it triggers the given extraction method; if no
+ extraction method is given, Memoize executes either the initial extraction
+ method \refmmz{extract=perl}, or the extraction method specified in
+ \reffile{memoize.cfg}. The possibility of invoking key \refmmz{extract} from
+ the document preamble makes it possible to bake Memoize into a user format
+ (with \refmmz{extract}|=|\refmmz{extract=no}), and trigger the internal
+ extraction manually.
+
+ Executing extraction method \refmmz{extract=perl} or \refmmz{extract=python}
+ has an additional effect of setting the \refmmz{mkdir command} to the
+ extraction script with option \refscript{memoize-extract.pl--mkdir}. This
+ obviates the need to include |mkdir| among the restricted shell commands if
+ one is using the restricted shell mode.
+\end{doc}
+
+\subsubsection{Perl- and Python-based extraction}
+\label{sec:ref:extraction-perl-python}
+
+Perl- and Python-based extraction is triggered by
+\refmmz{extract}|=|\refmmz{extract=perl} and
+\refmmz{extract}|=|\refmmz{extract=python}, respectively.
+
+\begin{doc}{
+ key={name=perl extraction command, par=\meta{system command},
+ desc={no default, initially \refscript{memoize-extract.pl}}},
+ key={name=perl extraction options, par=\meta{options},
+ desc={no default, initially: \code{\refscript[short]{memoize-extract.pl--format}
+ \meta{format} \string\jobname}}},
+ key={name=python extraction command, par=\meta{system command}, desc={no
+ default, initially \refscript{memoize-extract.py}}}, key={name=python
+ extraction options, par=\meta{options}, desc={no default, initially:
+ \code{\refscript[short]{memoize-extract.pl--format} \meta{format}
+ \string\jobname}}}
+ }
+
+ These keys determine the system calls used for invoking the extraction
+ scripts \refscript{memoize-extract.pl} and \refscript{memoize-extract.py}.
+ All the details below apply both to the Perl and the Python version.
+
+ Use |perl|\slash|python extraction command| to set the name of the extraction
+ script. If necessary, include the full path to the script, or |perl|\slash
+ |python| plus the path to the script. Whatever you set here must be allowed
+ by the shell escape mode.
+
+ Use |perl|\slash|python extraction options| to set the options that the
+ script will receive; consult the documentation of
+ \refscript{memoize-extract.pl} for their meaning. The initial value asks the
+ script to process \meta{document name}\dmmz, and informs it that it is
+ executed from within a \hologo{TeX} compilation of a document in the given
+ \meta{format} (|latex| for \hologo{LaTeX}, |plain| for \hologo{plainTeX},
+ |context| for \hologo{ConTeXt}); the latter makes the script yield any
+ warnings or errors in a form expected by the format.
+
+ During the execution of a system call, the values of these settings are fully
+ expanded. Furthermore, these keys were initialized using \pkg{pgfkeys}
+ handler |.initial|, so their values may be modified by handlers |.prefix|,
+ |.append|, etc. The initial value of the extraction options contains a space
+ on both sides, so that these handlers are easy to use. For example, write
+ \code{\refmmz{perl extraction
+ options}/.append=\refscript{memoize-extract.pl--quiet}} to ask
+ for less output.
+\end{doc}
+
+\begin{doc}{
+ script={name=memoize-extract.pl, par=[\meta{options}] \meta{name}\texttt{.mmz}},
+ script={name=memoize-extract.py, par=[\meta{options}] \meta{name}\texttt{.mmz}},
+ }
+ These scripts extract the new externs recorded in \meta{name}\dmmz from
+ \meta{name}|.pdf|. Memoize invokes them when loaded with package option
+ \refmmz{extract}|=|\refmmz{extract=perl} (the default) or
+ \refmmz{extract}|=|\refmmz{extract=python}.
+
+ Argument \meta{name}\dmmz may be given either in full (e.g.\ |doc.mmz|), or
+ merely as the stem (|doc|). In the latter case, |.mmz| is appended to the
+ given argument even if it already contains a suffix (e.g.\ |my.doc| will
+ result in |my.doc.mmz|); the exception is suffix |.tex|, which is replaced by
+ |.mmz|.
+
+ The script inspects the given record file, \meta{name}\dmmz, for lines of
+ form \refcmd{mmzNewExtern}\marg{extern filename}\marg{extern page
+ number}\braces{\meta{expected width}\texttt{pt}}\braces{\meta{expected
+ height}\texttt{pt}}. For each such line, page number \meta{extern page
+ number} is extracted from \meta{name}|.pdf| into \meta{extern filename}.
+ (The script also creates directories specified by \refcmd{mmzPrefix} lines.
+ Other lines are ignored, and so are commented invocations of
+ \refcmd{mmzNewExtern} and \refcmd{mmzPrefix}.)
+
+ The \meta{extern filename} may contain a (relative or absolute) path to the
+ new extern file. The relative paths are relative to the location of the
+ \dmmz file, even when the script is invoked from some other directory.
+
+ To guard against extracting a wrong page, the script checks whether the size
+ of each extracted page matches the \meta{expected width} and \meta{expected
+ height}.\footnote{\label{fn:tolerance}To avoid false positives, the match
+ need not be exact, a difference up to |0.01pt| is tolerated. Some PDF
+ tools, notably the |PDF::API2| library deployed by the Perl version of the
+ script, round the dimensions of a PDF page, recorded in |/MediaBox|, to two
+ digits.} If it does not, the script refuses to extract the page, yields a
+ warning and even removes the extern file if it exist.
+
+ The extraction script's paranoia extends further. It will refuse to extract
+ the page, yielding a warning, if a (c)c-memo associated to the extern does
+ not exist. And it will respect the |openin| and |openout| settings of
+ |texmf.cnf|; under the default configuration, it will therefore refuse
+ (yielding an error) to write to any file whose absolute path does not occur
+ under the current directory, the temporary directory set by
+ \docref{TEXMFOUTPUT} (in \reffile{texmf.cnf} or as an environment variable),
+ or the output directory (\docref{TEXMF_OUTPUT_DIRECTORY}). Furthermore, it
+ will refuse to write into the root directory (except on Windows, where
+ writing into a drive root might potentially make sense).
+
+ Exit codes: 0 = success, 1 = Perl/Python error, 2 = usage error, 10 =
+ extraction warning, 11 = extraction error.
+
+ \yadocset{ of=script:memoize-extract.pl, override/.style={ index
+ annotation={option of {%
+ \hypercolor{link}{gray}%
+ \hyperref[script:memoize-extract.pl]{\texttt{memoize-extract}}%
+ }}, }, }
+ \begin{doc}{
+ option={name=pdf, short name=P, par=\meta{pdf}}
+ }
+ Extract the externs from the given \meta{pdf} instead of the default
+ \meta{name}\texttt{.mmz}. Note that file \meta{pdf}, despite the different
+ name, should be produced by the same compilation that produced
+ \meta{name}\dmmz, otherwise wrong pages might be extracted.
+ \end{doc}
+ \begin{doc}{
+ option={name=prune, short name=p}
+ }
+ After extraction, remove the extracted extern pages from the document PDF.
+ \end{doc}
+ \begin{doc}{
+ option={name=keep, short name=k}
+ }
+ By default, the script comments out the \refcmd{mmzNewExtern} lines in the
+ \dmmz file, to prevent multiple extractions. Specifying this option
+ prevents this behaviour.
+ \end{doc}
+ \begin{doc}{
+ option={name=format, short name=F,
+ par=\code{latex}\Alt\code{plain}\Alt\code{context}}
+ }
+ When this option is given, the script assumes that it was called from
+ within a \hologo{TeX} compilation of a document in the given format
+ (|latex| for \hologo{LaTeX}, |plain| for \hologo{plainTeX}, |context| for
+ \hologo{ConTeXt}).
+
+ For one, the script will prefix all its output by the script name, to be
+ easily identifiable in the terminal output, but more importantly, it will
+ create a log file (\code{\string\jobname.mmz.log}) which will receive any
+ warnings and errors yielded by the script (in absence of this option, the
+ warnings and errors are printed to the standard error). The log contains
+ messages in a form recognized by the format (e.g.\ \cs{PackageError} for
+ \hologo{LaTeX}), so Memoize can then \cs{input}s this log to reproduce the
+ warnings and errors in the compilation.
+
+ There should be no need to use this option when executing the script form
+ the command line.
+ \end{doc}
+ \begin{doc}{
+ option={name=force, short name=f}
+ }
+ Force extern extraction even if the size check or (c)c-memo existence check
+ fails. The failure will still yield a warning.
+ \end{doc}
+ \begin{doc}{
+ option={name=quiet, short name=q}
+ }
+ Normally, the script prints what it is doing to the standard output; in
+ particular, it prints out the page number and the filename of each extern
+ it is extracting. This option disables this behaviour.
+ \end{doc}
+ \begin{doc}{
+ option={name=library, short name=L,
+ par=\texttt{PDF::API2\Alt PDF::Builder}, desc=Perl-only}
+ }
+ Use the specified library for PDF processing. If this option is not given,
+ the script uses the first available library (in the order given above).
+ \end{doc}
+ \begin{doc}{
+ option={name=mkdir, short name=m}
+ }
+ When given this option, the extraction script transforms into a paranoid
+ |mkdir| (|-p|). Argument \meta{name}|.mmz| is interpreted as a path to the
+ directory to create; all other options are ignored.
+
+ The ancestors of the directory are created as needed. The script will
+ refuse to create any directory whose absolute path does not occur under the
+ current directory or a directory listed in \docref{TEXMFOUTPUT} (set in
+ \reffile{texmf.cnf} or as an environment variable).
+
+ This option exists so that the author using the restricted shell mode does
+ not have to list |mkdir| among the restricted shell mode commands (and it
+ is also safer than a plain |mkdir|).
+
+ Memoize automatically uses this script to create the memo directory when
+ option \refmmz{extract} is given value \refmmz{extract=perl} or
+ \refmmz{extract=python}.
+ \end{doc}
+ \begin{doc}{
+ option={name=version, short name=V}
+ }
+ Show Memoize version.
+ \end{doc}
+ \begin{doc}{
+ option={name=help, short name=h}
+ }
+ Show help.
+ \end{doc}
+
+ Functionally, the Perl (|.pl|) and the Python (|.py|) version of the script
+ are almost equivalent. The minor differences, listed below, are mostly due
+ to the underlying PDF-processing library:
+ \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} in Perl, and
+ \hreftt{https://pypi.org/project/pdfrw2/}{pdfrw2} in Python.
+ \begin{itemize}
+ \item The Python script is about twice as fast as the Perl script. However,
+ both scripts are very fast compared to the \hologo{TeX}-based extraction.
+ On my computer, extracting 160 externs out of a 400-page book takes 1.4s
+ with Python, 3.7s with Perl, and 65s with \hologo{TeX}. But even when
+ using \hologo{TeX}-based extraction, externalization using Memoize is
+ extremely fast compared to the \TikZ's externalization library. Adding
+ the regular compilation time of one minute to the above numbers, we arrive
+ at the maximum externalization time of about two minues, whereas my
+ estimate for the production of all 160 externs using \TikZ's
+ externalization would be an \emph{hour} or more.
+ \item The Python script cannot extract externs out of PDFs created without
+ stream compression, i.e.\ with \refcmd{pdfvariable}
+ \docref{pdfvar:compresslevel} set to |0|.
+ \item Occasionally, the Perl script crashes during extraction externs; see
+ section~\ref{sec:troubleshooting} for details.
+ \end{itemize}
+\end{doc}
+
+\subsubsection{\texorpdfstring{\hologo{TeX}}{TeX}-based extraction}
+\label{sec:ref:extraction-tex}
+
+\hologo{TeX}-based extraction is triggered by
+\refmmz{extract}|=|\refmmz{extract=tex}.
+
+\begin{doc}{
+ key={name=tex extraction command, par=\meta{system command},
+ desc={no default, initially |pdftex|}},
+ key={name=tex extraction options, par=\meta{options},
+ desc={no default, initially: see below}},
+ key={name=tex extraction script, par=\meta{\hologo{TeX} code},
+ desc={no default, initially: see below}},
+ }
+ \yadocset{
+ before/.style={label prefix=cmd:te},
+ index annotation={defined at \hologo{TeX} extraction},
+ }
+
+ Together, these keys determine the system call used for invoking the
+ \hologo{TeX}-based extraction script, \refscript{memoize-extract-one.tex}.
+ (They were initialized using \pkg{pgfkeys} handler |.initial|, so their
+ values may be modified by handlers |.prefix|, |.append|, etc.)
+
+ Memoize uses the resulting system call string at the following occasions.
+ First, it executes it, once for each new extern in the \dmmz file, during the
+ internal extraction, i.e.\ when it is loaded with package option
+ \refmmz{extract}|=|\refmmz{extract=tex}. Second, it uses it to construct
+ record files of types \refmmz{record=sh}, \refmmz{record=bat} and
+ \refmmz{record=makefile} when recording of these types is requested.
+
+ Key \refmmz{tex extraction command} sets the \hologo{TeX} binary used for
+ \hologo{TeX}-based internal extraction. By default, the \hologo{pdfTeX}
+ engine |pdftex| is used; other sensible values for this key are |luatex| and
+ |xetex|, or the program name including the path. Note that shell escape mode
+ must be configured appropriately, and that
+ \refscript{memoize-extract-one.tex} must be compiled with the
+ \hologo{plainTeX} format.
+
+ The value of key \refmmz{tex extraction options} is passed as options to the
+ \hologo{plainTeX} binary. As shown in the initial value in the frame below,
+ it may use the temporary macro \docaux{cmd}{externbasepath}, which expands to
+ the path to the extern without the |.pdf| suffix. This macro is available
+ during internal \hologo{TeX}-based extraction and during the execution of
+ \refmmz[show keypath]{record/record type/new extern} key for and \meta{record
+ type}.
+
+ \makeexcerpt{tex-extraction-options}
+ \tcbinputexample[.tex][.excerpt]{%
+ listing only, one file, no attachment, title={The initial value of
+ \refmmz{tex extraction options}},
+ }
+
+ Key \refmmz{tex extraction script} defines the command-line script executed
+ to extract the extern. The default value, shown in the frame below, invokes
+ \refscript{memoize-extract-one.tex} after setting its parameter macros.
+ Temporary macros \docAux{cmd={name=pagenumber}, comma, cmd={name=expectedwidth},
+ and, cmd={name={expectedheight}}} are defined at the same occasions as
+ \refcmd{te:externbasepath} above, i.e.\ during internal \hologo{TeX}-based
+ extraction and during the execution of a \refmmz{record/record type/new
+ extern} key. The initial value requests the extern to be of the same version as
+ the main document, if possible;\footnote{As far as I know, it is impossible
+ to access the version of the PDF being produced in \hologo{XeTeX}, i.e.\
+ there are no registers \docref{reg:pdfmajorversion} and
+ \docref{reg:pdfminorversion}. To request production of a specific version of
+ PDF, \hologo{XeTeX} must be invoked by with command-line option
+ \code{-output-driver 'xdvipdfmx -V $N$'}.} note that Memoize defines
+ \docref{reg:pdfmajorversion} and \docref{reg:pdfminorversion} in \hologo{LuaTeX}.
+
+ \makeexcerpt{tex-extraction-script}
+ \tcbinputexample[.tex][.excerpt]{%
+ listing only, one file, title=The initial value of \refmmz{tex extraction
+ script}, no attachment}
+
+ As the value of \refmmz{tex extraction script} is fully expanded when used,
+ the initial value shown above must prevent the expansion of much code.
+ Furthermore, the initial value varies with the \hologo{TeX} format, as
+ indicated by the |.dtx| guards in the definition of
+ \refcmd{meo:warningtemplate}.
+\end{doc}
+
+\begin{doc}{
+ script={
+ head prefix={%
+ (\texttt{pdf}\alt\texttt{lua}\alt\texttt{xe})\texttt{tex -jobname}
+ \meta{extern filename}
+ \texttt{"}\meta{parameters}
+ \cs{input}\texttt{\textbraceleft}
+ },
+ name=memoize-extract-one.tex,
+ head infix=\texttt{\textbraceright"},
+ },
+ }
+ Compiling \refscript{memoize-extract-one.tex} with \hologo{plainTeX}
+ produces an extern file containing a single (extern) page extracted from the
+ document PDF.
+
+ Memoize invokes this script, once for each new extern appearing in the \dmmz
+ file, when loaded with package option
+ \refmmz{extract}|=|\refmmz{extract=tex}.
+
+ The desired \meta{extern filename} is given as the value of option |-jobname|
+ of the \hologo{TeX} binary. To set the extraction \meta{parameters}, define
+ the following macros before |\input|ting the file:
+ \yadocset{
+ index annotation=option of \texttt{memoize-extract-one.tex},
+ before/.style={head prefix=\cs{def}, label prefix=cmd:meo},
+ }
+ \begin{doc}{cmd={name=fromdocument, parameters=\marg{document pdf filename}}}
+ Defining this macro sets the filename of the PDF which the externs will be
+ extracted from. The filename is relative to the working directory.
+ \end{doc}
+ \begin{doc}{cmd={name=pagenumber, parameters=\marg{number}}}
+ Defining this macro sets the number of the page to extract. The first page
+ has number 1.
+ \end{doc}
+ \begin{doc}{
+ cmd={name=expectedwidth, parameters=\marg{dimension}, desc=optional},
+ cmd={name=expectedheight, parameters=\marg{dimension}, desc=optional},
+ }
+ Defining these macros sets the expected width and height of the extracted
+ page.
+
+ To guard against extracting a wrong page, the dimensions of the extracted
+ page are compared against the expected width and height. If the size check
+ fails,\footnote{The match need not be exact, see
+ footnote~\ref{fn:tolerance}.} the resulting extern PDF is empty (which
+ counts as non-existent when Memoize checks for its presence when it
+ attempts to utilize it), and a warning message (formatted via
+ \refcmd{meo:warningtemplate}) is printed to the log file, if logging was
+ requested via \refcmd{meo:logfile}.
+
+ If any of these macros is undefined, the size check will be skipped.
+ \end{doc}
+ \begin{doc}{cmd={name=logfile, parameters=\marg{filename}, desc=optional}}
+ Defining this macro sets the name of the log file. If not defined, no log
+ file will be produced.
+
+ The log file is intended to be used when the script is invoked from an
+ outer \hologo{TeX} compilation. In particular, it is intended to be
+ |\input| by that compilation to see whether the extraction was successful.
+ Upon a failed size check, it will contain a warning (formatted by
+ \refcmd{meo:warningtemplate}, if that macro is defined). The log file ends
+ with |\endinput| to signal that extraction actually took place.
+ \end{doc}
+ \begin{doc}{cmd={name=warningtemplate, parameters=\marg{code}, desc=optional}}
+ Defining this macro determines how to log the warning message in the case
+ of a failed size check. The macro should expand to a \hologo{TeX}
+ format-specific warning message code containing the warning text given in
+ \docAux{cmd={name=warningtext, label prefix=cmd:meo, into
+ index=false}}.\footnote{Macro \refcmd{meo:warningtemplate} is passed
+ the warning text by a macro rather than a formal parameter to avoid
+ category code problems with the parameter character when setting key
+ \refmmz{tex extraction script}.}
+
+ While the script formats the warning message \emph{text} on its own (``I
+ refuse to extract page \dots''), the warning message is not written into
+ the log unadorned. The log file is intended to be \cs{input} by the outer
+ \hologo{TeX} compilation, and the idea is that inputting it should yield a
+ warning in that compilation (in the case of a failed size
+ check). Therefore, the content of the log file must contain an invocation
+ of the command used to produce warning messages in the \hologo{TeX} format
+ used by the outer compilation.
+
+ For example, when this script is invoked from within a \hologo{LaTeX}
+ compilation, it makes sense to define something like
+ |\def\warningtemplate{\PackageWarning{memoize}{\warningtext}}|.
+ \end{doc}
+ \begin{doc}{cmd={name=force,
+ parameters=\bracestt{\texttt{true}\Alt\texttt{false}}, desc=optional}}
+ If this macro is defined to |true|, extern extraction will be carried out
+ even if the size-check fails. The failure will still be logged.
+ \end{doc}
+ \begin{doc}{
+ cmd={name=mmzpdfmajorversion, parameters=\marg{number}, desc=optional},
+ cmd={name=mmzpdfminorversion, parameters=\marg{number}, desc=optional},
+ }
+ Defining (one or both of) these macros requests that the extern PDF be
+ produced with the given major\slash minor PDF version, i.e.\ the extraction
+ script will set registers \docref{reg:pdfmajorversion} and
+ \docref{reg:pdfminorversion}.
+ \end{doc}
+
+ After extracting the extern, the script will end the compilation, i.e.\
+ intentionally, only one page documents can be produced.
+\end{doc}
+
+\subsubsection{The clean-up scripts}
+\label{sec:cleanup-scripts}
+
+\begin{doc}{
+ script={name=memoize-clean.pl, par=[\meta{options}] [\meta{name}\texttt{.mmz} \dots]},
+ script={name=memoize-clean.py, par=[\meta{options}] [\meta{name}\texttt{.mmz} \dots]},
+ }
+ This script removes memo and extern files whose filenames start with
+ \meta{prefix}es mentioned in the given \dmmz files or by the
+ \refscript{memoize-clean.pl--prefix} option. Unless option
+ \refscript{memoize-clean.pl--all} is given, the script only deletes the
+ \emph{stale} files, i.e.\ the files not mentioned in any of the given \dmmz
+ files.
+
+ A \meta{prefix} of a memo or an extern is what was set by key
+ \refmmz{prefix}, or more commonly, one of the shortcut keys \refmmz{memo dir}
+ and \refmmz{no memo dir}; see section~\ref{sec:ref:dirs} for details on the
+ form of a memo/extern filename.
+
+ In detail, the script scans the given \dmmz files for occurrences of
+ \refcmd{mmzPrefix}, and adds their \meta{prefix} arguments to the list of
+ prefixes given on the command line by option
+ \refscript{memoize-clean.pl--prefix}; a \meta{prefix} occurring in some \dmmz
+ file is interpreted relatively to the location of the \dmmz file. The script
+ removes all files whose full pathname (relative to the current directory)
+ matches pattern
+ \meta{prefix}\meta{md5sum}(|-|\meta{md5sum})(|.memo|\alt(|-|$N$)|.pdf|\alt|.log|),%
+ \footnote{The |.log| files are produced by the \hologo{TeX}-based extraction
+ script.} except those which occur as the \meta{filename} argument to one
+ of \refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo}, \refcmd{mmzUsedExtern},
+ \refcmd{mmzNewCMemo}, \refcmd{mmzNewCCMemo} and \refcmd{mmzNewExtern} in one
+ of the \dmmz files.
+
+ The script is fairly paranoid. It refuses to delete anything if a \dmmz file
+ is malformed in any way (but not if it doesn't exist or is completely empty,
+ which facilitates its usage in clean-up scripts), or if it would remove a
+ file not residing under the current directory. Before removing the files, it
+ lists the files to be removed and asks for confirmation.
+
+ Functionally, the Perl (|.pl|) and the Python (|.py|) version are completely
+ equivalent.
+
+ \yadocset{
+ of=script:memoize-clean.pl,
+ override/.style={
+ index annotation={option of {%
+ \hypercolor{link}{gray}%
+ \hyperref[script:memoize-clean.pl]{\texttt{memoize-clean}}%
+ }},
+ },
+ }
+ \begin{doc}{
+ option={name=prefix, short name=p, par=\meta{prefix}}
+ }
+ Add \meta{prefix} to the list of prefixes; the given prefix is
+ relative to the current directory. This option may be given multiple
+ times.
+ \end{doc}
+ \begin{doc}{
+ option={name=all, short name=a}
+ }
+ When given this option, the script removes \emph{all} memos and externs
+ belonging to the document, not just the stale ones, i.e.\ it effectively
+ ignores the occurences of \refcmd{mmzUsedCMemo} and friends in the \dmmz
+ file.
+ \end{doc}
+ \begin{doc}{
+ option={name=yes, short name=y}
+ }
+ When given this option, the script does not ask for confirmation before
+ removing the files.
+ \end{doc}
+ \begin{doc}{
+ option={name=quiet, short name=q}
+ }
+ Normally, the script prints what it is doing to the standard output; in
+ particular, it prints out the filename of each file as it is deleting it.
+ This option disables this behaviour.
+ \end{doc}
+ \begin{doc}{
+ option={name=help, short name=h}
+ }
+ Show help.
+ \end{doc}
+ \begin{doc}{
+ option={name=version, short name=V}
+ }
+ Show Memoize version.
+ \end{doc}
+\end{doc}
+
+\subsubsection{Record files}
+\label{sec:ref:record-files}
+
+\begin{doc}{
+ key={name=record, par=\marg{record type},
+ desc={cumulative, initially \refmmz{record=mmz}, no default}},
+ key={name=no record},
+ }
+ Memoize records which externs were produced and used in the compilation,
+ producing a record file of every type found in the record-type list. These
+ keys add \meta{record type} to the record-type list, or clear this list. See
+ section~\ref{sec:record-files} for details.
+
+ Note that passing an undefined \meta{record type} to this key will not yield
+ an error.
+
+ Out of the box, the following \meta{record type}s are recognized:
+ \yadocset{of=key:/mmz/record}
+ \begin{doc}{value={name=mmz}}
+ This record type produces a \dmmz file recording new/used
+ externs/c-memos/cc-memos and changes in the \refmmz{prefix} to these
+ files; see section~\ref{sec:.mmz} for details.
+
+ The produced file is named \docAux{file={name=mmz,
+ name prefix=\meta{jobname}.,
+ ref prefix=.,
+ index prefix=.,
+ }}. This name cannot be changed.
+
+ The \dmmz file is a \hologo{TeX} file, but uses only a simple subset of
+ the \hologo{TeX} syntax, to be easily parsable by the external scripts such
+ as \refscript{memoize-extract.pl}. Each line of the file consists of a
+ (possibly commented) invocation of one of the commands listed below; the
+ final line is |\endinput|. The \meta{prefix} below consists of the
+ path to memos/externs and the immutable \refmmz{prefix} of their filename.
+ \begin{doc}{
+ cmd={name=mmzUsedCMemo, par=\marg{filename}},
+ cmd={name=mmzUsedCCMemo, par=\marg{filename}},
+ cmd={name=mmzUsedExtern, par=\marg{filename}},
+ }
+ Record that the (c)c-memo or extern residing in file \meta{filename} was
+ utilized.
+ \end{doc}
+ \begin{doc}{
+ cmd={name=mmzNewCMemo, par=\marg{filename}},
+ cmd={name=mmzNewCCMemo, par=\marg{filename}},
+ }
+ Record that a new (c)c-memo residing in file \meta{filename} was
+ produced.
+ \end{doc}
+ \begin{doc}{cmd={name=mmzNewExtern, par=\marg{filename}\marg{page number}%
+ \marg{expected width}\marg{expected height}}}
+ Record that a new extern was produced and dumped as page \meta{page
+ number} into the document, that it should be extracted into file
+ \meta{filename}, and that it should be \meta{expected width} wide and
+ \meta{expected height} high (modulo tolerance of |0.01pt|, see
+ footnote~\ref{fn:tolerance}), where the height is the total height
+ comprising both \hologo{TeX} height and depth.
+ \end{doc}
+ \begin{doc}{cmd={name=mmzPrefix, par=\marg{prefix}}}
+ Record that the \refmmz{prefix} of memo and extern files was
+ changed.
+ \end{doc}
+ \end{doc}
+ \begin{doc}{value={name=makefile}}
+ This record type produces a makefile which, when processed by the |make|
+ utility, triggers \hologo{TeX}-based extraction of the new externs.
+ \begin{doc}{
+ key={name=makefile, par=\marg{filename}, desc={no default,
+ initially \texttt{memoize-extract.\string\jobname.makefile}}}
+ }
+ Use this key to change the filename of the produced makefile.
+ \end{doc}
+ \end{doc}
+ \begin{doc}{value={name=sh}, value={name=bat}}
+ These record types produce a shell script which, when executed, triggers
+ \hologo{TeX}-based extraction of the new externs.
+
+ Use \refmmz{record=sh} on Unix-like systems, and \refmmz{record=bat} on
+ Windows.
+
+ \begin{doc}{
+ key={name=sh, par=\marg{filename}, desc={no default,
+ initially \texttt{memoize-extract.\string\jobname.sh}}},
+ key={name=bat, par=\marg{filename}, desc={no default,
+ initially \texttt{memoize-extract.\string\jobname.bat}}}
+ }
+ Use these keys to change the filename of the produced shell script.
+ \end{doc}
+ \end{doc}
+\end{doc}
+
+\begin{doc}{
+ desc=definable,
+ keypath=/mmz/record/\meta{record type},
+ keypath label=/mmz/record/record type,
+ name prefix={\meta{record type}/}, ref prefix=,
+ index annotation={key in \texttt{/mmz/\meta{record type}}},
+ key={name=begin},
+ key={name=prefix, par=\marg{prefix}},
+ key={name=new extern, par=\marg{filename}},
+ key={name=new cmemo, par=\marg{filename}},
+ key={name=new ccmemo, par=\marg{filename}},
+ key={name=used extern, par=\marg{filename}},
+ key={name=used cmemo, par=\marg{filename}},
+ key={name=used ccmemo, par=\marg{filename}},
+ key={name=end},
+ }
+ A new record type can be implemented by defining these keys in keypath
+ \docAux{key path={name=mmz/record/\meta{record type}, label=mmz/record/record
+ type}} (using the standard \pkg{pgfkeys} handlers such as |.code| and
+ |.style|). The keys are invoked by Memoize where appropriate if recording
+ for the defined type is activated by \refmmz{record}|=|\meta{record type},
+ just as for the predefined types. Only those keys which are required for
+ implementing the desired functionality need to be defined.
+
+ The following macros are available during the execution of key
+ \refmmz{record/record type/new extern}:
+ \yadocset{before/.style={label prefix=cmd:ne},
+ index annotation=defined at \refmmz{record/record type/new extern}}
+ \begin{doc}{cmd={name=pagenumber}}
+ This macro holds the number of the extern page.
+ \end{doc}
+ \begin{doc}{
+ cmd={name=expectedwidth},
+ cmd={name=expectedheight},
+ }
+ These macros hold the width and the height of the extern page.
+ \end{doc}
+ \begin{doc}{cmd={name=externbasepath}}
+ This macro holds the filename of the extern, minus the |.pdf| suffix (but
+ including the path leading to the extern).
+ \end{doc}
+\end{doc}
+
+
+
+\subsection{Automemoization}
+\label{sec:ref:advicememoization}
+
+
+\subsubsection{Package Advice}
+\label{sec:ref:advice}
+
+Package Advice is a namesake of \Emacs's Advice. As such, it implements a
+generic framework for extending the functionality of selected commands and
+environments. Each \emph{advised} command and environment is assigned a piece
+of \emph{advice} --- a command which is executed instead of the advised command
+and environment, and which may, or may not, invoke the original command or
+environment during its execution. The package offers an elegant way of
+declaring advice, setting up the conditions upon which the advised command will
+actually be replaced by the advice, collecting the arguments of the advised
+command and invoking it, and (de)activating the advice.
+
+Before the advising framework can be used, it must be installed into a selected
+\pkg{pgfkeys} keypath (multiple installations into different keypaths are
+allowed, even if they handle the same commands). Memoize installs the
+framework into keypath \refkeypath{/mmz} and (primarily) uses it to
+automatically memoize the results of compilation of selected commands and
+environments.
+
+\begin{doc}{
+ keypath={/handlers}, desc=handler,
+ key={name=.install advice,
+ sort index=install advice,
+ par=\marg{configuration keylist},
+ },
+ }
+ This key is a \pkg{pgfkeys} key handler (see \PGFmanual{87.3.5}) which
+ installs the advising framework into the keypath which it was invoked from
+ --- henceforth, the \meta{namespace}.
+
+ For example, \code{\cs{pgfkeys}\bracestt{/my/\refkey{/handlers/.install
+ advice}}} installs the framework into keypath |/my|.
+
+ Argument \meta{configuration keylist} may contain the following keys:
+ \yadocset{keypath=/advice/install}
+ \begin{doc}{key={
+ name=setup key,
+ parameters=\marg{name},
+ description={no default, initially \texttt{advice}}},
+ }
+ This key determines the names of the user-interface keys used to setup
+ advice for commands and environments in \meta{namespace}.
+
+ The keys whose names are determined by this key are the following:
+ \meta{name}, \code{\meta{name} csname}, \code{\meta{name} key},
+ \code{\meta{name}'}, \code{\meta{name} csname'} and \code{\meta{name}
+ key'}. Memoize sets \refkey{/advice/install/setup key}|=auto|, and
+ thereby defines \refmmz{auto}, \refmmz{auto csname}, \refmmz{auto key},
+ \refmmz{auto'}, \refmmz{auto csname'} and \refmmz{auto key'}.
+ \end{doc}
+ \begin{doc}{key={
+ name=activation,
+ par=\meta{initial activation type},
+ desc={no default, initially \texttt{immediate}}
+ },
+ }
+ This key sets the \meta{initial activation type} for \meta{namespace}.
+
+ At the end of the installation, the system will execute
+ \meta{namespace}|/|\refmmz{activation}|=|\meta{initial activation type};
+ consequently, \meta{initial activation type} must be one of
+ \refmmz{activation=immediate} and \refmmz{activation=deferred}. In
+ Memoize, the \meta{initial activation type} is
+ \refmmz{activation=deferred}.
+
+ Setting the \refmmz{activation} type during the installation only matters
+ in \hologo{LaTeX}, where the installation ends by advising \refcmd{begin}
+ to implement advising of environments.
+ \end{doc}
+
+ \begin{tcolorbox}[warning]
+ Writing the documentation for Advice, I was faced with a dilemma. Should
+ the documentation reflect the fact that the full names of keys defined by
+ the package depend on the installed instance of the framework, in
+ particular on \meta{namespace} and \meta{setup key}? For example, should
+ the reference headers contain things like \meta{namespace}|/activate| and
+ \meta{namespace}|/|\meta{setup key} |csname|? In my opinion, this would
+ make the reference hard to read, so I decided to have the reference headers
+ refer to the Advice keys of the Memoize installation, where
+ \meta{namespace}=|/mmz| and \meta{setup key}=|auto|, resulting in
+ friendlier headers such as \refkeypath{/mmz}|/|\refmmz{activate} and
+ \refkeypath{/mmz}|/|\refmmz{auto csname}. (Consequently, it also made
+ sense to document Advice within the Memoize documentation.)
+
+ The bottomline: if you're reading this section with a non-Memoize
+ installation in mind, you have to mentally replace any \refkeypath{/mmz}
+ and \refmmz{auto} in the reference headers with the \meta{namespace} and
+ the \meta{setup key} selected by that installation.
+ (Section~\ref{sec:ref:advice:memoization} is another matter. Keys
+ described there are only available in Memoize.)
+ \end{tcolorbox}
+
+ In more detail, key handler \refkey{/handlers/.install advice} performs the
+ following actions (as explained in the box above, we assume that the
+ advising framework was installed into keypath \refkeypath{/mmz} with the
+ setup key named |auto|):
+ \begin{itemize}
+ \item \edef\origtolerance{\the\tolerance}\tolerance=1000
+ It defines the following keys in keypath \refkeypath{/mmz}:
+ \refmmz{auto},
+ \refmmz{auto csname},
+ \refmmz{auto key},
+ \refmmz{auto'},
+ \refmmz{auto csname'},
+ \refmmz{auto key'},
+ \refmmz{activation},
+ \refmmz{activate deferred},
+ \refmmz{activate},
+ \refmmz{deactivate},
+ \refmmz{activate csname},
+ \refmmz{deactivate csname}.
+ \refmmz{activate key},
+ \refmmz{deactivate key}.
+ \refmmz{force activate},
+ \refmmz{try activate}.
+ \item
+ It defines the following keys in keypath \refkeypath{/mmz/auto}:
+ \refmmzauto{run conditions},
+ \refmmzauto{outer handler},
+ \refmmzauto{bailout handler},
+ \refmmzauto{collector},
+ \refmmzauto{args},
+ \refmmzauto{collector options},
+ \refmmzauto{clear collector options},
+ \refmmzauto{raw collector options},
+ \refmmzauto{clear raw collector options},
+ \refmmzauto{inner handler},
+ \refmmzauto{options},
+ \refmmzauto{clear options},
+ \refmmzauto{reset}.
+ \item \tolerance\origtolerance\relax
+ It defines the |.unknown| key handler for \refkeypath{/mmz/auto}.
+ This handler appends any unknown keys (and their values) to
+ \refmmzauto{options}.
+ \item It executes \refmmz[show keypath]{activation}|=|\meta{initial
+ activation type}.
+ \item In \hologo{LaTeX}, it submits \refcmd{begin} to advising, thereby
+ enabling environment support in this format. Consequently, advising of
+ environments can switched off by writing \refmmz{deactivate}|=\begin|.
+ \end{itemize}
+\end{doc}
+
+\paragraph*{The keys installed into keypath \metabf{namespace}} are used to
+declare and (de)activate advice. In the documentation in this subsection,
+we assume that \meta{namespace}=\refkeypath{/mmz} and that \meta{setup
+ key}=\refmmz{auto}. In particular, this also applies to the reference
+headers.
+
+\begin{doc}{
+ key={name=activation, par=\docAux{of=key:/mmz/activation, value={name=immediate},
+ text=\Alt, value={name=deferred}}, desc={no default, initially set by \refkey{/handlers/.install advice}}},
+ key={name=activate deferred, desc={style}},
+ }
+ Key \refmmz{activation} selects the activation regime. Under the
+ \refmmz{activation=immediate} regime, keys \refmmz{activate},
+ \refmmz{deactivate}, \refmmz{force activate} and \refmmz{try activate} behave
+ as described in their documentation below. Under the
+ \refmmz{activation=deferred} regime, however, those keys are not executed;
+ rather, their invocations are appended to style \refmmz{activate deferred}.
+ For example, writing \refmmz{activate}|=\foo| in the deferred activation
+ regime appends \refmmz{activate}|=\foo| to \refmmz{activate deferred}. It is
+ up to the user if and when to execute the keys collected in \refmmz{activate
+ deferred}; see the documentation of \refmmz{manual} to learn what Memoize
+ does with the contents of this style.
+\end{doc}
+
+
+\begin{doc}{easy,
+ par=\marg{list of commands and/or environments},
+ desc={style},
+ key={name=activate},
+ key={name=deactivate},
+ }
+ These keys activate or deactivate the advice for the given commands and
+ environments. When the advice is activated, it replaces the advised command;
+ when it is deactivated, the command is reverted to its original definition.
+
+ In Memoize, these keys are most commonly used to activate or deactivate
+ automemoization for the given commands or environments. For example, write
+ \refmmz{deactivate}|={\tikz,tikzpicture}| to deactivate automemoization of
+ \TikZ pictures (which is declared and active by default). The curly braces
+ may be omitted if the list contains a single command or environment, e.g.\
+ \refmmz{deactivate}|=\tikz| or \refmmz{deactivate}|=tikzpicture|.
+
+ (De)activation of a piece of advice is completely orthogonal to its declaration with
+ \refmmz{auto}. For example, there is no need to deactivate a command before
+ redeclaring its advice, and reactivate it afterwards. A command may be
+ activated even before declaring its advice --- however, the command itself
+ must be defined at the time of activation.
+
+ As the advice is normally automatically activated upon declaration with
+ \refmmz{auto}, explicit activation is rarely needed, but see \refmmz{auto'}.
+ The effect of these keys under the deffered activation regime is described in
+ \refmmz{activation}.
+
+ Note that I sometimes speak of (de)activating a command, and sometimes of
+ (de)activating its advice. I mean the same thing.
+\end{doc}
+
+\begin{doc}{
+ par=\marg{control sequence name},
+ desc={style},
+ key={name=activate csname},
+ key={name=deactivate csname},
+ }
+ These keys activate and deactivate a command given by its \meta{control
+ sequence name}; for example, \refmmz{activate csname}|=foo| is equivalent
+ to \refmmz{activate}|=\foo|. Note that unlike the regular \refmmz{activate}
+ and \refmmz{deactivate}, their |csname| variants only accept a single command
+ at a time (otherwise, including a comma in the command name would be
+ impossible).
+\end{doc}
+
+\begin{doc}{
+ par=\marg{list of full key names},
+ desc={style},
+ key={name=activate key},
+ key={name=deactivate key},
+ }
+ These keys activate and deactivate |pgfkeys| keys. Note that \emph{full} key
+ names must be given, i.e.\ the names must include the keypath.
+
+ Under the hood, these keys merely execute \refmmz{activate} and
+ \refmmz{deactivate} on the internal macros corresponding to the given keys.
+\end{doc}
+
+\begin{doc}{
+ key={name=try activate, conditional=false},
+ }
+ When this conditional is set to true, \refmmz{activate} will not yield an
+ error if the advice is already activated, and \refmmz{deactivate} will not
+ yield an error if the advice is not yet activated.
+
+ This key applies to the next, and only to the next, invocation of key
+ \refmmz{activate} or \refmmz{deactivate}, i.e.\ it is reset back to |false|
+ after invoking \refmmz{activate} or \refmmz{deactivate}.
+\end{doc}
+
+\begin{doc}{
+ key={name=force activate, conditional=false},
+ }
+ When this conditional is set to true, \refmmz{activate} will activate even a
+ previously activated command, provided that, additionally, the command has
+ been redefined since the prior activation.
+
+ In more detail, the original definition of the advised command is saved upon
+ activation (to provide the possibility of both deactivation and the usage of
+ the original command by the handler). Consequently, activation of an already
+ activated command would result in the saved original definition being
+ overwritten by the redefinition made during the first activation. However,
+ if the handled command was meanwhile redefined by a third party, reactivation
+ makes sense, under the assumption that the former original definition is
+ obsolete and should be replaced by the (third party) redefinition. As a
+ safeguard, however, \refmmz{activate} requires such reactivation to be
+ explicitly requested using conditional \refmmz{force activate}.\footnote{A
+ potential problem, not (yet) addressed, is that the third party might be
+ another incarnation of the advising framework. In this case, forced
+ reactivation will result in the loss of the original command and a circular
+ dependency between the two pieces of advice.}
+
+ This key applies to the next, and only to the next, invocation of key
+ \refmmz{activate}, i.e.\ it is reset back to |false| after invoking
+ \refmmz{activate}. This key does not apply to \refmmz{deactivate}.
+\end{doc}
+
+\begin{doc}{easy,
+ key={name=auto, par=\marg{command or environment}{\marg{keylist}}, desc=style}}
+
+ This key sets up the advice for the given command or environment, or updates
+ the configuration of an existing piece of advice.
+
+ In Memoize, this key is most commonly used to submit a command or environment
+ to automemoization. For an environment (say, |bar|), it suffices to write
+ \refmmz{auto}|=|\bracestt{bar}\bracestt{\refmmzauto{memoize}}; for a command,
+ we usually need to include its argument specification:
+ \refmmz{auto}|=\foo|\bracestt{\refmmzauto{memoize},
+ \refmmzauto{args}=\bracestt{...}}. Another common usage is to prevent
+ memoization during the execution of a command or environment:
+ \refmmz{auto}|=|\bracestt{bar}\bracestt{\refmmzauto{nomemoize}}. For
+ details, see sections~\ref{sec:tut:automemoization},
+ \ref{sec:tut:working-on-a-picture} and~\ref{sec:tut:verbatim}.
+
+ Note that in \hologo{LaTeX}, advising an environment (say, |bar|) is
+ different than advising the internal command (|\bar|) containing the
+ \meta{begin-code} of the environment, created by \cs{newenvironment}.
+ Advising environment |bar| effectively replaces the entire
+ |\begin{bar}...\end{bar}| construction, so that \hologo{LaTeX}'s |\begin|
+ code is never executed. The advice of command |\bar|, on the other hand,
+ is executed after |\begin| initializes the environment; in particular, it
+ is executed within the group introduced by the environment.
+
+ The advice is configured by the given \meta{keylist}, which is executed with
+ the default keypath set to \docaux{key path}{mmz/auto}. Any unknown keys in
+ \meta{keylist} are passed on to key \refmmzauto{options}; for example, a
+ plain \refmmz{verbatim} or \refmmz{padding}|=2in| have the same effect as
+ \refmmzauto{options}|=|\refmmz{verbatim} or
+ \code{\refmmzauto{options}=\braces{\refmmz{padding}=2in}}.
+
+ This key automatically activates the declared advice, unless it is already
+ activated; under the deferred activation regime, the automatic activation is
+ deferred as well. Use variant \refmmz{auto'} when you don't want to
+ automatically activate the advice.
+
+ When this key is used on a command or environment with an existing piece of advice,
+ the advice is merely updated. This makes it easy to, for example,
+ temporarily switch to verbatim collection of an environment in Memoize:
+ \refmmz{auto}|=|\bracestt{tcolorbox}\braces{\refmmz{verbatim}}. Use key
+ \refmmzauto{reset} to setup the advice from scratch:
+ \refmmz{auto}|=|\bracestt{...}\bracestt{\refmmzauto{reset}, ...}.
+
+ A piece of advice consists of several interlocked components, declared by keys
+ residing in path \refkeypath{/mmz/auto}: \refmmzauto{run conditions},
+ \refmmzauto{bailout handler}, \refmmzauto{outer handler},
+ \refmmzauto{collector} and \refmmzauto{inner handler}. During the execution
+ of the advice, these components are available through the following macros:
+ \refcmd{AdviceRunConditions}, \refcmd{AdviceBailoutHandler},
+ \refcmd{AdviceOuterHandler}, \refcmd{AdviceCollector} and
+ \refcmd{AdviceInnerHandler}. These macros are also defined during setup, and
+ it is possible to change the configuration by modifying them directly; in
+ that case, it likely also makes sense to use the low-level variant of this
+ key, macro \refcmd{AdviceSetup}.
+
+ Control sequences used in the advice components do not need to be defined at
+ the time of invoking key \refmmz{auto}, or activating the advice; they must
+ only be defined at the time the advice is actually executed. It is thus
+ perfectly fine to declare \refmmzauto{inner handler}|=\myinnerhandler| before
+ defining |\myinnerhandler|, or to redefine |\myinnerhandler| between the
+ invocations of the advised command.
+
+ This key configures not only the components, but also the options of the
+ advice. These options are set by keys \refmmzauto{args},
+ \refmmzauto{collector options}, \refmmzauto{raw collector options} and
+ \refmmzauto{options}. Whether these options are used or not depends on the
+ advice components. (Same as the components, the options have their
+ corresponding low-level macros: \refcmd{AdviceArgs},
+ \refcmd{AdviceCollectorOptions}, \refcmd{AdviceRawCollectorOptions} and
+ \refcmd{AdviceOptions}.)
+
+ Parameter symbols (i.e.\ |#|) are not allowed in advice settings.
+
+ A command or environment may be submitted to several instances of the
+ advising framework, i.e.\ instances installed under different keypaths. The
+ effect of such chained advice depends on the order of activation. If advice
+ $A$ is activated before advice $B$, it will also be applied before $B$.
+
+ The advice setup takes place in a group. Use key \refmmzauto{after setup} to
+ execute code outside this group.
+
+ In general, the name of this key equals whatever was submitted to
+ \refkey{/advice/install/setup key} during the installation of the advising
+ framework via \refkey{/handlers/.install advice}; the initial value is
+ |advice| (and Memoize sets the value to |auto|).
+\end{doc}
+
+\begin{doc}{
+ key={name=auto csname, par=\marg{control sequence name}{\marg{keylist}}, desc=style}
+ }
+ This key is a variant of \refmmz{auto}, but with the command of the first
+ argument given as a control sequence name, i.e.\ \refmmz{auto
+ csname}|={foo}{...}| is equivalent to \refmmz{auto}|=\foo{...}|.
+\end{doc}
+
+\begin{doc}{
+ key={name=auto key, par=\marg{full key}{\marg{keylist}}, desc=style},
+ }
+ This key is a variant of \refmmz{auto}, but it works with \pkg{pgfkeys} keys.
+ The first argument should be a \meta{full key} like |/tcb/float|, i.e.\ it
+ must consist of both the keypath and the keyname.
+
+ This key sets up advice for the internal command corresponding to the given
+ \meta{full key}, and also properly initializes the collector, so that
+ \refmmzauto{inner handler} will ``just work.''
+\end{doc}
+
+\begin{doc}{
+ key={name=auto', par=\marg{command or environment}{\marg{keylist}}, desc=style},
+ key={name=auto csname', par=\marg{control sequence name}{\marg{keylist}}, desc=style},
+ key={name=auto key', par=\marg{full key}{\marg{keylist}}, desc=style},
+ }
+ These keys are variants of \refmmz{auto}, \refmmz{auto csname} and
+ \refmmz{auto key} which do not attempt to activate the command after setting
+ it up.
+\end{doc}
+
+\begin{doc}{
+ cmd={
+ name=AdviceSetup,
+ par=\marg{namespace}\marg{command or environment}{\marg{setup code}}
+ }}
+ This macro is the low-level variant of key \refmmz{auto}. The differences
+ between the two are the following:
+ \begin{itemize}
+ \item An invocation of the macro must provide the namespace (i.e.\ the
+ installation keypath) as the first argument.
+ \item There is no automatic activation at the end of the setup.
+ \item The final argument should not be a keylist (of keys belonging to
+ \refkeypath{/mmz/auto}) but \hologo{TeX} code adjusting the contents of the
+ settings macros \refcmd{AdviceRunConditions}, \refcmd{AdviceBailoutHandler},
+ \refcmd{AdviceOuterHandler}, etc. For the full list of available macros, see
+ the documentation of their corresponding keys below; the setting macros are
+ mentioned at the end of each entry.
+ \end{itemize}
+\end{doc}
+
+\begin{doc}{
+ cmd={name=AdviceTracingOn},
+ cmd={name=AdviceTracingOff},
+ }
+ Advice tracing is initially off. When it is on, Advice will show (on the
+ terminal and in the |.log| file) which advice components are executed, and
+ what arguments and options they have received.
+\end{doc}
+
+\paragraph*{The keys installed into keypath \metabf{namespace}\texttt{/}\metabf{setup key}}
+are used to configure advice. They may only occur within the second
+argument of the setup key. In the documentation in this subsection, we assume
+that \meta{namespace}=\refkeypath{/mmz} and that \meta{setup
+ key}=\refmmz{auto}. In particular, this also applies to the reference
+headers.
+
+\begingroup
+\yadocset{keypath=/mmz/auto}
+
+\begin{doc}{%
+ key={name=run conditions, par=\meta{\hologo{TeX} code},
+ desc=initially and default: \refcmd{AdviceRuntrue}},
+ }
+
+ This key declares the \meta{control sequence} as the run conditions component
+ of the advice.
+
+ The run conditions macro is executed at the very start of the advice. Its
+ function is to decide whether we should proceed to advise the command by
+ executing the outer handler, or execute the original command (after invoking
+ the bailout handler).
+
+ The run conditions macro should take no arguments. If it determines that the
+ run conditions are satisfied, it should set the \hologo{TeX} conditional
+ \docaux{cmd}{ifAdviceRun} to true by executing \docaux{cmd}{AdviceRuntrue}.
+ There is no need to execute \docaux{cmd}{AdviceRunfalse} when the run
+ conditions are not satisfied.
+
+ Initially, the run conditions are set to \refcmd{AdviceRuntrue}, translating to
+ ``always run.'' For two non-trivial examples, see \refmmzauto{run if
+ memoization is possible} and \refmmzauto{run if memoizing}. Executing this
+ key without a value restores it to the initial value.
+
+ During advising and advice setup, the run conditions of the advised command
+ are accessible through \docaux{cmd}{AdviceRunConditions}, a parameterless
+ macro expanding to the given \meta{\hologo{TeX} code}.
+\end{doc}
+
+\begin{doc}{%
+ key={name=bailout handler, par=\meta{\hologo{TeX} code},
+ desc=initially and default: \cs{relax}},
+ }
+
+ This key declares the \meta{\hologo{TeX} code} as the bailout handler
+ component of the advice.
+
+ The bailout handler is executed when the run conditions are not met, just prior
+ to executing the original definition of the advised command. The bailout
+ handler should take no arguments.
+
+ The initial bailout handler, |\relax|, does nothing. Memoize defines and
+ uses a bailout handler which clears the next-options. Executing this key
+ without a value restores it to the initial value.
+
+ During advising and advice setup, the bailout handler of the handled command is
+ accessible through \docaux{cmd}{AdviceBailoutHandler}, a parameterless macro
+ expanding to the given \meta{\hologo{TeX} code}.
+\end{doc}
+
+\begin{doc}{
+ key={name=outer handler,par=\meta{\hologo{TeX} code},
+ desc=initially and default: see below},
+ }
+
+ This key declares the \meta{\hologo{TeX} code} as the outer handler component
+ of the advice.
+
+ The outer handler can be safely imagined as the command which replaces the
+ handled command. This also holds for handled environments, but with a
+ caveat: for a \hologo{plainTeX} or \hologo{ConTeXt} environment |foo|, the
+ outer handler replaces |\foo| and |\startfoo|, respectively; in the case of a
+ \hologo{LaTeX} environment, it replaces |\begin{foo}|.%]
+
+ The outer handler is the first component which has the opportunity to inspect
+ the arguments given to the handled command. It is invoked just in front of
+ these arguments (which are, in case \hologo{TeX} hasn't seen them yet,
+ untokenized), and while it is expected that the advice will consume
+ the same arguments as the advised command itself would, how precisely that
+ happens may vary from situation to situation. In particular, the argument
+ structure of the outer handler is not prescribed.
+
+ In fact, the outer handler has complete control over the remainder of the
+ advising process. In situations where advising requires knowledge of the
+ advised command's arguments as a whole, the outer handler executes the
+ collector, which in turn invokes the inner handler, which does the real work;
+ see \refmmzauto{memoize} for the usage case which inspired this design.
+ Sometimes, however, it is the outer handler which does the real work (and
+ there is thus no inner handler). This is the case in situations when the
+ arguments of the handled command are irrelevant for the functioning of the
+ advice, or when the advice needs to inspect some individual argument of the
+ handled command; for examples of such situations, see \refmmzauto{abort} and
+ \refmmzauto{ref}.
+
+ To reiterate the argument situation of the outer handler, it sees the
+ arguments of the handled command as they were given. The arguments are
+ \emph{not} collected before invoking the outer handler --- in fact, avoiding
+ the argument collection is the raison d'être of the outer handler! (In the case
+ of an advised environment, the environment body can be seen as an argument of
+ \pkg{xparse} type \docref{xparse:+}\docref{xparse:b}.)
+
+ The outer handler (and any other component of the advice it invokes) has
+ access to the following auxiliary macros, defined by the framework:
+ \begin{itemize}
+ \item \tolerance 1000 the macros holding the configuration of the advised
+ command, as set up by \refmmz{auto}: \refcmd{AdviceRunConditions},
+ \refcmd{AdviceBailoutHandler} and \refcmd{AdviceOuterHandler} are probably
+ useless, as they refer to components already invoked, but the remaining
+ components (\refcmd{AdviceCollector} and \refcmd{AdviceInnerHandler}) and
+ their options (\refcmd{AdviceArgs}, \refcmd{AdviceCollectorOptions},
+ \refcmd{AdviceRawCollectorOptions} and \refcmd{AdviceOptions}) should be
+ commonly used.
+ \item the macros holding information about the namespace and the advised
+ command or environment: \refcmd{AdviceNamespace}, \refcmd{AdviceName},
+ \refcmd{AdviceCsname}, \refcmd{AdviceReplaced} and \refcmd{AdviceOriginal}.
+ (Commands \refcmd{AdviceGetOriginal} and \refcmd{AdviceCsnameGetOriginal}
+ might also be useful, although using \refcmd{AdviceOriginal} will likely be
+ more practical.)
+ \end{itemize}
+
+ This key is initially set to an internal control sequence which merely
+ invokes the collector by executing \refcmd{AdviceCollector}; in other words,
+ the initial outer handler leaves all the work to the collector and the inner
+ handler. There is no need to specifically set up the outer handler when
+ using the inner handler. Executing this key without a value restores it to
+ the initial value.
+
+ During advising and advice setup, the outer handler of the advised command is
+ accessible through \docaux{cmd}{AdviceOuterHandler}, a parameterless macro
+ expanding to the given \meta{\hologo{TeX} code}.
+\end{doc}
+
+\begin{doc}{
+ key={name=collector,par=\meta{\hologo{TeX} code},
+ desc=initially and default: see below},
+ }
+
+ This key declares the \meta{\hologo{TeX} code} as the collector component of
+ the advice.
+
+ The collector, if used, is invoked by the outer handler. It is invoked
+ immediately in front of the advised command's arguments (which are, in case
+ \hologo{TeX} hasn't seen them yet, untokenized), and its function is to
+ collect these arguments and pass them on, as a single argument, to the inner
+ handler.
+
+ While this manual occasionally states that the initial argument collector is
+ \refcmd{CollectArguments} of package CollArgs, this is, if we're precise,
+ incorrect on two counts. For one, the initial collector is not a CollArgs
+ command, but a macro which acts as the ``bridge'' between Advice
+ and CollArgs. Second, the initial collector does not really invoke
+ \refcmd{CollectArguments}, but its cousin, \refcmd{CollectArgumentsRaw},
+ which allows Advice (and Memoize) to fine tune its behaviour
+ using the fast low-level (``programmer interface'') commands rather than the
+ slower \pkg{pgfkeys} interface; clearly, the latter point also provides
+ raison d'être for \refmmzauto{raw collector options}. Summing up, this key
+ is initially set to an internal control sequence which compiles the settings
+ provided by \refmmzauto{args}, \refmmzauto{collector options} and
+ \refmmzauto{raw collector options} into an invocation of
+ \refcmd{CollectArgumentsRaw} of package CollArgs.\footnote{The initial
+ collector also sets the CollArgs' option \refcollargs{caller} to the
+ name of the advised command or environment.} Executing this key without a
+ value restores it to the initial value.
+
+ The above-mentioned collector settings were clearly tailored to suit
+ \refcmd{CollectArgumentsRaw}. In general, a collector might or might not use
+ them, and if it does, it may interpret them in any way. For example, Advice
+ ships with a \refcmd{tikz} collector, \docaux{cmd}{mmzCollectTikZArguments}, which
+ ignores them completely, as it knows everything about the idiosyncrasies of
+ that command anyway. Incidentally, \refcmd{mmzCollectTikZArguments} becomes
+ available upon loading \reffile{advice-tikz.code.tex} (which Memoize does
+ automatically in the presence of \TikZ).
+
+ The collector has access to the same auxiliary macros as the outer handler.
+ In particular, it will \emph{have} to use \refcmd{AdviceInnerHandler} (followed
+ by the braced collected arguments) to invoke the inner handler.
+
+ During advising and advice setup, the collector of the advised command is
+ accessible through \docaux{cmd}{AdviceCollector}, a parameterless macro
+ expanding to the given \meta{\hologo{TeX} code}.
+\end{doc}
+
+\begin{doc}{easy,
+ key={name=args,par=\meta{argument specification}, desc=initially and default: unset},
+ }
+
+ This key describes the \meta{argument specification} of the advised command.
+
+ Assuming that the initial value of \refmmzauto{collector} has not been
+ modified, the given \meta{argument specification} is eventually interpreted
+ by command \refcmd{CollectArguments} of package CollArgs, which expects an
+ argument specification in the format specified by package \pkg{xparse}; the
+ format is summarized in the frame below for convenience, for details, see the
+ \pkg{xparse} manual. If the specification is not given, the initial collector
+ assumes that the advised command was defined using \refcmd{NewDocumentCommand}
+ (or similar) of package \pkg{xparse}, and will attempt to retrieve the argument
+ specification automatically via \refcmd{GetDocumentCommandArgSpec}.
+
+ \begin{tcolorbox}[float, before float=\hfill,
+ title={The {\pkg[white]{xparse}} argument specification
+ (as understood by \refcmd[link color=white]{CollectArguments})}]
+ \begin{tabularx}{\linewidth}{>{\tt}lX}
+ \multicolumn{2}{l}{\rm\textbf{Mandatory argument types}}\\
+ m&standard (a single token or multiple tokens in braces)\\
+ r\meta{token$_1$}\meta{token$_2$}&delimited by
+ \meta{token$_1$} and \meta{token$_2$} \\
+ v&verbatim, in the style of \cs{verb}\\
+ b&the body of an environment\\
+ [1ex]\multicolumn{2}{l}{\rm\textbf{Optional argument types}}\\
+ o&in square brackets\\
+ d\meta{token$_1$}\meta{token$_2$}&delimited by
+ \meta{token$_1$} and \meta{token$_2$}\\
+ s&an optional star\\
+ t\meta{token}&an optional \meta{token}\\
+ e\marg{tokens}&a set of embellishments\\
+ [1ex]\multicolumn{2}{l}{\rm\textbf{Weird argument types}}\\
+ l&a mandatory argument until the first begin-group token\\
+ u\marg{tokens}&\hologo{TeX}'s delimited argument\\
+ g&an optional argument inside braces\\
+ [1ex]\multicolumn{2}{l}{\rm\textbf{Modifiers}}\\
+ +&allow the next argument to be long\\
+ !&disallow spaces before arguments of type \docref{xparse:d} and \docref{xparse:t}\\
+ >\marg{processor}&process the next argument\\
+ [1ex]\multicolumn{2}{l}{\rm\textbf{CollArgs extensions}}\\
+ \docref{xparse:b}\marg{name}&set the environment name for this environment\\
+ \docref{xparse:amp}\marg{options}&apply CollArgs options to the next argument\\
+ \docref{xparse:amp}\docref{xparse:amp}\marg{raw options}&apply raw CollArgs options to the next argument\\
+ \end{tabularx}
+
+ \smallskip
+
+ \refcmd{CollectArguments} can grab an argument of any type in the
+ \refcollargs{verbatim} mode.
+
+ As \refcmd{CollectArguments} does not use the arguments but only collects
+ them, it does not care about the default values of optional arguments.
+ Therefore, argument types with defaults (\docref{xparse:O},
+ \docref{xparse:D} and \docref{xparse:R}) may be substituted by their
+ \texttt{-NoValue-} counterparts (\docref{xparse:o}, \docref{xparse:d} and
+ \docref{xparse:r}) and are therefore not included in the above table.
+ \end{tcolorbox}
+
+ In general, however, an argument collector may this interpret this setting it
+ in any way it sees fit --- or not at all. For example, in Memoize the value
+ of \refmmzauto{args} is ignored for command \refcmd{tikz}, which requires a special
+ collector (\refcmd{mmzCollectTikZArguments}).
+
+ When setting up advice for a \emph{command}, this key is initially
+ ``unset,'' i.e.\ it holds a special value indicating that the argument
+ specification is not provided. Note that this special value is not an empty
+ string --- \refmmzauto{args}|={}|, or simply \refmmzauto{args}|=|, indicates
+ a command which takes no arguments. During the execution of the advice, one
+ may use the \hologo{LaTeX}-style conditional
+ \docaux{cmd}{AdviceIfArgs}\marg{true branch}\marg{false branch} to test
+ whether the argument specification was provided. Executing this key without
+ a value restores it to the initial, unset value.
+
+ When setting up the advice of an \emph{environment}, this key is
+ initialized to \docref{xparse:+}\docref{xparse:b} (a long environment body),
+ making it unnecessary to specify this value manually. Note that this holds
+ even for environments with arguments other that the environment body
+ ``argument'': those arguments will be caught as the start of the body even if
+ not explicitly specified.
+
+ During advising and advice setup, the argument specification of the advised
+ command is accessible through \docaux{cmd}{AdviceArgs}, a parameterless macro
+ expanding to the given \meta{argument specification}.
+\end{doc}
+
+
+\begin{doc}{
+ key={name=collector options, par=\marg{keylist},
+ desc={cumulative, initially empty, value required}},
+ key={name=raw collector options, par=\marg{code},
+ desc={cumulative, initially empty, value required}},
+ }
+ These keys append the given value to the list of user-friendly and raw
+ collector options, respectively. A comma is prefixed to the user-friendly
+ \meta{keylist} before appending it.
+
+ Both kinds of collector options are intended to be used by the collector,
+ which may interpret them in any way it sees fit --- or not at all. The
+ initial collector, which invokes \refcmd{CollectArgumentsRaw} of package
+ CollArgs, passes both lists to this command, which interprets
+ \refmmzauto{collector options} as a user-friendly \pkg{pgfkeys} keylist
+ (which therefore requires a bit of processing) and \refmmzauto{raw collector
+ options} as plain \hologo{TeX} code (expecting it to contain only the
+ allowed, ``programmer's interface'' macros).\footnote{Clearly,
+ \refmmzauto{raw collector options} are why Advice deploys
+ \refcmd{CollectArgumentsRaw} rather than \refcmd{CollectArguments}. But
+ how does it then pass the user-friendly \refmmzauto{collector options} to
+ that command? It embeds them in \refcmd{collargsSet}.} The raw variant is
+ used internally by both Advice and Memoize, and may be used by a package
+ deploying the advising framework which wants to save a few processing cycles.
+ In CollArgs, the two kinds of options are functionally equivalent; both are
+ documented in section~\ref{sec:ref:collargs}.
+
+ Initially, the list of collector options is empty, and for commands, so is
+ the list of raw collector options. For environments, however, the latter
+ list is initialized to set (the raw equivalent of) \refcollargs{environment}
+ to the environment name, and \refcollargs{end tag} to true. The rationale
+ for the latter is that the environment body containing the end tag (e.g.\
+ |\end{foo}|) is nicely compatible with \refcmd{AdviceReplaced} (which equals
+the begin tag, e.g.\ |\begin{foo}|) and \refcmd{AdviceOriginal} (which executes
+ the original definition of e.g.\ |\begin{foo}|). For example, thanks to
+ \refcollargs{end tag}, writing \refcmd{AdviceOriginal}|#1| in the inner
+ handler executes the original environment. Importantly, the original
+ environment can be executed without explicitly referring to the
+ environment's name, and with code that works not only for environments of
+ any \hologo{TeX} format, but is actually the same as the code which invokes
+ an original \emph{command}. Consequently, the same inner handler works for
+ both commands and environments, and in all \hologo{TeX} formats.
+
+ Furthermore, the initial collector also sets option \refcollargs{caller}
+ to the name of the advised command or environment (however,
+ \refcollargs{caller} never appears in any of the collector options
+ lists; it is simply prefixed to them while constructing the invocation of
+ \refcmd{CollectArgumentsRaw}). And in Memoize, using keys
+ \refmmz{verbatim}, \refmmz{verb} or \refmmz{no verbatim} triggers the
+ addition of the cognominal \refcollargs{verbatim}, \refcollargs{verb}
+ or \refcollargs{no verbatim} among the collector options.
+
+ Precious few CollArgs' options thus remain to be set by the author. For
+ memoization, the most likely candidates are \refcollargs{ignore nesting}
+ and \refcollargs{ignore other tags}, which could help deal with unusual
+ environments. Overriding the initial \refcollargs{end tag} by
+ \refcollargs{begin tag}, \refcollargs{end tag} and/or \refcollargs{tags}
+ might also be useful on occasion.
+
+ During advising and advice setup, the \pkg{pgfkeys} and the raw collector
+ options of the advised command are accessible through
+ \docaux{cmd}{AdviceCollectorOptions} and
+ \docaux{cmd}{AdviceRawCollectorOptions}, both a parameterless macro expanding
+ to the given \meta{keylist} and \meta{code}, respectively.
+\end{doc}
+
+\begin{doc}{
+ key={name=clear collector options},
+ key={name=clear raw collector options},
+ }
+ These keys empty the list of user-friendly and raw collector options,
+ respectively.
+\end{doc}
+
+\begin{doc}{
+ key={name=inner handler, par=\meta{\hologo{TeX} code}, long description=0.5,
+ desc=initially and default: see below},
+ }
+
+ This key declares the \meta{\hologo{TeX} code} as the inner handler component
+ of the advice.
+
+ The inner handler is intended to be used in situations which require
+ knowledge of the advised command's arguments as a whole. In such situations,
+ the outer handler will normally invoke the collector, which will in turn
+ execute the inner handler and provide it with a single (braced) argument,
+ containing the collected arguments of the advised command. See
+ \refmmzauto{memoize} for the usage case which inspired this design.
+
+ The simplest example of an inner handler is a (single-parameter) macro which
+ does nothing. Surprisingly enough, such an inner handler could be useful.
+ Defining |\def\Gobble#1{}| and setting
+ \code{\refmmz{auto}=\cs{foo}\bracestt{\refmmzauto{inner handler}=\cs{Gobble},
+ \refmmzauto{args}=\bracestt{...}}} with the argument structure
+ appropriate for |\foo| could be used to eradicate all invocations of |\foo|
+ from the document.
+
+ The inner handler has access to all the macros available to the outer
+ handler, but given that most of them have already fulfilled their function,
+ only the following will likely be useful in the inner handler:
+ \refcmd{AdviceNamespace}, \refcmd{AdviceName}, \refcmd{AdviceCsname},
+ \refcmd{AdviceReplaced}, \refcmd{AdviceOriginal}, and \refcmd{AdviceOptions}.
+
+ Because there is clearly no reasonable default for the inner handler, this key
+ is initially set to an internal control sequence producing an ``undefined
+ inner handler'' error. Note that it is not necessary to define a dummy inner
+ handler when handling is entirely performed by the outer handler, i.e.\ in
+ cases when the inner handler is not invoked. Executing this key without a
+ value restores it to the initial value.
+
+ During advising and advice setup, the inner handler of the advised command is
+ accessible through \docaux{cmd}{AdviceInnerHandler}, a parameterless macro
+ expanding to the given \meta{\hologo{TeX} code}.
+\end{doc}
+
+\begin{doc}{easy,
+ key={name=options,par=\marg{keylist},
+ desc={cumulative, initially empty, value required}},
+ key={name=clear options},
+ }
+
+ The first key appends the given \meta{keylist} to the list of advice options
+ (after prefixing it by a comma), and the second one empties this list. For a
+ \meta{key} undefined in keypath \refkeypath{/mmz/auto},
+ \meta{key}|=|\meta{value} has the same effect as
+ \refmmzauto{options}|=|\bracestt{\meta{key}=\meta{value}}.
+
+ In Memoize, the options set by this key are known as
+ \emph{auto-options} --- options which are applied (using \refcmd{mmzset}) at
+ every invocation of the advised command or environment. For example, the
+ \env{tcolorbox} environment of package \pkg{tcolorbox} is used extensively
+ for typesetting this manual, and I have submitted this environment to
+ automemoization. However, the \env{tcolorbox}es in this manual often include
+ code listings. To memoize such environments successfully, their bodies must
+ be grabbed verbatim. I have therefore submitted the \env{tcolorbox}
+ environment to automemoization like this:
+ \refmmz{auto}|={tcolorbox}|\bracestt{\refmmzauto{memoize},
+ \refmmzauto{options}=\refmmz{verbatim}}; the simpler
+ \refmmz{auto}|={tcolorbox}|\bracestt{\refmmzauto{memoize}, \refmmz{verbatim}}
+ would work as well.
+
+ In general, whether to use the options set by this key, and how, remains at
+ the sole discretion of the advice. Note that they might be used by either
+ the outer or the inner handler, or perhaps even the collector.
+
+ During advising and advice setup, the options of the advised command are
+ accessible through \docaux{cmd}{AdviceOptions}, a parameterless macro expanding
+ to the given \meta{keylist}.
+\end{doc}
+
+\begin{doc}{key={name=reset, desc=style}}
+ Executing this key restores all \refmmz{auto} keys to their initial values.
+
+ Invoking \refmmz{auto} on the same command or environment again
+ \emph{updates} the advice configuration. Use this key to start from scratch.
+\end{doc}
+
+\begin{doc}{key={name=after setup, desc={initially empty, cumulative}}}
+ The code given to this key will be executed after exiting the group opened by
+ \refmmz{auto}. The same effect may be achieved by appending to macro
+ \docaux{cmd}{AdviceAfterSetup}.
+
+ For example, \refmmzauto{integrated driver} uses this key to declare a new
+ conditional.
+\end{doc}
+
+\paragraph{Commands available during the execution of advice}
+
+With the exception of \refcmd{AdviceGetOriginal} and
+\refcmd{AdviceCsnameGetOriginal}, the commands listed below only become
+available in the outer handler, and if that handler does nothing funky, they
+should be available in the collector and the inner handler, as well. However,
+once the advice yields control to foreign code, these macros are not guaranteed
+to hold the expected values anymore, because the foreign code might trigger
+another piece of advice. Consequently, these macros should be expanded, once,
+before integrating them into arbitrary (non-advice) code; in particular, this
+applies to \refcmd{AdviceOriginal}.
+
+\begin{doc}{cmd={name=AdviceNamespace}}
+ This macro holds the \meta{namespace}, i.e.\ the keypath which this instance
+ of the advising framework was installed into.
+\end{doc}
+
+\begin{doc}{cmd={name=AdviceName}}
+ This macro holds the name of the advised command or environment, i.e.\ the
+ name which was used as the first argument to \refmmz{auto}. For a command,
+ this will be a control sequence, e.g.\ |\foo|; for environments (in any
+ \hologo{TeX} format), their name, e.g.\ |foo|.
+\end{doc}
+
+\begin{doc}{cmd={name=AdviceCsname}}
+ This macro holds the control sequence name of the advised command; it is
+ undefined for environments. For example, when \refcmd{AdviceName} contains
+ |\foo at bar|, this macro will hold |foo at bar|.
+\end{doc}
+
+\begin{doc}{cmd={name=AdviceReplaced}}
+ This macro holds the code which was replaced by the outer handler. For
+ commands, this will be the command itself, e.g.\ |\foo|, so
+ \refcmd{AdviceReplaced} will equal \refcmd{AdviceName}. For an environment
+ |foo|, \refcmd{AdviceReplaced} is set to |\begin{foo}| in \hologo{LaTeX},
+ |\foo| in \hologo{plainTeX} and |\startfoo| in \hologo{ConTeXt}.
+\end{doc}
+
+\begin{doc}{cmd={name=AdviceOriginal}}
+ This macro executes the original code of the advised command.
+
+ This macro is defined as \refcmd{AdviceGetOriginal}\marg{namespace}\marg{name},
+ and therefore acts as a shortcut for an explicit invocation of
+ \refcmd{AdviceGetOriginal}. When executing the original command directly from
+ the advice, one may safely write \refcmd{AdviceOriginal}. However, whenever
+ \refcmd{AdviceOriginal} is embedded in code which might contain other advised
+ commands, it should be pre-expanded, exactly once.
+\end{doc}
+
+\begin{doc}{cmd={name=AdviceGetOriginal, par=\marg{namespace}\marg{control sequence}}}
+ This command invokes the original definition of the \meta{control sequence}
+ advised by the \meta{namespace} instantiation of Advice; more precisely, the
+ full expansion of this macro produces the (internal) control sequence holding
+ the definition of \meta{control sequence} in effect when this control
+ sequence was activated in \meta{namespace}. This macro may be safely used
+ outside the advice, even if the advised command is not activated.
+
+ For example, upon executing key |/ns/|\refmmz{auto}|=\foo{...}|,
+ |\AdviceGetOriginal{/ns}{\foo}| will recall the original definition of |\foo|
+ if |\foo| is activated, and simply execute |\foo| otherwise.
+
+ The second argument of this command should \emph{not} be an environment name.
+ To execute the original environment |foo| in \hologo{plainTeX} or
+ \hologo{ConTeXt}, use \refcmd{AdviceGetOriginal} with the appropriate macro:
+ |\AdviceGetOriginal{/ns}{\foo}| or |\AdviceGetOriginal{/ns}{\startfoo}|. In
+ \hologo{LaTeX}, one should use |\AdviceGetOriginal{/ns}{\begin}{foo}|, which
+ executes the original \refcmd{begin} and provides it with the environment
+ name.
+
+ Within the advice, you will probably never have to use this command directly,
+ but will rather rely on the (plain or pre-expanded) \refcmd{AdviceOriginal}.
+ However, outside the advice, this command provides the only official means to
+ access the original definition of an advised command. (Unlike the
+ commands described above, this command is available throughout the
+ document.)
+
+ \begin{tcolorbox}[warning]
+ A typo in the invocation of this command may result in an infinite loop.
+ Assume that the advice for |\foo|, declared in namespace \refkeypath{/mmz},
+ executes \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, which incorrectly refers
+ to the non-existing namespace |/zzm|, and that command |\foo| is activated.
+ Executing |\foo| will eventually execute
+ \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, which won't find the original
+ definition of |\foo| in the non-existing namespace |/zzm| and will thus
+ execute macro |\foo| (again), which, being advised, will lead to another
+ \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, etc. My advice is to define an
+ abbreviation like |\def\mmzAdviceGetOriginal{\AdviceGetOriginal{/mmz}}|.
+ And note that the name\-space is a full keypath, which begins with a slash
+ (|/|), but has no slash at the end.
+ \end{tcolorbox}
+\end{doc}
+
+\begin{doc}{cmd={name=AdviceCsnameGetOriginal,
+ par=\marg{namespace}\marg{control sequence name}}}
+ This is a version of \refcmd{AdviceGetOriginal} which accepts a control
+ sequence name as the second argument.
+
+ This macro is used by auto-key \refmmzauto{to context} to include the meaning
+ of any command (even internal commands containing |@|, or \pkg{expl3}
+ commands) into the context.
+\end{doc}
+
+\paragraph{Support for specific packages}
+At the moment, Advice only implements specific support for \TikZ, by defining a
+\refmmzauto{collector} for command \refcmd{tikz}.
+
+\begin{doc}{
+ cmd={name=AdviceCollectTikZArguments},
+ }
+ This command collects the arguments in the format expected by \refcmd{tikz}, and
+ executes macro \refcmd{AdviceInnerHandler} with the collected arguments given
+ as a single braced argument. The collector supports both the group and the
+ semicolor invocation of |\tikz|, i.e.\ both |\tikz{...}| and |\tikz...;|.
+
+ This command is only available upon |\input|ting file
+ \docaux{file}{advice-tikz.code.tex}.
+\end{doc}
+
+\endgroup % keypath = /mmz/auto
+
+\subsubsection{Memoization-related additions to the advising framework}
+\label{sec:ref:advice:memoization}
+
+In section~\ref{sec:ref:advice}, we have seen that Memoize installs the
+advising framework into keypath \refkeypath{/mmz}, with setup key name
+\refmmz{auto}. This populates keypaths \refkeypath{/mmz} and
+\refkeypath{/mmz/auto} with various generic advice keys. However, Memoize
+installs further advice\slash automemoization-related keys into these keypaths.
+It is these keys which are described in this section.\footnote{One such key,
+ \refmmzauto{integrated driver}, is actually documented in
+ section~\ref{sec:ref:memoization}.}
+
+Therefore, in contrast to section~\ref{sec:ref:advice}, \refkeypath{/mmz} and
+\refmmz{auto} have no secret generic meaning here, i.e.\ they should \emph{not}
+be generalized to \meta{namespace} and \meta{setup key} of
+\refkey{/handlers/.install advice}.
+
+\paragraph{Keys residing in \refkeypath{/mmz}}
+
+\begingroup
+\setlength\textfloatsep{20pt plus 2pt minus 4pt}% back to the default
+
+\begin{doc}{easy,
+ key={name=manual, description={preamble-only,{ }}, conditional=false},
+ }
+ When this conditional is set to true, no commands are \refmmz{activate}d at
+ the beginning of the document. The list of commands and environments advised
+ and activated out of the box can be found in Table~\ref{tab:advised-commands}.
+
+ The auto-framework allows the activation to be deferred (see
+ \refmmz{activation} and \refkey{/handlers/.install advice}) but leaves it
+ open to the specific instance of the framework to use the deferred activation
+ commands as it sees fit. Normally, Memoize switches to immediate activation
+ at the end of the preamble (hook \refmmz{begindocument/before}) and issues
+ \refmmz{activate deferred} at the beginning of the document, more precisely
+ in hook \refmmz{begindocument/end} (afterwards, \refmmz{activate deferred} is
+ emptied). However, when \refmmz{manual} is in effect, the deferred
+ activation is suppressed (though it may be still carried out by the user by
+ executing \refmmz{activate deferred}).
+
+ In \hologo{LaTeX}, \refmmz{manual} affects the (internal) activation of
+ \refcmd{begin} as well, which effectively deactivates handling of all environments.
+\end{doc}
+
+\begin{table}
+ \centering
+ \begin{tabularx}{\linewidth}{llX}
+ \toprule
+ command/environment&handler¬es\\
+ \midrule
+ \refcmd{begin}&custom&Only in \hologo{LaTeX}; declared by Advice.\\
+ \refcmd{errmessage}&\refmmzauto{abort}&Not available in \hologo{LuaTeX},
+ where better error-detection is implemented.\\
+ \refenv{forest}&\refmmzauto{memoize}\\
+ \refcmd{Forest}&\refmmzauto{memoize}\\
+ \refcmd{index}&\refmmzauto{replicate}&The argument is expanded prior to replication.\\
+ \refcmd{label}&custom&\refmmzauto{run if memoizing}; globally appends
+ \refcmd{mmzLabel}\marg{label key}\marg{current label}
+ to register \refcmd{mmzCCMemo}.\\
+ \refcmd{pageref}&\refmmzauto{ref}\\
+ \refcmd[into index=false]{pdfsavepos}&\refmmzauto{abort}&Not available in \hologo{LuaTeX}.\\
+ \refcmd{pgfsys at getposition}&\refmmzauto{abort}&Available only in \TikZ is
+ loaded, it aborts memoization of a picture which gets accidentally marked
+ as
+ ``remembered''.\\
+ \refcmd{ref}&\refmmzauto{ref}\\
+ \refcmd[short]{savepos}&\refmmzauto{abort}&Available only in \hologo{LuaTeX}.\\
+ \refcmd{tikz}&\refmmzauto{memoize tikz}\\
+ \refenv{tikzpicture}&\refmmzauto{memoize tikz}\\
+ \bottomrule
+ \end{tabularx}
+ \caption{Commands advised by Memoize}
+ \label{tab:advised-commands}
+\end{table}
+
+\begin{doc}{easy,
+ key={name=ignore spaces, conditional=false},
+ }
+ Ignore any spaces following \emph{auto}memoized code. This key has no effect
+ for manual memoization, i.e.\ command \refcmd{mmz} and environment
+ \refenv{memoize}.
+
+ It is common practice to conclude the definition of a command by \hologo{TeX}
+ primitive \refcmd{ignorespaces}, which consumes any following spaces, to
+ prevent unintended blank space after the command's invocation. Automemoizing
+ such a command disrupts this behaviour.\footnote{It is clear that
+ \refcmd{ignorespaces} is disrupted during utilization; in this case, the
+ original command, including the concluding \cs{ignorespaces}, is never even
+ executed. However, the disruption also occurs during memoization, and even
+ during regular compilation. In both cases, the memoized code is embedded
+ in some internal Memoize code. Therefore, the original \cs{ignorespaces}
+ does not occur directly in front of the rest of the document.} The
+ workaround is to use this key, normally as an option in the \refmmz{auto}
+ declaration; it will work both for automemoized macros and environments.
+\end{doc}
+
+\paragraph{Keys residing in \refkeypath{/mmz/auto}}
+
+\begingroup
+\yadocset{keypath=/mmz/auto}
+
+\begin{doc}{easy,key={name=memoize, desc=style}}
+ This key sets up advice which triggers memoization of the command or
+ environment whenever it is encountered; we often refer to such a command as
+ ``automemoized,'' or say that it was ``submitted to automemoization.''
+
+ An automemoized command will consume the next-options, whether memoization
+ actually occurs or not.
+
+ Under the hood, this key declares both an \refmmzauto{outer handler} and an
+ \refmmzauto{inner handler}. The outer handler opens the memoization group
+ (so that options can be applied locally), applies the auto-options (given by
+ \refmmzauto{options} within \refmmz{auto}) and the next-options (given by
+ \refcmd{mmznext}) by executing \refmmzauto{apply options}, and appends the
+ verbatim keys to \refmmzauto{collector options} if necessary. The inner
+ handler invokes \refcmd{Memoize} (which closes the group opened by the outer
+ handler): the first argument is \refcmd{AdviceReplaced}, expanded once and
+ followed by the arguments of the handled command; the second argument is
+ \refcmd{AdviceOriginal}, also expanded once and followed by the arguments of
+ the handled command. The inner handler also makes sure that \refmmz{ignore
+ spaces} is respected.
+\end{doc}
+
+\begin{doc}{easy,key={name=nomemoize,desc=style}}
+ This key installs advice which disables memoization for the space of the
+ command or environment; we sometimes refer to such commands as
+ ``autodisabled.''
+
+ This key is merely an abbreviation for \refmmzauto{noop}|,|
+ \refmmzauto{options}|=|\refmmz{disable}. See the documentation of
+ \refmmzauto{noop} for further details.
+\end{doc}
+
+\begin{doc}{key={name=noop,desc=style}}
+ This key sets up advice which does nothing.
+
+ Ok, not nothing at all. The installed handler applies the auto-options and
+ the next-options by executing \refmmzauto{apply options}, and makes sure that
+ \refmmz{verbatim} and \refmmz{ignore spaces} are respected.
+
+ For commands and non-\hologo{LaTeX} environments, this key declares the same
+ outer handler as \refmmzauto{memoize}, while the inner handler merely
+ executes the original command (respecting the potential verbatim mode),
+ closes the group opened by the outer handler, and makes sure that
+ \refmmz{ignore spaces} is respected.
+
+ For \hologo{LaTeX} environments, which open the group necessary for the local
+ application of options themselves, this key declares an outer handler which
+ adds the relevant code into the next hook |env/|\meta{environment
+ name}|/begin|. There is no need to open a group, collect the environment
+ body, or make special provisions for the verbatim mode.
+\end{doc}
+
+
+\begin{doc}{key={name=apply options,desc=style}}
+ This style, used by \refmmzauto{memoize}, \refmmzauto{nomemoize} and
+ \refmmzauto{noop} described above, installs two handlers:
+ \begin{itemize}
+ \item an outer handler which opens a group, applies auto-options and
+ next-options by executing \refcmd{mmzAutoInit}, and executes the collector;
+ and
+ \item a bailout handler which clears the next-options.
+ \end{itemize}
+\end{doc}
+
+
+\begin{doc}{cmd={name=mmzAutoInit}}
+ This macro applies the auto-options and the next-options.
+
+ Additionally, if \refmmz{verbatim}, \refmmz{verb} or \refmmz{no verbatim} was
+ previously executed, this style appends the corresponding CollArgs key
+ (\refcollargs{verbatim}, \refcollargs{verb} or \refcollargs{no
+ verbatim}) to \refcmd{AdviceRawCollectorOptions}. In case several of the
+ verbatim keys were executed, the final one takes effect.
+\end{doc}
+
+
+\begin{doc}{easy, key={name=abort,desc=style}}
+ This key sets up advice which aborts any ongoing memoization.
+
+ Under the hood, the advice merely executes \refcmd{mmzAbort} followed by
+ \refcmd{AdviceOriginal}. The advised command does \emph{not} consume the
+ next-options. Out of the box, we submit two control sequences to this
+ handler:
+ \begin{itemize}
+ \item \refcmd{errmessage}: this allows us to detect and abort upon at least some
+ errors.
+ \item \refcmd[into index=false]{pdfsavepos} (in \hologo{LuaTeX},
+ \refcmd[short]{savepos}): one common effect is that memoization of any
+ \TikZ picture with \refkey{/tikz/remember picture} set is aborted.
+ \end{itemize}
+\end{doc}
+
+
+\begin{doc}{key={name=unmemoizable,desc=style}}
+ This key sets up advice which aborts the ongoing memoization and marks the
+ automemoized code as unmemoizable, so that it will be henceforth compiled
+ regularly.
+
+ Under the hood, the advice merely executes \refcmd{mmzUnmemoizable} followed
+ by \refcmd{AdviceOriginal}. The advised command does \emph{not} consume the
+ next-options.
+
+ Out of the box, we submit no control sequences to this advice, but it might
+ make sense to submit \refcmd[into index=false]{pdfsavepos}\slash
+ \refcmd[short]{savepos}. Keys \refmmzauto{abort}|=|\refcmd[short]{savepos}
+ and \refmmzauto{unmemoizable}|=|\refcmd[short]{savepos} will most often have
+ the same effect, as far as the author is concerned; the former was chosen as
+ the default because it does not produce a c-memo; see
+ \refcmd{mmzUnmemoizable} for a situation where \refmmzauto{unmemoizable} is
+ preferred.
+\end{doc}
+
+\begin{doc}{key={name=memoize tikz,desc=style}}
+ This key is used to declare memoization of \TikZ pictures; it may also be
+ used for PGF pictures. Besides executing \refmmzauto{memoize}, it uses
+ \refmmz{at begin memoization} and \refmmz{at end memoization} to add the code
+ which measures the increase of the PGF picture ID during memoization of a
+ picture, and increases this ID for the measured amount upon the utilization
+ of the extern, thereby making sure that a utilized \TikZ\slash\PGF extern
+ advances the PGF picture ID as if the picture was compiled.
+\end{doc}
+
+\begin{doc}{easy,
+ key={name=ref,desc=style},
+ key={name=force ref,desc=style},
+ }
+
+ These keys set up advice which adds the reference key to the context
+ expression. They are intended to be used with cross-referencing commands
+ such as \refcmd{ref} and \refcmd{pageref}.
+
+ Indeed, \refcmd{ref} and \refcmd{pageref} are submitted to this advice by
+ Memoize, with the effect that standard cross-referencing inside memoized code
+ ``just works.'' Note that the stabilization of the document after changing
+ the reference takes three compilation cycles, i.e.\ one cycle more than
+ without memoization.
+
+ The advice set up by \refmmzauto{ref} aborts memoization if the reference
+ key is undefined, the rationale being that the produced memo and extern would
+ most often be useless, and could even obscure an undefined reference. The
+ \refmmzauto{force ref} handler produces the memo and the extern even when the
+ reference is undefined.
+
+ The reference produced by the advised command should be fully expandable
+ (because it will be expanded as a part of the context expression).
+
+ Typically, a \refcmd{ref} command takes a single argument, the reference key.
+ However, some packages may define a reference command which takes optional
+ arguments, as well; in particular, the \pkg{hyperref}'s incarnation of \refcmd{ref}
+ takes an optional star. This advice does not care: it will accept any number
+ of any kind of optional arguments, as long as the reference key is the first
+ braced argument following the advised command; for example, |\ref*{key}|,
+ |\ref[opt]{key}|, |\ref*[opt]{key}| etc.\ will all be handled correctly,
+ while |\ref{mand}{key}| will not work. Effectively, it is as if we had set
+ \refmmzauto{args}|=lm| --- and with the same downside, namely that an
+ unlikely unbraced single-token reference key, like |\ref k|, will not work.
+
+ Under the hood, these two pieces of advice pass the reference key to macros
+ \refcmd{mmzNoRef} and \refcmd{mmzForceNoRef}, and it is these commands ---
+ which may also be used in user-defined advice or the document itself ---
+ which actually add the reference key to the context expression.
+\end{doc}
+
+\begin{doc}{easy,
+ key={name=refrange,desc=style},
+ key={name=force refrange,desc=style},
+ }
+ These keys have the same function as \refmmzauto{ref} and \refmmzauto{force
+ ref}, but they operate on reference-range commands, such as \pkg{cleveref}'s
+ \refcmd{crefrange}, which take two arguments (the starting and the ending reference
+ key).
+\end{doc}
+
+\begin{doc}{easy,
+ key={name=multiref,desc=style},
+ key={name=force multiref,desc=style},
+ }
+ These keys have the same function as \refmmzauto{ref} and \refmmzauto{force
+ ref}, but they operate on ``multireference'' commands, such as
+ \pkg{cleveref}'s \refcmd{cref}, which allow the author to list several
+ comma-separated reference keys in a single argument.
+\end{doc}
+
+\begin{doc}{
+ key={name=to context, desc=style}
+ }
+ This key sets up advice which appends the original meaning of the advised
+ command to the context. It invokes \refmmzauto{run if memoizing}, so that
+ the command is only advised during memoization.
+
+ It is safe to apply this key to internal commands such as commands whose name
+ contains |@|, or \pkg{expl3} commands. It is not necessary to provide the
+ argument structure of the advised command (using \refmmzauto{args}).
+
+\end{doc}
+
+\begin{doc}{
+ key={name=replicate, desc=style}
+ }
+ This key sets up advice which replicates the invocation of the command in
+ the cc-memo during memoization.
+
+ When using this key, it is necessary to set \refmmzauto{args} as well. For
+ \refcmd{index}, Memoize executes
+ \refmmz{auto}|=\index|\bracestt{\refmmzauto{args}=m, \refmmzauto{replicate}}.
+
+ This key takes an auto-option, \docAux{keypath=/mmz/auto/replicate,
+ key={name=expanded}}.\footnote{This option is unrelated to Memoize's options,
+ settable by \refcmd{mmzset}.} If given, the collected arguments will be
+ expanded before replicating them in the cc-memo; in \hologo{LaTeX}, this
+ expansion is |\protect|ed.
+
+ In \hologo{LaTeX}, Memoize submits \refcmd{index} to this handler (with
+ expansion). Therefore, any |\index{key}| in the memoized code gets copied
+ into the cc-memo. Effectively, indexing from within the memoized code ``just
+ works.''
+
+ Note that \refcmd{label}, despite essentially requiring replication, cannot
+ use this advice, because it needs to replicate not only the label key but
+ \refcmd{@currentlabel} as well.
+\end{doc}
+
+\begin{doc}{key={name=run if memoization is possible, desc=style}}
+ Under the run conditions installed by this key, a command is only advised if
+ Memoize is enabled but we're not already ``within Memoize,'' i.e.\ memoizing
+ or normally compiling some code submitted to memoization. In code:
+ \refcmd{ifmemoize}\refcmd{ifinmemoize}|\else|\refcmd{AdviceRuntrue}|\fi\fi|.
+
+ Internally, this key is used by \refmmzauto{memoize} and \refmmzauto{noop}.
+\end{doc}
+
+\begin{doc}{key={name=run if memoizing, desc=style}}
+ Under the run conditions installed by this key, a command is only advised
+ during memoization. In code:
+ \refcmd{ifmemoize}\refcmd{ifmemoizing}\refcmd{AdviceRuntrue}|\fi\fi|.
+
+ Internally, this key is used by \refmmzauto{abort}, \refmmzauto{replicate},
+ and \refmmzauto{ref} and friends.
+\end{doc}
+
+\endgroup % keypath = /mmz/auto
+\endgroup
+
+
+
+\subsubsection{Package CollArgs}
+\label{sec:ref:collargs}
+
+\begingroup
+\yadocset{keypath=/collargs}
+
+\begin{doc}{cmd={
+ name=CollectArguments,
+ par=\oarg{options}\marg{argument specification}\marg{next-code}%
+ \textcolor{gray}{\meta{tokens}}}}
+
+ This command determines the extent to which the \meta{tokens} following the
+ the three formal arguments of the command conform to the given \meta{argument
+ specification}, effectively splitting \meta{tokens} into \meta{argument
+ tokens} and the \meta{rest} of the tokens, and then executes
+ \meta{next-code} with the \meta{argument tokens} provided as a single, braced
+ argument:
+ \begin{center}
+ \meta{next-code}\marg{argument tokens}\textcolor{gray}{\meta{rest}}
+ \end{center}
+ If the initial part of \meta{tokens} does not conform to \meta{argument
+ specification}, \refcmd{CollectArguments} throws an error. (In this case,
+ \meta{next-code} is not executed, and the \meta{tokens} collected until the
+ error are thrown away.)
+
+ The optional \meta{options} are processed using the \pkg{pgfkeys} utility of
+ PGF/\TikZ (see \PGFmanual{87}), with the default path set to
+ \docaux{key path}{collargs}. The given options apply to all the arguments in
+ \meta{argument specification}. The recognized keys are listed in the rest of
+ the section.
+
+ The \meta{argument specification} should be given in the \pkg{xparse} format
+ (we summarize this format in the documentation for \refmmzauto{args} in
+ section~\ref{sec:ref:advice}), with several extensions:\footnote{Collargs
+ internally uses a dot (|.|) to delimit the argument specification from the
+ following argument tokens. Therefore, the dot really counts as an extra
+ argument type, in the sense that Collargs will stop working if the dot
+ becomes an argument type or a modifier in some future release of
+ \pkg{xparse}.}
+ \begin{itemize}
+ \item We introduce modifier \docAux{xparse modifier={name=\&,label=amp,index
+ annotation/.prefix={additional\ }, index annotation/.append={\
+ (options)} }} taking a mandatory argument specifying the options to
+ apply to the following argument in the specification. Options given here
+ override the \meta{options} given as the optional argument.
+ \item The environment body type \docAux{xparse type={name=b,index
+ annotation/.append={\ (environment body)}}} may be followed by an optional
+ \emph{braced} argument providing the name of the environment to collect.
+ The name given here overrides the name given by the
+ \refcollargs{environment} option.
+ \item The number of collected ``arguments'' is unlimited.
+ \end{itemize}
+
+ Also note that the effect of \docref{xparse:O}\marg{default} is the same as
+ the effect of \docref{xparse:o}, and similarly for other pairs of types with
+ and without defaults (\docref{xparse:R} and \docref{xparse:r},
+ \docref{xparse:D} and \docref{xparse:d}, and \docref{xparse:E} and
+ \docref{xparse:e}). CollArgs is dedicated to collecting the argument tokens
+ precisely as they are given: if an optional argument is missing, its default
+ value is \emph{not} inserted among the collected arguments --- consequently,
+ \refcmd{CollectArguments} is utterly uninterested in the default value.
+
+ Collection of environments automatically adapts to the format, i.e.\ given
+ environment body name |foo|, \refcmd{CollectArguments} knows to search for
+ \refcmd{begin}\bracestt{foo} |...| \refcmd{end}\bracestt{foo} in
+ \hologo{LaTeX}, \cs{foo} |...| \cs{endfoo} in \hologo{plainTeX}, and
+ \cs{startfoo} |...| \cs{stopfoo} in \hologo{ConTeXt}. For further
+ information on environment collection, see keys \refcollargs{ignore
+ nesting} and \refcollargs{tags}.
+\end{doc}
+
+\begin{doc}{cmd={
+ name=CollectArgumentsRaw,
+ par=\marg{option-setting code}\marg{argument specification}\marg{next-code}%
+ \textcolor{gray}{\meta{tokens}}}}
+
+ This command is the programmer's interface to CollArgs, intended to be used
+ instead of \refcmd{CollectArguments} when compilation speed is an issue. The
+ two commands only differ in how they deal with options.
+
+ One difference is that for \refcmd{CollectArgumentsRaw}, the options form a
+ mandatory rather than an optional argument. More importantly, however, they
+ do not take the form of a keylist, but should be composed out of low-level
+ option-setting commands. Each key documented in this section has a
+ corresponding low-level macro; these macros are listed in footnotes alongside
+ the keys. The name of the macro starts with |\collargs| and continues with
+ the name of the key, without spaces, each word capitalized; if the key is
+ boolean, this convention applies to the base of the \hologo{TeX}
+ conditional. For example,
+ \begin{tcblisting}{listing only, listing options app={escapechar=|}}
+\CollectArguments[caller=\foo, tags, verbatim]|\marg{argument specification}\marg{next-code}|
+ \end{tcblisting}
+ is equivalent to
+ \begin{tcblisting}{listing only, listing options app={escapechar=|}}
+\CollectArgumentsRaw{%
+ \collargsCaller{\foo}%
+ \collargsBeginTagtrue\collargsEndTagtrue
+ \collargsVerbatim
+}|\marg{argument specification}\marg{next-code}|
+ \end{tcblisting}
+
+ Withing the option-setting code, the programmer may also deploy macro
+ \docaux{cmd}{collargsSet}, which processes the \meta{options} in the keylist
+ format. One idea could be to execute this macro at the end of the low-level
+ options; this would set the ``defaults'' using the fast programmer's
+ interface, but still allow for user customization.
+\end{doc}
+
+
+\begin{doc}[
+ pi={\docaux{cmd}{collargsCaller}},
+ ]{
+ key={name=caller, par=\meta{control sequence (name)},
+ desc={no default, initially \cs{CollectArguments}}},
+ }
+ Set the control sequence to refer to in error messages.
+
+ If \meta{tokens} do not match the \meta{argument specification},
+ \refcmd{CollectArguments} throws an error. By default, the error message
+ contains a reference to \refcmd{CollectArguments} itself, for example %
+ |! Argument of \CollectArguments has an extra }.| However, this might not be
+very informative to the author. When \refcollargs{caller}|=\cs| is in effect,
+the error messages will refer to the given |\cs| instead.
+
+If the value of this key is not a control sequence, it is assumed to be an
+environment name, but as the caller must be a macro, this name will be
+converted into a control sequence. Setting \refcollargs{caller}|=foo| will
+result in error messages referencing |\foo| in \hologo{plainTeX}, |\startfoo|
+in \hologo{ConTeXt} and |\begin{foo}| (a single control sequence!) in
+ \hologo{LaTeX}.
+\end{doc}
+
+\begin{doc}[
+ pi=\docaux{cmd}{collargsEnvironment},
+ ]{key={name=environment, par=\meta{environment name},
+ desc={applicable to type \docref{xparse:b}, no default, initially empty}}
+ }
+
+ Set the name of the environment collected by argument type
+ \docref{xparse:b}.
+\end{doc}
+
+\begin{doc}[
+ pi={\docaux{cmd}{ifcollargsBeginTag}, \docaux{cmd}{ifcollargsEndTag};
+ \refcollargs{tags} has no corresponding low-level command}
+ ]{
+ key={name=begin tag, desc={applicable to type \docref{xparse:b},{ }}, conditional=false},
+ key={name=end tag, desc={applicable to type \docref{xparse:b},{ }}, conditional=false},
+ key={name=tags, par=\meta{boolean}, % not a conditional
+ desc={applicable to type \docref{xparse:b}, style, default |true|}},
+ }
+
+ When \refcollargs{begin tag}\slash \refcollargs{end tag} is in effect, the
+ begin\slash end tag will be will be prepended/appended to the collected
+ environment body. Style \refcollargs{tags} is a shortcut for setting
+ \refcollargs{begin tag} and \refcollargs{end tag} simultaneously.
+
+ In \hologo{LaTeX}, using \refcollargs{tags} will thus dress up the
+ collected body in a pair or \refcmd{begin}\marg{environment name} and
+ \refcmd{end}\marg{environment name}. CollArgs will automatically use the tags
+ appropriate to the format.
+
+ In the verbatim modes, the added tags are verbatim as well, with the detail
+ that in \hologo{LaTeX}, there is a slight difference between the full
+ \refcollargs{verbatim} and the partial \refcollargs{verb} mode. In the
+ full verbatim mode, the braces surrounding the environment name are verbatim
+ (the characters used as braces are actually determined by key
+ \refcollargs{braces}). In the partial verbatim, as well as the
+ non-verbatim mode, the environment name is surrounded by a pair of actual
+ braces of category 1 and 2, regardless of which characters are of these
+ categories in the calling code.
+\end{doc}
+
+\begin{doc}[
+ pi=\docaux{cmd}{ifcollargsIgnoreNesting},
+ ]{
+ key={name=ignore nesting, desc={applicable to type \docref{xparse:b},{ }},
+ conditional=false},
+ }
+ When this key is \emph{not} in effect, CollArgs respects the hierarchical
+ structure created by tag pairs such as \refcmd{begin}\bracestt{foo} and
+ \refcmd{end}\bracestt{foo}. Given the situation below on the
+ left, argument type |b{foo}| will collect everything up until the
+ \emph{second} |\end{foo}|. Now this is what we usually want, because
+\hologo{LaTeX} keeps track of environment embedding as well. However, all
+\emph{verbatim} environments I know of, starting with the standard
+\hologo{LaTeX} \refenv{verbatim}, will ignore the nesting and simply scoop up
+everything up to the first \cs{end}|{verbatim}|. In CollArgs, we can replicate
+their behaviour by setting \refcollargs{ignore nesting}, as shown below on the
+right. (Of course we also need to set \refcollargs{verbatim} if we want to
+grab the environment body in the verbatim mode.)
+
+ \begin{tcbraster}[raster columns=2]
+ \begin{tcblisting}{listing only, mark region={1}{5},
+ example title=\texttt{ignore nesting=false}}
+ ...
+ \begin{foo}
+ ...
+ \end{foo}
+ ...
+\end{foo}
+ \end{tcblisting}
+ \ExampleName{_collargs-verbatim}
+ \makeexample{\examplename.tex.c1}
+ \tcbinputexample{listing only, no attachment, mark region={1}{3},
+ example title=\texttt{ignore nesting=true}}
+ \end{tcbraster}
+
+ This key applies not only to argument type \docref{xparse:b} (in either
+ normal or verbatim mode), but also to the verbatim argument type
+ \docref{xparse:v} and to argument types \docref{xparse:m} and
+ \docref{xparse:g} in the \refcollargs{verbatim} (but not normal, or
+ \refcollargs{verb}) mode. With these keys, the relevant structure markers
+ are braces, |{| and |}|.
+\end{doc}
+
+\begin{doc}[
+ pi=\docaux{cmd}{ifcollargsIgnoreOtherTags},
+ ]{
+ key={name=ignore other tags, desc={applicable to type \docref{xparse:b},{ }},
+ conditional=false},
+ }
+ In \hologo{LaTeX}, the environment tags, \refcmd{begin}\marg{name} and
+ \refcmd{end}\marg{name}, contain braces, which retain their usual category codes
+ in the non-verbatim and in the partial verbatim mode. Consequently, CollArgs
+ cannot easily search for the full tags to delimit the
+ environment.
+
+ When this key is \emph{not} in effect, CollArgs takes the easy path, and
+ determines the end of the environment only by inspection of \cs{begin}s and
+ \cs{end}s, without reference to what \meta{name} they begin or end. Only
+ when this key \emph{is} in effect does CollArgs inspect these \meta{name}s,
+ effectively ignoring any \cs{begin}s and \cs{end}s not followed by the name of
+ environment being collected. The effect of absence vs.\ presence of this key
+ is shown below, where the shaded area marks the code collected into
+ environment |foo|.
+
+ \begin{tcbraster}
+ \ExampleName{_collargs-ignore-other-tags}
+ \makeexample{\examplename.tex.c1 N=2}
+ \tcbinputexample{listing only, no attachment, mark region={3}{3},
+ example title=\texttt{ignore other tags=false}}
+ \tcbinputexample[.tex][.c2]{listing only, no attachment, mark region={3}{5},
+ example title=\texttt{ignore other tags=true}}
+ \end{tcbraster}
+
+ This key does not have any effect the full verbatim mode, which always
+ behaves as if this key was set to true, because braces are of category
+ ``other'' as well. Similarly, it is as if this key was always true in
+ \hologo{plainTeX} and \hologo{ConTeXt}, simply because environment tags in
+ these formats don't contain braces.
+\end{doc}
+
+\begin{doc}[
+ pi={\raggedright
+ \docaux{cmd}{collargsAppendPreprocessor},
+ \docaux{cmd}{collargsPrependPreprocessor},
+ \docaux{cmd}{collargsAppendPostprocessor},
+ \docaux{cmd}{collargsPrependPostprocessor}},
+ ]{
+ key={name=append preprocessor, par=\meta{code}, desc={style, no default}},
+ key={name=prepend preprocessor, par=\meta{code}, desc={style, no default}},
+ key={name=append postprocessor, par=\meta{code}, desc={style, no default}},
+ key={name=prepend postprocessor, par=\meta{code}, desc={style, no default}},
+ }
+ These keys declare processors which will transform the collected argument
+ before appending it to the argument list.
+
+ A collected argument undergoes the following transformations:
+ \begin{itemize}
+ \item First, the argument is processed by any \emph{pre}processors, in the
+ order indicated by |append| and |prepend|.
+ \item Next, the processed argument is dressed up in the delimiters according
+ to its type. For example, an optional argument of type \docref{xparse:o}
+ will be surrounded by square brackets.
+ \item Finally, the delimited argument is processed by any
+ \emph{post}processors, again in the order indicated by |append| and
+ |prepend|.
+ \end{itemize}
+
+ \meta{code} will typically consist of a single control sequence pointing to a
+ one-argument macro, which will receive the collected argument (possibly
+ modified by the processors already applied). In general, however, the value
+ of this key may be any code; Collargs will execute \meta{code}\marg{collected
+ argument}.
+
+ The processed argument should be returned by storing it into token register
+ \docaux{cmd}{collargsArg}.
+
+ The following example illustrates how one could go about reimplementing Bruno
+ Le Floch ingenious package \pkg{cprotect}.\footnote{This example is merely a
+ proof of concept. For the bells and whistles which would make it useful in
+ real life, see the documentation of \pkg{cprotect}.} We define processor
+ |\writetofile| which dumps the argument into a file, replacing it with the
+ |\input| statement. (Of course, to allow for verbatim content in the
+ footnote, we also have to mark the argument as \refcollargs{verbatim}.
+ And we use \refcollargs{no delimiters} to get rid of the braces around the
+ footnote text.)
+
+ \ExampleName{collargs-processor}
+ \makeexample{\examplename.tex.c1}
+ \tcbinputexample{listing and text, listing file=\examplepath.tex.c1}
+\end{doc}
+
+\begin{doc}[
+ pi={\docaux{cmd}{collargsClearPreprocessors},
+ \docaux{cmd}{collargsClearPostprocessors}},
+ ]{
+ key={name=clear preprocessors}, desc=style,
+ key={name=clear postprocessors}, desc=style,
+ }
+ Clear the list of pre- or post-processors.
+\end{doc}
+
+\begin{doc}[
+ pi={\docaux{cmd}{collargsAppendExpandablePreprocessor},
+ \docaux{cmd}{collargsPrependExpandablePreprocessor},
+ \docaux{cmd}{collargsAppendExpandablePostprocessor},
+ \docaux{cmd}{collargsPrependExpandablePostprocessor}
+ },
+ ]{
+ key={name=append expandable preprocessor, par=\meta{code}, desc={style, no default}},
+ key={name=prepend expandable preprocessor, par=\meta{code}, desc={style, no default}},
+ key={name=append expandable postprocessor, par=\meta{code}, desc={style, no default}},
+ key={name=prepend expandable postprocessor, par=\meta{code}, desc={style, no default}},
+ }
+ These keys may be used to add fully expandable processors. A processor added
+ with one of these keys will end up among the processors declared by
+ \refcollargs{append preprocessor} et al.
+
+ A processor declared by one of these keys will define the processed argument
+ as the full expansion (|\edef|) of \meta{code}\marg{collected argument}.
+ \meta{code} will typically consist of a single control sequence pointing to a
+ fully expandable one-argument macro.
+
+ For example, |\trim at spaces@noexp| from package \pkg{trimspaces} could be used
+ as an expandable processor of environment body to remove the spaces around
+ the grabbed environment body.
+\end{doc}
+
+\begin{doc}[
+ pi={\docaux{cmd}{collargsAppendPrewrap},
+ \docaux{cmd}{collargsPrependPrewrap},
+ \docaux{cmd}{collargsAppendPostwrap},
+ \docaux{cmd}{collargsPrependPostwrap}},
+ ]{
+ key={name=append prewrap, par=\meta{macro definition}, desc={style, no default}},
+ key={name=prepend prewrap, par=\meta{macro definition}, desc={style, no default}},
+ key={name=append postwrap, par=\meta{macro definition}, desc={style, no default}},
+ key={name=prepend postwrap, par=\meta{macro definition}, desc={style, no default}},
+ }
+ These keys add processors which transform the collected argument in a single
+ expansion.
+
+ The declared processor will use \meta{macro definition} to define a temporary
+ one-argument \meta{macro}, and then set the \meta{processed argument} to be
+ the single expansion of \meta{macro}\marg{collected argument}.
+
+ For example, to add quotes around the collected argument, write
+ \refcollargs{append prewrap}|={``#1''}| (doubling the hash when executing
+ \refcmd{CollectArguments} from a macro, of course). Or, perhaps more
+ usefully, \refcollargs{append prewrap}|={\scantokens{#1}}| can be used to
+ retokenize a verbatim argument (during the execution of the
+ \meta{next-code}).
+\end{doc}
+
+
+\begin{doc}[
+ pi=\docaux{cmd}{ifcollargsNoDelimiters}
+ ]{
+ key={name=no delimiters, conditional=false},
+ }
+ When this key is in effect, the collected argument will not be dressed up
+ into delimiters that it was dressed up in \meta{argument tokens}. For
+ example, an optional argument, encountered as [\meta{argument}] inside
+ \meta{argument tokens}, will be spit out simply as \meta{argument}.
+
+ Any user-specified pre- or post-processing will still be applied.
+ \ExampleName{collargs-nodelimiters}
+ \makeexample{\examplename.tex.c1}
+ \tcbinputexample{comment=\input{\examplepath.tex.c1}}
+\end{doc}
+
+\begin{doc}[
+ pi=\docaux{cmd}{ifcollargsBraceCollected}
+ ]{
+ key={name=brace collected, conditional=true},
+ }
+ When this conditional is set to false, the collected arguments are not
+ enclosed in braces when passed on to \meta{next-code}. This probably only
+ makes sense when wrapping the individual arguments, e.g.\ by
+ \refcollargs{append postwrap}|={{#1}}|.
+\end{doc}
+
+\begin{doc}[
+ pi={\docaux{cmd}{collargsVerbatim},
+ \docaux{cmd}{collargsVerb},
+ \docaux{cmd}{collargsNoVerbatim}.
+ To ensure the same effect as with the keys,
+ place these macros at the end of the option code},
+ ]{
+ key={name=verbatim, desc=style},
+ key={name=verb, desc=style},
+ key={name=no verbatim, desc={style, the initial mode}},
+ }
+ Select the full verbatim, the partial verbatim, or the non-verbatim mode of
+ argument collection.
+
+ In the full \refcollargs{verbatim} mode, the arguments are collected under
+ a category code regime in which all characters are of category 12, ``other''.
+ The same goes for the partial \refcollargs{verb} mode, except that in this
+ case, the grouping characters --- usually the braces |{| and |}| --- retain
+ their usual category codes 1 and 2. Key \refcollargs{no verbatim} selects
+ the normal, non-verbatim mode.
+
+ The partial \refcollargs{verb} mode can be useful for verbatim collection
+ of an optional argument. To pass |]| as an optional argument to command
+|\foo|, we normally enclose it in braces: ˇ\foo[{~]~}]ˇ. However, if we try to
+collect |[{]}]| with \refcmd{CollectArguments}|[|\refmmz{verbatim}|]{o}|, we
+will get |{| (and most likely an error, as well), because in the
+ \refmmz{verbatim} mode, braces do not have their grouping function. Using
+ the \refmmz{verb} mode solves the problem: occuring within braces, the first
+ |]| is ``invisible'' to \refcmd{CollectArguments}|[|\refmmz{verb}|]|, so the
+optional argument is correctly recognized as ending at the second |]|.
+
+The partial \refcollargs{verb} mode is also useful for collecting the bodies
+of \emph{\hologo{LaTeX}} environments. The full \refcollargs{verbatim} mode
+will only correctly collect these bodies when the relevant \refcmd{begin}
+and\slash or \refcmd{end} control sequences are followed by the grouped
+environment name without any intervening spaces. The partial
+\refcollargs{verb} mode has no such restriction.
+
+In the verbatim modes, modifier \docref{xparse:+} has no effect. The arguments
+are always collected as if they were long.
+
+To correctly collect arguments in the verbatim modes, CollArgs has to mimic the
+many details of \hologo{TeX}'s tokenization and argument delineation. These
+details depend on the category code regime, and CollArgs automatically adapts
+to the ``outside'' category code regime, i.e.\ the regime in effect at the time
+of invoking \refcmd{CollectArguments}. In particular, CollArgs remembers which
+characters were of category codes 0, 1, 2, 5, 10 and 11, and adapts the
+argument collection accordingly. For example, it will correctly pick up a
+control sequence as a single-token \docref{xparse:m}-type (\hologo{TeX}'s undelimited)
+argument even when it begins with a non-standard character of category code 0.
+The single caveat is that only a single pair of characters can function as the
+grouping characters in the full verbatim mode; to compensate for the
+deficiency, this character pair is customizable via key
+\refcollargs{braces}.
+\end{doc}
+
+\begin{doc}[
+ pi={\docaux{cmd}{collargsFixFromVerbatim},
+ \docaux{cmd}{collargsFixFromVerb},
+ \docaux{cmd}{collargsFixFromNoVerbatim}},
+ ]{
+ key={name=fix from verbatim, desc=style},
+ key={name=fix from verb, desc=style},
+ key={name=fix from no verbatim, desc=style},
+ }
+ Key \refcollargs{fix from no verbatim} should be used when the first
+ argument should be collected in a verbatim mode, but the outside code has
+ already tokenized the first character of the subsequent input stream (most
+ probably by a |\futurelet|) in the non-verbatim category code regime. Using
+ this key will trigger a CollArgs' ``mode transition'' (described below) which
+ will fix the situation. (This key is used in the implementation of
+ \refcmd{mmz}.)
+
+ The other two keys should be used in the unlikely reverse situation, where
+ the outside code has tokenized the following character in a verb(atim) mode,
+ while CollArgs is requested to collect the first argument in the non-verbatim
+ mode.
+\end{doc}
+
+\begin{doc}[
+ pi=\docaux{cmd}{collargsBraces}
+ ]{
+ key={name=braces, par={\meta{begin-group char}\meta{end-group char}},
+ desc={no default, for the initial value see below}},
+ }% here
+ This key sets the verbatim begin-group and end-group characters. The setting
+ affects collection of argument types \docref{xparse:m}, \docref{xparse:g},
+ \docref{xparse:v}, \docref{xparse:l}, \docref{xparse:e} and (in
+ \hologo{LaTeX}) \docref{xparse:b} in the \emph{full} verbatim
+ mode.\footnote{The choice of the verbatim grouping characters also affects
+ the effect of \refcollargs{begin tag} and\slash or \refcollargs{end tag};
+ see the documentation of these keys for details.}
+
+ For example, in the non-verbatim and the partial verbatim mode, an
+ \docref{xparse:m}-type argument may be delimited by any characters of
+ category code 1 (``begin-group'') and 2 (``end-group''). In the full
+ verbatim mode, there are of course no characters of these categories, so
+ CollArgs internally assigns the grouping function to some pair of characters.
+ When entering the full verbatim mode, CollArgs automatically sets the
+ verbatim grouping characters to characters which were of categories 1 and 2
+ in the ``outside'' category code regime, i.e.\ the regime in effect at the
+ time of invoking \refcmd{CollectArguments}. However, in contrast to
+ \hologo{TeX}'s internal argument parser, only one pair of characters may
+ serve as the begin-group and the end-group character in CollArgs' full
+ verbatim mode. In case multiple characters were of category 1 or 2 on the
+ outside, CollArgs therefore has to make a choice, and it chooses the
+ candidate with the lowest character code. This choice may be overridden by
+ the user by invoking key \refcollargs{braces}; the user may even choose
+ characters which did not belong to categories 1 and 2 in the outside regime.
+
+ When \meta{begin-group char} and \meta{end-group char} are of categories 1
+ and 2 in the outside category regime, they must be enclosed in a triple
+ group. For example, if both |()| and |{}| have the grouping function on the
+ outside, and the user wants to select |{}| as the verbatim grouping
+ characters (CollArgs would go for |()|, as this pair has lower character
+ codes), the correct way to invoke this key is |braces={{{{}}}}| or
+ |braces=((({})))|. \footnote{This complication is due to the details of
+ \pkg{pgfkeys}' keylist processing, and does not apply to
+ \refcmd{collargsBraces}.}
+\end{doc}
+
+\begin{doc}[
+ pi=\docaux{cmd}{collargsVerbatimRanges}
+ ]{
+ key={name=verbatim ranges,
+ par={\bracestt{\meta{from}\texttt{-}\meta{to}(\texttt{,} \meta{from}\texttt{-}\meta{to})*}},
+ desc={no default, initially \texttt{0-255}}},
+ }
+ If run under the \hologo{pdfTeX} or \hologo{XeTeX} engine, this key
+ determines which characters will be assigned category code 12 in the verbatim
+ mode. In \hologo{pdfTeX}, the range should remain at the initial |0-255|,
+ but in \hologo{XeTeX}, some rare situations might require extending this
+ range (don't attempt to set the full range of |0-1114111|, as this would be
+ very slow and you would most likely run out of save stack).
+
+ In \hologo{LuaTeX}, we switch the category code regime using category code
+ tables, so this key has another meaning: it determines the range in which
+ CollArgs will scan for characters of category codes 1, 2 and 14, whose
+ identity it needs to know, for internal reasons.
+\end{doc}
+
+
+
+\paragraph{Mode transition limitations}
+
+\refcmd{CollectArguments} has some minor limitations regarding the transition
+from a verbatim into non-verbatim mode, or vice versa. The gist of the issue
+is best illustrated with the optional argument type \docref{xparse:o} collected
+in the verbatim mode. CollArgs determines whether an argument of this type is
+present by peeking ahead (using \hologo{TeX}'s |\futurelet| primitive) into the
+input stream. If the argument is present (i.e.\ if the input stream continues
+with an open bracket, |[|%]
+), all is well. But when the optional argument is absent, the peek-ahead will
+tokenize the following character, which presents a problem when no more
+arguments are present in the input stream, like in the example below, where the
+verbatim \docref{xparse:o} is the (only and) final type in the argument
+specification. In this case, the peek-ahead ``incorrectly'' assigns category
+code 12 (``other'') to the first |$|. %$
+This character was intended to be tokenized as the math shift character of
+category 3, to start the math mode after \refcmd{CollectArguments} is finished,
+but having been assigned category code 12, it cannot perform this function,
+resulting in error |! Missing $ inserted| %$
+once \hologo{TeX} encounters the superscript character |^|.
+
+\ExampleName{collargs-transition-ok}
+\makeexample{\examplename.tex.c1}
+\tcbinputexample{sidebyside, lefthand ratio=0.6,
+ comment=\input{\examplepath.tex.c1}}
+
+Well --- this is what \emph{would} happen if CollArgs didn't address the
+transition issue described above. In fact, the above example compiles just
+fine, because CollArgs \emph{does} address this issue, but unfortunately,
+certain transition problems simply cannot be resolved --- read on to learn what
+can go wrong.
+
+For example, you can typeset the name of the document author via
+\hologo{LaTeX}'s internal command |\@author|, but to use this command in the
+document, you have to precede it by |\makeatletter|. As shown by the first
+line of the example below, this works rather nicely: |\makeatletter| sets the
+category code of |@| to 11 (``letter''), so |@| may help form the control word
+|\@author| --- importantly, |\makeatletter| sets the category code of |@|
+\emph{before} control sequence |\@author| is constructed, even if it precedes
+it immediately.
+
+\ExampleName{collargs-transition-cs}
+\makeexample{\examplename.tex.c1}
+\long\def\ShowArgs#1{Collected: ``{\color{red}\tt\detokenize{#1}}''}
+\tcbinputexample{comment=\input{\examplepath.tex.c1}}
+
+In the second line of the example, our clever invocation of |\@author| is
+immediately preceded by a call to \refcmd{CollectArguments}, which tries to
+collect a verbatim argument of type \docref{xparse:o}. It doesn't find it, which results in
+the wrong, verbatim tokenization of the escape character of |\makeletter|.
+CollArgs realizes the problem and tries to fix it. But while it is searching
+for the end of control sequence |\makeletter| (which it successfully
+constructs), it triggers the tokenization of what follows --- which, as |@| is
+at that point of category 12 (``other''), yields the control symbol |\@| (later
+followed by word ``author'', typeset in the example).
+
+In short, the solution has created another, delayed instance of the problem ---
+an instance which cannot be addressed any further. But we're nevertheless
+better off, as this particular issue will bite only in the case when the
+``corrupted'' control sequence immediately following the invocation
+\refcmd{CollectArguments} changes category codes in a way that affects the
+tokenization of what immediately follows it.
+
+This was an example of what can go wrong in the transition from the
+\meta{argument tokens} to the \meta{rest} of the tokens following an invocation
+of \refcmd{CollectArguments}. As the ``outside world'' is non-verbatim, this
+transition can only be problematic if the argument which corrupted the first of
+the \meta{rest} tokens was verbatim, so if the transition was from the verbatim
+to the non-verbatim mode. Such a transition can also occur within the
+\meta{argument tokens}, but the good news here is that CollArgs successfully
+solves any problems that occur there, so you should only worry about the
+end-of-arguments situation.
+
+The other direction of the transition, from the non-verbatim to the verbatim
+mode, however, can affect both the internal and the external transitions. Let
+us illustrate the problem with the internal transition. Say you want to
+collect an optional argument (\docref{xparse:o}) in the non-verbatim mode, and
+then a mandatory argument (\docref{xparse:m}) in (either full or partial)
+verbatim mode. In the first invocation of \refcmd{CollectArguments} below, the
+optional argument is present, and we get what we expect: the percent sign is
+collected, verbatim, into the mandatory argument. In the second invocation,
+however, the percent character retains its usual commenting function ---
+despite the fact that we have requested verbatim mode for the mandatory
+argument --- which results in the group in the second line being picked up as
+the mandatory argument. Again, this happens because CollArgs has to peek ahead
+in the input stream when determining whether the optional argument is present.
+Having requested non-verbatim mode for the optional argument, the peeking is
+performed in the non-verbatim mode, and as the optional argument is not
+present, it finds the comment character, which fulfills its regular function of
+disappearing along with the rest of the line. Once CollArgs sets to find the
+(verbatim) mandatory argument, the rest of the line is already gone, so it
+searches for, and finds, this argument in the next line.
+
+\ExampleName{collargs-transition-comment}
+\makeexample{\examplename.tex.c1}
+\long\def\ShowArgs#1{Collected: ``{\color{red}\tt\detokenize{#1}}''}
+\tcbinputexample{listing and text}
+
+Nothing can be done here --- commenting deletes information, irrevocably ---
+and in a similar fashion, nothing can be done to catch a verbatim end-of-line
+character when preceded by an absent optional argument in the non-verbatim mode
+(because it was already tokenized into a space token).
+
+Finally, the transition issues are not limited to transits from argument type
+\docref{xparse:o}. The full list of argument types which give rise to
+transition problems (when transiting \emph{from} arguments of these types) is
+as follows: \docref{xparse:o}, \docref{xparse:d}, \docref{xparse:s},
+\docref{xparse:t}, \docref{xparse:g}, \docref{xparse:e}.
+
+
+\endgroup % /collargs
+
+
+\section{Varia}
+
+\subsectionclearpagefalse
+
+\subsection{Known issues}
+\label{sec:known-issues}
+
+\paragraph*{Bitmap graphics export} is coming up in the next release of the
+package.
+
+
+\paragraph{An error occurs, but disappears in the next compilation}
+If a non-fatal internal \hologo{TeX} error occurs during memoization, the memos
+and externs may be nevertheless produced and utilized in subsequent
+compilations. In such a case, the erroneous code won't be compiled again, and
+therefore won't yield any errors, giving the mistaken impression that the code
+is error-free.
+
+This problem does not apply to errors which trigger \refcmd{errmessage},
+because that control sequence is advised by \refmmzauto{abort}.
+Note that internal \hologo{TeX} errors like |Undefined control sequence| are
+\emph{not} reported through \refcmd{errmessage}, and will therefore cause the
+issue.
+
+This problem does not affect \hologo{LuaTeX}, because this engine allows
+Memoize to detect errors and abort memoization if it encounters any.
+
+
+\paragraph{A minimal issue with \hologo{XeLaTeX}}
+If the very first page of a document of class |minimal|, compiled by
+\hologo{XeLaTeX}, happens to be an extern page, we have a problem: all the
+regular pages of the document will be of the same size as that extern page.
+\hologo{pdfLaTeX} and \hologo{LuaLaTeX} do not exhibit this behaviour, nor do
+\hologo{LaTeX} classes other than |minimal|, even when compiled with
+\hologo{XeLaTeX}.
+
+
+\paragraph{CollArgs}
+
+Due to an unfortunate design decision, CollArgs does not accept a dot |.| as
+the \meta{token} argument of types, \docref{xparse:r}, \docref{xparse:R},
+\docref{xparse:d}, \docref{xparse:D}, and \docref{xparse:t}. Furthermore,
+\refcollargs{append prewrap} and friends (and their macro counterparts) do not
+accept a parameter symbol consistently. These issues will be fixed in the next
+release.
+
+
+\subsection{Troubleshooting}
+\label{sec:troubleshooting}
+
+\paragraph{Extern extraction does not work}
+\label{sec:fadings-problem}
+\label{sec:loading-order}
+
+Upon an unsuccessful extern extraction, you should get one of the errors or
+warnings listed below. They should appear both in the terminal output and in
+the log file. All but the final error can also appear when extern extraction
+is performed externally.
+
+
+Errors:
+
+\begin{enumerate}
+
+\item \code{Error: Python module 'pdfrw' was not found'},\\
+ \code{Error: Perl module 'PDF::API' was not found"}, or\\
+ \code{Error: Perl module 'PDF::Builder' was not found"}
+
+ You have not installed the PDF processing library, at least not
+ successfully. The installation instructions can be found in
+ section~\ref{sec:install}. Perhaps you have multiple instances of Perl\slash
+ Python on your system and you have installed the library into the wrong
+ instance?
+
+\item\label{item:warning:extract:cannot-read} \code{Error: File
+ "\meta{jobname}.pdf" seems corrupted. Perhaps you have to load Memoize
+ earlier in the preamble?}
+
+ While the PDF could be corrupted for various reasons, the most probable
+ reason is the package loading order, as indicated in the error message
+ itself. (Note that the absence of \meta{jobname}|.pdf| does not yield this
+ error. This is so that a fatal error unrelated to Memoize doesn't make
+ Memoize throw another error in the next compilation.) Embedded extern
+ extraction requires an intact document PDF from the previous compilation, so
+ Memoize must be loaded before the document PDF is opened for writing the
+ results of the ongoing compilation. In particular, the PDF is opened by \PGF
+ library |fadings|, included by \TikZ's libraries |fadings| and |shadows|, so
+ Memoize must be loaded before any of these libraries. With \pkg{beamer}, the
+ problem is particularly acute because the PDF is opened while loading the
+ class. In this case, simply moving |\usepackage{memoize}| up the preamble,
+ as suggested, won't help: you have to write |\RequirePackage{memoize}|
+ \emph{before} |\documentclass{beamer}|!
+
+ It is also possible that the PDF processing library deployed by the
+ extraction script chokes on your particular document. Inspecting the error
+ message original error message produced by Perl\slash Python, which can be
+ found at the end of the long error message in the log, should help you see
+ whether this is the case. For example, a couple of times, I got
+ \code{%\myfontexpansionon
+ Invalid dictionary key at /.../perl5/site\_perl/PDF/API2/Basic/PDF/File.pm
+ line $N$\myfontexpansionoff} when extracting with the Perl-based script.
+ The issue might be related to PDF version --- by default, \hologo{TeX}
+ produces PDFs of version 1.5, while the PDF library
+ \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} officially only
+ supports versions up to 1.4 --- but I'm afraid I haven't identified the exact
+ circumstances yet (possibly, the externalizing a picture containing an
+ embedded PDF file might be the culprit). In general, the workaround is to
+ use another extraction method or PDF processing library, see \refmmz{extract}
+ and \refscript{memoize-extract.pl--library}.
+
+\item\label{item:error:openin_any}
+ \code{Error: I'm not allowed to write to '{\meta{filename}}' (openout\_any =
+ {\meta{mode}})"}
+
+ Your \hologo{TeX} distribution does not allow this file to be written to.
+ The relevant setting is called \docref{openin_any} in \hologo{TeX}Live and
+ |[Core]AllowUnsafeOutputFiles| in MiK\hologo{TeX}.
+
+\item \code{Error: I'm not allowed to read from '{\meta{filename}}' (openin\_any
+ = {\meta{mode}})"}
+
+ Your \hologo{TeX} distribution does not allow this file to be read from. The
+ relevant setting is called \docref{openout_any} in \hologo{TeX}Live and
+ |[Core]AllowUnsafeInputFiles| in MiK\hologo{TeX}.
+
+\item \code{Error: "Semi-absolute" paths are disallowed: '\meta{filename}'}
+
+ This error can only occur on Windows. The extraction scripts are even more
+ paranoid than \hologo{TeX} and don't allow paths such as |C:foo\bar| or
+ |\foo\bar|, neither as path to a \dmmz, memo or extern file nor as an output
+ or temporary directory.
+
+\item \code{Error: I cannot extract page {\meta{n}} '\meta{filename}.pdf', as
+ it contains only \meta{N} pages}
+
+ The document PDF is too short. This could happen if you are accidentally
+ triggering the extern extraction twice, and the first extraction was called
+ with \refscript{memoize-extract.pl--prune}.
+
+\item\label{item:error:runtime} \code{Error: Python error: \meta{error
+ message}} or \code{Error: Perl error: \meta{error message}}
+
+ A runtime error occurred during the execution of the extraction script. The
+ error message should be followed by the traceback information which includes
+ the line number (in the extraction script) where the error occurred.
+
+ One possible source of this error are insufficient filesystem permissions.
+ For example, it will occur if the current directory is marked as read-only
+ and the temporary output directory (\texttt{\$TEXMFOUTPUT}) is not set.
+
+\item \code{Error: Extraction of externs from document "\meta{jobname}.pdf"
+ using method "\meta{extraction method}" was unsuccessful.}
+
+ This is a generic error produced when the extraction script was either not
+ executed at all, or the execution didn't finish properly.\footnote{How does
+ Memoize know whether this happened? When invoked given option
+ \refscript{memoize-extract.pl--format}, the extraction script is supposed
+ to write a log (actually, status) file called
+ % todo: \mbox is a workaround preventing "font expansion: using fonts with
+ % different limit of expansion in one paragraph is not allowed" error,
+ % which arises in combination with "pi=..." in the CollArgs section.
+ \mbox{\meta{jobname}}|.mmz.log|. Upon a clean completion, the final line
+ of this log reads \cs{endinput}. If this marker is missing, Memoize
+ produces the error under discussion. Feel free to see what's up with this
+ file if all else fails.}
+
+ A couple of reasons for the failure to execute the script:
+ \begin{itemize}
+ \item Is the shell escape configured properly? It should be, as both
+ Memoize's extraction scripts are listed among restricted shell escape
+ commands in both \hologo{TeX}Live and MiK\hologo{TeX}, but it never hurts
+ to check.
+
+ The easiest way to see if shell escape is the culprit is to compile the
+ document with command option \docref{-shell-escape} (on \hologo{TeX}Live)
+ or \docref{--enable-write18} (on MiK\hologo{TeX}). If this resolves the
+ problem, check your shell escape mode; see
+ footnote~\ref{fn:shell-escape-commands} on
+ page~\pageref{fn:shell-escape-commands} for how to do this.
+ \item Is environment variable PATH set correctly, so that the system can find
+ the extraction script?
+ \item Is Perl\slash Python installed on your system, and accessible to
+ \hologo{TeX}?
+ \end{itemize}
+
+ Normally, you should get error~\ref{item:error:runtime} if a runtime error
+ occurs during the execution of the extraction script. You can only get the
+ generic error if the runtime error occurred before the script could set up
+ the log file, perhaps due to insufficient filesystem permissions in
+ combination with the paranoid \docref{openout_any} setting (which is the
+ default).
+
+ If the source of the error remains a mistery, I suggest inspecting the
+ following sources of information, to help you with your investigation:
+ \begin{itemize}
+ \item Can you can run the extraction script by hand? Open the terminal, go
+ into the directory containing your document, write |memoize-extract.pl
+ |\meta{document name} (or |memoize-extract.py |\meta{document name}), and
+ see what happens.
+ \item Inspect \meta{jobname}|.log| --- search for
+ \code{runsystem(memoize-extract.pl ...)}, it will tell you whether the
+ script was executed.
+ \item Inspect the \hologo{TeX} terminal output --- if the script was
+ executed, it should've announced itself by \code{Extracting externs from
+ \meta{jobname}.pdf}; are there any further messages between this header
+ and the error message?
+ \item Inspect \meta{the path to the extern}|.log|, if you are using
+ \hologo{TeX}-based extraction
+ (\code{\refmmz{extract}=\refmmz{extract=tex}}).
+ \end{itemize}
+
+\end{enumerate}
+
+Not every failure to extract externs results in an error. When a warning is
+produced, the compilation will succeed, it's just that as Memoize cannot
+extract the externs, they will be produced, and dumped into your document, at
+each and every compilation. For warnings other than the missing document PDF,
+Memoize extracts as many externs as possible.
+
+\begin{enumerate}
+
+\item \code{Warning: Cannot open '\meta{filename}.pdf'}
+
+ This is not an error because what if you deleted the document PDF on purpose?
+
+\item\label{item:warning:extract:unexpected-size} \code{Warning: I refuse to
+ extract page $n$ from "\meta{jobname}.pdf", because its size
+ ($\meta{width}\times\meta{height}$) is not what I expected\\
+ ($\meta{expected width}\times\meta{expected height}$)}
+
+ If the compilation which produced the offending extern pages yielded any
+ errors, you should probably disregard this warning, fix the errors, and
+ compile again. Otherwise, you have somehow winded up with mismatched
+ \meta{jobname}|.pdf| and \meta{jobname}\dmmz (the latter file contains
+ instructions on which pages to extract, complete with the expected
+ dimensions). Are you sure that they were produced by the same compilation,
+ and have remained untouched since? Are you perhaps trying to perform the
+ extraction the second time, after the first extraction
+ \refscript{memoize-extract.pl--prune}d the PDF?
+
+ If the warning stubbornly persists, but you are sure that the page the script
+ is refusing to extract is correct, you can force the extraction by adding
+ option \refscript{memoize-extract.pl--force} to the script invocation, which
+ can be set by \refmmz{perl extraction options}. However, as such a situation
+ probably indicates a bug in Memoize, please let me know about it.
+
+\item \code{Warning: I refuse to extract page $n$ into extern \meta{extern
+ filename}, because the associated (c)c-memo does not exist.}
+
+ Assuming that you haven't deleted the memos (|.memo| files), either manually
+ or via \refscript{memoize-clean.pl}, could it be that they were never created
+ in the first place? Check where they should be written to (the configuration
+ commands are listed in section~\ref{sec:ref:dirs}). Is that directory
+ writeable, both in the sense of the system and for \hologo{TeX} (the
+ |openout| setting in |texmf.cnf|)?
+
+
+\end{enumerate}
+
+
+% warning('I failed to execute "kpsewhich"; assuming openin_any = openout_any =
+% "p" '
+
+\paragraph{An extern won't be included}
+
+Did you receive a warning or error message?
+\begin{enumerate}
+\item \code{Package memoize Warning: Unexpected size of extern "\meta{extern
+ path}.pdf"; expected\\ $\meta{expected width}\times\meta{expected
+ height}$, got $\meta{width}\times\meta{height}$}
+
+ This warning is related to warning~\ref{item:warning:extract:unexpected-size}
+ above, only that it occurs once the extern is extracted. The same
+ investigative methods apply.
+\item \code{!pdfTeX error: pdflatex (file \meta{extern path}.pdf): reading
+ image file failed}, or something similar for engines other than
+ \hologo{pdfTeX}
+
+ This is a fatal error. The extern file got corrupted, somehow --- inexistent
+ and even empty extern files merely trigger recompilation.
+\item \code{pdfTeX warning: pdflatex (file \meta{extern path}.pdf): PDF
+ inclusion: found PDF version <$m$>, but at most version <$n$> allowed}
+
+ When you produced the externs, a higher \docref{reg:pdfmajorversion} and/or
+ \docref{reg:pdfminorversion} was in effect than now. I guess you shouldn't worry
+ about this warning if the output looks fine.
+\end{enumerate}
+
+If there was no warning or error --- are you certain that Memoize is
+\refmmz{enable}d, and that it is not in the \refmmz{recompile} mode? Remember
+that these settings can also apply only to a part of the document; search for
+any stray \refcmd{mmzset} or \refcmd{mmznext} commands.
+
+
+\paragraph*{Warnings about duplicate labels, indices, etc.} may be safely
+disregarded.
+
+Externalization causes any (non-immediate) |\write| commands in the extern to
+be executed twice, once upon the shipout of the regular page, and once upon the
+shipout of the extern page. This results in warnings about doubly defined
+labels, hyperreferences, indices, etc. For example, you might get
+\texttt{LaTeX Warning: Label `<name>' multiply defined} or \texttt{warning (pdf
+ backend): ignoring duplicate destination with the name '<name>'}. You can
+safely disregard these warnings; they will disappear once the extern is
+utilized.
+
+
+\paragraph{Memoization was aborted}
+
+This warning means that either:
+\begin{itemize}
+\item you are trying to (auto)memoize a \refenv{tikzpicture} with
+ \refkey{/tikz/remember picture} set, or more generally, some code which
+ contains \refcmd{savepos} --- this can't be done, see
+ section~\ref{sec:tut:good-to-know}; or
+\item an error occurred during memoization --- in this case, Memoize cowardly
+ refuses to proceed with memoization, see section~\ref{sec:known-issues} for
+ details.
+\end{itemize}
+
+
+\subsection{License}
+
+Copyright \textcopyright\ 2020- Sašo Živanović.
+
+This work may be distributed and/or modified under the conditions of the
+\hologo{LaTeX} Project Public License, either version 1.3c of this license or
+(at your option) any later version. The latest version of this license is
+in \url{https://www.latex-project.org/lppl.txt} and version 1.3 or later is
+part of all distributions of \hologo{LaTeX} version 2008 or later.
+
+This work comprises of the sources, generated files, accompanying scripts,
+auxiliary files and scripts, documentation, documentation sources, examples and
+example sources of packages |memoize|, |nomemoize|, |memoizable|, |auto| and
+|collargs|. The files belonging to this work and covered by LPPL are listed in
+\meta{texmf}|/doc/generic/memoize/FILES|.
+
+This work has the LPPL maintenance status `maintained.' The Current Maintainer
+of this work is Sašo Živanović. The work is available on CTAN
+at \url{https://ctan.org/pkg/memoize} and on GitHub
+at \url{https://github.com/sasozivanovic/memoize}.
+
+
+\subsection{Acknowledgments}
+
+First and foremost, my gratitude goes to Stefan Müller of Language Science
+Press for his encouragement and patience --- without him, this package might've
+remained an idea forever!
+
+But there were others as well. Shunsaku Hirata of the \hologo{XeTeX} team
+promptly resolved an issue with forward references in |xdvipdfmx|, and thereby
+made Memoize work with \hologo{XeTeX}. Joseph Wright kindly didn't object to
+me misappropriating \pkg{etoolbox} for other formats. Petra Rübe-Pugliese
+(CTAN) and Karl Berry (\hologo{TeX}Live) offered good advice on packaging the
+package; I would never get to call Advice Advice without their advice. Karl
+also performed a security review of the extraction scripts, providing some very
+useful comments along the way, and agreed to include the scripts among the
+shell escape commands allowed in \hologo{TeX}Live. Christian Schenk worked
+hard to support Memoize in MiK\hologo{TeX}. After the package was first
+published, amonakov, mbert, Qrrbrbirlbel, and Ulrike Fischer provided valuable
+feedback, suggestions and\slash or bug reports; cfr did all that, and also
+advertised the package extensively.
+
+Thank you all!
+
+
+\printindex
+
+\end{document}
+
+
+% Local Variables:
+% TeX-engine: luatex
+% TeX-master: t
+% End:
Property changes on: trunk/Master/texmf-dist/doc/generic/memoize/memoize-doc.tex
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Modified: trunk/Master/texmf-dist/doc/generic/memoize/memoize-extract.1.md
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize-extract.1.md 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize-extract.1.md 2024-01-03 21:18:51 UTC (rev 69285)
@@ -2,8 +2,8 @@
title: memoize-extract
section: 1
header: User Manual
-footer: memoize-extract of Memoize v1.0.0
-date: October 10, 2023
+footer: memoize-extract of Memoize v1.1.0
+date: January 02, 2024
hyphenate: false
---
@@ -34,17 +34,17 @@
*document.mmz*, which records which pages in the produced document are extern
pages and to which extern files they should be extracted. Therefore, after
compiling *document.tex*, the externs should be extracted by
-**memoize-extract** *document.mmz*.
+**memoize-extract** *document.mmz*.
+*document.mmz* may also be given as document* or *document.tex*. When
+environment variable *TEXMF_OUTPUT_DIRECTORY* is set, this filename is relative
+to the output directory specified by this variable.
+
*document.mmz* also records the expected width and height of each extern. In
case of a mismatch, **memoize-extract** refuses to extract the page and removes
the extern file if it already exist, and prints a warning message to the
-standard error. The script furthermore refuses to extract the page if a
-(c)c-memo associated to the extern does not exist, and to write to any file
-whose absolute path does not occur under the current directory or the directory
-set by TEXMFOUTPUT (in *texmf.cnf* or as an environment variable); TEXMFOUTPUT
-is also not allowed to point to the root directory, except on Windows, where it
-can only point to a drive root.
+standard error. The script also refuses to extract the page if a (c)c-memo
+associated to the extern does not exist. See also section SECURITY.
The Perl (.pl) and the Python (.py) version of the script are functionally
equivalent. The Perl script requires library
@@ -64,31 +64,69 @@
: Do not modify the *document.mmz* to mark the externs as extracted. By
default, they are commented out to prevent double extraction.
+**-F, \--format** *latex*|*plain*|*context*
+: When this option is given, the script assumes that it was called from within
+ a TeX compilation of a document in the given format: it prefixes all output
+ by the script name, and creates a log file *document.mmz.log*, which receives
+ any extraction-related warnings and errors.
+
**-f, \--force**
: Extract the extern even if the size-check fails.
-**-l, \--log** *filename*
-: Log size mismatches to *filename* rather than to the standard error.
-
-**-w, \--warning-template** *string*
-: Wrap the size mismatch warning text into the given template: \\warningtext in
- the template will be replaced by the warning text.
-
**-q, \--quiet**
: Don't describe what's happening.
-**-e, \--embedded**
-: Prefix all messages to the standard output with the script name.
-
**-m, \--mkdir**
-: A paranoid *mkdir -p*. (No extraction occurs, *document.mmz* is interpreted as a directory name.)
+: A paranoid *mkdir -p*. (No extraction occurs, *document.mmz* is interpreted as a directory name, which may end in any suffix; no suffix mangling is performed.)
+**-V, \--version**
+: Show the Memoize version number and exit.
+
**-h, \--help**
: Show help and exit.
-**-V, \--version**
-: Show the Memoize version number and exit.
+# SECURITY
+This script respects the restrictions on file input and output imposed by the
+TeX configuration, more precisely, the variables *openin_any* and *openout_any*
+of the **kpathsea** library (https://tug.org/kpathsea). You can inspect the
+values of these variables by executing '**kpsewhich** -var-value=openin_any' and
+'**kpsewhich** -var-value=openout_any'. The interpretation is as follows:
+
+**a** (or **y** or **1**) any
+: Allows any file to be opened.
+
+**r** (or **n** or **0**) restricted
+: Means disallowing special file names.
+
+**p** (or any other value) paranoid
+: Means being really paranoid: disallowing special file names and restricting
+ input/output files to be in or below the working directory or the directory
+ specified by *TEXMFOUTPUT* or *TEXMF_OUTPUT_DIRECTORY*. *TEXMFOUTPUT* may be
+ set either in *texmf.cnf* (e.g. by **tlmgr**) or as an environment
+ variable. *TEXMF_OUTPUT_DIRECTORY* may only be set as an environment
+ variables; it is automatically set by TeX when called by *-output-directory*
+ option (starting in TeXLive 2024).
+
+# EXIT STATUS
+
+**0**
+: The externs were successfully extracted. This exit code is returned even if
+ no externs need to be extracted, or if *document.mmz* does not exist.
+
+**10**
+: A warning also reported back to the compilation when given option
+ **\--format**. Currently, either: (i) size-mismatch; (ii) a non-existing
+ associated (c)c-memo file; or (iii) unavailable *kpsewhich*.
+
+**11**
+: An error also reported back to the compilation when given option
+ **\--format**. Currently, either: (i) a currupted document PDF, or (ii)
+ a kpathsea permission error.
+
+Other exit codes are as produced by the underlying scripting language (Perl of
+Python).
+
# SEE ALSO
[Memoize manual](https://ctan.org/pkg/memoize), section 6.6.1.
Deleted: trunk/Master/texmf-dist/doc/generic/memoize/memoize.mst
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize.mst 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize.mst 2024-01-03 21:18:51 UTC (rev 69285)
@@ -1 +0,0 @@
-encap '`'
Deleted: trunk/Master/texmf-dist/doc/generic/memoize/memoize.pdf
===================================================================
(Binary files differ)
Deleted: trunk/Master/texmf-dist/doc/generic/memoize/memoize.tex
===================================================================
--- trunk/Master/texmf-dist/doc/generic/memoize/memoize.tex 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/generic/memoize/memoize.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -1,9094 +0,0 @@
-\documentclass[a4paper,11pt]{article}
-\usepackage[margin=2cm,top=1cm,bottom=1cm,nohead,includefoot]{geometry}
-
-\usepackage{nomemoize}
-\usepackage{memoize-doc}
-\mmzset{
- % recompile,
- % readonly,
- % disable,
- % inverse search=false,
- % include context in ccmemo,
- % direct ccmemo input,
- % trace,
-}
-\def\exampledir{examples/}
-
-\title{\pkg[white]{Memoize}}
-\author{Sašo Živanović\\[2mm]
- \emailsymbol~\url(mailto:){saso.zivanovic at guest.arnes.si}\\
- \homepagesymbol~\url(http://){spj.ff.uni-lj.si/zivanovic}\\
- \faGithub~\url(https://){github.com/sasozivanovic}}
-\datefrompackageversion{memoize}
-\preto\packagever{v}
-
-\hypersetup{
- pdftitle={Memoize},
- pdfauthor={Sašo Živanović},
- pdfsubject={externalization},
- pdfkeywords={LaTeX, externalization, memoization}
-}
-
-\usepackage{makeidx}
-\pdfsystem{makeindex -s memoize.mst \jobname.idx}
-\makeindex
-\usepackage[hangindent=1em,justific=raggedright,font=small]{idxlayout}
-\appto\indexfont{\def\pkgcolor{gray}}
-
-\AddToHook{begindocument}{% so that the region compilation takes these in
- \docForeign{
- scope={
- after/.style={index annotation/.prefix={\hologo{LaTeX}\space}},
- cmd={name=begin}, cmd={name=end},
- cmd={name=label}, cmd={name=@currentlabel},
- cmd={name=ref}, cmd={name=pageref},
- cmd={name=index},
- cmd={name=NewDocumentCommand}, cmd={name=GetDocumentCommandArgSpec},
- env={name=verbatim},
- cmd={name=ReadonlyShipoutCounter},
- hook={name=begindocument},
- cmd={name=usepackage},
- },
- scope={
- before/.style={index annotation=\hologo{TeX} primitive},
- cmd={name=shipout},
- cmd={name=hbox}, cmd={name=vbox},
- cmd={name=ignorespaces}, cmd={name=unskip},
- cmd={name=errmessage},
- cmd={name=catcode},
- cmd={name=mag},
- },
- scope={
- before/.style={index annotation=\hologo{eTeX} primitive},
- cmd={name=scantokens},
- cmd={name=currentgrouplevel},
- },
- scope={
- before/.style={index annotation=\hologo{pdfTeX} primitive},
- cmd={name=quitvmode},
- },
- scope={
- before/.style={index annotation=\hologo{LuaTeX} primitive},
- pdfcmd={name=savepos},
- cmd={name=pdfvariable},
- },
- pdfvariable={name=compresslevel},
- pdfvariable={name=horigin},
- pdfvariable={name=vorigin},
- scope={
- after/.style={index annotation/.prefix={\hologo{ConTeXt}\space}},
- cmd={name=realpageno},
- },
- pdfcmd={name=primitive, index annotation=\hologo{LuaTeX}/\hologo{XeTeX} primitive},
- register={name=pdfmajorversion},
- register={name=pdfminorversion},
- scope={
- after/.style={index annotation/.prefix={\hologo{plainTeX}\space}},
- cmd={name=llap}, cmd={name=rlap},
- },
- scope={
- pkg=forest,
- env={name=forest},
- cmd={name=Forest},
- },
- scope={
- pkg=tikz, keypath=/tikz,
- cmd={name=tikz},
- env={name=tikzpicture},
- key={name=remember picture},
- key={name=overlay},
- cmd={name=tikzexternalize},
- cmd={name=pgfsys at getposition},
- },
- file={name=texmf.cnf},
- env={name=tcblisting, pkg=tcolorbox},
- cmd={name=vref, pkg=varioref},
- cmd={name=cref, pkg=cleveref},
- cmd={name=crefrange, pkg=cleveref},
- xparse type={name=l, index annotation/.append={\ (up to begin-group)}},
- xparse type={name=u, index annotation/.append={\ (until tokens)}},
- xparse type={name=r, index annotation/.append={\ (required delimited)}},
- xparse type={name=R, index annotation/.append={\ (required delimited with default)}},
- xparse type={name=d, index annotation/.append={\ (optional delimited)}},
- xparse type={name=D, index annotation/.append={\ (optional delimited with default)}},
- xparse type={name=o, index annotation/.append={\ (standard optional)}},
- xparse type={name=O, index annotation/.append={\ (standard optional with default)}},
- xparse type={name=t, index annotation/.append={\ (optional token)}},
- xparse type={name=s, index annotation/.append={\ (optional star)}},
- xparse type={name=m, index annotation/.append={\ (mandatory)}},
- xparse type={name=g, index annotation/.append={\ (optional group)}},
- xparse type={name=G, index annotation/.append={\ (optional group with default)}},
- xparse type={name=v, index annotation/.append={\ (verbatim)}},
- xparse type={name=e, index annotation/.append={\ (embellishments)}},
- xparse type={name=E, index annotation/.append={\ (embellishments with defaults)}},
- xparse modifier={name=+, index annotation/.append={\ (long)}},
- xparse modifier={name=!, index annotation/.append={\ (don't ignore spaces)}},
- xparse modifier={name=>, index annotation/.append={\ (argument processor)}},
- generic={name=shell-escape, name prefix=-, label prefix:=-,
- index annotation=option of \hologo{TeX} binaries},
- generic={name=enable-write18, name prefix=--, label prefix:=--,
- index annotation=option of \hologo{TeX} binaries},
- generic={name=TEXMFOUTPUT, index annotation={variable in
- \reffile[into index=false]{texmf.cnf}}},
- }%
- \docindex{%
- cmd={name=pdfprimitive, see={\refcmd[into index=false,short]{primitive}}},
- cmd={name=pdfsavepos, see={\refcmd[into index=false,short]{savepos}}},
- cmd={name=pdfhorigin, see={\docref[into index=false]{pdfvar:horigin}}},
- cmd={name=pdfvorigin, see={\docref[into index=false]{pdfvar:vorigin}}},
- cmd={name=pdfcompresslevel, see={\docref[into index=false]{pdfvar:compresslevel}}},
- pdfvariable={name=majorversion, see={\docref[into index=false,long]{reg:pdfmajorversion}}},
- pdfvariable={name=minorversion, see={\docref[into index=false,long]{reg:pdfminorversion}}},
- }%
-}
-
-\begin{document}
-
-
-\maketitle
-\thispagestyle{empty}
-
-\begin{abstract}
- Memoize is a package for externalization of graphics and memoization of
- compilation results in general, allowing the author to reuse the results of
- compilation-intensive code.
- Memoize
- \begin{enumerate*}[(i)]
- \item induces very little overhead, as all externalized graphics is produced
- in a single compilation. It features
- \item automatic recompilation upon the change of code or user-adjustable
- context, and
- \item automatic externalization of \TikZ pictures and Forest trees, easily
- extensible to other commands and environments. Furthermore, Memoize
- \item supports cross-referencing, \TikZ overlays and Beamer,
- \item works with all major engines and formats, and
- \item is adaptable to any workflow.
- \end{enumerate*}
-\end{abstract}
-
-\bigskip
-
-\ExampleName{titlepage}
-\makeexample{\examplename.pdf N=2} % N=2, because we show the .mmz file later
-\begingroup
-\tcbset{page box/.style={
- box align=center,boxsep=2mm,top=0pt,bottom=0pt,left=0pt,right=0pt,
- enhanced, remember as=#1,
- }}%
-\begin{tcboxedraster}
- [raster columns=1]
- {title=The two steps of externalization of graphics in Memoize}
- \tcbinputexample{
- title=\tt doc.tex,
- listing only,
- remember as=tex,
- }
- \begin{tcolorbox}[example title=doc.pdf, no attachment, remember as=pdf1]
- \includeexamplepdf[page box=extern1]{page=1,trim=1in 1in 1in 1in}\hfill
- \includeexamplepdf[page box=extern2]{page=2,trim=1in 1in 1in 1in}\hfill
- \includeexamplepdf[page box=extern3]{page=3,trim=1in 1in 1in 1in}\hfill
- \includeexamplepdf[page box=docpage]{page=4}%
- \end{tcolorbox}
- \vspace{10mm}
- \begin{tcboxedraster}[raster columns=3, raster column skip=1cm]{blankest}
- \begin{tcolorbox}[example title=.pdf]\centering
- \includeexamplepdf[page box=externfile1]{page=1,trim=1in 1in 1in 1in}\hfill
- \end{tcolorbox}
- \begin{tcolorbox}[example title=.pdf]\centering
- \includeexamplepdf[page box=externfile2]{page=2,trim=1in 1in 1in 1in}\hfill
- \end{tcolorbox}
- \begin{tcolorbox}[example title=.pdf]\centering
- \includeexamplepdf[page box=externfile3]{page=3,trim=1in 1in 1in 1in}\hfill
- \end{tcolorbox}
- \end{tcboxedraster}
-\end{tcboxedraster}
-
-\medskip
-
-\begin{tcolorbox}[title=Using the externalized graphics]
- \begin{tcolorbox}[example title=doc.pdf,
- halign=center, remember as=pdf2]
- \tcbox[remember, enhanced]{\footnotesize
- We all love Ti\emph{k}Zlings!
- \def\mytikzling#1#2{%
- \tcbox[blankest, baseline=1mm, remember as=used extern #1]
- {\tikz[x=0.3cm,y=0.3cm]{#2;}}}%
- \mytikzling1{\penguin} is a penguin,
- \mytikzling2{\koala} is a koala, and
- \mytikzling3{\owl} is an owl.
- }
- \end{tcolorbox}
-\end{tcolorbox}
-
-\begin{tikzpicture}[remember picture, overlay]
- \draw[->, snake arrow, thick, red] (tex.west) to[out=-90-60, in=90+60] (pdf1.west);
- \draw[->, snake arrow, thick, red] (tex.east) to[out=-90+19, in=90-19] (pdf2.east);
- \draw[->, thick, red] (extern1.south) to[out=south, in=north] (externfile1);
- \draw[->, thick, red] (extern2.south) to[out=south, out looseness=1.1, in=north, in looseness=0.9] (externfile2);
- \draw[->, thick, red] (extern3.south) to[out=south, out looseness=0.7, in=north, in looseness=0.9] (externfile3);
- \draw[->, thick,blue] (externfile1) to[out=south, in=north] ([yshift=0.8mm]used extern 1.north);
- \draw[->, thick,blue] (externfile2) to[out=south, in=north] (used extern 2.north);
- \draw[->, thick,blue] (externfile3) to[out=south, in=north] ([yshift=0.3mm]used extern 3.north);
-\end{tikzpicture}
-\endgroup
-
-\begin{tcolorbox}[colback=emphcolor, fontupper=\footnotesize]
- This manual also documents packages Advice
- ({\datefrompackageversion{advice}v\packagever}) and CollArgs
- ({\datefrompackageversion{collargs}v\packagever}). These are auxiliary
- packages which were developed alongside Memoize, but are distributed as
- independent packages as they might be useful outside the context of Memoize,
- as well. See sections~\ref{sec:advice} and \ref{sec:ref:advice} for Advice, and
- sections~\ref{sec:collargs} and \ref{sec:ref:collargs} for CollArgs.
-\end{tcolorbox}
-
-
-
-\section*{Introduction}
-\label{sec:intro}
-
-\begin{tcolorbox}[
- title={\introboxtitle{What} is externalization and why you might want it?},
- ]
- If you have ever worked on a long document full of \TikZ pictures and maybe
- Forest trees, you have probably had some compilation-enforced coffee breaks
- --- even on modern computers, compiling pictures, trees and such takes a lot
- of time. And you might have wondered, why do I need to compile these
- pictures over and over again? --- after all, I'm not changing them anymore!
- Enter \emph{externalization}, a mechanism designed to deal precisely with
- your issue, by saving the produced pictures into separate PDFs and including
- those PDFs in subsequent compilations --- in no time at all!
-\end{tcolorbox}
-
-\begin{tcolorbox}[
- title={\introboxtitle{Why} yet another externalization package?},
- ]
- \TikZ, the popular and all-powerful graphics language for \TeX, ships
- including an externalization library (described in \PGFmanual{53}). \TikZ's
- library does an excellent job, but with one caveat. Assume you're using it
- for the first time (or after a clean-up) on that long document full of \TikZ
- pictures and Forest trees. It will take ages to produce all the externalized
- graphics. Why? Because to get you up to speed, your document has to be
- compiled many many times --- once for each and every externalized picture.
- Even with \TikZ's advanced mechanism for skipping the parts of the document
- irrelevant for the picture at hand, the first externalization can be a
- daunting task.
-\end{tcolorbox}
-
-\begin{tcolorbox}[
- title={\introboxtitle{How} does Memoize save your time?},
- ]
- There is a reason why \TikZ uses an entire compilation cycle to produce a
- single externalized picture: \TeX\ itself can only produce a single PDF per
- compilation (at least at the moment). Memoize evades this limitation by
- dumping the externalized pictures right in the middle of the document
- (ouch!). More precisely, an externalized picture occurs in the PDF twice,
- first on a special page of its own and then on a regular page, where you
- intended it to be. The daring dump obviously necessitates a second step of
- the procedure, when those special pages are \emph{extracted} into separate
- PDFs, called \emph{externs}, which are then included into the document in
- subsequent compilations.
-
- This two-step procedure, illustrated on the cover page of this manual, is
- very fast. The first step, which externalizes \emph{all} the pictures into
- the document itself (the squiggly red arrow), takes virtually no more time
- than a regular compilation. The time needed for the second step, extraction
- (the normal red arrows), depends on the system setup, but it ranges from
- little to almost none.
-\end{tcolorbox}
-
-\begin{tcolorbox}[
- title={\introboxtitle{When} should I use Memoize?},
- ]
- In short, whenever you are writing a document containing lots of \TikZ
- pictures, Forest trees, or other time-consuming constructs, and you are bored
- waiting for the compilation to finish.
-
- Using Memoize on a paper containing a single picture does not make much of a
- difference. But with more complex documents, the speed-up can be immense.
- For example, the compilation time of my 400-page book containing about 160
- Forest trees was reduced by more than half, and the compilation time of a
- 260-page Beamer presentation with a hundred complex dynamic trees went from
- four minutes to a mere half minute!
-\end{tcolorbox}
-
-
-\begin{tcolorbox}[
- title={\introboxtitle{How} much extra work does Memoize require?},
- ]
- In principle, none.
-
- For one, while allowing for manual memoization of selected document chunks
- (\S\ref{sec:tut:memoize}), Memoize features a system which automatically
- triggers memoization at each invocation of selected commands and
- environments. Out of the box, Memoize \emph{automemoizes} \TikZ
- pictures\noprint{\refenv{tikzpicture}} and Forest
- trees\noprint{\refenv{forest}}, but the author can easily submit (almost) any
- command or environment to automemoization (\S\ref{sec:tut:automemoization}).
- Memoize also does its best to automatically prevent memoization of code that
- cannot be externalized, like \TikZ pictures with \refkey{/tikz/remember
- picture}, and to abort memoization in case the memoized code yields any
- errors.
-\end{tcolorbox}
-
-
-\begin{tcolorbox}[
- title={\introboxtitle{Why} is Memoize not called Externalize?},
- ]
- Fundamentally, Memoize is about producing and utilizing \emph{memos} ---
- pieces of \hologo{TeX} code replicating the effect of the compilation of a
- document chunk in a computationally less intensive manner. Typically, each
- memo has an associated extern, which is where the effect of
- \emph{typesetting} is stored, but conceptually, memos come first. For
- example, the extern is included back into the document by the memo, and a
- memo may be associated with any number of externs, including zero.
-
- Memos solve several externalization-related problems in a generic fashion,
- allowing for a multitude of applications. For example, they store the
- information about the associated externs, so that an extern can be integrated
- back into the document as a box with the original orientation, width, height
- and depth. They solve the problem of cross-referencing from and into the
- memoized code by storing its \emph{context} (\S\ref{sec:cross-referencing})
- and replicating any \cs{label}s which occur in it. They are also crucial for
- externalizing pictures in Beamer frames overlay by overlay.
-
- Incidentally, the term ``memoization'' is used with some programming
- languages to refer to the process of remembering the result of the function,
- along with the given arguments, so that on subsequent invocations of the
- function with the same arguments, the result can be returned from memory
- rather than recomputing it. I would say that, give or take the functions,
- what Memoize does fits the bill.
-\end{tcolorbox}
-
-
-\begin{tcolorbox}[
- title={\introboxtitle{How} can I make my command (auto)memoizable?},
- ]
- Any command which interacts with the rest of the document, like a command
- which produces a float, must receive special treatment. Some issues can be
- resolved from within Memoize. Other issues require a memoization-compatible
- (re)implementation of the command. It is in the hope that package writers
- will adapt their ``difficult'' commands to Memoize that this package offers a
- documented interface to the memoization process, fully described in
- sections~\ref{sec:tut:package}, \ref{sec:memoization-drivers}
- and~\ref{sec:tut:automemoization-details}.
-
- An advanced user might also want to know that Memoize ships with two
- auxiliary packages, which form the base of Memoize's automemoization feature.
- Package Advice implements a generic framework for extending the functionality
- of selected commands and environments, while package CollArgs provides a
- command which can determine the argument scope of any command whose argument
- structure conforms to \pkg{xparse}'s argument specification.
-\end{tcolorbox}
-
-\begin{tcolorbox}[
- title={\introboxtitle{What else} is out there?},
- ]
- Not long before submitting Memoize to CTAN, I became aware of another new
- externalization package, \pkg{robust-externalize}, and it seems that the same
- happened to the author of that package \Smiley, who found the
- proof-of-concept version of Memoize, which was available
- at \href{https://github.com/sasozivanovic/memoize}{GitHub} for a while.
-
- The key idea behind \pkg{robust-externalize} seems to be to extract the code
- submitted to externalization into separate files, and add the necessary
- preamble. While a compilation from scratch takes more time than with Memoize
- (but less than with \TikZ library), the approach allows for parallel
- compilation of externs --- nice!
-\end{tcolorbox}
-
-\newpage
-\tableofcontents
-
-
-\section{Before you start}
-\label{sec:setup}
-
-\subsection{Installing the extern extraction software}
-\label{sec:install}
-
-Good news: using Memoize can be as easy as writing
-\EmphVerbatim{\usepackage{memoize}} in the preamble.
-
-Bad news: Memoize won't work out of the box. The culprit is the extern
-extraction --- the process which ships the externalized graphics from the main
-document into separate extern files; for details, see the title page
-illustration and the ``How'' box in the Introduction. For the extraction to
-work, you will probably have to install some additional software, and you might
-also have to allow your \hologo{TeX} to execute the extraction script. But
-there's a silver lining: once Memoize is set up, it is set up for good.
-
-\begin{tcolorbox}[title=What do I have to do?]
-
- In principle, all you have to do for Memoize to work \emph{under the default
- configuration} is install Perl libraries
- \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} and
- \hreftt{https://metacpan.org/pod/Path::Class}{Path::Class}; see the Perl
- section below for the installation guideline.
-
-\end{tcolorbox}
-
-Consult section~\ref{sec:extraction} if you want to use an extraction method
-other than the default \refmmz{extract=perl}-based method or adapt Memoize to
-your particular workflow (for example, if you're compiling via a Makefile).
-
-If you installed Memoize through the package manager of your \hologo{TeX}
-distribution, your system should be already set up to allow the execution of
-Memoize's extraction scripts. If this is not the case, please contact either
-me or the maintainer of your distribution; until the issue is resolved, you
-have to either
-\begin{enumerate}[(a)]
-\item compile documents loading Memoize with command-line option
- \docref{-shell-escape} (on \hologo{TeX}Live) or
- \docref{--enable-write18} (on MiK\hologo{TeX}), or
-\item set up the restricted shell escape mode to allow for the execution of
- \refscript{memoize-extract.pl}.\footnote{\label{fn:shell-escape-commands}On
- \hologo{TeX}Live, execute \code{tlmgr conf texmf shell\_escape\_commands}
- to get the list of currently allowed commands \meta{current}, then add the
- script by executing \code{tlmgr conf texmf shell\_escape\_commands
- \meta{current},memoize-extract.pl}. On MiK\hologo{TeX}, you get the
- \meta{current} list by \code{initexmf
- --show-config-value=[Core]AllowedShellCommands[]}, and add to it by
- \code{initexmf
- --set-config-value=[Core]AllowedShellCommands[]=\meta{current};memoize-extract.pl}.}
-\end{enumerate}
-
-Once you have set up your system, I advise you to follow the instructions in
-section~\ref{sec:tut:test} to test if the setup was successful.
-
-
-\begingroup
-\setlength\intextsep{0pt}
-
-\paragraph{Perl} If you're running GNU/Linux or macOS, Perl is most likely
-already installed on your system. On Windows, you might have to install it. I
-tested Memoize with Strawberry Perl, available
-at \url(https://){strawberryperl.com}; see \url(https://){www.perl.org} for
-other options.
-
-\begin{wrapfigure}[3]{r}{0.25\linewidth}
- \raisebox{0.5mm}[\dimexpr\height-0.5mm][\depth]{%
- \mmznext{no verbatim}
- \begin{tcolorbox}[
- % example title=Installing the required libraries in Perl,
- % attach shifted boxed title to top right,
- % top=2.5mm,
- fontupper=\small,
- left=2mm,
- force nobeforeafter,
- ]
- |cpan PDF::API2|
-
- |cpan Path::Class|
- \end{tcolorbox}%
- }%
-\end{wrapfigure}
-Once Perl is installed on your system, you will need to install two Perl
-libraries as well: the PDF processing library
-\hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2}, and the cross-platform
-path specification manipulation library
-\hreftt{https://metacpan.org/pod/Path::Class}{Path::Class}. On some GNU/Linux
-distributions, these libraries are included as packages --- just use your
-package manager to install them. Otherwise, install them from
-\hreftt{https://www.cpan.org}{CPAN} using
-\hreftt{https://metacpan.org/pod/CPAN}{cpan} tool, as shown in the box.
-
-
-\paragraph{Python} Installing (Python and) the required Python library is only
-necessary if you decide to use the Python-based extraction script; see
-section~\ref{sec:extraction}.
-
-If you're running GNU/Linux, install Python using your package manager.
-Otherwise, download the installer from \url(https://){www.python.org}. You can
-install Python even if you don't have administrator privileges: simply uncheck
-the ``Install launcher for all users'' when running the installer.
-
-\begin{wrapfigure}[2]{r}{0.25\linewidth}
- \raisebox{0.2mm}[\dimexpr\height-0.2mm][\depth]{%
- \mmznext{no verbatim}
- \begin{tcolorbox}[
- % example title=Installing the required library in Python,
- % attach shifted boxed title to top right,
- % top=2.5mm,
- fontupper=\small,
- left=2mm,
- nobeforeafter,
- ]
- |pip install pdfrw2|
- \end{tcolorbox}%
- }%
-\end{wrapfigure}
-
-Once Python is installed on your system, you will also need to install the
-|pdfrw2| library (or its predecessor, |pdfrw|, which will work just as well).
-Some GNU/Linux distributions offer this library as a package; if this is not
-the case, and on other operating systems, install it
-from \href{https://pypi.org}{The Python Package Index}
-using \href{https://pip.pypa.io}{pip} tool (if you
-run \href{https://pip.pypa.io}{pip} as a superuser, it will install the library
-system-wide, otherwise locally), as shown in the box.
-
-\endgroup % \intextsep
-
-
-\subsection{The configuration commands}
-\label{sec:mmzset}
-
-Memoize can be configured using command \Emph{\refcmd{mmzset}\marg{keylist}}
-(and friends). The \meta{keylist} argument is a comma-separated list of
-configuration settings, each setting having the form \meta{key}|=|\meta{value},
-or sometimes just \meta{key}. The \meta{keylist} argument is processed by package
-\pkg{pgfkeys} (see \PGFmanual{88}), so you can use all the bells and whistles
-of that fantastic PGF utility (like easily defining your own styles).
-
-Here's some examples of \refcmd{mmzset}. For one, to whet your appetite to learn
-about the various keys in the \refkeypath{/mmz} path, but more importantly now, to show
-you that white-space is irrelevant in the \meta{keylist} argument, so you can
-format the keylist as you wish --- as long as it does not contain an empty
-line.\footnote{I like to add a comma after the final key as well, as shown in
- the bottom left example, because if I don't, I often forget to insert it when
- I add more keys.}
-
-\begin{tcbraster}[raster columns=4, raster valign=bottom]
- \begin{tcboxedraster}[raster columns=1]{blankest, raster multicolumn=3, valign=bottom}
- \begin{tcblisting}{listing only}
-~\mmzset~{memo dir,recompile,memoize=circuitikz}
- \end{tcblisting}
- \begin{tcblisting}{listing only}
-~\mmzset~{memo dir, recompile, memoize=circuitikz}
- \end{tcblisting}
- \end{tcboxedraster}
- \begin{tcblisting}{listing only}
-~\mmzset~{
- memo dir,
- recompile,
- padding=2in}
- \end{tcblisting}
-\end{tcbraster}\par
-\begin{tcbraster}[raster columns=4, raster valign=top]
- \begin{tcblisting}{listing only, raster multicolumn=2}
-~\mmzset~{
- memo dir, readonly,
- memoize=\qrcode{om},
- deactivate=\label,
- disable,
-}
- \end{tcblisting}
- \begin{tcblisting}{listing only}
-~\mmzset~{
- memo dir,
- recompile,
- % comment
- padding=2in
-}
- \end{tcblisting}
- \begin{tcblisting}{listing only, bad}
-~\mmzset~{
- memo dir,
- recompile,
-
- padding=2in
-}
- \end{tcblisting}
-\end{tcbraster}
-
-Command \refcmd{mmzset} can be used any time after loading the package. It is
-very common in the preamble, but also useful in the document body, where its
-effect is local to the \hologo{TeX} group.\footnote{Any keys with a non-local
- effect are explicitly marked as such in the reference section.} For example,
-you can use the idiom on the left below to force recompilation of a single
-Forest tree somewhere in the middle of the document (in the code listings
-below, highlighting marks the resulting scope of the \refmmz{recompile}
-directive). However, as applying some setting to a single piece of
-automatically memoized code is common, Memoize provides a special command for
-the occasion: the keys given as the argument of \Emph{\refcmd{mmznext}} will be
-applied only at the next instance of automemoization, overriding any keys set
-by \refcmd{mmzset} in case of a conflict. If the command is given more than
-once, only the final invocation takes effect.
-
-\begin{tcbraster}[raster columns=2, raster equal height=rows, raster force size=false]
-\begin{tcblisting}{listing only, mark region={3}{6}}
-~{~
- ~\mmzset~{recompile}
- \begin{forest}
- [VP[V][DP]]
- \end{forest}
- % ...
-~}~
-\end{tcblisting}
-\begin{tcblisting}{listing only, add to width=-1em, mark region={3}{5}}
-% ...
-~\mmznext~{recompile}
-\begin{forest}
- [VP[V][DP]]
-\end{forest}
-% ...
-
-\end{tcblisting}
-\end{tcbraster}
-
-I like to follow |\usepackage{memoize}| by \refcmd{mmzset}, but if you prefer, you
-can also provide the document-wide configuration as \Emph{package options.}
-Note, however, that \hologo{LaTeX} removes spaces from package
-options, so keys such as \refmmz{memo dir} won't work. That said, the following are
-equivalent --- both will force re-externalization of all the externs in the
-document.
-
-\begin{tcbraster}[raster columns=2, raster equal height=rows]
- \begin{tcblisting}{listing only}
-\usepackage{memoize}
-\mmzset{recompile}
- \end{tcblisting}
- \begin{tcblisting}{listing only}
-\usepackage[recompile]{memoize}
- \end{tcblisting}
-\end{tcbraster}
-
-
-\subsection{The configuration file}
-\label{sec:memoize.cfg}
-
-Memoize allows you to configure settings which apply to more than just one
-document. It does that by attempting to load file \Emph{\reffile{memoize.cfg}}
-just before executing package options. Given how \hologo{TeX} searches for
-files, the location of this file determines whether it applies system-wide,
-user-wide or directory-wide. The directory-wide location is clearly the
-directory itself. The user-wide and system-wide location depend on the
-\hologo{TeX} distribution and which format(s) you want to use
-\reffile{memoize.cfg} with:\footnote{The |memoize| subfolder is not
- obligatory.}
-
-\begin{center}
- \meta{the relevant texmf tree root}|/tex/|\meta{format}|/memoize/memoize.cfg|
-\end{center}
-
-You can say |generic| for \meta{format} if you want the configuration file to
-be accessible by all formats, otherwise \meta{format} should be one of the
-formats supported by Memoize: |plain|, |latex| or |context|. The texmf root
-directory depends on the distribution, here's how to figure out what it is:
-
-\begin{tcboxedraster}
- [raster columns=1, raster valign=top, raster force size=false]
- {title=The roots of TEXMF trees}
-\begin{tcolorbox}[title={\hologo{TeX} Live\hfill\url[white](https://){tug.org/texlive}}]
- \begin{description}[itemsep=0pt]
- \item[user-wide]
- |tlmgr conf texmf TEXMFHOME|
- or |kpsewhich -var-value TEXMFHOME|\\
- the default (on Linux): |/home/|\meta{username}|/texmf|
- \item[system-wide]
- |tlmgr conf texmf TEXMFLOCAL|
- or |kpsewhich -var-value TEXMFLOCAL|\\
- the default (on Linux): |/usr/local/texlive/texmf-local|
- \end{description}
-
- \smallskip
- Don't forget to run |texhash| or |mktexlsr| after creating \reffile{memoize.cfg}.
-\end{tcolorbox}
-\begin{tcolorbox}[title={MiK\hologo{TeX}\hfill\url[white](https://){miktex.org}}, before upper={\parskip0.3\baselineskip plus 2pt\relax},]
- Open the MiK\hologo{TeX} Console, select the ``Settings'' page and then the
- ``Directories'' tab.
-
- If there is a folder marked with the ``Generic'' purpose with attribute
- ``User'' (for a user-wide \reffile{memoize.cfg}) or ``Common'' (for a system-wide
- \reffile{memoize.cfg}), that's the folder you are looking for. Otherwise, create a
- folder following MiK\hologo{TeX}'s instructions and add it to the list.
-
- Don't forget to click ``Refresh file name database'' (in the ``Tools'' menu
- of the MiK\hologo{TeX} Console) after creating \reffile{memoize.cfg}.
-\end{tcolorbox}
-\end{tcboxedraster}
-
-Note that a directory config file will override the user config, and the user
-config will in turn override the system-wide one. This should not concern you
-too much, because you will probably only want to use the user-wide config
-anyway,\cprotect\footnote{If you want to have a directory-wide configuration
- based on (rather than overriding) the user-wide configuration, your could
- write down the real user-wide config in, say, |memoize.user.cfg| (located
- user-wide), and then |\input| this file by both the user-wide and the
- directory-wide \reffile{memoize.cfg}. Of course, the same logic can be used to base
- a user-wide config on the system-wide one.} which might look something like
-this:
-
-\ExampleName{memoize}
-\makeexample{\examplename.cfg.c1}
-\tcbinputexample[.cfg]{listing only}
-
-I recommend including \refmmz{memo dir} in your \reffile{memoize.cfg},
-as shown in the first line. It reduces the clutter in the document
-directories; see section~\ref{sec:tut:memodir} for details.
-
-The \refmmz{extract} line concludes the story on ``permanently'' selecting the
-extraction method, started in chapter~\ref{sec:setup}. As the final note,
-observe that key \refmmz{extract} only makes sense as a package option or as a
-\refcmd{mmzset} key in \reffile{memoize.cfg}. It will have no effect as a
-\refcmd{mmzset} key in the document, because the extraction happens while the
-package is loaded.
-
-
-
-\section{Your first memoized documents}
-\label{sec:tut}
-
-\subsection{Let's see if it works!}
-\label{sec:tut:test}
-
-\ExampleName{test}
-\noprint{\refcmd{tikz}\refenv{tikzpicture}\refenv{forest}}%
-
-Take example file \texttt{\examplename.tex},\footnote{Where can you find
- the example files? For one, they are integrated into this manual, so if your
- PDF viewer supports attachments, you can simply click on the paperclip icon
- on the top right of the example box (even if you're offline). Otherwise,
- visit the |examples| subdirectory of wherever you found this document
- \Smiley. Online, the Memoize documentation can be found at CTAN:
- \url{https://ctan.org/pkg/memoize}; and if your \hologo{TeX} installation
- includes the documentation, you should also find it in directory \meta{the
- root of your \hologo{TeX} installation}|/doc/generic/memoize|.
-
- Incidentally, while we present a full example document in this section, many
- code listings will only present the parts of the file relevant for the
- discussion, for brevity. The example \emph{files}, however, will remain
- full, compilable documents, including the document preamble etc.} or some
-simple document containing a \TikZ picture or a Forest tree, add
-\EmphVerbatim{\usepackage{memoize}} to the preamble,\footnote{Memoize likes to
- be loaded early. If you get the error \texttt{Extern extraction from
- document \meta{jobname}.pdf was unsuccessful}, move
- \cs{usepackage}\bracestt{memoize} up the preamble; see
- section~\ref{sec:loading-order} for details.} and compile it twice.
-
-\begin{itemize}
-\item The first compilation of the example should produce a three-page PDF.
- The first two pages are the \emph{extern pages} holding the externalized
- graphics, while the final page is the (sole page of the) real document. Note
- that you can see each extern twice: first on a page of its own, and then
- wherever it belongs to in the real document.
-\item At the second compilation, the extern pages should have disappeared from
- the PDF, meaning they were successfully extracted into extern files, which
- are now embedded into the main document.
-\end{itemize}
-
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- sidebyside,
- comment={\centering
- \includeexamplepdf[extern page,left=2mm,right=2mm]{page=1,trim=1in 1in 1in 1in}\quad
- \includeexamplepdf[extern page,left=2mm,right=2mm]{page=2,trim=1in 1in 1in 1in}\\[2ex]
- \includeexamplepdf[document page]{page=3}
- },
-}
-
-You might want to play with this example a bit now.
-For example, if you reverse the order of the \TikZ picture and the Forest
-tree, you should notice that the externs don't get recompiled. You won't see
-any extern pages again until you change the actual code of the picture or the
-tree --- or until you add some other picture or tree, of course.
-
-If you \emph{don't} want to automatically memoize \TikZ pictures and/or Forest
-trees, you can switch off their \emph{automemoization} using key
-\Emph{\refmmz{deactivate}}. This key takes a list of command and environment
-names. As you can see below, the command and the environment must be
-deactivated separately.
-
-\begin{tcblisting}{listing only}
-\mmzset{
- deactivate={\tikz, tikzpicture}, % deactivate automemoization of all TikZ pictures
- deactivate=\tikz, % deactivate only the command
- deactivate=tikzpicture, % deactivate only the environment
-}
-\end{tcblisting}
-
-
-\subsection{Memoizing by hand}
-\label{sec:tut:memoize}
-
-In the previous section, we have compiled our very first document which used
-Memoize. In that document, we only had to load the package, as Memoize knows
-how to externalize \TikZ pictures and Forest trees without any help from the
-author. But what if you want to externalize some other code? The manual way of
-doing this is by surrounding the code by a \Emph{\refenv{memoize}} environment,
-or by making it the argument of the \Emph{\refcmd{mmz}} command. The only
-difference between the two is that the environment, but not the command,
-ignores any spaces surrounding the given code.
-
-\ExampleName{manual}
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- sidebyside, lefthand ratio=0.528,
- comment={\centering
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\hfill
- \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in}
- },
-}
-
-Both the \refenv{memoize} environment and the \refcmd{mmz} command take a
-configuration keylist as the optional argument, so their full syntax is
-\cs{begin}\marg{\refenv{memoize}}\oarg{keylist}\meta{code to be
- externalized}\cs{end}\marg{\refenv{memoize}} and and
-\refcmd{mmz}\oarg{keylist}\marg{code to be externalized}. The keys given in
-this optional argument take precedence over the keys set by \refcmd{mmzset}.
-Note that \refcmd{mmznext} does not apply to manual memoization.
-
-Manual memoization is great for one-shot memoizations, but you can use it
-within your own macros as well. For example, assume that you don't want to
-externalize \TikZ pictures in general (so you have \refmmz{deactivate}d
-automemoization of the \cs{tikz} command, as explained at the end of
-section~\ref{sec:tut:test}), but that you want to easily memoize selected
-pictures. You could define a memoized variant of the \cs{tikz} command, as shown
-below (and similary for the environment). (Note the |%| comment characters in
-the definition of |\mmztikz|. The definition was intentionally broken into
-several lines to remind you that the spaces around the argument of \refcmd{mmz}
-matter.)
-
-\ExampleName{mmztikz}
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- comment={\centering
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
- \quad
- \includeexamplepdf[document page]{page=2}
- },
-}
-
-\clearpage
-\subsection{Memoizing automatically}
-\label{sec:tut:automemoization}
-
-Out of the box, Memoize automatically externalizes \TikZ pictures and Forest
-trees. Let us see how other commands and environments can be submitted to this
-\emph{automemoization} process.
-
-We start with the simpler case of environments (fortunately, externalizing
-environments also makes sense more often than externalizing commands). You can
-submit an environment to automemoization by writing
-\EmphTT{\refmmz{auto}=\marg{environment name}\braces{\refmmzauto{memoize}}} (as
-a key in \refcmd{mmzset}). The natural (but not the only possible) location
-for this instruction is the preamble. Below, we automemoize environment
-\env{circuitikz} of package \pkg{circuitikz}, used for drawing electronic
-circuits.\footnote{In section~\ref{sec:tut:test}, we learned that
- automemoization can be switched off using key \refmmz{deactivate}. Memoize
- also offers key \refmmz{activate}, but you probably won't have to use it, as
- an \refmmz{auto} declaration automatically activates the submitted command.}
-
-\ExampleName{automemoize-environment}
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- sidebyside, lefthand ratio=0.6,
- comment={\centering
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in,scale=0.8}
- },
-}
-
-Commands are a bit harder to automemoize, because Memoize cannot possibly know
-how far the arguments of a command extend (in contrast, the end of an
-environment is clearly marked). With commands, we must inform Memoize about
-their argument structure, which we achieve using key \refmmzauto{args} in the
-second argument of key \refmmz{auto}:
-\EmphTT{\refmmz{auto}=\meta{command}\braces{\refmmzauto{memoize},
- \refmmzauto{args}=\marg{argument specification}}}. We can only leave out
-\refmmzauto{args} if the command was defined by \refcmd{NewDocumentCommand} or
-similar; in this case, Memoize can retrieve the argument specification on its
-own.
-
-And how does the \meta{argument specification} look like? It is a sequence of
-letters, each letter determining an argument type. Memoize recognizes the same
-argument types and their letters as package \pkg{xparse} (which defines
-\refcmd{NewDocumentCommand} and friends), so you should look at (section 1 of)
-the documentation of that package for details, if and when you need them.
-Here, we focus on the two most commonly used types, \docref{xparse:m} and \docref{xparse:o}, and add the
-optional star for good measure:
-\begin{center}
- \begin{tabular}{lll}
- \toprule
- letter & argument type & example\\
- \midrule
- \docref{xparse:m} & mandatory argument \\
- & --- either surrounded with braces & |\foo{arg}|\\
- & --- or a single token & |\foo a|\\
- \docref{xparse:o} & optional argument, surrounded with brackets & |\foo[arg]|\\
- \docref{xparse:s} & optional star & |\foo*|\\
- \bottomrule
- \end{tabular}
-\end{center}
-
-Below, we write |args=om| because command \cs{qrcode} (of package \pkg{qrcode})
-takes two arguments: a bracketed optional argument, followed by a mandatory
-argument (in braces).
-
-\ExampleName{automemoize-command}
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- sidebyside, lefthand ratio=0.603, top=0mm,
- comment={\centering
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad
- \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in}
- },
-}
-
-This should get you started with automemoization. We'll provide some further
-basic information in sections~\ref{sec:tut:working-on-a-picture}
-and~\ref{sec:tut:verbatim} of the tutorial, but only package writers will
-probably ever need the gory details from
-section~\ref{sec:tut:automemoization-details}. It is my sincere hope that they
-will support Memoize in their packages, where necessary, so that you don't even
-have to write the \refmmz{auto} declarations, but there is one thing you should
-know if you encounter a package supporting Memoize: you should load it
-\emph{after} Memoize!
-
-
-
-\subsection{Working on a picture}
-\label{sec:tut:working-on-a-picture}
-
-
-Memoize automatically recompiles a picture when the code producing the picture
-changes. However, sometimes we can modify a picture without changing its code,
-like when we modify the definition of a command used in the code. In the
-example below, a predefined style |emph| is applied to the node, producing a
-node with a red background. Let's say we compile the document (with
-memoization) and then change this style to set the yellow background.
-Curiously, the node will remain red.
-
-\ExampleName{recompile}
-\makeexample{\examplename.pdf N=3}
-\begin{tcbraster}[raster columns=2, raster valign=top, raster column skip=4mm]
- \tcbinputexample{
- after title pre={\ (version~1)},
- comment={\centering
- \includeexamplepdf[document page]{page=2}
- },
- }
- \tcbinputexample[.tex][.c2]{
- no attachment,
- after title pre={\ (version~2)},
- comment={\centering
- \includeexamplepdf[document page]{page=1}
- },
- }
-\end{tcbraster}
-
-The curious thing happens (or rather, doesn't happen) because Memoize doesn't
-keep track of how commands and styles are defined; it just uses the extern file
-it created when the old style was in effect. To get a yellow node, we must ask
-Memoize to reexternalize the picture. The simplest way to do that is by using
-the \Emph{\refmmz{recompile}} key; below, we write
-\refcmd{mmznext}\bracestt{\refmmz{recompile}} just before the picture and
-compile the document again (remember from section~\ref{sec:mmzset} that
-whatever keys we provide through \refcmd{mmznext} only apply to the instance of
-automemoization). After the compilation, we may (and should) remove the
-\refmmz{recompile} directive (otherwise, Memoize will produce the extern page
-again and again).
-
-\tcbinputexample[.tex][.c3]{
- no attachment,
- after title pre={\ (version~3)},
- sidebyside, lefthand ratio=0.6,
- comment={\centering
- \includeexamplepdf[document page]{page=2}
- },
-}
-
-It is also common to put (again, for the space of a single compilation)
-\refcmd{mmzset}\braces{\refmmz{recompile}} in the preamble, or to use
-\refmmz{recompile} as the package option. Either will remake all the
-externalized graphics in the document, so you can be sure all of them use the
-latest version of your macros and styles.
-
-We'll revisit the issue of memoized code depending on macros and styles defined
-elsewhere in section~\ref{sec:tut:redefinitions}. In this section, we will
-learn how the issue can be \emph{avoided}, at least to some extent. One idea
-is to turn off memoization for the picture(s) we are currently working on;
-another idea is to let Memoize know which definitions the picture relies on.
-The simplest way to achieve the former is by using key \Emph{\refmmz{disable}};
-by putting it into a \hologo{TeX} group, we can localize its effect to the
-selected pictures.
-
-\ExampleName{disable}
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- sidebyside,
- lefthand ratio=0.6,
- top=0mm, % manual top, bottoms and \\[...] to fit the page
- comment={\centering
- \includeexamplepdf[extern page,bottom=0.8mm]{page=1,trim=1in 1in 1in 1in}\\[0.5mm]
- \includeexamplepdf[extern page,bottom=0.8mm]{page=2,trim=1in 1in 1in 1in}\\[0.5mm]
- \includeexamplepdf[document page,bottom=0.8mm]{page=3}
- },
-}
-
-\begin{tcolorbox}[warning]
- As you can imagine, key \refmmz{disable} is complemented by \refmmz{enable},
- but it is perhaps worth mentioning a problem that can arise if you disable
- memoization for a part of your document by enclosing it in a pair of
- \refcmd{mmzset}\braces{\refmmz{disable}} and
- \refcmd{mmzset}\braces{\refmmz{enable}}. Yes, it might work at the moment,
- but say you later (e.g.\ when you are preparing the final version of the
- document) decide to disable memoization for the entire document, and say you
- try to do this by writing \refcmd{mmzset}\braces{\refmmz{disable}} in the
- preamble. As shown below on the left, you're in for a surprise: memoization
- will still be enabled in the part of the document following
- \refcmd{mmzset}\braces{\refmmz{enable}}! The solution is to always disable
- memoization for a part of the document by using
- \refcmd{mmzset}\braces{\refmmz{disable}} in a \hologo{TeX} group (i.e.\ the
- braces), as shown on the right. (In the examples below, the shaded areas
- mark the parts of the document where memoization is \emph{disabled}.)
-\end{tcolorbox}
-
-\makeexample{disable-bad.tex.c1}
-\makeexample{disable-good.tex.c1}
-\makeexample{disable-nomemoize.tex.c1}
-\makeexample{disable-nommz.tex.c1}
-\makeexample{disable-auto-cmd.tex.c1}
-\makeexample{disable-auto-env.tex.c1}
-
-\begin{tcbraster}[raster columns=2, raster equal height=rows]
- \tcbinputexample(disable-bad){
- listing and comment, bad, mark region={4}{7},
- comment={\raggedright The upper \refcmd{mmzset}\braces{\refmmz{disable}}
- does not have the intended effect,
- i.e.\ it doesn't apply to the whole document!}}
- \tcbinputexample(disable-good){
- listing and comment, mark region={4}{10},
- comment={\raggedright The upper \refcmd{mmzset}\braces{\refmmz{disable}} applies
- to the entire document, as expected.}}
-\end{tcbraster}
-
-In fact, it might be better to disable memoization using environment
-\Emph{\refenv{nomemoize}} or macro \Emph{\refcmd{nommz}}. I also like these
-commands because it is easy to add and remove prefix |no| to switch manual
-memoization (triggered using environment \refenv{memoize} or macro
-\refcmd{mmz}) off and on.
-
-\begin{tcbraster}[raster columns=2, raster equal height=rows]
- \tcbinputexample(disable-nomemoize){listing and comment, mark region={2}{2},
- comment=\raggedright Disable using the dedicated environment.}
- \tcbinputexample(disable-nommz){listing and comment, mark region={2}{2},
- comment=\raggedright Disable using the dedicated command.}
-\end{tcbraster}
-
-It is also possible to disable memoization for all occurrences of a selected
-command or environment. In fact, we're already familiar with the procedure
-from section \ref{sec:tut:automemoization}, where we used key
-\refmmzauto{memoize} inside the second argument of \refmmz{auto} to
-automatically memoize all instances of the command or environment given as the
-first argument. All we have to do to auto-disable rather than auto-memoize, is
-substitute \refmmzauto{nomemoize} for \refmmzauto{memoize}. Note that this
-prevents memoization of not only the given command or environment, but also of
-any (manual or automatic) memoization which would otherwise occur during its
-execution; for example, if |\foo| executes \cs{tikz} under the hood,
-autodisabling |\foo| prevents memoization of the inner \cs{tikz}, even though
-that command is normally automemoized.
-
-\begin{tcbraster}[raster columns=2, raster equal height=rows]
- \tcbinputexample(disable-auto-cmd){
- listing and comment,
- mark region={3}{3}, mark region={5}{5},
- comment=\raggedright Autodisable within a command.,
- }
- \tcbinputexample(disable-auto-env){
- listing and comment,
- mark region={3}{5},
- comment=\raggedright Autodisable within an environment.,
- }
-\end{tcbraster}
-
-All that said, Memoize actually offers a neater way to switch off the
-externalization for the picture I'm currently working on. The
-\Emph{\refmmz{readonly}} key instructs Memoize to use whatever externs it had
-already produced (thereby reducing the document compilation time), but to
-abstain from producing any new externs. In effect, the stuff we are currently
-working on does not undergo memoization and therefore does not produce the
-clutter which can potentially lead to trouble described in the \refmmz{recompile}
-examples above.
-
-What I like to do is load the package using |\usepackage[readonly]{memoize}|,
-work on stuff, and once I'm happy with the most recent pictures, remove
-\refmmz{readonly} from the package options for one compilation.
-
-\ExampleName{readonly}
-\makeexample{\examplename.pdf N=2}
-
-\begin{tcbraster}[raster columns=5, raster valign=top, raster column skip=4mm]
- \tcbinputexample{
- raster multicolumn=2,
- after title pre={\ (work in progress)},
- comment={\centering
- \includeexamplepdf[document page]{page=1}
- },
- }
- \tcbinputexample[.tex][.c2]{
- raster multicolumn=3,
- no attachment,
- title={\tt readonly.tex} (the final version),
- comment={\centering
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad
- \includeexamplepdf[document page]{page=2}
- },
- }
-\end{tcbraster}
-
-We're now ready to tackle a completely different way of avoiding the issue, by
-informing Memoize which definitions the externalized picture depends on. We do
-this by appending these definitions to \emph{context} --- when the context of a
-picture changes, Memoize recompiles the picture, same as if the code of the
-picture itself was changed. (We will talk about context in more detail in
-section~\ref{sec:cross-referencing}.)
-
-A command can be added as a dependency using key \Emph{\refmmz{meaning to
- context}}. Below, we make the following externalized picture depend on the
-definition of macro \cs{answer}; changing this definition will result in the
-recompilation of the extern. In general, \refmmz{meaning to context} accepts a
-comma-separated list of command and environment names, e.g.\
-\code{\refmmz{meaning to context}=\bracestt{\cs{foo},bar}} (note the braces).
-
-Memoize offers several variants of \refmmz{meaning to context}, applicable to
-various types of commands. For example, the easiest way of making the picture
-depend on the definition of a \pkg{pgfkeys} style is to use handler
-\Emph{\refkey{/handlers/.meaning to context}} --- note the dot in the name, and
-observe that \code{emph/.meaning to context} below is executed within
-\cs{tikzset}, not \refcmd{mmzset}; see \PGFmanual{87} to learn about key
-handlers.
-
-\ExampleName{meaning-to-context}
-\makeexample{\examplename.pdf N=2}
-\tcbinputexample{
- after title pre={\ (version~1)},
- sidebyside,
- comment={\centering
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
- },
-}
-
-\tcbinputexample[.tex][.c2]{
- after title pre={\ (version~2)},
- sidebyside,
- comment={\centering
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
- },
- no attachment,
-}
-
-All variants of \refmmz{meaning to context} (see reference
-section~\ref{sec:ref:memoization:basic} for the full list) may be used within
-the externalized picture itself, e.g.\ \code{\cs{node}[emph,
- emph/\refkey{/handlers/.meaning to context}]\bracestt{...}} is perfectly
-valid --- and also handy when you want to limit the effect of the handler to a
-single picture, as \refkey{/handlers/.meaning to context} cannot be used within
-\refcmd{mmznext}.
-
-
-
-\subsection{Keeping a clean house}
-\label{sec:tut:memodir}
-
-Memoize produces lots of auxiliary files. For each piece of memoized code, it
-produces two \emph{memo} files (we will learn more about these in
-section~\ref{sec:memos}), which will be joined by the extern PDF upon the next
-compilation.\footnote{If you're using the \hologo{TeX}-based extraction, each
- extern (|.pdf|) is also accompanied by a log file (|.log|) produced during
- the compilation that extracted the extern.} You can recognize these files
-easily: their names start with the name of your document and include one or two
-long hexadecimal numbers.
-
-\ExampleName{dirty-house}
-\makeexample{\examplename.pdf}
-{
- \relaxmmzcommands
- \edef\marshal{%
- \noexpand\def\noexpand\mynewextern.//\examplename
- }\marshal.#1-#2.pdf\relax{%
- \forestset{
- append={[\examplename.#1.memo\vphantom{p},pointcolor]},
- append={[\examplename.#1-#2.memo\vphantom{p},pointcolor]},
- append={[\examplename.#1-#2.pdf,pointcolor]},
- }%
- }
- \tcbinputexample{
- comment={%
- \begin{tcolorbox}[reset, left=2mm, bottom=0mm,
- my boxed title=the folder contents (after two compilations),
- attach shifted boxed title to top right,
- ]
- \begin{forest}
- before typesetting nodes={for tree={
- font=\scriptsize\tt, grow'=0, folder, s sep=0}},
- [\meta{the document folder}, inner xsep=0,
- TeX={%
- \def\mmzNewExtern##1##2##3##4##5{\mynewextern##1\relax}%
- \input{\examplepath.mmz}%
- },
- [\examplename.tex]
- [\examplename.pdf]
- [\normalsize\dots, no edge]
- [\examplename.mmz, tikz={\draw[dotted](.east)--+(5em,0ex) node [anchor=south west, yshift=-1ex, align=justify, text width=14em, font=\footnotesize]{This is another auxiliary file produced by Memoize. We will mention it at the end of the section.};}]
- ]
- \end{forest}
- \end{tcolorbox}
- },
- }
-}
-
-To top it off, changing the memoized code will produce new memo (and extern)
-files, with the old files staying in place. This is all by design --- the
-first hexadecimal number in these filenames is the MD5 sum of the memoized code
-and that's how Memoize knows which memo belongs to which piece of
-code\footnote{The second hexadecimal number in the memo and extern filenames is
- the MD5 sum of the \emph{context}. The context is crucial for properly
- externalizing code containing cross-references, see
- section~\ref{sec:cross-referencing} for details.} --- but it has the downside
-that the folder containing your document can get quite cluttered (imagine the
-directory listing as above, but for a document with a hundred externalized
-pictures which you have been working on for a month).
-
-\ExampleName{clean-house}
-
-I like to keep a clean house by instructing Memoize to put memos and friends
-into their own directory. This can be achieved by writing
-\refcmd{mmzset}\braces{\Emph{\refmmz{memo dir}}} into the preamble (anytime
-after loading the package).\footnote{The \refmmz{memo dir} key is in fact
- merely an abbreviation for a sequence of \refmmzpath{dir} and
- \refmmzpath{prefix} within \refmmz{path}; use these keys if you need more
- control over the name and location of the auxiliary files. Furthermore, there
- is also the \refmmz{no memo dir} key, which reverts the configuration back to
- the dirty default.} This will put the memo and the extern files into folder
-\meta{document name}|.memo.dir| (and it will also omit the \meta{document name}
-prefix in their filenames, because it makes no sense to repeat it there).
-
-\makeexample{\examplename.pdf}
-{%
- \relaxmmzcommands
- \edef\marshal{%
- \noexpand\def\noexpand\mynewextern./\examplename
- }\marshal.memo.dir/#1-#2.pdf\relax{%
- \forestset{
- append={[#1.memo\vphantom{p},pointcolor]},
- append={[#1-#2.memo\vphantom{p},pointcolor]},
- append={[#1-#2.pdf,pointcolor]},
- }%
- }
- \tcbinputexample{
- comment={%
- \begin{tcolorbox}[reset, left=2mm, bottom=0mm,
- my boxed title=the folder contents (after two compilations),
- attach shifted boxed title to top right,
- ]
- \begin{forest}
- before typesetting nodes={for tree={
- font=\scriptsize\tt, grow'=0, folder, s sep=0}},
- [\meta{the document folder}, inner xsep=0
- [\examplename.tex]
- [\examplename.pdf]
- [\normalsize\dots, no edge]
- [\examplename.mmz]
- [\examplename.memo.dir,pointcolor,
- % tikz={\draw[dotted](.east)--+(9em,0ex) node [anchor=south west, yshift=-1ex, font=\footnotesize, emphshade=black!5!white, text width=15em, align=justify]{You need to create this directory by hand, or allow Memoize to create it by adding |mkdir| to the list of restricted shell escape commands.};},
- tikz={\draw[dotted](.east)--+(9em,0ex) node [anchor=west, font=\footnotesize, emphshade=black!5!white]{This directory must be created, somehow.};},
- TeX={%
- \def\mmzNewExtern##1##2##3##4##5{\mynewextern##1\relax}%
- \input{\examplepath.mmz}%
- },
- ]
- ]
- \end{forest}
- \end{tcolorbox}
- },
- }
-}
-
-\begin{tcolorbox}[title={Why is \refmmz[link color=white]{memo dir} not the default?}]
- The \code{clean-house} example most likely compiled just fine, and you are
- wondering why \refmmz{memo dir} is not in effect by default. Well, out of
- the box, \hologo{TeX} cannot create directories, so it is the fact that
- Memoize \emph{can} create them (at least under the default settings) which
- requires explanation. By default, Memoize triggers extraction by executing
- the Perl extraction script \refscript{memoize-extract.pl}, and it is this
- script which actually creates the memo directory. However, not everybody
- will necessarily use this script \dots\ so \refmmz{memo dir} should not be
- the default.
-
- The Python extraction script \refscript{memoize-extract.pl}, used when
- \refmmz{extract}|=|\refmmz{extract=python}, works the same as the Perl
- variant. With \hologo{TeX}-based extraction
- \refmmz{extract}|=|\refmmz{extract=tex}, things are different. If you are
- compiling the document with a full shell escape mode
- (\docref{-shell-escape}), Memoize successfully creates the directory with the
- system command |mkdir|.\footnote{|mkdir| is the default value of key
- \refmmz{mkdir command}, but executing the extraction method
- \refmmz{extract=perl} or \refmmz{extract=python} overrides this default.}
- However, if you're using the restricted shell escape mode, the attempt to
- create the directory won't succeed unless you include |mkdir| among the
- restricted shell escape commands (see footnote~\ref{fn:shell-escape-commands}
- on page~\pageref{fn:shell-escape-commands} for how to do this, but note that
- it is not recommended).
-
- If you are using external extraction, you have to create directory
- \texttt{\examplename.memo.dir} by hand, prior to the first compilation of the
- document (with Memoize). This is the case even if you are performing the
- extraction using one of the shipped extraction methods, and it is due to the
- fact that Memoize needs the memo directory to be present even before extern
- extraction, because it writes the |.memo| files into the same directory.
- (When Memoize uses the extraction script to create the memo directory, it
- does so completely independently of extraction, and prior to creating any
- |.memo| files.)
-\end{tcolorbox}
-
-I actually suggest adding \refcmd{mmzset}\braces{\refmmz{memo dir}} into your
-user-wide \reffile{memoize.cfg} (see section~\ref{sec:memoize.cfg} for details
-on this file). This will keep all your houses clean --- without work! --- as
-Memoize will automatically use the memo directory for any document your create.
-
-It is always safe to delete memos (|.memo|) and externs (|.pdf|s residing next
-to memos), in the sense that you cannot lose data this way.\footnote{The same
- goes for the extern |.log| files produced by the \hologo{TeX}-based
- extraction.}
-
-\begin{itemize}
-
-\item Many memos and externs are typically stale anyway, i.e.\ they reflect
- some previous state of your document and are not needed anymore. These files
- can be deleted without any repercussions whatsoever (unless you later revert
- to a previous version of the document, of course). In fact, you might
- \emph{want} to delete them periodically, or at least once you finish writing
- the document. As it is hard to figure out which memos/externs are stale,
- Memoize ships with a clean-up script: writing \refscript{memoize-clean.pl}
- \meta{document name}\dmmz (replace |.pl| with |.py| if you use Python rather
- than Perl) into the command line will delete all the \emph{stale} auxiliary
- files belonging to the document.
-
-\item If you delete a memo or an extern currently in use, you will trigger
- recompilation of their code --- so deleting a memo or an extern is actually a
- perfectly legal alternative to using the \refmmz{recompile} key!\footnote{For the
- users of the \hologo{TeX}-based extraction: deleting the |.log| file does
- \emph{not} trigger recompilation.}
-
-\end{itemize}
-
-It is also safe to delete the \dmmz file (or any other kind of record file, see
-section~\ref{sec:record-files}) residing next to your document's |.pdf|. The
-\dmmz file contains the information about which externs should be extracted
-from the |.pdf|. Deleting it before this is done (by default, before compiling
-the document again) will prevent the extraction (same as if providing the
-package option \code{\refmmz{extract}=\refmmz{extract=no}}) and ultimately
-result in the recompilation of the externs produced in the previous run.
-Deleting it after the extraction will have almost no effect: it will only only
-prevent the clean-up script from working (the \dmmz file also lists the
-currently active memos and externs, and thereby indirectly informs the clean-up
-script which files are stale). For further information on the \dmmz file, see
-section~\ref{sec:.mmz}.
-
-
-
-\subsection{Writing a book?}
-\label{sec:tut:multifile}
-
-Books and other long documents are usually produced from sources which reside
-in more than a single file, and to speed up the editing process, authors
-usually use some system which allows them to compile each chapter separately.
-Can Memoize --- designed for virtually the same task of speeding up the editing
-process --- work sensibly in this kind of situation? More precisely, can the
-book and the individual chapters share the memos and the externs? Yes they
-can! If we instruct Memoize to use the same memo directory for both the book
-job and the chapter jobs, then we can externalize graphics when compiling a
-chapter and have the externs included when compiling the book (and vice
-versa).\footnote{Package \pkg{docmute} makes \hologo{LaTeX} ignore the preamble of
- the chapter file when including this file into the main document.} All we
-need to do is use our old friend \refmmz{memo dir} from section~\ref{sec:tut:memodir}
---- we see now that this setting is good for more than just keeping a clean
-house!
-
-\ExampleName{book}
-\makeexample{\examplename.tex.c1}
-\makeexample{chapters/chapter1.tex.c1}
-
-\begin{tcbraster}[raster columns=2, raster valign=top]
- \tcbinputexample{listing only}
- \tcbinputexample(chapters/chapter1){listing only}
-\end{tcbraster}
-
-In the above example, the individual chapters reside in files stored in the
-|chapters| subdirectory, and that's why the |book.tex| preamble uses
-\code{\refmmz{memo dir}=chapters/book} (rather than \code{\refmmz{memo
- dir}=book} or just \refmmz{memo dir}). However, Memoize has no trouble
-with a situation where the main file and the chapters reside in the same
-folder; the setup is even simpler, as we then say \code{\refmmz{memo dir}=book}
-in both the book and the chapter preamble. The more complicated situation was
-chosen to point out the following potential problem with the setup where the
-chapters reside in a subdirectory.
-
-If you're anything like me, you would first go for having a memo directory
-immediately contained in the project directory (so |examples/book.memo.dir|
-above) and set up \refmmz{memo dir} as shown below. Well, this won't work, or
-at least it won't work with the vanilla \hologo{TeX} Live, because \hologo{TeX}
-will refuse to \emph{write} into (memo) files outside the directory where it
-was executed,\cprotect\footnote{In \hologo{TeX} Live, the \reffile{texmf.cnf}
- option controlling this behaviour is called |openout_any|. By default, it is
- set to |p| (paranoid), which ``disallow[s] opening dot files [and]
- \emph{going to parent directories}, and restrict[s] absolute paths to be
- under \texttt{\$TEXMFOUTPUT}'' (emphasis mine).} and this is precisely what
-the chapter compilation is asked to do below.
-
-\begin{tcboxedraster}[raster columns=2, raster valign=top]{blankest, bad}
- \begin{tcblisting}{example title, title=the main file, listing only}
-\mmzset{~memo dir=book~}
-% ...
-\input{chapters/chapter1.tex}
- \end{tcblisting}
- \begin{tcblisting}{example title, title=a chapter file, listing only}
-\mmzset{~memo dir=../book~}
- \end{tcblisting}
-\end{tcboxedraster}
-
-\pagebreak % manual
-
-Section~\ref{sec:tut:working-on-a-picture} presented some ideas on how to work
-on a single picture. Those ideas can be all easily applied to the multi-file
-situation. For example, you could use \refmmz{readonly} on the chapter that you're
-working on (and that chapter only). This way, the preview of the chapter will
-not be tarnished by the extern pages, and if you periodically compile it
-without \refmmz{readonly}, or compile the book (which does not have the \refmmz{readonly}
-set), you will have a reasonably up-to-date set of externs.
-
-\begin{tcboxedraster}[raster columns=2, raster valign=top]{blankest}
- \begin{tcblisting}{example title, title=the main file, listing only}
-\mmzset{memo dir=chapters/book}
-% ...
-\input{chapters/chapter1.tex}
- \end{tcblisting}
- \begin{tcblisting}{example title, title=the current chapter file, listing only}
-\mmzset{memo dir=book, ~readonly~}
- \end{tcblisting}
-\end{tcboxedraster}
-
-\ExampleName{memoize.cfg._region_}
-\makeexample{\examplename.tex.c1}
-\begin{tcolorbox}[title=For \Emacs users]
- I often use this \refmmz{readonly} trick myself, but with a twist. As an Emacs user,
- I don't use a \hologo{TeX}-based mechanism (such as the \pkg{docmute} package) to
- compile a chapter, but rely on the region compilation feature of Emacs'
- AUC\hologo{TeX} package. AUC\hologo{TeX} offers a way to compile the current
- buffer (if you don't know what an Emacs buffer is, read ``file'') or region
- (roughly speaking, the selected text). It does that by putting the buffer or
- the region into a file called |_region_.tex| while dressing it up in the
- preamble of the original document (when I'm working on a multi-file document,
- it correctly pulls the preamble from the main document). This results in a
- compilable region file. My trick is to detect whether I'm compiling a region
- (this is the job of |\ifregion|), and if so, put Memoize into the \refmmz{readonly}
- mode (an alternative trick would be to \refmmz{disable} it).
-
- This is the trick in a nutshell, but to make it really work we have to
- address one further issue: the original document and the region have to share
- memos and externs. This happens automatically if the original document sets
- \refmmz{memo dir} explicitly (e.g.\ if a document called |doc.tex| contains
- \code{\refmmz{memo dir}=doc} in the preamble), but I'm lazy and don't want to
- write this in every document --- if I have to do that, what's the point of
- \refmmz{memo dir} I put into my \reffile{memoize.cfg} in
- section~\ref{sec:tut:memodir}? Fortunately, the region file starts with
- |\message{ !name(|\meta{original document name}|.tex)}| to indicate the
- origin. The complicated part of the code below (everything following
- \refcmd{mmzset}\braces{\refmmz{readonly}} parses this header to extract the
- \meta{original document name}, which is then fed to \refmmz{memo dir}. Now,
- the trick works automatically for any document.\footnote{The assumption here
- is that \refmmz{memo dir} is in effect for the original document. If not,
- the trick can be adapted to use (\refmmzpath{dir} and) \refmmzpath{prefix}
- within \refmmz{path}.}
-
- \tcbinputexample{
- title=\tt memoize.cfg,
- attachment name=memoize-region.cfg,
- listing only,
- }
-\end{tcolorbox}
-
-
-\subsection{Writing a presentation?}
-\label{sec:tut:beamer}
-
-Memoize ships with built-in support for the most widespread \hologo{LaTeX}
-presentation class, Beamer, in the sense that it can externalize a picture
-which changes from overlay to overlay. Before we learn how to use that
-functionality, however, there's a peculiarity about loading Memoize in Beamer
-to address.
-
-\ExampleName{beamer}
-\makeexample{\examplename.pdf}
-
-\tcbinputexample{
- comment and listing,
- listing options app={lastline=2},
- warning, no attachment, title=,
- %
- comment={Beamer opens the document PDF while loading the class, while
- Memoize requires the PDF from the previous compilation intact in order to
- extract the externs (when extraction is triggered internally, which is the
- default setting). The solution is to load Memoize (a package) before
- Beamer (a class), which can be done by using \cs{RequirePackage} instead of
- the usual \cs{usepackage}. Easy, if hacky.},
- %
-}
-
-To memoize a piece of code which produces different results on different
-overlays --- by virtue of containing |\pause|, |\only|, and\slash or related
-commands --- apply key \Emph{\refmmz{per overlay}}. Without this key,
-externalization of the picture will end badly, with a single extern (the final
-one) appearing on all overlays. The key may be invoked either from a prior
-\refcmd{mmznext} command,\footnote{Of course, \refmmz{per overlay} may also be
- invoked from \refcmd{mmzset}, but I guess this won't make sense often. For
- example, if you set it for the entire presentation, and the presentation
- contains static memoized pictures as well, you will compile those pictures
- more times than necessary: once for each overlay, whereas once per frame
- would suffice. It might occasionally make sense, however, to use \refmmz{per
- overlay} as an \refmmz{auto} option --- consult
- section~\ref{sec:tut:verbatim} to learn what that is.} or executed in the
-memoized code itself. The example below illustrates the latter option, and
-also shows that we may invoke it via its full path,
-\refkeypath{/mmz}|/|\refmmz{per overlay}, when listed among options processed by
-\pkg{pgfkeys}.\footnote{Read section~\ref{sec:per-overlay} to learn how the
- Beamer support is implemented. The implementation only uses Memoize's public
- interface, so understanding it should help if you need to support some other
- presentation package.}
-
-\tcbinputexample{
- listing options app={firstline=3},
- middle=1mm,
- comment={\def\dpwidth{0.27\linewidth}\def\epwidth{0.15\linewidth}%
- \raisebox{-\height}{\includeexamplepdf[extern page,left=0mm,right=0mm]{width=\epwidth,page=1,trim=1in 1in 0.4in 0.6in}}\hfill
- \raisebox{-\height}{\includeexamplepdf[document page,left=1mm,right=1mm]{page=2,width=\dpwidth}}\hfill
- \raisebox{-\height}{\includeexamplepdf[extern page,left=0mm,right=0mm]{width=\epwidth,page=3,trim=1in 1in 0.4in 0.6in}}\hfill
- \raisebox{-\height}{\includeexamplepdf[document page,left=1mm,right=1mm]{page=4,width=\dpwidth}}%
- },
-}
-
-\begin{tcolorbox}[warning]
- If the memoized code changes the value of Beamer's pause counter
- |beamerpauses|, e.g.\ by issuing a |\pause|, take care that (i) \refmmz{per
- overlay} is executed prior to any changes of |beamerpauses|, and that (ii)
- the final value of this counter in the memoized code is the same for all
- overlays.
-\end{tcolorbox}
-
-
-\subsection{When stuff sticks out}
-\label{sec:tut:padding}
-
-Some constructs --- like plain \hologo{TeX}'s \refcmd{llap} and \refcmd{rlap},
-and, notably, \TikZ overlays --- fool \hologo{TeX} into thinking that the
-``size'' of the typeset material is different than what it actually is. This
-can cause trouble for externalization: a piece of your picture might disappear!
-In a sentence, the solution is to manually set the \emph{padding} of the
-externs, but let's slow down a bit.
-
-The \TikZ picture in the following example consists of node with a pin on the
-right, but let's say we want to horizontally center this picture so that only
-the node rather than the entire picture (including the pin) will be centered.
-This can be achieved by adding key \refkey{/tikz/overlay} to the pin (actually,
-we need to add it to both the pin and its edge). \TikZ normally updates the
-extents (called the bounding box) of the picture every time it puts something
-in it; when \refkey{/tikz/overlay} is in effect, however, these updates are
-temporarily disabled. In effect, the \refkey{/tikz/overlay} key on the pin
-below will fool \hologo{TeX} into thinking that the node is all there is to the
-picture, so centering will work as desired.
-
-\ExampleName{overlay}
-\makeexample{\examplename.pdf N=4}
-\tcbinputexample{
- after title pre={\ (no memoization)},
- comment={\centering
- % \includeexamplepdf[document page]{page=1,trim=1.8in 7.4in 1.8in 1.7in}
- \includeexamplepdf[document page]{page=1}
- },
-}
-
-What happens when we try to externalize this picture? The example below shows
-what would happen if Memoize had no concept of \emph{padding} --- which we
-simulate by setting \code{\refmmz{padding}=0pt}.\footnote{Unlike in the rest of
- the manual, the extern pages in this section are shown without trimming the
- whitespace.} Along with the rest of \hologo{TeX}, Memoize would be fooled
-into thinking that the picture comprises of the node only, so the pin would
-never make it into the extern. You would end up with a document missing the
-pin!\footnote{On the first compilation, the document page containing the figure
- without padding looks fine, as it uses the result of the compilation rather
- than the extern file. But on the second compilation, when Memoize actually
- uses the extern, the pin disappears.}
-
-\tcbinputexample[.tex][.c2]{
- after title pre={\ (memoization without padding)},
- attachment name=\examplename-no-padding.tex,
- sidebyside, lefthand ratio=0.3,
- comment={\centering
- \includeexamplepdf[size=minimal,boxrule=0.5mm,extern page,
- attach boxed title to top right={xshift=1.5mm,yshift=-1mm},
- ]{page=1}
- },
-}
-
-
-By default, Memoize puts an inch of space around (what it thinks is) the
-externalized picture, and if the overlayed parts of the picture fit into this
-inch of space, you will find them in the extern and therefore also in the
-document. In our example, however, the default padding is not enough --- the
-pin is only partially visible.\footnote{You might wonder why I didn't make the
- default padding much bigger, like 10 inches. \hologo{TeX} wouldn't be
- bothered (unless the resulting extern size exceeded its maximum dimension),
- but you might be, because with such a large default padding, all the externs
- would be huge, most often bigger than the document pages, and remember that
- the externs are first dumped into the document, where they can bother you.}
-
-\tcbinputexample[.tex][.c3]{
- after title pre={\ (memoization with default padding)},
- attachment name=\examplename-default-padding.tex,
- listing and comment,
- middle=0.9mm,
- comment={\centering
- \includeexamplepdf[extern page,size=minimal,boxrule=0.5mm]{page=1}
- },
-}
-
-The solution is to set the padding manually. Below, I used \refmmz{padding
- right} to only increase the padding on the right side (clearly, we also have
-\refmmz{padding left}, \refmmz{padding top} and \refmmz{padding bottom}), but
-if you're not bothered by a large extern, you can just use
-\Emph{\refmmz{padding}}, which sets all four sides at once. By the way, having
-too much padding (almost) never hurts, and as you see, you can use (simple)
-arithmetics in the value of these keys.
-
-\tcbinputexample[.tex][.c4]{
- after title pre={\ (memoization with extra padding)},
- attachment name=\examplename-extra-padding.tex,
- listing and comment,
- middle=0.9mm,
- comment={\centering
- \includeexamplepdf[extern page,size=minimal,boxrule=0.5mm]{page=1}
- },
-}
-
-\enlargethispage{1ex} % manual
-Incidentally, the \refmmz{padding} keys only change how
-the externalized picture is \emph{stored}. Memoize remembers the size of the
-extern as seen by \hologo{TeX} (e.g.\ the bounding box of the picture as
-reported by \TikZ, with overlayed parts of the picture protruding out of it),
-and it uses that size when integrating the extern into the document --- so
-everything works as it should!
-
-
-
-\subsection{The verbatim mode}
-\label{sec:tut:verbatim}
-
-Not all code will peacefully submit to memoization. In particular, this is the
-case for environments which process the environment body verbatim (or perform
-some other kind of \refcmd{catcode} magic). A simple environment of this kind is the
-standard \hologo{LaTeX} \refenv{verbatim}, but let us illustrate the issue with
-\refenv{tcblisting}, which typesets a code listing alongside its compiled effect.
-(This environment is defined by the |listings| library of package \pkg{tcolorbox}
-and was used extensively during the production of this manual.) To manually
-memoize a \refenv{tcblisting} environment, we enclose it in a \refenv{memoize}
-environment with a \Emph{\refmmz{verbatim}} key in the optional argument ---
-without this key, the example below would produce nothing but
-errors.\footnote{Memoize also offers a \emph{partial} verbatim mode, triggered
- by key \refmmz{verb}; in this mode, the braces retain their usual category
- codes. Also note that the effect of \refmmz{verbatim} can be ``undone'' by
- key \refmmz{no verbatim}.}
-
-\ExampleName{verbatim-manual}
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- sidebyside,
- comment={\centering
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\\[1ex]
- \footnotesize
- (The document page is the same as for the \texttt{verbatim-auto} example below.)
- },
-}
-
-Using \refmmz{verbatim} from \refcmd{mmzset} or \refcmd{mmznext} works just as
-well, and the latter can be very useful with automemoization, when some
-environment (say, \env{tcolorbox}) generally does not require the verbatim mode,
-but a specific occurrence does (say, because it contains some verbatim
-construction such as \verb!|!\meta{verbatim text}\verb!|! of the \pkg{ltxdoc}
-class).
-
-However, for an environment such as |tcblisting|, it makes the most sense to
-declare it verbatim in general, so that all instances of the environment will
-be processed in the verbatim mode. This is simple to do: add \refmmz{verbatim}
-to the \refmmz{auto} keylist.
-
-\ExampleName{verbatim-auto}
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- sidebyside, % lefthand ratio=0.62,
- comment={\centering
- \footnotesize
- (The extern page is the same as for the \texttt{verbatim-manual} example above.)\\[1ex]
- \includeexamplepdf[document page]{page=2}\\
- },
-}
-
-In fact, you can add any \refkeypath{/mmz} key to the \refmmz{auto} keylist,
-and the key will be applied to all occurrences of the command or the
-environment. For example, adding \refmmz{recompile} to the declaration of
-|tcblisting| above would recompile all and only the |tcblisting| environments;
-and as an \refmmz{auto} declaration only updates (rather than completely
-replaces) a previous declaration, you can also say things like
-\code{\refmmz{auto}=\refcmd{tikz}\braces{\refmmz{recompile}}} to recompile all
-\TikZ pictures produced by the \cs{tikz} command (handy, as you don't know how
-automemoization for \cs{tikz} was declared unless you've read
-section~\ref{sec:tut:automemoization-details} or looked at the Memoize's source
-code).
-
-
-\subsection{The final version of your document}
-\label{sec:tut:final}
-
-Bluntly put, you might want to disable Memoize when compiling the final version
-of your document, at least if you intend to distribute it in electronic form,
-for two reasons:
-\begin{itemize}
-\item An externalized picture cannot contain hyperlinks. Any hyperlinks (or
- hyperlink anchors) contained in the original picture will silently disappear
- during the production of the extern.
-\item When the document contains many externs, the size of the resulting PDF
- can be several times the size of the PDF compiled without externalization.
-\end{itemize}
-
-Below, we list several ways of fully disabling Memoize. You're of course
-already familar with the first two ways, but what's this \refpkg{nomemoize}
-package? The rationale behind this package is that if you want to be
-absolutely sure that there is no trace of memoization in your document (for
-example, see the \refmmz{disable} -- \refmmz{enable} pitfall in
-section~\ref{sec:tut:working-on-a-picture}), the best thing to do is to not
-load the package at all. However, you have all those \refcmd{mmzset}s etc.\ in
-your source, so the document won't compile without
-\cs{usepackage}\braces{\refpkg{memoize}}, right? Right, but wrong. Enter
-\Emph{\refpkg{nomemoize}}, a dummy package which accepts all the commands that
-Memoize does, but does nothing. In effect, your document will compile, but you
-can be sure that not a single memo or extern was loaded or produced.
-
-\begin{tcbraster}[raster columns=3, raster force size=false, raster valign=top]
- \begin{tcblisting}{listing only, add to width=2.4em, enhanced}
-\usepackage[~disable~]{memoize}
- \end{tcblisting}
- \begin{tcblisting}{listing only, add to width=-1.6em, enhanced}
-\usepackage{memoize}
-\mmzset{~disable~}
- \end{tcblisting}
- \begin{tcblisting}{listing only, add to width=-0.8em, enhanced}
-\usepackage{~no~memoize}
- \end{tcblisting}
-\end{tcbraster}
-
-There is one issue you might need to resolve manually before package
-\refpkg{nomemoize} works as intended, though. If you have used any
-\refkeypath{/mmz} keys outside \refcmd{mmzset}, you need to list them in
-\refcmd{nommzkeys}. For example, if you used \refmmz{per overlay} in the manner
-illustrated in section~\ref{sec:tut:beamer}, i.e.\ as
-\refkeypath{/mmz}|/|\refmmz{per overlay} among the \TikZ keys, you need to
-write \refcmd{nommzkeys}\braces{\refmmz{per overlay}} into the document
-preamble.
-
-Another thing you might want to do once you have produced the final version of
-the document (in fact, just before you disable Memoize for good) is clean up.
-As we saw in sections~\ref{sec:tut:memodir} and~\ref{sec:tut:redefinitions},
-Memoize produces a lot of auxiliary files (memos and externs) and it keeps the
-old versions around! Once your document is prepared, you can reduce the
-clutter (and save some disk space) by deleting memos and externs belonging to
-the work-in-progress versions of your document, and keep only those used in the
-final version.
-
-You could achieve this by deleting all the memos and externs (if you're using
-the \refmmz{memo dir} directive, this amounts to the entire contents of the
-memo directory) and compiling your document for the final couple of times.
-However, there is an easier (but \hologo{TeX}-external) way: on the command
-line, change into the directory containing your (main) document and write
-\EmphTT{\refscript{memoize-clean.pl}} \meta{document name}\dmmz (substitute
-|.py| for |.pl| to use Python rather than Perl). The script will inspect the
-contents of the \dmmz record file to see which memos and externs were used in
-the final compilation, and delete all other memos and externs belonging to the
-given document.
-
-\begin{tcolorbox}[warning]
- Deleting memos and externs is never an irreversible operation, as you can
- always recreate them, but it is still wise to be cautious when cleaning up.
- For one, avoid cleaning up after a compilation which produced errors; a
- failed compilation can lead to an incomplete \dmmz file, which can in turn
- lead to over-deletion. Another bad idea is cleaning up after disabling
- Memoize for a part of a document, for the same reason.
-
- All that said, Memoize takes some precautions itself. It will cowardly
- refuse to perform the clean-up when the \dmmz file is missing the
- end-of-file marker (|\endinput|), assuming that this indicates a fatal error
- in the previous compilation. It will do the same in case the \dmmz file is
- absent or empty. The latter is assumed to be a result of a globally
- \refmmz{disable}d memoization, but note that clean-up will be performed if
- memoization was disabled using package \refpkg{nomemoize}: that package does not
- touch the \dmmz file, so cleaning up should work as intended.
-\end{tcolorbox}
-
-As the final note, memos and externs (cleaned-up or not) may be copied (along
-the document source) to another directory or machine, where they should be
-picked up by Memoize. There is no need to copy the \dmmz file (assuming that
-the document PDF contains no extern pages waiting for extraction).
-
-
-\section{Digging deeper}
-\label{sec:potential-pitfalls}
-
-
-
-\subsection{Good to know}
-\label{sec:tut:good-to-know}
-
-
-\paragraph{Line- and page-breaking} An extern can't be broken across lines or
-pages.
-
-Externalization of a chunk of code produces a PDF, which is included into the
-document at subsequent compilations as a picture --- an unbreakable object (a
-horizontal box) with fixed width and height. Therefore, the original code
-should produce an unbreakable object as well. For example, this means that you
-cannot externalize some text and expect \hologo{TeX} to break it across lines
-or pages on subsequent compilations. If you try, the compilation will succeed
---- without an error! --- but your externalized text will end up in a single
-line, as shown below.
-
-\ExampleName{no-linebreaking}
-\makeexample{\examplename.pdf N=2}
-\tcbinputexample{
- bad,
- comment={\centering
- \includeexamplepdf[extern page,to be continued on right]
- {page=1,trim=1in 1in 14in 1in}\\[1ex]
- \includeexamplepdf[document page]{page=2}\\[1ex]
- \includeexamplepdf[document page,title=the expected document page]
- [\examplepath.c2.pdf]{page=1}
- },
-}
-
-That said, you \emph{can} externalize a paragraph or some other vertical mode
-material using \refmmz{capture}|=|\refmmz{capture=vbox}, but beware that the
-vertical spacing between the memoized material and its surroundings might
-change.
-
-
-\paragraph{\env{remember picture}}
-\TikZ pictures using this key cannot be externalized.
-
-Memoize will silently refuse to externalize any \TikZ picture using
-\refkey{/tikz/remember picture} (see \PGFmanual{17.13}). Such pictures
-interact with the outside world --- they either reference or are referenced by
-other pictures --- and are as such unsuitable for externalization. For
-example, while the colored boxes in this manual are generally externalized ---
-out of principle \Smiley\ --- the title page illustration is not, and it cannot
-be, because of the arrows connecting the various \TikZ pictures composing that
-illustration. Some packages use the \refkey{/tikz/remember picture} mechanism
-under the hood, and are thus subject to this limitation; one example is package
-\pkg{todonotes}, but in general, any package dealing with absolute positions on the
-page will be limited in this way.
-
-How does Memoize deal with this situation? Well, by cowardly refusing to
-externalize any code which uses \refkey{/tikz/remember picture} or a similar
-mechanism for dealing with absolute positions. Luckily, any such mechanism
-eventually boils down to the \hologo{TeX} primitive \refcmd{savepos}, so
-Memoize hacks --- or as we will say in this manual, \emph{advises} --- this
-primitive to abort any ongoing memoization. Initializing and then aborting the
-memoization takes some time, to be sure, but the overhead is negligible,
-especially in the light of the fact that \emph{not} aborting wreaks real havoc.
-
-Memoize actually provides a user interface for aborting memoization.
-Memoization can be aborted either manually, by executing \refcmd{mmzAbort}, or
-automatically. The latter is a generalization of the automemoization idea: a
-command such as \refcmd{savepos} can be advised to abort memoization by
-\refmmz{auto}|=|\refcmd{savepos}\bracestt{\refmmzauto{abort}}. In Memoize,
-commands take your advice seriously, so memoization will be aborted whenever
-the advised command or environment is encountered.
-
-
-\paragraph{Indirectly embedded environments} Such environments cannot be
-memoized.
-
-Read this if you got an error message such as |Environment "tikzpicture" ended
-as "foo"|.
-
-Some environments are defined so that they embed another environment using the
-idiom shown on the left: the begin-code of the outer environment opens the
-inner environment, and the end-code of the outer environment closes the inner
-environment. While this is a fine, and common, idiom, it messes up the
-memoization of the inner environment. In the example on the right, trying to
-\emph{auto}memoize a \env{minipage} environment (not recommended at all!) causes
-trouble with package \pkg{sectionbox}.\footnote{This is a \texttt{Package
- collargs Error} because Memoize outsources the actual work of collecting
- the environment body to the auxiliary package CollArgs, described in
- section~\ref{sec:collargs}.}\fncomma\cprotect\footnote{Why does this happen?
- As mentioned in section~\ref{sec:tut:memodir}, Memoize keeps track of memos
- and externs by the MD5 sum of the memoized code. But to compute that sum for
- an environment, Memoize has to \emph{grab} the environment body, meaning it
- has to collect the body in advance. This presents no problem when
- \refcmd{end}\meta{environment name} is already present in the input stream at the
-time \refcmd{begin}\meta{environment name} is executed, like when you use the
- environment normally in your document, or when some macro expands so that it
- produces both \cs{begin}\meta{environment name} and \cs{end}\meta{environment
- name} simultaneously --- so there would be no problem above if
- |\end{minipage}| occured in the beginning code of \env{sectionbox}. The idiom
-presented above is problematic for memoization because at the time \hologo{TeX}
-executes |\begin{sectionbox}|, putting |\begin{minipage}| into the input
- stream, |\end{sectionbox}| is not yet executed and remains as it is. The
- input stream therefore contains a pair of |\begin{minipage}| and
- |\end{sectionbox}|. In the normal, non-memoizing course of events this
- would not be a problem, because |\end{sectionbox}| would eventually expand to
-|\end{minipage}|. During memoization, however, this \emph{is} a problem,
-because, as we said, Memoize needs to grab the environment body: upon
-encountering |\begin{minipage}|, it looks in the input stream for
- |\end{minipage}| --- but there is no |\end{minipage}| in the input stream,
-there is only |\end{sectionbox}|, and this results in the |Environment
-"minipage" ended as "sectionbox"| error.}
-
-\begingroup
-\ExampleName{sectionbox}
-\makeexample{\examplename.tex.c1}
-\mmzset{disable}
-\begin{tcbraster}[raster equal height]
-\begin{tcblisting}{listing only, bad, valign=center}
-\newenvironment{foo}% ... args
-{% the begin-code of foo
- % ...
- \begin{tikzpicture}
- % ...
-}
-{% the end-code of foo
- % ...
- \end{tikzpicture}
- % ...
-}
-\end{tcblisting}
-\tcbinputexample{
- bad,
- comment={\small\tt
- ! Package collargs Error: Environment\\"minipage" ended as "sectionbox".
- },
- }
-\end{tcbraster}
-\endgroup
-
-What are your options in this kind of a situation?
-\begin{enumerate}
-\item The only way to perform any memoization here is to memoize the
- \emph{outer} environment --- if that makes sense.\footnote{This avoids the
- error because Memoize grabs and memoizes the outer environment, and while
- it is memoizing it, further memoization is switched off.} You can do this
- either on a case-by-case basis, by enclosing it in the \refenv{memoize}
- environment, or automemoize it: \code{\refmmz{auto}=\marg{outer
- environment}\braces{\refmmzauto{memoize}}}.
-\item But what if memoizing the outer environment is out of the question?
- Then, the only way to avoid the error is to prevent the automemoization of
- the inner environment.
- \begin{enumerate}
- \item If you are facing a single occurrence of the problem, it is perhaps
- easiest to use key \refmmz{disable} just before the start of the outer
- environment.
- \item Otherwise, you can automatically disable memoization for the span of
- each occurrence of the outer environment: \code{\refmmz{auto}=\marg{outer
- environment}%
- \braces{\refmmzauto{nomemoize}}}.
- \item To deactivate the automemoization of the inner environment for the span
- of the outer environment, but otherwise allow for memoization inside the
- outer environment: \code{\refmmz{auto}=\marg{outer environment}%
- \braces{\refmmzauto{noop}, \refmmz{deactivate}=\meta{inner
- environment}}}. Key \refmmzauto{noop} does nothing but apply the
- given options (in this case, \refmmz{deactivate}=\meta{inner environment})
- to the advised command or environment.
- \end{enumerate}
-\end{enumerate}
-
-\subsection{Extraction methods and modes}
-\label{sec:extraction}
-
-Remember that in Memoize, externalization is a two-step process. First,
-externs are typeset on separate pages of the main document, called extern
-pages. Then, these extern pages are extracted out of the main document PDF
-into extern files. The process is illustrated on the title page.
-
-Memoize is flexible in terms of which piece of software is used to perform
-extern extraction. It ships with three \emph{extraction methods}:
-
-\begin{description}
-\item[\refmmz{extract=perl}] A Perl script, \refscript{memoize-extract.pl}.
- This method is the default because it is fast and because Perl is usually
- already installed on a system running \hologo{TeX}. However, you will most
- likely still need to install two pieces of software that the script depends
- on: Perl libraries \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} and
- the \hreftt{https://metacpan.org/pod/Path::Class}{Path::Class}, the
- installation guidelines can be found in section~\ref{sec:install}.
-\item[\refmmz{extract=python}] A Python script, \refscript{memoize-extract.py}.
- This method is even faster than the Perl script, though not by much. Try it
- out if you have problems installing Perl or the required libraries, or if the
- Perl script chokes on your document (see section~\ref{sec:troubleshooting}
- for the list of known issues). Besides Python ($\geq 3.8$), you will also
- need the Python library \hreftt{https://pypi.org/project/pdfrw/}{pdfrw} or
- \hreftt{https://pypi.org/project/pdfrw2/}{pdfrw2}. For the installation
- guidelines, see section~\ref{sec:install}.
-\item[\refmmz{extract=tex}] \hologo{TeX}-based extraction requires no
- additional software, but it is much slower than the scripts. As \hologo{TeX}
- can only produce a single PDF per compilation, an instance of \hologo{TeX}
- (loading the entire document PDF) has to be invoked for each extern, and this
- takes time (although the entire process is still much faster than the
- venerable \TikZ externalization library).
-\end{description}
-
-Memoize is also flexible in terms of how extern extraction is triggered,
-providing two \emph{extraction modes}:
-
-\begin{description}
-\item[internal]\label{item:setup:who} By default, extern extraction is
- triggered internally, i.e.\ by Memoize during the compilation of the
- document; more precisely, any externs produced in a compilation are extracted
- during the next compilation. To choose an extraction method other than the
- default Perl script, load Memoize with the package option
- \EmphTT{\refmmz{extract}=\meta{extraction method}}.
-
-\item[external] Loading Memoize with with package option
- \EmphTT{\refmmz{extract}=\refmmz{extract=no}} instructs Memoize to \emph{not}
- trigger the (internal) extraction. When instructed to use extraction
- ``method'' \refmmz{extract=no}, Memoize expects you to trigger the extraction
- yourself, in any way that is convenient to you: manually from the command
- line, or automatically through your editor, a Makefile, etc. --- all Memoize
- cares about is that the extraction takes place before the next compilation of
- the document.
-\end{description}
-
-Summing up, the extraction mode and method are selected by providing the
-appropriate value to package option key \refmmz{extract}; the possible values
-are listed in the table below. Note that this key can only be used as a
-package option, or in \refcmd{mmzset} within \reffile{memoize.cfg}. In
-particular, it is disabled in the document preamble, because Memoize performs
-extraction while it is loaded.
-
-\medskip
-\begin{center}
- \begin{tabular}{lll}
- \toprule
- extraction method&external program&Memoize invocation\\
- \midrule
- \refmmz{extract=perl}&\refscript{memoize-extract.pl}&\refcmd{usepackage}\braces{\refpkg{memoize}}\footnotemark\\
- \refmmz{extract=python}&\refscript{memoize-extract.py}&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=python}}\braces{\refpkg{memoize}}\\
- \refmmz{extract=tex}&|pdftex|&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=tex}}\braces{\refpkg{memoize}}\\
- \refmmz{extract=no}&none (external extraction)&\refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=no}}\braces{\refpkg{memoize}}\\
- \bottomrule
- \end{tabular}%
- \footnotetext{Or
- \refcmd{usepackage}\bracketstt{\refmmz{extract}=\refmmz{extract=perl}}%
- \braces{\refpkg{memoize}}. This is useful if you have changed the default
- using \reffile{memoize.cfg}.}%
-\end{center}
-\smallskip
-
-For internal extraction, \hologo{TeX} must be allowed to execute the external
-program implementing the chosen extraction method. Both |memoize-extract|
-scripts should be listed among restricted shell escape mode commands in your
-\hologo{TeX} distribution; their execution should therefore be allowed under
-the default, restricted shell escape mode. However, the |pdftex| program,
-executed by extraction method \refmmz{extract=tex}, is not listed there, nor
-should it be. If you are forced to use this fallback method, I suggest you
-compile documents loading Memoize under the full shell escape mode, by adding
-command-line option \docref{-shell-escape} (on \hologo{TeX}Live) or
-\docref{--enable-write18} (on MiK\hologo{TeX}) to the invocation of the
-\hologo{TeX} program. The answers linked from question
-``\href{https://tex.stackexchange.com/q/598818/16819}{How can I enable
- shell-escape?}'' on \href{https://tex.stackexchange.com}{\hologo{TeX}
- StackExchange} will tell you how you can ask your editor to do this for you.
-
-You may use any extraction method to perform external extraction. The simplest
-option is to use the Perl or the Python script. Supposing you are doing this
-manually from the command line, change into the directory containing your
-document, which should contain the auxiliary \dmmz file produced by Memoize,
-and execute:
-
-\begin{enumerate}[(a)]
-\item \refscript{memoize-extract.pl} \meta{document name}\dmmz \hfill (for
- the Perl script)
-\item \refscript{memoize-extract.py} \meta{document name}\dmmz \hfill (for
- the Python script)
-\end{enumerate}
-
-See sections~\ref{sec:.mmz} and \ref{sec:ref:extraction-perl-python} for
-further details on the \dmmz file and the extraction scripts.
-
-Things are a bit more complicated if you want to use the \hologo{TeX}-based
-extraction method externally, because an instance of |pdftex| needs to be
-invoked for each extern (and these have unwieldy names and can be many in
-number), but Memoize can help you here by producing a shell script or a
-makefile, executing which will extract all the externs at once. To have Memoize
-produce a shell script, use package option
-\code{\refmmz{record}=\refmmz{record=sh}} (or
-\code{\refmmz{record}=\refmmz{record=bat}} on Windows); package option
-\code{\refmmz{record}=\refmmz{record=makefile}} will make a makefile. By
-default, these files are named |memoize-extract.|\meta{document name} plus the
-|.sh|, |.bat| or |.makefile| suffix. If neither a shell script nor a makefile
-works for you, you can also define your own kind of \emph{record file}, to be
-processed by the external tool of your choice (and implementation) in order to
-extract the externs; see section~\ref{sec:new-record-file} to learn how to do
-this.
-
-
-\subsection{From cross-references to the context}
-\label{sec:cross-referencing}
-
-Cross-referencing presents a challenge to externalization, because without
-special provisions, the ``communication channel'' between the \cs{label} and the
-\cs{ref} is broken once we start utilizing the extern.
-
-One direction of the issue occurs when a \refcmd{label} within the memoized
-code is referenced by a \cs{ref} on the outside. Without the (built-in)
-workaround, the \cs{label} command would only be executed when the extern is
-being produced, but not on subsequent compilations of the document, when it is
-merely included. Memoize addresses this problem by generalizing
-externalization (which can only produce a picture, the extern) to memoization
-(which can additionally produce arbitrary code). When Memoize is externalizing
-code which contains a \cs{label}, it automatically replicates it into the
-\emph{memo}, which is input into the document on subsequent compilations. In
-effect, the memo--extern team will continue to produce the label even when it
-is utilized rather than compiled. As far as the author is concerned, \cs{label}s
-in memoized code ``just work,'' without any observable differences to the
-situation without memoization. This is why we will not discuss this direction
-of the issue here; a reader interested in how precisely the system works is
-invited to read section~\ref{sec:memos}.
-
-The other direction of the issue occurs when a \refcmd{ref} within the memoized
-code references \cs{label} on the outside. In this situation, the extern
-should be recompiled when the value of the label it refers to changes. Again,
-Memoize addresses this problem in full generality, by associating with each
-extern a \emph{context}, and recompiling the extern whenever the value of the
-context changes.\footnote{The dependency of an extern upon prior definitions
- and such can also be addressed in a more \emph{ad hoc} manner, by recompiling
- manually; we have already touched upon this subject in
- section~\ref{sec:tut:working-on-a-picture}, and will revisit it in
- section~\ref{sec:tut:redefinitions}.} All that needs to be done for \cs{ref}
-and friends, specifically, is to advise them to add their reference keys to the
-context.
-
-As we shall see presently, for the author, the only difference between a
-non-memoized and a memoized \cs{ref} is that the latter will take one more
-compilation cycle to ``stabilize'' the resulting document. (More precisely,
-the memoized situation will take one more cycle if the reference is undefined
-on the first compilation.) Then, we will show how we can teach Memoize about
-cross-referencing commands other than \cs{ref} and \cs{pageref}. Finally, we will
-learn about key \refmmz{context}, the backbone of the cross-referencing support
-in Memoize. (The inner workings of the context are further explained in
-section~\ref{sec:c-memos}.)
-
-\ExampleName{ref}
-\makeexample{\examplename.pdf N=7}
-
-When the memoized code contains a \cs{ref} referring to a label given in another
-part of the document, the code is recompiled when (and only when) the reference
-changes. Let us look at the following example, jumping in at the point where
-it was already compiled enough times that the resulting PDF had stabilized into
-a single (document) page with correct references. (Environment
-\refenv{nomemoize} disables memoization
-of \href{https://ctan.org/pkg/tikzlings}{Ti\emph{k}Zlings}, so that their
-externs don't disturb us, and we can focus on the \cs{tikz} command, which does
-get externalized and contains a \cs{ref}.)
-
-\tcbinputexample[.tex][.c3]{
- after title pre={\ (with stable output after three compilations)},
- middle=1mm,
- comment={\centering
- \includeexamplepdf[document page]{page=1}
- }
-}
-
-Let us add an owl in front of the penguin. In the next compilation, neither
-the ``normal'' nor the memoized reference is yet updated, as expected --- in
-this compilation, the new value of the penguin label only makes it into the
-|.aux| file.
-
-\tcbinputexample[.tex][.c4]{
- after title pre={\ (after the first compilation with the added owl)},
- no attachment,
- comment={\centering
- \includeexamplepdf[document page]{page=1}
- }
-}
-
-During the following compilation, the \refcmd{ref}s pick up the new value of the
-penguin label, and the \cs{ref} inside the automemoized \cs{tikz} command
-forces recompilation of the extern (\emph{how} this is done will be explained
-later).
-
-\tcbinputexample[.tex][.c5]{
- after title pre={\ (after the second compilation with the added owl)},
- comment only, no attachment,
- comment={\centering
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}\quad
- \includeexamplepdf[document page]{page=2}
- }
-}
-
-In the next compilation, the resulting PDF is finally stabilized, as the
-updated extern is (extracted and) included into the document.
-
-\tcbinputexample[.tex][.c6]{
- after title pre={\ (after the third compilation with the added owl)},
- comment only, no attachment,
- comment={\centering
- \includeexamplepdf[document page]{page=1}
- }
-}
-
-The message to take home? When some memoized code contains a reference and
-that reference changes, it will take three compilation cycles (so, one more
-cycle than without memoization) for the resulting document to ``stabilize.''
-
-Out of the box, Memoize supports the standard \hologo{LaTeX} cross-referencing
-commands \cs{ref} and \refcmd{pageref}. To automatically recompile code
-containing some other cross-referencing command, like \refcmd{vref} of package
-\pkg{varioref}, we use the advising framework implemented by package Advice.
-This framework is a generalization of automemoization: we use the familiar
-\refmmz{auto}, but with advice offered by \Emph{\refmmzauto{ref}} rather than
-\refmmzauto{memoize}.
-
-{\ExampleName{vref}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{listing only}}
-
-Key \refmmzauto{ref} only works for commands which operate on a single
-reference key. However, that single key (which must be enclosed in braces) may
-be preceded by optional argumen(s) of any kind. Extensions to \refcmd{ref},
-e.g.\ the \pkg{hyperref}'s variant, which accepts an optional |*|, work out of
-the box. Furthermore, Memoize offers support for cross-referencing commands
-which work on multireferences and reference ranges, such as \pkg{cleveref}'s
-\refcmd{cref} and \refcmd{crefrange}. Those commands should be advised by
-\refmmz{auto} keys \refmmzauto{multiref} and \refmmzauto{refrange},
-respectively.
-
-We have jumped into first example of this section with the assumption that it
-had already been compiled several times, allowing the resulting PDF to
-stabilize. Let us now take a look at what happens at the very first, fresh
-compilation of our original example (the one without the owl). (Removing the
-|.aux| file before compiling the example again will start afresh.) The curious
-thing is that we don't get the extern page containing
-\tikz[baseline]\node[draw=red,thick,fill=yellow,anchor=base,font=\bf]{??};.
-This is so because by default, Memoize aborts a memoization containing an
-undefined reference.
-
-\tcbinputexample[.tex][.c1]{
- after title pre={\ (after the fresh compilation of the original example)},
- comment only, no attachment,
- comment={\centering
- \includeexamplepdf[document page]{page=2}
- }
-}
-
-Now sometimes you might want to produce an extern even if it contains an
-undefined reference --- for example, you might intend to write the code
-containing the \cs{label} much later but enjoy the speed-up offered by Memoize
-until then. In that case, apply the \refmmz{auto} key \Emph{\refmmzauto{force
- ref}} to \cs{ref}.
-
-\tcbinputexample[.tex][.c7]{
- after title pre={\ (after the fresh compilation with \refmmzauto{force ref})},
- attachment name=\examplename-force.tex,
- comment={\centering
- \includeexamplepdf[extern page][\examplepath.c1.pdf]{page=1,trim=1in 1in 1in 1in}\quad
- \includeexamplepdf[document page][\examplepath.c1.pdf]{page=2}
- }
-}
-
-However, note that when you use \refmmzauto{force ref}, \hologo{LaTeX} will
-\emph{not} complain about the undefined reference once the extern containing it
-is included (unless that reference also occurs in some non-memoized piece of
-code). Using \refmmzauto{force ref} is therefore a tiny bit dangerous, and
-this is why \refmmzauto{ref}, with the abortion mechanism, is the default
-handler for \cs{ref} and \refcmd{pageref}.
-
-As already noted in the previous section, \refcmd{ref} works by appending the
-cross-reference to the \emph{context}, the change of which triggers
-recompilation. Memoize initializes the context to contain the four
-\refmmz{padding}s --- as a result, an extern recompiles if we change the
-padding --- but we can append stuff to the context by ourselves, as well.
-Below, we use key \Emph{\refmmz{context}} to append the font size (we'll talk
-about the value given to this key a bit later); as a result, the picture is
-recompiled whenever the font size changes. Below, we change the font size
-using command |\small|; changing the default size with a class option such as
-|12pt| works as well.
-
-\ExampleName{fontsize}
-\makeexample{\examplename.pdf N=2}
-\tcbinputexample{
- sidebyside, lefthand ratio=0.7, after title pre={\ (the first version)},
- comment={
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
- }
-}
-
-\tcbinputexample[.tex][.c2]{
- sidebyside, lefthand ratio=0.7, after title pre={\ (the second version)},
- no attachment,
- comment={
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in}
- }
-}
-
-How does this work? Key \refmmz{context} appends the given tokens to the
-\emph{context expression}. When creating an extern or trying to use it,
-Memoize (fully) expands this expression and computes the MD5 sum of the
-expansion. This \emph{context MD5 sum} then serves as a part of the extern's
-filename (see sections~\ref{sec:tut:memodir} and~\ref{sec:memos}). In effect,
-Memoize will only find (and utilize) the extern if \emph{the context MD5 sum
- computed during (attempted) utilization matches the one computed during
- memoization}.
-
-As revealed by looking at the \hologo{LaTeX} source code, \hologo{LaTeX} holds
-the current font size in macro |\f at size|, and above, we have effectively added
-the contents of this macro to the context. Now, why didn't we simply write
-\refmmz{context}|=\f at size|? First, we used |\csname ... \endcsname| because we
-were under the normal \hologo{LaTeX} catcode regime, where |@| cannot be a part
-of the command name. Of course, we could have temporarily changed the catcode
-of |@| using |\makeatletter| and |\makeatother|, but I would advise against
-that, because the approach does not work in general: it fails when key
-\refmmz{context} is used \emph{within} memoized code (we will explain why in
-section~\ref{sec:memos}). Another reason why I recommend the |\csname
-... \endcsname| approach is that it does not result in an error when the
-control sequence is not defined (|\csname ... \endcsname| will expand to
-|\relax| then); this behaviour is handy for undefined cross-references, for
-example. Second, why did I write |fsize={...}| around the control sequence?
-Well, because I'm being paranoid, really. Writing \refmmz{context}|={\csname
- f at size\endcsname}| would work just as well, but I like to explicitly
-``announce'' the value to prevent any possibility of a conflict with an
-alternative context. Imagine that we don't use the ``announcements'' and we
-decide to add some other dimension instead of the font size to the context.
-Now if that dimension happened to have the same value as the font size, Memoize
-would incorrectly pick up the ``font size extern'' instead of producing a new
-one.
-
-It bears emphasizing that whatever you add to the context expression must be
-fully expandable, and also not merely declared as robust. So writing
-\code{\refmmz{context}=\cs{ref}\marg{key}}, for example, would be unwise, since
-it would not work as intended when package \pkg{hyperref} is loaded. (This
-package declares \refcmd{ref} as robust, so it won't expand to the
-cross-reference value.) You have to look up where the cross-references are
-stored internally; the cross-reference for \meta{key} turns out to be stored in
-the internal control sequence |\r@|\meta{key}, so it is |\csname
-r@|\meta{key}|\endcsname| that the \refmmzauto{ref} handler actually appends to
-the context.
-
-The padding and font-size contexts are useful quite generally. However, the
-context can be pretty command-specific, as well. Consider the \pkg{skak}
-package used for typesetting chess games. The board is drawn using command
-|\showboard|, but this command has no arguments, because it draws the state of
-the board that is reached by the moves given by command |\mainline|. Memoizing
-|\showboard| as such will therefore yield the wrong result --- all the boards
-will be one and the same board! The solution is to provide the correct
-context: we dig into the \pkg{skak} sources and realize that the current
-board is stored in macro |\csname chessgame.skak.mainline\endcsname|.
-
-\ExampleName{skak}
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- sidebyside, lefthand ratio=0.48,
- comment={
- \includeexamplepdf[extern page]{page=1,trim=1in 1in 1in 1in,scale=0.45}\quad
- \includeexamplepdf[extern page]{page=2,trim=1in 1in 1in 1in,scale=0.45}
- }
-}
-
-If you remove \code{\refmmz{context}=\bracestt{...}} from the code above, you
-will end up with a document where the final board drawn takes place of all the
-boards. This is so because in that case, all externs are written into the same
-file, so the final extern overwrites the previous ones, but note that you will
-only observe this after the second compilation, when the externs are actually
-used.
-
-
-
-\subsection{More on redefinitions and stale externs}
-\label{sec:tut:redefinitions}
-
-In this subsection, we elaborate on an issue touched upon at the beginning of
-section~\ref{sec:tut:working-on-a-picture}: what happens if the memoized code
-depends on some macro or style which gets redefined? The answer was
-``nothing,'' and one solution was to \refmmz{recompile} the code. Let us take
-the example from that section a bit further. We will propose no new solution
-or workaround, but deepen our understanding of the issue.
-
-\ExampleName{redefinitions}
-\makeexample{\examplename.pdf N=6}
-\tcbset{
- recompile step/.style={
- sidebyside, lefthand ratio=0.74, title=,
- comment={\includeexamplepdf[blankest][\examplepath.c\theenumi.pdf]{#1}},
- /utils/exec={\addtocounter{enumi}{1}},
- enhanced, overlay app={\node[at=(frame.north west), font=\scriptsize, circle, fill=black!80!white, text=white, inner sep=1pt]{\theenumi};},
- },
-}
-\begin{tcboxedraster}[raster columns=1]{%
- title=Working on \texttt{\examplename.tex}, enhanced, breakable}
- \setcounter{enumi}{0}%
- \tcbox[blankest]{I like red. My emphasized nodes will have red background.}
- \tcbinputexample[.tex][.c1]{recompile step={page=2}}
- \tcbox[blankest]{Hmm, this particular node is really important,
- let me put the text in italics as well!}
- \tcbinputexample[.tex][.c2]{recompile step={page=2}, no attachment}
- \tcbox[blankest]{You know what? Perhaps yellow background would work better
- --- in general.}
- \tcbinputexample[.tex][.c3]{recompile step={page=1}, bad, no attachment}
- \tcbox[blankest]{How come my node is still red?! Oh yes, I changed the
- style, so I have to recompile the extern!}
- \tcbinputexample[.tex][.c4]{recompile step={page=2}, no attachment}
- \tcbox[blankest]{Ahh, yellow background, that's much better. But you know
- what, this double emphasis won't do after all, let me go back to the
- upright shape.}
- \tcbinputexample[.tex][.c5]{recompile step={page=1}, bad, no attachment}
- \tcbox[blankest]{Red???!???!? Ok, I know that recompiling will help, but what
- happened here?}
- \tcbinputexample[.tex][.c6]{recompile step={page=2}, no attachment}
-\end{tcboxedraster}
-
-What happened is that the externs from steps~1 and~5 share the very same code.
-In step~1, this code was compiled when the red |emph| style was in effect, and
-that extern lingered and was eventually picked up again in step~5, Memoize
-having no idea that it is including an extern produced with the obsolete
-definition of the style.
-
-There are two points to this story. First (and forgetting for a moment about
-the context, which we started discussing in
-section~\ref{sec:cross-referencing}), Memoize identifies externs (and memos) by
-the code that produced them --- or more precisely, by the MD5 sum of the code,
-as each piece of code has a unique (well, unique enough) MD5 sum. Each extern
-is saved in a file whose name contains this MD5 sum; see
-section~\ref{sec:tut:memodir} for illustration. Generally, this is a very
-useful feature. You can move your picture to another location in the document,
-insert some other (externalized) picture in front of it, and so on, all without
-triggering recompilation of the extern(s). (None of this is possible with
-\TikZ's externalization library, which identifies the externs by the order in
-which they appear in the document.)
-
-The downside of the MD5 sum approach is the potential pitfall illustrated
-above, and the downside comes about because of the second point of the story:
-Memoize does not attempt to delete the ``old'' externs. (However, as described
-in section~\ref{sec:tut:memodir}, stale memos and externs can be easily removed
-using the \refscript{memoize-clean.pl} script.) That would be not only
-dangerous (as any deletion inherently is) but also potentially wasteful: what
-if you have only temporarily removed some code, or compiled only a portion of
-the document --- you surely wouldn't want your hard-won externs to disappear in
-such a situation!
-
-The pitfall described above applies to any command which depends on parameters
-which can be set prior to the invocation of the command, like \TikZ pictures,
-which depend on the settings given in \cs{tikzset}. After customizing the
-settings, you will have to recompile the externs:
-\refcmd{mmznext}\braces{\refmmz{recompile}} is useful when you only have to
-recompile a single extern; use \refcmd{mmzset}\braces{\refmmz{recompile}}, or
-the package option \refmmz{recompile}, to recompile all externs in the
-document; and there is also the middle road: if you have changed only Forest's
-settings, you can write
-\code{\refmmz{auto}=\braces{forest}\braces{\refmmz{recompile}}} to recompile
-all and only the Forest trees.
-
-Above, we have seen the ``same code, same extern'' issue manifested ``through
-time,'' i.e.\ Memoize was (incorrectly) reusing externs produced in previous
-compilations, but the issue can also manifest ``through space.'' This can
-happen if the same code appears twice in the same document --- but, crucially,
-with some parameters which it depends on changed from one occurrence to the
-next. Observe what happens in the following example, where the settings for
-|\progressbar| are changed by |\progressbarchange|. After the first
-compilation, everything looks fine. But as both extern pages were produced by
-the same code, they will be stored into the same file, the second one
-overwriting the first one. The second compilation pulls in the externs --- or
-rather, the single extern --- resulting in the document containing the second
-progressbar in the place of the first one as well.
-
-\ExampleName{progressbar}
-\makeexample{\examplename.pdf N=2}
-\tcbinputexample{
- bad, float,
- comment={
- \begin{tcbitemize}[raster columns=3,raster equal height,halign=center,valign=center]
- \tcbitem[title=After 1\textsuperscript{st} compilation]
- \includeexamplepdf[extern page,remember as=extern1]{page=1,trim=1in 1in 1in 1in}\\
- \includeexamplepdf[extern page,remember as=extern2]{page=2,trim=1in 1in 1in 1in}\\
- \includeexamplepdf[document page]{page=3}
- \tcbitem[title=After extern extraction\vphantom{p\textsuperscript{nd}}]
- \includeexamplepdf[my boxed title=the extern file,attach shifted boxed title to top right,remember as=externfile]{page=2,trim=1in 1in 1in 1in}
- \tcbitem[title=After 2\textsuperscript{nd} compilation]
- \includeexamplepdf[document page,remember as=doc][\examplepath.c2.pdf]{page=1}
- \end{tcbitemize}
- \begin{tikzpicture}[remember picture,overlay]
- \draw[->, thick, red] (extern1.east) to[out=east, in=west] ([yshift=0.5ex]externfile.west);
- \draw[->, thick, red] (extern2.east) to[out=east, in=west] ([yshift=-0.5ex]externfile.west);
- \draw[->, thick, blue] (externfile.east) to[out=east, in=west] (doc.west);
- \end{tikzpicture}
- },
-}
-
-The same ``extern duplication'' can arise due to how a particular command is
-implemented. Say we deactivate automemoization of Forest trees
-(\code{\refmmz{deactivate}=forest}), but keep on automemoizing \TikZ pictures.
-Forest uses \refenv{tikzpicture}\noprint{\refenv{forest}} under the hood (a
-lot); in particular, the tree itself is typeset as a \env{tikzpicture}
-environment. But the code that typesets it is the same for all trees,
-regardless of their content (the actual content of the tree is hidden in
-various macros and boxes, rather than ``pasted'' into the \env{tikzpicture}).
-Consequently, the final tree of the document will overwrite all other trees in
-the document, just as the second (and thus final) progress bar overwrote the
-first one above.\footnote{That is assuming that \hologo{TeX} doesn't simply
- spew a bunch of errors. This can happen as well. In the interest of full
- disclosure, compiling a Forest tree in the situation described above would
- actually also produce --- but only in the first compilation --- a number of
- small empty extern pages, one for each node of the tree. A promise: Forest
- will soon fully support Memoize and (among other things) avoid this pitfall.
- But the principle will remain.} Ouch!
-
-Generally speaking, this final sort of extern duplication issue can arise
-whenever we have an ``outer'' command that we don't want to (auto)memoize which
-uses an ``inner'' command that we \emph{do} want to automemoize. The solution
-is to use the \refmmz{auto} key \refmmzauto{nomemoize} on the outer command;
-remember that this key disables memoization for the space of the command or
-environment. For example, the correct way to ``deactivate'' automemoization of
-\env{forest} environments (but keep automemoizing \TikZ pictures) is
-\code{\refmmz{auto}=\bracestt{forest}\braces{\refmmzauto{nomemoize}}}.
-
-
-
-
-\subsection{Supporting Memoize in your package}
-\label{sec:tut:package}
-
-\subsubsection{Loading Memoize?}
-\label{sec:loading-memoize}
-
-So you want to support Memoize in your package? That's great!
-
-What form precisely this support will take of course depends on your package.
-For some commands, a simple \refmmz{auto} declaration will suffice; for other
-commands, you might need to write a dedicated \emph{memoization driver}, as
-explained in section~\ref{sec:memoization-drivers}. However, one thing is
-clear: you \emph{don't} want to require Memoize's presence by
-\cs{RequirePackage}\braces{\refpkg{memoize}} in your package. That would
-trigger memoization, but triggering memoization should be left at the sole
-discretion of the author. The question is, if you're not allowed to load
-Memoize, how can you even issue the \refmmz{auto} declaration?
-
-Well, it's not that you really want to \emph{memoize} anything; you want to
-make the commands of your package \emph{memoizable}. So:
-\cs{RequirePackage}\braces{\Emph{\refpkg{memoizable}}} --- and note that in
-`memoizable', the final `e' of `memoize' is dropped, apparently this is the
-correct way to spell it.
-
-Loading \refpkg{memoizable} does nothing if Memoize is already loaded, and
-behaves like package \refpkg{nomemoize} otherwise --- remember from
-section~\ref{sec:tut:final} that \refpkg{nomemoize} is a dummy package which
-accepts all the commands that Memoize does, but does nothing.
-
-I have decided to require that Memoize must be loaded before any package that
-supports it. Allowing for an arbitrary loading order would complicate the
-implementation (and possibly even turn out to be problematic), and furthermore,
-Memoize likes to be loaded early anyaway, because it needs to be loaded before
-the document PDF is opened if it is to perform the embedded extern extraction.
-I don't think the ordering requirement will cause any problems --- let me know
-if it does! --- but perhaps it is wise to inform the author about it in the
-documentation of your package (I did so at the end of
-section~\ref{sec:tut:automemoization}). Anyway, I have enforced the
-requirement by raising an error and refusing to load the package in case
-Memoize detects \refpkg{memoizable} to be loaded.
-
-Note that the loading order requirement implies that you can use
-|\@ifpackageloaded{memoize}| to specifically react to the presence Memoize, if
-necessary.
-
-
-\subsubsection{Memoizable design}
-\label{sec:memoizable-design}
-
-Many commands and environments can be submitted to externalization with a
-single-line \refmmz{auto} declaration, as illustrated in
-section~\ref{sec:tut:automemoization}, perhaps requiring an addition to the
-context (section~\ref{sec:cross-referencing}), or some pre- or post-memoization
-code (section~\ref{sec:tut:beamer}). In some situations, however, these simple
-approaches won't work. Most often, this will happen when the extern must be
-integrated into the document in some special way. For example, a command might
-internally create floating material, or surround the core typeset material with
-some stretchable space.\footnote{Commands and environments of package
- \pkg{tcolorbox} exhibit both these issues (see \pkg{tcolorbox} options |float|,
- |before| and |after|), and were in fact the inspiration for several technical
- details of Memoize.} None of these behaviours can be replicated by merely
-including the extern; with respect to the stretchable space, remember that an
-extern, being a picture, has fixed size, so if our extra space ended up in the
-extern, it would lose the stretchability.
-
-The key to successful memoization of problematic commands is their design. In
-a nutshell, the idea is to \emph{break up the command's definition into two
- parts, the outer command and the inner command, and only submit the inner
- command to automemoization}. We will illustrate this with a simple
-environment --- |poormansbox| --- which produces a potentially framed box of
-the given width, and surrounds this box with some configurable material --- by
-default, this material will be stretchable vertical space, and this will be the
-source of our memoization problem. (In terms of user experience, the solution
-in this section will leave something to be desired, but we will revisit the
-example in section~\ref{sec:memoization-complex-single-driver} and make things
-right.)
-
-Let us first take a look at a document using our to-be-developed box
-environment. The |poormansbox| environment takes one optional argument, a
-keylist of options, which can also be set with the |\pmbset| command. This
-being a \textbf{p}oor \textbf{m}an's \textbf{b}ox, it doesn't recognize many
-options. One can set the |width| of the box, or request that it occurs in a
-|frame|, and the surrounding material can be configured using keys |before| and
-|after|. As we will see later in the listing of the package, the box is
-|\linewidth| wide by default, has no frame, and is surrounded by vertical glue
-|\vskip 2ex plus 1ex minus 1ex| (|2ex| of natural vertical space which may both
-stretch and shrink for |1ex|); furthermore, the default value of |before|
-contains |\centering| to center the box horizontally (centering is of course
-only observable when we change the width of the box).
-
-\ExampleName{poormansbox}
-\makeexample{\examplename.sty}
-\makeexample{\examplename.pdf}
-\tcbinputexample{
- sidebyside, lefthand ratio=0.482,
- comment={\centering
- \includeexamplepdf[document page,left=0pt,right=0pt]
- {page=1,scale=0.5,trim=0mm 3mm 0mm 3mm}
- },
-}
-
-You might want to play with the example to see that the surrounding vertical
-space is indeed stretchable. The example is set up so that the surrounding
-space is shrunk a bit to fit all the material onto one page. But if you remove
-the final |\lipsum[144]|, the natural amount of all vertical space can be
-accommodated on the page, so you should observe an increase of vertical
-spacing.
-
-You might have noticed that the example contains nested |poormansbox|es: the
-second box (the one which contains |\lipsum[66]|) is nested within the first
-one (between |\lipsum[101]| and |\lipsum[75]|). This is intentional: when we
-will revisit the |poormansbox| example in
-section~\ref{sec:memoization-complex-single-driver}, the implementation will
-have to pay special attention to nesting (which presents no problem to the
-implementation in this section).
-
-As you can see in the package listing below (\texttt{\examplename.sty}), the
-implementation of our environment is straightforward. We first define the
-configuration command |\pmbset| and the option keys (we're using
-\pkg{pgfkeys}), and set the option defaults. Then, we move on to the
-environment itself: we apply the given options, execute the pre-code, typeset
-the box (which is a |minipage| of the given width, potentially wrapped in a
-|\fbox|), and execute the post-code.
-
-\tcbinputexample[.sty]{listing only, float}
-
-\ExampleName{poormansbox-memoizable}
-Now let's make our |poormansbox| externalizable (\texttt{\examplename.sty}). As
-announced above, the idea is to split the definition of the environment into
-the outer part (below, the user-level environment |poormansbox|), which
-(applies the options and) executes the pre- and the post-code, and the inner
-part (below, the macro |\@poormansbox|), which typesets the actual box. If we
-then then submit |\@poormansbox|, rather than |poormansbox|, to
-automemoization, the outer part will be executed at every compilation (giving
-us stretchable space if we request it), while the inner command will be
-executed (and memoized) at the first compilation, and substituted for the
-extern (the fixed-size box) at subsequent compilations.\cprotect\footnote{You
- might have wondered why our definition of the |poormansbox| environment grabs
- the body into an argument (|+b|, yielding |#2|), necessitating the use of
- |\NewDocumentEnvironment| over the venerable |\newenvironment|. One reason
- was that having the environment body as an argument simplifies wrapping the
- |\fbox| around the |minipage|, but there is a more important reason. If we
- did not grab the environment body, we would have to implement the internal
- part of the definition as an environment (|@poormansbox|) as well, and embed
- it into the user-level environment using the following idiom:
- |\newenvironment{poormansbox}[2][]{...\begin{@poormansbox}}{\end{@poormansbox}...}|.
- However, as illustrated in section~\ref{sec:tut:good-to-know}, automemoizing
- an environment indirectly embedded in such a way produces an error, because
- Memoize is prevented from collecting the environment body.}
-
-\makeexample{\examplename.sty}
-\makeexample{\examplename.pdf}% just to know if it compiles fine
-\tcbinputexample[.sty]{listing only, float}
-
-Looking at the definition of the internal |\@poormansbox| command, it might
-strike you as weird that we have equipped this command with an optional
-argument (|#1|) it never uses. However, this optional argument is crucial for
-memoization. It will become a part of the memoized code (note
-\code{\refmmzauto{args}=om} in the \refmmz{auto} declaration) and thereby
-ensure that Memoize will produce separate externs for invocations of
-|\@poormansbox| with the same environment body but different options; or in
-other words, it will ensure that changing the options recompiles the
-extern.\cprotect\footnote{Of course, this only holds for options given in the
- optional arguments; if the user changes an option value using a prior
- \cs{pmbset} (and that option does not occur in the optional argument),
- Memoize won't detect the change. But the end-user knows about this issue, as
- it was addressed in sections~\ref{sec:tut:working-on-a-picture}
- and~\ref{sec:tut:redefinitions}, and she is also aware of two workarounds:
- manual recompilation, or setting the context
- (section~\ref{sec:cross-referencing}).
-
- While we're on the subject of the context, note that it is also possible to
- deploy context to trigger recompilation of the inner command upon change of
- parameters it depends on. We could simply omit the optional argument of
- \cs{@poormansbox} and add
- \refmmz{context}|={width=\pmb at width,frame=\ifpmb at frame true\else false\fi}|,
- to the \refmmz{auto} declaration. The advantage of such an approach is that
- Memoize reacts to the change of parameters regardless of whether they are set
- using the optional argument or \cs{pmbset}. However, the approach is
- unfeasible for commands depending on many parameters: can you imagine listing
- all the \TikZ options in the context? Not to mention that a particular
- picture usually only depends on a small subset of these options --- by and
- large, \TikZ externs would get recompiled too often if the context contained
- all \TikZ options.}\footnote{I have toyed with the idea of
- splitting (using \pkg{pgfkeys} key filtering) the given options into outer
- options, relevant for the outer command, and inner options, relevant for the
- inner command, and only passing the inner options to the inner command. The
- thought was that would (i) avoid recompiling the extern when only outer
- options change, as these options don't affect the inner command, and (ii)
- avoid applying the inner options when utilizing the extern, as these options
- don't affect the outer command. However, it then hit me that the end-user
- might define a style which incorporated both inner and outer options --- I
- know I do this with my |tcolorbox|es.}
-
-The downside to automemoizing an internal command is that this might be
-counter-intuitive for the author. For example, to deactivate automemoization
-of |poormansbox|, the author will have to write
-\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{@poormansbox}} (note the
-|\@|), but they will have no clue they have to do this unless they have
-carefully read |poormansbox|'s documentation. Even worse, the above
-\refcmd{mmzset} command will not work unless surrounded by |\makeatletter| and
-|\makeatother|, as it refers to an internal control sequence containing |@|.
-Well, Memoize offers \refmmz{auto csname}, \refmmz{activate csname} and
-\refmmz{deactivate csname}, so that |@| catagory code manipulations can be
-omitted by writing \refcmd{mmzset}\bracestt{\refmmz{deactivate
- csname}|=@poormansbox|}, but still.
-
-Another downside could occur when you use the same (automemoized) internal
-command in service of several user interface commands. For the sake of
-illustration, assume we have also defined an UI-macro |\pmb| which again relies
-on |\@poormansbox|. How is the author to deactivate automemoization of |\pmb|
-but leave the |poormansbox| environment intact? This is how:
-\refcmd{mmzset}\bracestt{\refmmz{auto}=\cs{pmb}\bracestt{\refmmzauto{args}=m,
- \refmmzauto{nomemoize}}}. Again, counter-intuitive; the author expects
-\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{pmb}} to work.
-
-One other consequence of this approach is that the code included in the c-memo
-(if \refmmz{include source in cmemo} is in effect) will not faithfully reflect
-the source: as shown in the c-memo listing below, it will contain
-|\@poormansbox{...}| instead of |\begin{poormansbox}...\end{poormansbox}| ---
-even if this might actually count as an advantage, as the discrepancy will at
-least inform the author who refuses to read the fine material accompanying our
-|poormansbox| that something funky is going on.
-
-\begingroup
-\relaxmmzcommands
-\def\mmzNewCMemo#1{% fetch the last c-memo filename
- \def\mycmemo{#1}%
-}
-\input{\examplepath.mmz.c1}
-\sed{%
- s/[]]\lbrace/]\n\space\space\space\space\space\space\space
- \space\space\space\space\space\space\space\lbrace/;
-}{\exampledir\mycmemo}
-\tcbinputlisting{
- listing only,
- listing file=\exampledir\mycmemo,
- example title,
- title=the c-memo of the last \texttt{poormansbox} environment,
- left=0.5em, right=0em,
-}
-\endgroup
-
-In a nutshell, automemoizing an internal command might be counter-intuitive for
-the author. But the core idea --- to support memoization of a resistant
-command by splitting its definition into the outer and the inner command --- is
-sound, and we will elaborate on this idea in
-section~\ref{sec:memoization-complex-single-driver}, where we will revisit our
-|poormansbox| example and develop a variant of this environment which is both
-memoizable and user-friendly.
-
-
-\section{Under the hood}
-\label{sec:under-the-hood}
-
-This chapter is written for three audiences: a curious user who wants to
-know how Memoize does what it does; a package writer who wants to support
-Memoize in a tricky situation; and myself, lest I forget why I made the design
-decisions that I made.
-
-\subsection{The entry point}
-\label{sec:Memoize}
-
-
-From the author's perspective, the functionality of this package is entered
-either through the manual memoization commands (macro \refcmd{mmz} and
-environment \refenv{memoize}), or via automemoization. And while that is
-correct, those user interface entry points merely determine what code is
-submitted to memoization, and set any options specific to the upcoming
-memoization. The real fun starts with command \Emph{\refcmd{Memoize}}, which
-is eventually executed by both manual and automatic memoization.
-
-Not every call to \refcmd{Memoize} results in memoization. Calling this macro
-has three possible outcomes. It can result in \emph{memoization}, which
-produces the memos and externs; in \emph{utilization} of the result of an
-earlier memoization (which boils down to inputting the memos); or in
-\emph{regular compilation}, whereby the code is compiled
-(almost)\footnote{\label{fn:regular-compilation-almost}This is absolutely true
- for memoized code which is ``contained'' in the sense of not peeking into the
- input stream following the memoized code. In general, code which fails to
- satisfy this containment requirement is most likely simply not memoizable;
- but there are borderline cases. For example, \refcmd{ignorespaces} at the
- end of some code will have the expected effect in the absence of Memoize, but
- no effect when executed either during memoization or regular compilation
- under Memoize, simply because it will hit some code belonging to Memoize
- rather than the continuation of the document. Memoize offers the
- \refmmz{ignore spaces} provision to work around this specific problem.} as if
-Memoize was not there. Which outcome obtains depends on several factors. The
-decision logic is depicted below, and note that you can \refmmz{trace} the
-action on the terminal.
-
-\begin{center}
- \begin{forest}
- yes/.style={edge=->,edge label={node[font=\scriptsize,pos=0.4,left,#1]{yes}}},
- no/.style={edge=->,edge label={node[font=\scriptsize,pos=0.4,right,#1]{no}}},
- use/.style={content=utilization\\(of the cc-memo)\vphantom{dj},fill=orange,align=center},
- memoize/.style={content=memoization\vphantom{dj},fill=green},
- compile/.style={content=regular compilation\vphantom{dj},fill=emphcolor},
- for tree={
- l sep*=1.5,
- if n children={0}{}{align=center},
- child anchor=north,
- },
- [Is Memoize enabled?\\(\refcmd{ifmemoize}),
- [Are we currently undergoing memoization?\\(\refcmd{ifmemoizing}), yes
- [,compile, yes]
- [Is the\\\refmmz{recompile}\\mode in effect?, no
- [,memoize, yes]
- [Do the relevant\\memos and externs exist?, no
- [,use, yes]
- [Is the\\\refmmz{readonly}\\mode in effect?,no
- [,compile, yes]
- [,memoize, no]
- ]
- ]
- ]
- ]
- [,compile, no]
- ]
- \end{forest}
-\end{center}
-
-As the memoization options were already set by the user interface entry points,
-you might expect, quite reasonably, that \refcmd{Memoize} takes a single
-argument, the code submitted to memoization. After all, what more does it
-need? Clearly, executing this code is what produces the typeset material, and
-to detect whether the code has ``changed'' (in order to recompile the memos and
-externs), we compute the MD5 sum of this very code, don't we? Well, the
-reality is a bit more complicated. When it comes to automemoized commands, the
-code which the MD5 sum is computed off of (and which is displayed in the c-memo
-if \refmmz{include source in cmemo} is in effect) is not exactly the same as
-the code we compile (during either memoization or regular compilation). We'll
-see what the difference is in section~\ref{sec:tut:automemoization-details};
-what matters here is that we must provide \refcmd{Memoize} with both and that
-this macro therefore takes two arguments: the \emph{identification code}, which
-the MD5 sum is computed off of, and the \emph{executable code}, which, well, is
-the code that gets executed during memoization (or regular compilation).
-
-Let's illustrate this with an example which is probably entirely useless (but
-don't worry, we'll get to a realistic example in
-section~\ref{sec:tut:automemoization-details}). We first memoize some text
-manually, using command \refcmd{mmz}, and then do something very stupid: we use
-this very text as the identification code for the following \refcmd{Memoize},
-even if the executable code of that command is completely different. The
-second line of the typeset output should convince you that the first argument
-to that command was really used to produce the extern; and one further
-compilation should convince you that the first argument was indeed used to
-identify the extern: the extern produced by \refcmd{mmz} was overwritten by the
-extern produced by \refcmd{Memoize}, in the fashion of the |progressbar|
-example from section~\ref{sec:tut:redefinitions}.
-
-\ExampleName{memoize-internal}
-\makeexample{\examplename.pdf N=2}
-\tcbinputexample{
- comment={%
- \includeexamplepdf[document page,
- after title pre={\ (after the first compilation)}
- ]{page=3}\hfill
- \includeexamplepdf[document page,
- after title pre={\ (after the second compilation)}
- ][\examplepath.c2.pdf]{page=1}
- },
-}
-
-The example above also illustrates a(nother) peculiar feature of
-\refcmd{Memoize}. \refcmd{Memoize} does not open a new \hologo{TeX} group, but
-it \emph{expects a group to be opened prior to calling it}, as it will issue an
-|\endgroup| at some point. Specifically, the memoization group will be closed
-before regular compilation or utilization, but after memoization. If you want
-to know why, read the boxed text below.
-
-\begin{tcolorbox}[title=\string\Memoize\ and grouping, enhanced, breakable]
- One important desideratum behind the design of Memoize was that using the
- package should disrupt the original, Memoize-less compilation as little as
- possible. In particular, if the memoized code contains local assignments
- whose effect (in the original compilation) persists into the rest of the
- document (until the end of the surrounding \hologo{TeX} group, of course),
- wouldn't one want these local effects to persist when Memoize is around, as
- well? Fortunately, most memoized code does not have persistent local effects
- (at least for me, it is usually environments, like \env{tikzpicture}s, that I
- want to memoize, and environments introduce a group anyway) --- fortunately,
- because there are design reasons for enclosing memoization in a \hologo{TeX}
- group (or two), and this enclosure will of course cancel the effect of local
- assignments in the memoized code.
-
- For one, the user interface memoization commands, such as \refcmd{mmz} and
- automemoized commands, allow for options specific to a particular piece of
- memoized code (the options given as the optional argument to manual
- memoization, the next-options and the auto-options), and to delimit their
- effect, it makes most sense to apply them in a group. I have toyed with the
- idea of working around the introduction of a group by manually saving and
- restoring all the options, but I quickly gave up on this line of thought.
- For one, manually saving and restoring the options would be cumbersome and
- error-prone, and probably also slower than using the group. But even worse,
- all that work would not really solve the problem of the persistence of local
- effects, because memoization itself introduces a group, as well: during
- memoization, the typeset material is collected into a box, and opening a box
- introduces a group. In some particular situations, this could be avoided by
- typesetting the memoized code as-is and collecting the resulting material
- using |\lastbox|, but this approach cannot work in general. In general,
- memoization will take place in a group, so the issue of local effects must be
- addressed in some other way. Memoize offers the following workaround: during
- memoization, the memoized code can (globally) add code to the \refmmz{after
- memoization} hook, which gets executed immediately after closing the
- memoization group.
-
- Does this mean it would be best if the user interface memoization commands
- straightforwardly surrounded \refcmd{Memoize} by |\begingroup| and
- |\endgroup|? For example, \refcmd{mmz} would open the memoization group, let
- \refcmd{Memoize} do its work, and then close the group. Not really.
- Remember that memoization is not the only possible outcome of calling
- \refcmd{Memoize}. Perhaps we can at least retain the local effects of a
- regular compilation, and of utilization?
-
- We can, by finely tuning the timing of the memoization group closure within
- \refcmd{Memoize}. This command is designed to close the memoization group
- after memoization, but before regular compilation and utilization. Closing
- the group after memoization makes sure that the given options are in effect
- during this process. By closing the group prior to regular compilation,
- regular compilation of the memoized code (which takes place when Memoize is
- disabled, for example) is guaranteed to have (almost, see
- footnote~\ref{fn:regular-compilation-almost}) exactly the same effect as the
- compilation of that code in absence of Memoize; in particular, the effect of
- any local assignments will persist into the rest of the document. Finally,
- closing the group before utilization simplifies the construction of the memo
- in the cases where we need to replicate local effects of the memoized code
- --- the group closed, there is no need to smuggle local assignments out of a
- memo.
-\end{tcolorbox}
-
-\subsection{Memos}
-\label{sec:memos}
-
-Up until now, we have pretended that there is a single kind of a memo file. In
-truth, there's two kinds: \emph{code memos}, or \emph{c-memos} for short; and
-\emph{code--context memos}, or \emph{cc-memos} for short. In this section, we
-will learn what they are for, and how they look like --- and also a bit on how
-they are produced, even if the details on that will have to wait until
-section~\ref{sec:memoization-drivers}.
-
-We will see that when Memoize utilizes memos, c-memos are processed first. But
-conceptually, cc-memos are more important, so we will start the discussion with
-these.
-
-\subsubsection{Cc-memos (and extern inclusion)}
-\label{sec:cc-memos}
-
-When it is input, a cc-memo replicates the effect of the memoized code. This
-includes the reproduction of its visual output, which takes the form of
-inclusion of any externs produced by memoization. And yes, you got the
-implication right: a cc-memo can have any number of associated externs,
-including zero, even if the most common case is that of exactly one extern per
-cc-memo. The number of externs mostly depends on the memoization driver (see
-section~\ref{sec:memoization-drivers}); the default driver always produces
-exactly one extern.
-
-A cc-memo is located in the directory given via subkey \refmmzpath{dir} (and
-\refmmzpath{relative}) of key \refmmz{path}. You can recognize it by its filename,
-which has the following form (\meta{prefix} is set via subkey \refmmzpath{prefix}):
-\begin{center}
- \code{\meta{prefix}\meta{code md5sum}-\meta{context md5sum}.memo}
-\end{center}
-
-In fact, this is how Memoize recognizes --- or rather, searches for --- a
-cc-memo as well: Memoize will utilize a cc-memo when the code and the context
-MD5 sum computed during an attempted utilization match the code and the context
-MD5 sum computed during some previous memoization (for details on the context
-MD5 sum, see section~\ref{sec:c-memos}). In detail, a cc-memo is created at
-the end of memoization, at which point Memoize computes the MD5 sum of the
-memoized code and the MD5 sum of the context, and writes the results of
-memoization into the cc-memo identified by (the prefix and) these two MD5 sums.
-And when Memoize, on a subsequent compilation, encounters a piece of memoized
-code, it again computes the MD5 sum of that code and the MD5 sum of the
-context, and tries to input the cc-memo identified by (the prefix and) these
-two MD5 sums. If the inputting is successful, we have utilized the cc-memo
-(which in the typical case amounts to including the one associated extern); if
-the cc-memo cannot be found, Memoize starts the memoization process, which
-creates the memos and the externs.
-
-Let us take a look at the contents of a cc-memo in detail. Here's a typical
-cc-memo (it belongs to the titlepage penguin):
-
-\begingroup
-\relaxmmzcommands
-\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
- \def\myccmemo{#1}%
- \endinput
-}
-\input{\exampledir titlepage.mmz.c1}
-\sed{%
- s/\cmd{quitvmode} \cmd{mmzIncludeExtern}/\cmd{quitvmode}\n\cmd{mmzIncludeExtern}/;
- s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/;
-}{\exampledir\myccmemo}
-\tcbinputlisting{
- listing only,
- listing file=\exampledir\myccmemo,
- example title=\myccmemo,
-}
-\endgroup
-
-A cc-memo begins by listing the externs which the memo will (actually, might)
-attempt to include into the document. When the cc-memo is input, each
-\refcmd{mmzResource} command checks if the given extern exists. If some
-existence check fails, Memoize enters the memoization mode, same as if the
-cc-memo itself did not exist. If all the resources pass the existence check,
-Memoize inputs the core of the cc-memo, i.e.\ everything following the
-\refcmd{mmzMemo} marker.
-
-The core might contain arbitrary code, but most often, it will consist of only
-two commands. The first one is \refcmd{quitvmode} and it is included if the
-extern was \refmmz{capture}d into a horizontal box (which is the usual
-situation). The second one is \refcmd{mmzIncludeExtern}, and it is this
-command which actually includes the extern into the document upon inputting the
-cc-memo. The core code is executed without introducing any groups, i.e.\ the
-effect of any local assignments in the cc-memo will persist into the code
-following the memoized code.
-
-Command \refcmd{mmzIncludeExtern} takes nine parameters. The first is the
-sequential number of the extern associated with the cc-memo, starting with 0;
-usually, this is simply 0 as most memos are associated with a single extern.
-The second one is a \refcmd{hbox} or \refcmd{vbox}, noting the type of the box
-the memoized code was externalized into. The next three numbers are the
-expected width, height and the depth of the extern. Finally, we have the four
-\refmmz{padding} amounts (left, bottom, right and top). We should arrive at
-the expected size after trimming the extern PDF by the
-padding amounts; Memoize will complain if we don't.
-
-Let's look at a more interesting cc-memo. Using the advising framework,
-described in section~\ref{sec:tut:automemoization-details}, Memoize hacks
-\refcmd{label} to support \cs{label}s inside memoized code --- the following
-code ``just works.''
-
-\ExampleName{label}
-\makeexample{\examplename.pdf N=3}
-\tcbinputexample{
- comment={\centering
- \includeexamplepdf[document page, after title pre={\ (compilation 1)},
- left=1.5mm, right=1mm]{page=2}\quad
- \includeexamplepdf[document page, after title pre={\ (compilation 2)},
- left=1.5mm, right=1mm][\examplepath.c2.pdf]{page=1}\quad
- \includeexamplepdf[document page, after title pre={\ (compilation 3)},
- left=1.5mm, right=1mm][\examplepath.c3.pdf]{page=1}
- },
-}
-
-Everything seems normal --- after the first compilation, we get ``\textbf{??}''
-because the label has not made it into the |.aux| file yet, but in subsequent
-compilations, we learn where the penguin lives --- but it is far from normal
-under the hood. If we de-hacked \cs{label} by writing
-\refcmd{mmzset}\bracestt{\refmmz{deactivate}=\cs{label}}, the third compilation
-(and subsequent compilations) would revert to ``\textbf{??}''. Why would that
-happen? The memoized code containing the \cs{label}s is only executed in the
-first compilation; in the subsequent compilations, we're simply inputting the
-cc-memo, so the memoized code, including any \cs{label}s in contains, is not
-compiled, and the labels don't get into the |.aux| file anymore.
-
-The \cs{label} hack deploys Memoize's ability to put arbitrary code into the
-cc-memo. During memoization, the memoized code may add arbitrary code to
-register \refcmd{mmzCCMemo}, and the contents of this register at the end of
-the memoization form the free-form part of the cc-memo.\footnote{This is also
- how the above-described code containing \refcmd{mmzIncludeExtern} gets into
- the cc-memo. The code is produced by \refcmd{mmzExternalizeBox} and appended
- to \refcmd{mmzCCMemo} by the default memoization driver
- \refcmd{mmzSingleExternDriver}; see section~\ref{sec:memoization-drivers} for
- details.} When the hacked \cs{label} is encountered during memoization, it
-appends \refcmd{mmzLabel}\marg{label name}\marg{current label value} to
-\refcmd{mmzCCMemo}, so this command winds up in the cc-memo. It is then a
-simple job for \refcmd{mmzLabel}, executed when the cc-memo is input at
-subsequent compilations, to temporarily store \meta{current label value} (i.e.\
-the contents of \refcmd{@currentlabel} at the time the \cs{label} was invoked) back
-into \cs{@currentlabel} and to execute \cs{label}\marg{label name}. In effect, any
-\cs{label} command contained within the memoized code is executed at every
-compilation, even if the memoized code itself is not compiled.
-
-\begingroup
-\relaxmmzcommands
-\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
- \def\myccmemo{#1}%
- \endinput
-}
-\input{\examplepath.mmz.c1}
-\sed{%
- s/~//g;
- s/\cmd{quitvmode} \cmd{mmzLabel}/\cmd{quitvmode}\n\cmd{mmzLabel}/;
- s/\(\marg\)\cmd{mmzIncludeExtern}/\1\n\cmd{mmzIncludeExtern}/;
- s/\(\cmd{mmzLabel}\) *\(\marg\marg\) */~\1\2~\space/g;
- s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/;
-}{\exampledir\myccmemo}
-\tcbinputlisting{float,
- listing only, right=1.5mm,
- listing file=\exampledir\myccmemo,
- example title=\myccmemo,
-}
-\endgroup
-
-We will continue the discussion of \refcmd{label} in section~\ref{sec:label+} using
-a funkier example.
-
-
-\subsubsection{C-memos (and context)}
-\label{sec:c-memos}
-
-As explained in the previous section, a cc-memo belonging to a piece of
-memoized code is identified by two MD5 sums: the MD5 sum of the memoized code,
-and the MD5 sum of the associated context. However, when Memoize encounters
-some code submitted to memoization, the context expression is not yet fully
-known, as it may be adjusted by the memoized code itself during memoization ---
-and this potential adjustment is crucial for \cs{ref} and friends to work as
-advertised (see section~\ref{sec:cross-referencing}). Upon being invoked,
-Memoize therefore cannot immediately attempt to input the cc-memo; it needs to
-first learn about the context adjustments. Here's where c-memos enter the
-picture: \emph{the primary job of a c-memo is to store the context adjustments
- made by the memoized code}. Let's see how this works in detail.
-
-Same as cc-memos, c-memos are located in the directory given via subkey
-\refmmzpath{dir} (and \refmmzpath{relative}) of key \refmmz{path}. However, a c-memo
-belonging to some memoized code is identified by the MD5 sum of that code
-alone; its filename has the following form, where \meta{prefix} is again set
-via subkey \refmmzpath{prefix}:
-\begin{center}
- \code{\meta{prefix}\meta{code md5sum}.memo}
-\end{center}
-
-The c-memo is created at the end of the memoization process. At that time, the
-context expression is fully known, as the memoized code was already processed.
-Even more, Memoize keeps track of both the state of the context expression
-prior to memoization, stored in token register \refcmd{mmzContext}, and of the
-\emph{additions} to the context expression made by the memoized code, which are
-stored in token register \refcmd{mmzContextExtra}. (Incidentally, key
-\refmmz{context} automatically adapts to the situation by appending to
-\refcmd{mmzContext} outside memoization and to \refcmd{mmzContextExtra} during
-memoization.) The complete context expression is the concatenation of the
-contents of these two registers, but it is only the context expression
-additions, i.e.\ the contents of \refcmd{mmzContextExtra}, which Memoize stores
-into the c-memo, with the idea that during subsequent compilations, the initial
-context (\refcmd{mmzContext}) will be set up again via ``normal'' compilation,
-while inputting the c-memo will restore the additions, jointly reconstructing
-the complete context expression associated with a piece of memoized code to
-what it was at the end of memoization.
-
-We can now complete the picture of a utilization attempt started in
-section~\ref{sec:cc-memos}. Memoize begins by trying to input the c-memo; this
-can be done as the c-memo can be identified based solely on (the MD5 sum of)
-the memoized code. If the c-memo does not exist, Memoize starts the
-memoization process, which will produce the memos and the externs. But if it
-does exist, inputting it reconstructs the context expression to the state at
-the end of memoization. Therefore, as the MD5 sum of the \emph{expansion} of
-the context expression \emph{at the end of memoization} is baked into the
-cc-memo filename, trying to load the cc-memo identified by (the prefix, the
-code MD5 sum and) the MD5 sum of the \emph{expansion} of the context expression
-\emph{at attempted utilization} will succeed precisely when the context
-remained unchanged from memoization to attempted utilization.
-
-All this might have sounded very complicated, but in the end, most c-memos are
-quite boring, the titlepage penguin's c-memo shown below being no exception. A
-c-memo starts with the \refcmd{mmzMemo} marker, which is always followed by a
-(global) assignment to token register \refcmd{mmzContextExtra}, holding the
-context expression additions. As promised, the c-memo below is boring: it
-assigns an empty token list to this register, leaving the context expression
-as-is. Next comes the free-form part of the memo. Below, it is boringly empty
-as well (just the percent sign), but in principle, it will contain any code
-gathered in register \refcmd{mmzCMemo} during memoization; see
-\ref{sec:per-overlay} for an example. A c-memo is concluded by an optional
-part consisting of the \refcmd{mmzSource} marker, followed by the memoized
-code. The source code section is not used by Memoize in any way and can be
-switched off by \code{\refmmz{include source in cmemo}=false}; it is included
-by default so that an interested user can know which code produced which memo,
-which can be useful if one wants to trigger recompilation of an extern by
-deleting the corresponding memo. Incidentally, any newlines in the source code
-are lost in the c-memo replica (unless \refmmz{verbatim} is in effect), but we
-will only see this once we arrive at the \pkg{beamer} example below.
-
-\begingroup
-\relaxmmzcommands
-\def\mmzNewCMemo#1{% fetch the first c-memo filename
- \def\mycmemo{#1}%
- \endinput
-}
-\input{\exampledir titlepage.mmz.c1}
-\tcbinputlisting{
- listing only,
- listing file=\exampledir\mycmemo,
- example title=\mycmemo,
-}
-\endgroup
-
-A c-memo of code containing a cross-reference will prove more interesting. The
-following c-memo was produced by the |ref| example from section
-\ref{sec:cross-referencing}. As we know from that section, when a \refcmd{ref} (as
-hacked by Memoize's \refmmzauto{ref} key) occurs in some memoized code, it
-appends the cross-reference to the context. In the c-memo below, this is
-reflected by the (global) assignment of an expression containing the
-cross-reference macro to token register \refcmd{mmzContextExtra}, holding
-the context expression additions.
-
-\begingroup
-\relaxmmzcommands
-\def\mmzNewCMemo#1{% fetch the first c-memo filename
- \def\mycmemo{#1}%
- \endinput
-}
-\input{\exampledir ref.mmz.c1}
-\sed{%
- s/~//g;
- s/\cmd{global}.*\marg/~\0~/;
-}{\exampledir\mycmemo}
-\tcbinputlisting{
- listing only,
- listing file=\exampledir\mycmemo,
- example title=\mycmemo,
-}
-\endgroup
-
-
-\subsubsection{More on \texorpdfstring{\cs{label}}{\textbackslash label}}
-\label{sec:label+}
-
-A \refcmd{label} inside memoized code works out of the box in the usual situation
-when label value is fully determined by the memoized code, as in the example in
-section~\ref{sec:cc-memos}, where the memoized code contained the outermost
-(and only) \env{enumerate} environment. However, the out of the box approach does
-not work if the label value is (fully or partially) determined outside the
-memoized code. To illustrate the problem, and some potential solutions, we
-define two very simple enumeration environments, |listi| and |listii|, which
-use counters |counti| and |countii|, and which are intended as the outer and
-the inner environment, respectively. Our interest here is in the inner
-environment, |listii|. While it prefixes each item by an indented
-|\thecountii)|, the label is a composite of both counters:
-|\thecounti\thecountii|. The label is stored into \refcmd{@currentlabel}, so
-referencing works as usual. However, problems arise when we automemoize the
-inner environment.
-
-\ExampleName{label+}
-\makeexample{\examplename.tex.c1 N=6}
-\makeexample{\examplename.pdf N=2}
-\tcbinputexample{
- sidebyside, lefthand ratio=0.53,
- comment={\centering\includeexamplepdf[document page][\examplepath.c2.pdf]{page=1}},
-}
-
-While the result looks fine at first, changing the order of |listii|
-environments, for example by moving ``pets'' below ``domestic'', will result in a
-problem: the reference at the bottom will remain unchanged. This is so because
-the reference text is baked into the cc-memo, as shown below.
-
-\begingroup
-\relaxmmzcommands
-\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
- \def\myccmemo{#1}%
- \endinput
-}
-\input{\examplepath.mmz.c1}
-\sed{%
- s/~//g;
- s/\cmd{mmzLabel} \marg\marg/~\0~/;
- s/\(\marg\)\(\cmd{mmzIncludeExtern}\)/\1 \2/;
- s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/;
-}{\exampledir\myccmemo}
-\tcbinputlisting{
- listing only,
- listing file=\exampledir\myccmemo,
- example title=\myccmemo,
-}
-\endgroup
-
-How can we remedy this? The manual option is to force the recompilation of the
-extern by putting an (invisible) reference to the outer item into the inner
-item: add |\label{item:pets}| to item ``pets'' and refer to it at ``dog'' by
-\refcmd{mmzNoRef}|{item:pets}|.%
-\attachexample[\examplename mmzNoRef.tex][\examplepath.tex.c3.attachment]
-
-An automatic variant of the recompilation solution is to add \cs{@currentlabel}
-to the context upon memoizing |listii|. This can be achieved by adding
-\refmmz{context}|={@currentlabel={\csuse{@currentlabel}}}| to the \refmmz{auto}
-declaration for |listii|. The downside of this approach is that every |listii|
-will get reexternalized upon movement, whether it actually contains a label or
-not.\attachexample[\examplename context.tex][\examplepath.tex.c4.attachment]
-
-In fact, given that the externs produced by the inner environment do not
-contain the value of the outer counter, it seems wasteful to recompile any
-extern just to change the reference. And indeed, it is possible to avoid this,
-but the approach unfortunately requires adapting the inner environment code
-(and this is why I have not illustrated the problem using an environment of an
-elaborate package like \pkg{enumitem}). The idea is to ``unbake'' the
-reference to the outer item in the cc-memo. We can achieve this by changing
-|listii| to define \cs{@currentlabel} to be |\unexpanded{\thecounti}\thecountii|.
-Under this definition, the cc-memo will contain |\mmzLabel
-{item:dog}{\thecounti a}|, and rearranging |listii| environments will produce
-(upon two compilations, of course) the correct reference without recompiling
-the extern. Note again, however, that this solution can only work when the
-value of the outer counter does not appear in the extern, i.e.\ it would not
-work the ``dog'' item was prefixed by |1a)| rather than simply |a)|. In those
-cases, one should deploy one of the other solutions.%
-\attachexample[\examplename listii.tex][\examplepath.tex.c5.attachment]
-
-The final solution, presented below, is an elaboration on the second one.
-Rather than append \cs{@currentlabel} to the context immediately upon beginning
-to memoize environment |listii|, we will at that point redefine \cs{label} to do
-that. In effect, changing the location of |listii| will only recompile it if
-it contains a \cs{label}.
-
-At announced, we redefine \cs{label} once the memoization of |listii| begins,
-so within \refmmz{at begin memoization}.\footnote{Another generally good
- location for such redefinitions is among the auto-options of |listii|. We
- could include an \refmmz{auto}\cs{label}\marg{...} there, or a |/utils/exec|
- with \refcmd{AdviceSetup}. However, in this particular case this would be
- wasteful, as it would be applied regardless of whether memoization will take
- place or not, whereas we only need the redefined \cs{label} when memoizing.}
-However, we do not redefine \cs{label} directly, as Memoize advises this
-control sequence out of the box (see
-section~\ref{sec:tut:automemoization-details} for details). What we redefine
-is the command which the advising framework executes instead of \cs{label} ---
-its so-called \refmmzauto{outer handler} --- and we do this by calling
-\refcmd{AdviceSetup}, the low-level variant of the familiar key
-\refmmz{auto}.\footnote{We could have also used \refmmz{auto}, but we don't,
- because (a) \refcmd{AdviceSetup} is faster, (b) it is easier to prepend
- material to a handler using the low-level interface, and (c) I wanted to
- showcase \refcmd{AdviceSetup}.} The first argument of \refcmd{AdviceSetup}
-is the installation path (|/mmz|), the second one the command or environment we
-are submitting to the framework (\cs{label}), and the third one the setup
-\emph{code} --- here lies the biggest difference between \refmmz{auto} and
-\refcmd{AdviceSetup}: the former expects a keylist, and the latter \hologo{TeX}
-code which directly manipulates settings macros like
-\refcmd{AdviceOuterHandler} (for the full list, see
-section~\ref{sec:tut:automemoization-details} or~\ref{sec:ref:advice}).
-
-\tcbinputexample[.tex][.c6]{listing only, float,
- attachment name=\examplename auto.tex,
-}
-
-Within \refcmd{AdviceSetup}, we prefix (using macro \cs{preto} of package
-\pkg{etoolbox}) the original outer handler \refcmd{AdviceOuterHandler} by code
-which causes \cs{outerlabeltocontext} to be executed at the end of memoization
-(by globally appending this macro to \refcmd{mmzAtEndMemoizationExtra};
-``Extra'' because we're appending during memoization). It is
-\cs{outerlabeltocontext} which then appends \refcmd{@currentlabel} to the
-context (\refcmd{mmzContextExtra}; again, ``Extra'' because we're appending
-during memoization),\footnote{As a courtesy, we clear out macro
- \cs{outerlabeltocontext} once it did it's job, so that multiple \cs{label}s
- do not include multiple \cs{@currentlabel}s into the context. But the code
- would work even without this addendum, can you see why?} and it is crucial
-that this happens at the end of memoization rather than when \cs{label} is
-executed. When \cs{label} is executed, we're inside an inner item, and
-\cs{@currentlabel} refers to that item, while at the end of memoization, the
-value of this macro equals the value at the beginning of memoization, namely
-the label of the outer item. While the outer list remains unshuffled, the
-value of \refcmd{@currentlabel} that contributes to the context MD5 sum during
-the utilization attempt will therefore match the value which contributed to the
-context MD5 sum during memoization, resulting in matching MD5 sums and
-therefore in actual utilization of the extern; once the outer list is shuffled,
-this will cease to be the case and the extern will be recompiled.
-
-
-\subsubsection{The Beamer support explained}
-\label{sec:per-overlay}
-
-The implementation of \refmmz{per overlay}, which makes memoization sensitive
-to Beamer overlays, provides an example of a complex interaction between
-various components of memoization. At the core, the Beamer support works by
-adjusting the context, but we will also have the occasion to observe the
-free-form part of the c-memo, add a bit of extra code to the cc-memo, and
-deploy several memoization hooks. We will show the complete Beamer support
-code later on; let us build our understanding of that code step by step.
-(Before you read on, you might want to refresh your memory about the \pkg{beamer}
-example from section~\ref{sec:tut:beamer}, as we will refer to it in the
-present section.)
-
-The core idea behind \refmmz{per overlay} is to append the current beamer
-overlay number to the context:\cprotect\footnote{As we saw in
- section~\ref{sec:tut:beamer}, it is convenient to execute \refmmz{per
- overlay} inside memoized code. But remember, from
- section~\ref{sec:cross-referencing}, that when \refmmz{context} is executed
- from within the memoized code, its argument winds up in the c-memo. As the
- c-memo is processed under the normal category code regime, where |@| is not a
- letter, we have to access |\beamer at overlaynumber| using the |\csname
- ... \endcsname| construct.} \refmmz{context}|={overlay=\csname
- beamer at overlaynumber\endcsname}|. This makes Memoize produce a separate
-extern for each overlay. However, only the first of these externs will get
-utilized on subsequent compilations, in general at least. Even worse, we will
-lose each frame whose creation is driven solely by the memoized code. We will
-lose the second overlay in our example (i.e.\ in the \pkg{beamer} example from
-section~\ref{sec:tut:beamer}) as the second overlay was only created because
-Beamer encountered |only={2}{...}| (resolving to |\only<2>{...}| under the
-hood) inside the picture code; once we utilize the extern instead of compiling
-the picture on the first overlay, the |\only| command is not executed anymore,
-so Beamer thinks it is done with the frame.
-
-As the compilation of our picture is substituted by utilization of its
-cc-memos, we have to somehow drive the creation of the necessary overlays from
-these files. An easy way to achieve this is to furnish them with a dummy
-|\only<|\meta{final overlay number}|>{}|,\footnote{`The final overlay' here
- should be understood as relative to our memoized picture, i.e.\ as the final
- overlay containing the memoized picture.}\footnote{Actually, putting this
- \cs{only} command only into the first cc-memo would suffice, but would be
- harder to implement.} but there is a problem: the final overlay number is
-unknown when we're memoizing our picture --- it is unknown even when we're
-memoizing the picture on final overlay itself (we simply don't know yet that
-this overlay will end up being the final one), let alone during the memoization
-on the first overlay.
-
-The solution exploits the fact that \emph{the c-memo is rewritten at each
- memoization}: at each memoization of our picture, we store the the
-\emph{current} overlay number to the c-memo; after all memoizations, the c-memo
-will thus contain the number of the \emph{final} overlay containing our
-memoized picture. To access this number from the cc-memo, we store it as a
-macro definition, and then use the defined macro, |\mmzBeamerOverlays|, in the
-overlay specification of the dummy |\only|. Below, you can see all this in
-code, as the argument to \refmmz{at begin memoization}.
-
-\ExampleName{per-overlay-v1}
-\makeexample{\examplename.sty}
-\tcbinputexample[.sty]{
- listing only,
- title={The implementation of \refmmz{per overlay} (first attempt)},
-}
-
-A couple of remarks are in order here. First, the definition of
-|\mmzBeamerOverlays| in the c-memo is global, because it will be accessed from
-the cc-memo, but the cc-memo is input after closing the memoize group (which
-the c-memo \emph{is} processed in). Second, using \refmmz{at begin
- memoization} makes it possible to use \refmmz{per overlay} both outside and
-during memoization: if \refmmz{at begin memoization} is executed outside
-memoization, its argument is (locally) stored into hook
-\refcmd{mmzAtBeginMemoization}, to be called at the beginning of each
-memoization; if the key executed outside memoization, the argument is executed
-immediately. Third, the Memoize keys in the definition of \refmmz{per overlay}
-are prefixed with \code{\refkeypath{/mmz}/}, so that this key can be called from
-\pkg{pgfkeys} option lists of other packages, for example the option list of
-the \env{tikzpicture} environment, as shown in the example in
-section~\ref{sec:tut:beamer}.
-
-I used the above version of \refmmz{per overlay} for quite a while. In
-general, it worked as I expected, but there were glitches. Occasionally, the
-picture would appear on the wrong overlay, or I would get an extra overlay, or
-perhaps lose an overlay. Eventually, I figured out this happens when I play
-with the overlay structure of the frame: when I add or remove a \cs{pause} or
-similar. In hindsight, it is easy to see what was happening. Once the picture
-is memoized, it is fixed, forever, which extern will appear on which overlay.
-I cannot expect the extern--overlay correlation to change just because I added
-a \cs{pause} in front of the picture. Furthermore, the number of overlays the
-memos will drive to be created is fixed as well. If I memoize the picture
-while it follows a \cs{pause}, and the picture creates 10 overlays, the c-memo
-will define |\mmzBeamerOverlays| to 11. So what, if I then remove that
-\cs{pause}! The c-memo will still define |\mmzBeamerOverlays| to 11, and drive
-the creation of 11 overlays --- one too many.
-
-By now, the road ahead is probably clear --- we put the |beamerpauses| counter
-into the context --- but we will see there are still obstacles on the way. The
-issue is that the context is evaluated at the \emph{end} of memoization (so
-that those cross-references from section~\ref{sec:cross-referencing} actually
-get into it). However, the memoized code might contain a \cs{pause} or similar
-itself, and change the value of |beamerpauses|. For one, this means that we
-have to write down the changed value of |beamerpauses| into the cc-memo; below,
-we do this using key \refmmz{at end memoization} (the code given code to this
-key is executed after the driver but before Memoize writes down the memos and
-ships out the extern pages; the key itself may be executed either before or
-during memoization). Furthermore, if the memoized code changes the value of
-|beamerpauses|, the value of |beamerpauses| at the attempted utilization, which
-would nicely match the value from the start of memoization, will never match
-the changed, final value from the end of memoization, in effect preventing our
-hard-won externs from ever getting utilized.
-
-We therefore have to invent a way to get the memoization-\emph{initial} value
-of |beamerpauses| into the context.\footnote{This imposes a requirement on the
- in-code usage of \refmmz{per overlay}, namely, that it should be executed
- prior to any changes of |beamerpauses|.} Fine, we store it in a
-macro,\footnote{We must define \cs{mmzBeamerPauses} globally, because
- \refmmz{per overlay} can be arbitrarily deeply embedded in the memoized
- code.} |\mmzBeamerPauses|, when we start the memoization (\refmmz{at begin
- memoization}), and put |\mmzBeamerPauses| rather than |beamerpauses| into the
-context. Will this work? Not yet, because |\mmzBeamerPauses| is undefined at
-utilization. We need to set up the context expression so that it will expand
-to the value of |\mmzBeamerPauses| at (the end of) memoization, and to the
-value of |beamerpauses| at utilization. This leads to the
-|pauses=\ifmemoizing...| part of the context expression in (the final version
- of) the Beamer support code below.
-
-\medskip % manual, to avoid an orphan
-
-\ExampleName{per-overlay}
-\makeexcerpt{per-overlay}
-\tcbinputexample[.tex][.excerpt]{%
- listing only, one file,
- left=0pt, % manual indentation gobble
- title={The implementation of \refmmz{per overlay}},
-}
-
-Are we done? Almost. The final issue is that once we have introduced support
-for pauses, we have to relativize |\mmzBeamerOverlays| (the final overlay
-number) to |beamerpause|. So instead of a simple |\gdef\mmzBeamerOverlays| in
-the first version, we define |\mmzSetBeamerOverlays|\nohyphen\marg{beamer
- pauses}\marg{final overlay number}, which sets |\mmzBeamerOverlays| only if
-\meta{beamer pauses} argument matches the value of |beamerpauses| (at its
-invocation in the c-memo). Well, the macro has some other housekeeping to do
-as well: it is self-replicating, so that during potential memoization, the
-|\mmzBeamerOverlays| values belonging to non-current |beamerpauses| values get
-rewritten into the c-memo.\footnote{As replication should only occur during
- memoization (actually, it \emph{can} only occur then, anyway), the
- instruction to append to the c-memo is appended to the
- \cs{mmzAtBeginMemoization} hook (the low-level interface to \refmmz{at begin
- memoization}). Note that the assignment to this hook must be local (once
- local, always local), and it \emph{can} be local because of a little
- implementation detail: while the c-memo is processed in the memoize
- \hologo{TeX} group, we don't open an additional group to process it; so the
- local effects from c-memo will persist into memoization (but not into
- utilization, because remember that the memoize group is closed before
- inputting the cc-memo).} (Fine, there is another fine detail, regarding
-anti-pollution: the macro also ensures that, relative to \marg{beamer pauses},
-only the instance with the greatest \meta{final overlay number} is replicated.)
-
-We are now truly done, and we can look at the final result, the c-memo and the
-cc-memo belonging to the extern on the first overlay of the example from
-section~\ref{sec:tut:beamer}. Specifically, look at the |\mmzSetBeamerOverlays
-{1}{2}|, which says that the extern chain started when |beamerpauses| equals 1
-should continue up to overlay 2, and at the (expanded) context included at the
-end of the cc-memo, courtesy of \refmmz{include context in ccmemo}, where you
-can see that the cc-memo will be used when on the first overlay (|overlay=1|)
-when preceded by no \cs{pause} command (|pauses=1|).
-
-\begingroup
-\relaxmmzcommands
-\def\mmzNewCMemo#1{% fetch the first c-memo filename
- \def\mycmemo{#1}%
-}
-\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
- \def\myccmemo{#1}%
- \endinput
-}
-\input{\exampledir beamer.mmz.c1}
-\sed{%
- s/~//g;
- s/\cmd{mmzSetBeamerOverlays} \marg\marg/~\0~/;
-}{\exampledir\mycmemo}
-\sed{%
- s/~//g;
- s/overlay=[0-9]*/~\0~/;
- s/pauses=[0-9]*/~\0~/;
- s/\(\cmd{quitvmode}\) \(\cmd{only}\)/\1\n\2/;
- s/\rbrace\(\cmd{mmzIncludeExtern}\)/\rbrace\percentchar\n\1/;
- s/\rbrace\(\cmd{setcounter}\)/\rbrace\percentchar\n\1/;
- s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/;
-}{\exampledir\myccmemo}
-
-\tcbinputlisting{
- listing only,
- listing file=\exampledir\mycmemo,
- example title=\mycmemo,
-}
-
-\tcbinputlisting{
- listing only,
- listing file=\exampledir\myccmemo,
- example title=\myccmemo,
-}
-\endgroup
-
-
-\subsection{Record files}
-\label{sec:record-files}
-
-We have seen that externalization is a two-step process in Memoize: as it is
-impossible for \hologo{TeX} to create multiple PDFs during a single
-compilation, the externs are first dumped into the document PDF as special
-extern pages, and only later extracted from the main document into separate PDF
-files. But extraction requires a complete PDF, which is unavailable even at
-the very end of the compilation which produces the externs. The externs can
-therefore only be extracted \emph{after} that compilation (either before or at
-the beginning of the next one), and this necessitates some form of
-communication whereby the memoization step informs the extraction step which
-pages should be extracted from the document PDF and into which (PDF) files they
-should be stored. This communication is implemented through auxiliary files
-called \emph{record files}.
-
-\subsubsection{The \texttt{.mmz} file}
-\label{sec:.mmz}
-
-By default, Memoize records the information needed for the extraction in a file
-named \meta{document name}\dmmz,\footnote{For \hologo{TeX}perts: the
- \meta{document name} is of course the expansion of \cs{jobname}.} henceforth
-a \dmmz file. In fact, this file contains more than information on externs
-created during the last compilation: it records which memos and externs were
-either used or created during the compilation. The full information contained
-in the \dmmz file is used by the clean-up script \refscript{memoize-clean.pl}
-to safely remove stale memos and externs. Let us take a look at the \dmmz file
-produced by the titlepage illustration. In fact, we have two versions of this
-file, as it changes upon the second compilation.
-
-\ExampleName{titlepage}
-\makeexample{\examplename.pdf N=2}
-\sed{%
- s/~//g;
- s/\cmd{mmzNewExtern}/~\0~/;
-}{\examplepath.mmz.c1}
-\tcbinputexample[.mmz][.c1]{
- listing only, one file, no attachment, float,
- after title pre={\ (after the first compilation)},
- listing options app={breakatwhitespace=false,prebreak=\coloredpercentchar},
-}
-\tcbinputexample[.mmz][.c2]{
- listing only, one file, no attachment, float,
- after title pre={\ (after subsequent compilations)},
- listing options app={breakatwhitespace=false,prebreak=\coloredpercentchar},
-}
-
-As you can see, the \dmmz file takes the form of a \hologo{TeX} script (the
-format was chosen because it facilitated the implementation of the internally
-triggered \hologo{TeX}-based extraction). The crucial lines in this file, and
-the only lines used by the extraction script, occur in the first version of the
-file: they contain command \refcmd{mmzNewExtern}, which informs the extraction
-script that it should extract the document page given by the second argument
-into the extern file given by the first argument.\footnote{If you look at the
- \dmmz file after extracting the externs using \refscript{memoize-extract.pl}
- without the \refscript{memoize-extract.pl--keep} option, you will find that
- the \refcmd{mmzNewExtern} commands are commented out; this is to prevent
- multiple extractions (even if they are harmless).} (The following two
-arguments provide the expected width and height of the extern; the
-extraction script may check whether the extern size conforms to these
-expectations, but this is not crucial, as the extern size is checked every time
-it is included anyway.)
-
-A \dmmz file also contains a record of the memos (both c-memos and cc-memos)
-created in the last compilation; this information is provided by the sole
-argument of commands \refcmd{mmzNewCMemo} and \refcmd{mmzNewCCMemo}. And once
-memos and externs get used in subsequent compilations, the \dmmz file will
-reflect this with \refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo} and
-\refcmd{mmzUsedExtern}, as shown in the second version of the file above.
-
-Finally, both versions illustrate that a \dmmz file always begins with command
-\refcmd{mmzPrefix} and ends with the |\endinput| marker. The argument of
-\refcmd{mmzPrefix} is the path prefix to the memo and extern files, as
-determined by the invocation of key \refmmz{path} (and its subkeys
-\refmmzpath{relative}, \refmmzpath{dir} and \refmmzpath{prefix}). The initial
-\refcmd{mmzPrefix} line is written to the \dmmz file at the beginning of the
-document, but an additional \refcmd{mmzPrefix} line will occur for every
-invocation of \refmmz{path} in the document body. Finally, the |\endinput|
-marker signals that the \dmmz file is complete.
-
-As mentioned above, the full contingent of \dmmz file commands is only used by
-the clean-up script \refscript{memoize-clean.pl}. By default, this script
-removes all memos and externs with the prefix given by \refcmd{mmzPrefix}
-(relative to the directory hosting the \dmmz file) but those listed by any of
-\refcmd{mmzNewCMemo}, \refcmd{mmzNewCCMemo}, \refcmd{mmzNewExtern},
-\refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo} and \refcmd{mmzUsedExtern}.
-Furthermore, the clean-up script will cowardly refuse to delete anything if the
-\dmmz file does not end with |\endinput|, as this means that the compilation
-ended prematurely and that the \dmmz file might not mention all memos and
-externs actually used in the document. If given option
-\refscript{memoize-clean.pl--all}, the clean-up script removes even the memos
-and externs mentioned in the \dmmz file, and as this option is intended to
-bring Memoize to a clean slate after any ``disasters,'' the
-\refscript{memoize-clean.pl--all} mode also ignores the potential absence of
-the |\endinput| marker. Incidentally, the \refscript{memoize-clean.pl--all}
-mode is also the raison d'être for \refcmd{mmzPrefix}: while the prefix is
-usually recognizable from \refcmd{mmzNewCMemo} and friends, these commands
-might not make it into the \dmmz file in a fatally failed compilation, but it
-is precisely such compilations that could occasionally require the full
-clean-up.
-
-\subsubsection{Defining a new record type}
-\label{sec:new-record-file}
-
-The \dmmz file is not the only kind of a record file that can be produced by
-Memoize. Out of the box, it can also write down the extraction instructions
-into a makefile or a shell script. These are useful on systems which have to
-employ the \hologo{TeX}-based extraction but cannot trigger it internally.
-Running the \hologo{TeX}-based extraction manually would be painful, as it must
-be done on extern-by-extern basis, so Memoize offers to automate the extraction
-by a makefile or a shell script; here, the record file is named
-\code{memoize-extract.\meta{document name}.\meta{record type}} by default,
-where \meta{record type} is either \refmmz{record=makefile},
-\refmmz{record=sh} (for shell scripts on Linux), or \refmmz{record=bat} (for
-shell scripts on Windows).
-
-To turn on recording of an alternate record type, use key
-\refmmz{record}|=|\meta{record type}. Memoize can record any number of files
-simultaneously, so saying \code{\refmmz{record}=\refmmz{record=sh}} will
-produce the shell script alongside \dmmz (Memoize internally executes
-\code{\refmmz{record}=\refmmz{record=mmz}} to start recording the \dmmz file);
-this should not be a problem, but if you really want to disable the \dmmz file
-production, you can say \refmmz{no record}.
-
-The predefined record types are defined through a generic system open to the
-user. To define an additional record type, one needs to define, using
-\pkg{pgfkeys}, the relevant hooks of the form \refkeypath{/mmz/record/record
- type}|/|\meta{hook}. The following \meta{hook}s can be defined (the hooks
-not needed for the record file type may be left undefined):
-\begin{itemize}
-\item Key \refmmz{record/record type/begin} will be executed at the beginning
- of the document; it will receive no argument. Use it to open the record
- file.
-\item Key \refmmz{record/record type/end} will be executed at the end of the
- document; it will receive no argument. Use it to close the record file.
-\item Key \refmmz{record/record type/prefix} will be executed at the end of the
- document and at every invocation of key \refmmz{path} in the document body;
- it will receive a single argument, the path prefix determined by key
- \refmmz{path}.
-\item Keys \refmmz{record/record type/new cmemo}, \refmmz{record/record
- type/used cmemo}, \refmmz{record/record type/new ccmemo} and
- \refmmz{record/record type/used ccmemo} will be executed after creating or
- inputting a memo; they will receive a single argument, the full path to the
- memo.
-\item Key \refmmz{record/record type/used extern} will be executed after an
- extern was included into the document; it will receive a single argument, the
- full path to the extern.
-\item Key \refmmz{record/record type/new extern} will be executed after
- creating creating an extern, more precisely at the end of memoization, right
- after shipping out the extern page. It will receive a single argument, the
- full path to the extern, but additionally, Memoize prepares the following
- macros:
- \begin{itemize}
- \item \refcmd{ne:externbasepath} holds the full path to the extern, but (unlike |#1|)
- without the |.pdf| suffix;
- \item \refcmd{ne:pagenumber} holds the ``physical'' page number of the extern page in
- the document (the numbering starts by $1$);
- \item \refcmd{ne:expectedwidth} and \refcmd{ne:expectedheight} hold the width and
- the height (total height, i.e.\ the sum of \hologo{TeX}'s height and depth)
- of the extern page.
- \end{itemize}
-\end{itemize}
-
-Below, we present two simple examples of a record file. The first type simply
-records the names of all memos and externs used or created by Memoize; the
-resulting file could be included by |.gitignore| to have
-\hreftt{https://git-scm.com}{git} automatically ignore all files produced by
-Memoize. The second type lists the new externs, each preceded by its page
-number in the |.pdf|; this file could be fed to a custom extern extraction
-tool.
-
-\ExampleName{record-files}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{listing only}
-
-\ExampleName{record-extern-pages}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{listing only}
-
-Finally, note that (unlike memos and externs) record files are auxiliary files
-and may be deleted at any time after the extraction of the externs produced in
-the final compilation --- actually, even if these externs were not yet
-extracted, deleting the record file(s) will merely force their recompilation.
-
-
-\subsection{The memoization process}
-\label{sec:memoization-drivers}
-
-We now turn to the memoization process itself. The job of memoization is to,
-while compiling the given code in a regular fashion, prepare the cc-memo
-(which, when it is input, will replicate the effect of the given code),
-alongside any externs that the cc-memo will include (these hold the typeset
-material to be replicated). Clearly, merely compiling the code cannot have
-this effect (unless that code was written specifically to support memoization;
-more on this later), and this is why the memoized code is typically wrapped by
-a \emph{memoization driver}, which can be set using key \Emph{\refmmz{driver}}.
-We'll inspect the default memoization driver, \refcmd{mmzSingleExternDriver},
-in the first subsection, and we will learn how to write specialized drivers in
-the remaining subsections. But first, let us say some words about a
-grouping-related \hologo{TeX}nical detail we need to take care about during
-memoization.
-
-During memoization, we have to collect certain information, like build the
-contents of the cc-memo. Some of that information might be contributed by the
-memoized code itself. For example, a \cs{label} ``adds itself'' to the cc-memo
-(by appending to token register \refcmd{mmzCCMemo}); a
-\refkey{/tikz/remember picture} aborts memoization (by issuing
-\refcmd{mmzAbort}); etc. The issue is that the memoized code might open any
-number of \hologo{TeX} groups; we have no idea how deeply embedded the
-\cs{label} or \refkey{/tikz/remember picture} might be. Therefore, we have
-to collect all the information about the ongoing memoization \emph{globally}:
-all assignments to \refcmd{mmzCCMemo} must be global; \refcmd{mmzAbort} sets
-the underlying conditional globally; etc. (Clearly, all these global variables
-are initialized at the start of memoization.)
-
-This was the easy part. An additional complication arises with some options
-which may be set either outside memoization, or during this process. For
-example, you can append the font size to the context expression in the preamble
-(see section~\ref{sec:cross-referencing}), so that the externs will be
-automatically recompiled when the font size changes, and clearly, this context
-adjustment should respect \hologo{TeX} grouping; but a \cs{ref} or some other
-cross-referencing command in the memoized code needs to append to the context
-as well, and as this \cs{ref} occurs \emph{within} the memoized code, the
-assignment must be global, as explained above.
-
-Mixing the local and global assignments to the token register
-\refcmd{mmzContext}, which holds the (in the actual implementation, local)
-context expression, will not do. For one, we \emph{do} want to restore the
-pre-memoization context expression after we have memoized the code, and
-furthermore, mixing local and global assignments to the same variable is not
-recommended for save stack reasons anyway.
-
-Memoize addresses this issue by having \emph{two} context registers,
-\refcmd{mmzContext} and \refcmd{mmzContextExtra} --- when computing the context
-MD5 sum (which happens at the end of memoization), the two registers are
-concatenated (the local one comes first). A package writer should know when to
-use which register, and how. Outside memoization, one should assign to
-\refcmd{mmzContext} --- \emph{locally}. During memoization, one should assign
-to \refcmd{mmzContextExtra} --- \emph{globally}. The user interface key
-\refmmz{context} respects this requirement automatically: it locally appends to
-\refcmd{mmzContext} outside memoization, and it globally appends to
-\refcmd{mmzContextExtra} during memoization. (The same idea is applied to the
-post-memoization hooks \refmmz{at end memoization} and \refmmz{after
- memoization}.)
-
-
-\subsubsection{The default memoization driver}
-\label{sec:memoization-driver-default}
-
-The default memoization driver, \refcmd{mmzSingleExternDriver}, produces
-exactly one extern, which contains whatever is typeset by the code submitted to
-memoization. The driver compiles the code into a horizontal or vertical box
-depending on the value of key \refmmz{capture}. Let us look at the definition
-of the driver line by line:
-
-\makeexcerpt{single-extern-driver}
-\tcbinputexample[.tex][.excerpt]{%
- listing only, one file,
- listing options app={numbers=left, numberstyle=\tiny, numbersep=0.5em},
- title=The default memoization driver,
- float,
-}
-
-\begin{enumerate}
-\item Macro \refcmd{mmzSingleExternDriver} (and in fact any memoization
- driver) takes a single argument, the code to compile. Memoize will call the
- driver with the code given as the second argument to \refcmd{Memoize}, but
- wrapped in a macro which re-reads it using \refcmd{scantokens} when
- \refmmz{verbatim} is in effect.
-\item If we're capturing into a horizontal box
- (\refmmz{capture}|=|\refmmz{capture=hbox}), we put \refcmd{quitvmode} into
- the cc-memo --- putting it to the very beginning should make sure that any
- replicated \cs{label} and \cs{index} commands refer to the correct page.
-\item We compile the given code, storing the typeset material into a box
- (above, a temporary box called |\mmz at box|). |\mmz at capture| resolves into a
- box construction command, depending on the value \refmmz{capture}.
-\item Macro \refcmd{mmzExternalizeBox} instructs Memoize to externalize the box
- given as its first argument. However, this macro does not directly produce
- an extern page or write any instructions into the cc-memo; the road to this
- final destination is indirect. \refcmd{mmzExternalizeBox} has two effects.
- First, it adds the contents of the given box (above, |\mmz at box|) to an
- internal box dedicated to holding all the externs produced in this
- memoization (the contents of |\mmz at box| remain as they are) --- it is only at
- the end of memoization that the contents of this internal box are shipped off
- to extern pages. Second, \refcmd{mmzExternalizeBox} produces the code which
- will include the extern into the document on subsequent compilations (this
- will be a call to \refcmd{mmzIncludeExtern}, potentially prefixed by
- \refcmd{quitvmode}; see section~\ref{sec:cc-memos} for details). This code
- is stored into the token register gives as the second argument (above,
- |\mmz at temptoks|), and it is the responsibility of the driver to include it
- into the cc-memo. (In the interest of full disclosure,
- \refcmd{mmzExternalizeBox} also updates the list of externs produced in this
- memoization. At the end of memoization, this list is written to the
- beginning of the cc-memo, resulting in the \refcmd{mmzResource} lines
- preceding the \refcmd{mmzMemo} marker.)
-\item The construction of the cc-memo is indirect as well. In the third line
- of the definition, we globally append the extern-inclusion code residing in
- |\mmz at temptoks| to token register \refcmd{mmzCCMemo}. At the end of
- memoization, the contents of \refcmd{mmzCCMemo} are written into the cc-memo,
- preceded by the \refcmd{mmzMemo} marker.
-\item We put the typeset material into the document, again preceded by
- \refcmd{quitvmode} when capturing in a horizontal box.
-\end{enumerate}
-
-You might wonder why the construction of the extern pages and the cc-memo (and
-actually, of the c-memo as well) is indirect, as described above.
-\begin{itemize}
-\item For one, the indirect construction facilitates potential abortion of
- memoization (see section~\ref{sec:tut:good-to-know}). With the indirect route,
- aborting is easy --- as nothing was permanently written anywhere yet, Memoize
- simply skips the final part of the process, where extern boxes are shipped
- into extern pages and the memo registers written into memo files --- and also
- clean: if \refcmd{mmzExternalizeBox} immediately shipped out the extern
- pages, these pages would remain in the document even in the case of abortion.
-\item Even more importantly, the cc-memo filename contains the \meta{context
- md5sum} (see section~\ref{sec:cc-memos}), but the context expression is not
- yet fully known when memoization starts --- remember (from
- section~\ref{sec:cross-referencing}) that a \cs{ref} in the memoized code will
- update the context! The cc-memo can therefore only be opened at the end of
- memoization, which necessitates a buffer (i.e.\ the \refcmd{mmzCCMemo}
- register) for storing its contents during memoization.
-\end{itemize}
-
-
-\subsubsection{Pure memoization}
-\label{sec:pure-memoization}
-
-The default memoization driver discussed above is really an externalization
-driver: it produces a single extern. We now move to examples of drivers with
-other functions, starting with a pure memoization driver, which does not
-externalize any typeset output --- simply because it does not call
-\refcmd{mmzExternalizeBox} at any point --- but rather remembers the result
-of a (pgfmath) computation (let's pretend that the computation is
-time-consuming).
-
-\ExampleName{pgfmathparse}
-\makeexample{\examplename.pdf}
-\begingroup
-\relaxmmzcommands
-\def\mmzNewCCMemo#1{% fetch the first cc-memo filename
- \def\myccmemo{#1}%
- \endinput
-}
-\input{\examplepath.mmz.c1}
-\tcbinputexample{%
- comment={%
- \tcbinputlisting{
- listing only, width=.4\linewidth, nobeforeafter,
- listing file=\exampledir\myccmemo,
- example title, title=the cc-memo,
- }\quad
- \includeexamplepdf[document page]{page=1}
- },
-}
-\endgroup
-
-Command \refcmd{mmz} above memoizes its mandatory argument with the memoization
-\refmmz{driver} set to the previously defined macro \cs{mmzPgfmathDriver}.
-Just as the default driver above, \cs{mmzPgfmathDriver} first executes the
-given code. However, there is no need to do this in the context of a
-|\setbox|, as the memoized code, which is obviously expected to consist of a
-single \cs{pgfmathparse} call, does not typeset anything: \cs{pgfmathparse}
-evaluates the given expression and stores the result into macro
-\cs{pgfmathresult}. The driver has two jobs: first, it must store this result
-into the cc-memo, to be utilized in subsequent compilations; second, because
-the assignment to \cs{pgfmathresult} (within \cs{pgfmathparse}) is local, the
-driver also needs to somehow smuggle the result out of the \cs{endgroup} issued
-by \refcmd{Memoize} and thereby make it into the following code (the final line
-of the example, which typesets the equation). Both jobs are easy enough: the
-expansion of |\def\noexpand\pgfmathresult{\pgfmathresult}| (in this case,
-|\def\pgfmathresult{42.0}|) is (globally) appended both to the token register
-\refcmd{mmzCCMemo}, which Memoize later writes into the cc-memo, and to the
-macro underlying the \refmmz{after memoization} hook, whose contents are
-executed \emph{after} closing the memoization group.
-
-Let us consider an alternative implementation of the same goal of memoizing the
-result of a pgfmath computation, showcasing a couple of useful tricks.
-
-\ExampleName{pgfmathparse-embellished}
-\makeexample{\examplename.pdf}
-\tcbinputexample{listing only}
-
-For one, this ``embellished'' example reminds us that we can list the
-\refmmz{driver} key among the auto-options (even if I don't really recommend
-automemoizing \cs{pgfmathparse}). But even more importantly, the example shows
-that the driver consist of more than a single control sequence; the only
-requirement is that the given driver code will consume the memoized code. In
-this example, we have developed a generic smuggling driver and applied it to
-\cs{pgfmathresult} in particular --- \cs{pgfmathresult} will become the first
-argument of \cs{mmzSmuggleOneDriver}, and the memoized code will become its
-second argument.
-
-In the first version of the example, we have appended the same code to macro
-\refcmd{mmzCCMemo} and to macro \refcmd{mmzAfterMemoizationExtra} --- no
-surprise here, as we want the effect of memoization and utilization to be the
-same. In the embellished version, we advertise another way to achieve the same
-effect, a way which might be useful for complicated drivers: we simply smuggle
-out the entire cc-memo. The idea works even when memoization procudes externs;
-in that case, however, the driver also has to say \cs{mmzkeepexternstrue} ---
-conditional \refcmd{ifmmzkeepexterns} decides whether Memoize keeps the externs
-around, in memory, even after shipping them out (but they are always gone at
-the start of the next memoization).
-
-Finally, remember that the default context expression contains the
-\refmmz{padding} values. However, these really have no place in the context
-expression of some purely memoized code. We have therefore emptied out the
-context expression using \refmmz{clear context}.
-
-
-\subsubsection{Multiple externs per memo}
-\label{sec:memoization-multiple-externs}
-
-In the next example, we show how to produce multiple externs for a single piece
-of memoized code. The usage case I find most appealing is breaking the typeset
-material, like a table, across pages --- but of course, table-breaking is too
-complicated an example, so we illustrate the idea by defining command
-|\countdown|, which counts down from the given number, typesetting each number
-into its own line. Clearly, if we were to externalize a call to this command
-using the default memoization driver, page breaking would stop working, as the
-entire countdown would be seen as a single, unbreakable box. To externalize it
-properly, the chunks of the countdown that should appear on separate pages must
-be externalized into separate externs, as shown below.
-
-\ExampleName{countdown}
-\makeexample{\examplename.sty N=2}
-\makeexample{\examplename.pdf}
-\tcbinputexample{%
- listing options app={
- basicstyle=\ttfamily\footnotesize,
- },
- middle=1mm, bottom=0mm,
- comment={\centering
- \relaxmmzcommands
- \def\mmzUsedExtern##1{%
- \saveexpandmode\fullexpandarg
- \StrGobbleRight{##1}{4}[\mytemp]%
- \StrRight{\mytemp}{5}[\mytemp]%
- \restoreexpandmode
- \relaxmmzcommands
- }%
- \def\mmzNewExtern##1##2##3##4##5{\mmzUsedExtern{##1}}
- \def\myexternname##1{extern \dots\texttt{\mytemp##1.pdf}}%
- \input{\examplepath.mmz}%
- \begin{tabular}{ccc}
- \includeexamplepdf[extern page,title/.expand once=\myexternname{}]
- {page=2,trim=1in 1in 1in 1in,scale=0.333}&
- \includeexamplepdf[extern page,title/.expand once=\myexternname{-1}]
- {page=3,trim=1in 1in 1in 1in,scale=0.333}&
- \includeexamplepdf[extern page,title/.expand once=\myexternname{-2}]
- {page=4,trim=1in 1in 1in 1in,scale=0.333}\\
- \includeexamplepdf[document page]{page=1,scale=0.333}&
- \includeexamplepdf[document page]{page=5,scale=0.333}&
- \includeexamplepdf[document page]{page=6,scale=0.333}
- \end{tabular}
- },
-}
-
-To achieve this, we will have to integrate the memoization driver into the very
-code of |\countdown|. This approach contrasts sharply with the standard
-memoization driver, which is simply wrapped around the memoized code. Let's
-say we have implemented a non-memoization-aware variant of |\countdown| as a
-loop which gathers the countdown lines into a vertical box, and periodically,
-when this box holds all the material that will fit onto the page, places it
-into the main vertical list (i.e.\ on the page).\footnote{Of course,
- implementing \cs{countdown} this way would be idiotic; a sane implementation
- would simply spit out the countdown lines one by one, and let \hologo{TeX}
- deal with page-breaking. However, remember that we are pretending that we
- are typesetting (and page-breaking) some complex material, like a table; in
- such a case, the loop outlined in the main text would make perfect sense.}
-To have this command support memoization, we have to externalize our box every
-time we're placing it into the main vertical list. This is precisely what we
-do in the definition of |\countdowntypeset| below:\footnote{We omit the
- definition of the core algorithm of \cs{countdown} in the listing, because it
- is mostly irrelevant for our discussion, and only show the
- memoization-related code.} the final line of this macro adds the material to
-the main vertical list, and the preceding lines externalize it (the two lines
-inside \cs{ifmemoizingcountdown} should be familiar from the definition of the
-standard memoization driver; we'll explain about the conditional below); note
-that Memoize automatically deals with the fact that our box is vertical. As a
-result of having our memoization driver integrated into the loop of the core
-command, we can create as many externs as necessary, complete with the code in
-the cc-memo for including each and every one of them on subsequent
-compilations. Each extern eventually makes it into its own extern file, and
-note that the filenames of the non-first externs have their sequential number
-(we start counting at $0$) appended to the basename, as shown in the
-example.\cprotect\footnote{The \refmmz{auto} declaration of |\countdown| adds
- some relevant parameters to the context (see
- section~\ref{sec:cross-referencing}). The countdown will be recompiled upon
- change of either the font size (|f at size|), the text height (|\textheight|),
- or the height of the material in the main vertical list collected so far
- (|\pagetotal|). The |\pagetotal| parameter is especially important;
- including it makes sure that the countdown will be recompiled when it is
- pushed up or down the page. Also note that we want the context to record the
- value of |\pagetotal| when the (automemoized) |\countdown| is encountered
- (rather than at the end of memoization), so we expand it when applying the
- auto-options.}
-
-\tcbinputexample[.sty]{listing only, after title pre={\ (version 1)}}
-
-Of course, the chunks of the countdown should only be externalized when the
-code is actually being memoized, and not, say, when Memoize is disabled or
-performing regular compilation. (Note that this is a problem that only affects
-integrated drivers and not wrapped drivers such as the default driver.) The
-first thought is to detect whether we're undergoing memoization using
-conditional \refcmd{ifmemoizing}, which Memoize sets to true at the start of
-every memoization. This conditional is used in the run conditions of advice
-for \cs{ref} and \cs{label}, the idea being that they should add stuff to the
-context (\cs{ref}) and the cc-memo (\cs{label}) only when undergoing
-memoization. However, deploying \refcmd{ifmemoizing} in the current example
-would not be exactly right. It would work well with the main document as it
-is, but it would fail if |\countdown| was called from a piece of code that was
-independently submitted to memoization, e.g.\
-\refcmd{mmz}|{\countdown{30}}|.\footnote{Such embedding occurs more often than
- you might think. For example, \env{forest} calls \env{tikzpicture} under the
- hood, and both environments are automemoized.} In that case, both the
-\refcmd{mmz} driver and the |\countdown| integrated driver would get executed,
-resulting in the creation (and in subsequent compilations, utilization) of four
-externs: first, the |\countdown| driver would externalize each countdown chunk
-separately, and then, the \refcmd{mmz} driver would externalize them, all
-together, yet again. You can try this out by replacing
-\cs{ifmemoizingcountdown} in \cs{countdowntypeset} by \refcmd{ifmemoizing} (and
-wrapping the \cs{countdown} call in \cs{mmz}).
-
-The solution to the \refcmd{ifmemoizing} problem deployed in the example is to
-declare a new, \cs{countdown}-specific memoization conditional, and set it to
-true in \cs{countdown}'s formal driver, i.e.\ the macro set as the
-\refmmz{driver} in the \refmmz{auto} declaration for \cs{countdown}. In fact,
-Memoize can do most of this for you: when we write \refmmzauto{integrated
- driver}|=countdown|, Memoize creates the countdown-specific memoization
-conditional and declares the formal driver which sets this conditional to true;
-you only have to access this conditional in your code, and you should do this
-using the \hologo{LaTeX}-style conditional \refcmd{IfMemoizing}, as shown
-below.\footnote{You shouldn't directly use the plain \hologo{TeX}
- countdown-specific conditional created by \refmmzauto{integrated driver} ---
- to prevent accidental access, Memoize doesn't actually name it
- \cs{ifmemoizingcountdown} --- because this conditional is undefined when
- Memoize is not loaded, i.e.\ when only package \refpkg{memoizable} is in
- effect. Furthermore, \refcmd{IfMemoizing} addresses a problem faced by
- integrated drivers of potentially recursive commands; we will talk about this
- in section~\ref{sec:memoization-complex-single-driver}.}
-
-\tcbinputexample[.sty][.c2]{listing only, after title pre={\ (version 2)},
- attachment name=\examplename-integrated-driver.sty,
-}
-
-
-\subsubsection{Driver-based memoizable design}
-\label{sec:memoization-complex-single-driver}
-
-In the previous section, we used the integrated driver approach to produce
-memos including multiple externs, but the approach can be useful for one-extern
-memos as well, when the extern must be integrated into the document in some
-special way. We already discussed such situations in
-section~\ref{sec:memoizable-design}, where we suggested to split a
-``difficult'' command into the outer command and the inner command, and only
-submit the inner command to automemoization. However, the vanilla flavour of
-this approach had a negative impact on the user interface to automemoization.
-In this section, we will deploy the memoization driver to overcome the issue.
-
-\ExampleName{poormansbox-driver}
-\makeexample{\examplename.sty}
-\makeexample{\examplename.pdf N=2}
-
-Let us revisit the |poormansbox| example from
-section~\ref{sec:memoizable-design}. Remember that that environment produced
-a potentially framed box of a certain width, surrounded by some pre- and
-post-code, and that the issue was that the pre- and the post-code should not be
-memoized, but rather executed at every invocation of the command, as it was
-primarily intended to put some stretchable vertical space around the box.
-
-The document source\attachexample\ and the resulting PDF of the example are the
-same as in section~\ref{sec:memoizable-design}, so we will not repeat them
-here, but jump directly into a revised definition of the environment. We will
-retain the core idea from the original implementation: the outer command will
-execute the pre- and the post-code, and the inner command will typeset the box.
-But unlike in the original implementation, we will not automemoize the inner,
-internal command (this was the source of the author's discomfort) but the outer,
-user-level command --- and we will equip it with a custom memoization driver.
-The major idea here is to have the driver compose a cc-memo which not only
-includes the extern, but also executes the outer command.
-
-\tcbinputexample[.sty]{listing only, float}
-
-In detail, the implementation (partially shown in the |.sty| listing) is as
-follows. The outer command (|\poormansbox at outer|) first applies the options
-(|#1|) and then wraps the pre-code (|\pmb at before|) and the post-code
-(|\pmb at after|) around some arbitrary code (|#2|). During memoization or
-regular compilation, the outer command is invoked through the |poormansbox|
-environment, and you can see that in the definition of that environment, the
-second argument to |\poormansbox at outer| is a call to the inner command
-(|\poormansbox at inner|; this command takes two arguments, the options and the
-environment body). During utilization, the outer command is invoked from the
-cc-memo,\cprotect\footnote{As you can see, in the cc-memo the outer command is invoked
- by |\csuse{poormansbox at outer}|. A straight |\poormansbox at outer| would not
- work because we're in the middle of the document where |@| is not a letter,
- and including a |\makeatletter| in front of it (and in a group) only works if
- \refmmz{direct ccmemo input} was in effect. Under the default, indirect
- cc-memo input regime, the core cc-memo is tokenized before |\makeatletter| can
- take effect.} and as you can see in the cc-memo listing below, the second
-argument to |\poormansbox at outer| there is a call to \refcmd{mmzIncludeExtern}.
-
-\begingroup
-\relaxmmzcommands
-\def\mmzNewCCMemo#1{%
- \def\mmzNewCCMemo##1{% fetch the second cc-memo filename
- \def\myccmemo{##1}%
- \endinput
- }%
-}
-\input{\examplepath.mmz.c1}
-\sed{%
- s/~//g;
- s/\(\cmd{csuse} *{poormansbox at outer}\) *\({\nobrace*\marg\nobrace*}\)\({.*}\rbrace\)/~\1~\2~\3~/;
- s/\(\marg\marg\marg\)\(\marg\marg\marg\marg\marg\)/\1\space\2/;
-}{\exampledir\myccmemo}
-\tcbinputlisting{
- listing only, left=1.5mm, right=1mm,
- listing options app={basicstyle=\tt\small},
- listing file=\exampledir\myccmemo,
- example title, title=the second poor man's box's cc-memo,
-}%
-\endgroup
-
-And how does |\poormansbox at outer| get into the cc-memo, which normally only
-includes a call to \refcmd{mmzIncludeExtern}, you ask? This is the job of the
-memoization driver, which is in this case integrated into the inner command.
-The overall shape of the driver is the same as the shape of the standard
-driver, discussed in section~\ref{sec:memoization-driver-default}: typeset the
-extern material into a box, externalize this box, append the extern-inclusion
-code to the cc-memo, and put the extern box into the document. It is the
-cc-memo part which interests us right now: unlike the standard driver, we don't
-simply append the contents of |\mmz at temptoks|, i.e.\ a
-\refcmd{mmzIncludeExtern} call; we rather append a call to
-|\poormansbox at outer|, which gets the \refcmd{mmzIncludeExtern} call as its
-second argument (and the options as its first argument).
-
-The core part of the driver, which externalizes the box and appends to the
-cc-memo, is embedded inside the true branch of conditional
-\refcmd{IfMemoizing}|[1]{pmb}|. We already used this conditional in
-section~\ref{sec:memoization-multiple-externs}, but without the optional
-argument. Such usage will not work here, because it is not recursion-safe.
-Unlike in the \cs{countdown} situation, one |poormansbox| environment can be
-embedded in another one (and, in our example, it is). If we deployed
-\refcmd{IfMemoizing}|{pmb}| in the inner command, the driver would be executed for
-\emph{both} the outer and the inner instance of the environment, whereas it
-should really be executed only for the outer instance.
-
-When used in a recursion-safe way, i.e.\ with the optional argument,
-\refcmd{IfMemoizing} first tests whether the auxiliary command-specific
-conditional from the previous section is true, and then proceeds to compare the
-current group level (\hologo{eTeX}'s \refcmd{currentgrouplevel}) to the group
-level at the start of memoization (which Memoize stored in
-\refcmd{memoizinggrouplevel}). Only if these group levels match do we know
-that we're working on the outer instance of the environment, and that we should
-therefore execute the memoization driver. Importantly, though, the two group
-levels are compared modulo the offset, given as the optional parameter to
-\refcmd{IfMemoizing}: in our example, the offset is 1, because the driver is
-located inside the |poormansbox| environment, which opens a group --- note that
-0 zero (no offset) is not the default optional parameter; the absence of the
-optional parameter indicates that the non-recursion safe method should be
-used.\footnote{Even this approach is not completely bullet-proof. It will only
- work when the inner instance of the command is guaranteed to occur in an
- additional group, i.e.\ when our command opens up a group for any free-form
- code. I will assume that situations which require externalization of a
- potentially recursive command which, for some reason, cannot open the group
- before processing a free-form argument, are rare enough to not warrant a
- generic solution here.}
-
-
-\FloatBarrier
-
-
-\subsubsection{Shipout}
-
-Memoize is a hypocrite: when it is creating extern pages, it uses
-\refcmd{primitive}\refcmd{shipout} to bypass the regular shipout routine of the
-format, but it is offended if anyone else does that.
-
-Memoize bypasses the regular shipout because the extern pages should really not
-be modified or discarded by a foreign package. But using the primitive
-\refcmd{shipout} means that extern shipouts can't be detected by another
-package, at all. To facilitate peaceful coexistence with a potential package
-which needs to know about our extern pages, we offer public counter
-\refcmd{mmzExternPages} holding the number of externs shipped out so far. And
-if anyone really needs to \emph{do} something at every extern shipout, they can
-always (ab)use \refmmz[show keypath]{record/record type/new extern} as a
-post-extern-shipout hook.
-
-The other side of the story is about Memoize needing to know the ``physical''
-page numbers of its externs in the document PDF --- how else are we to extract
-them? Memoize computes these page numbers by adding the values of several
-counters: \refcmd{mmzRegularPages}, which holds the number of regular shipouts;
-the above-mentioned \refcmd{mmzExternPages}, which holds the number of extern
-shipouts; and \refcmd{mmzExtraPages}, which holds the number of other shipouts.
-The latter counter should be advanced by a package which, like Memoize,
-bypasses the regular shipout routine.
-
-\hologo{LaTeX} and \hologo{ConTeXt} kindly provide the number of regular
-shipouts as publicly a accessible counter, so we define
-\refcmd{mmzRegularPages} as synonymous with their
-\refcmd{ReadonlyShipoutCounter} and \refcmd{realpageno}. In \hologo{plainTeX}, we
-have to hijack the \refcmd{shipout} control sequence and count regular shipouts
-ourselves; as we have to hijack it while it still refers to the
-\refcmd{shipout} primitive, this format provides another reason for preferring
-Memoize to be loaded early.
-
-
-\subsection{Automemoization}
-\label{sec:tut:automemoization-details}
-
-Automemoization is a mechanism that automatically memoizes the result of the
-compilation of certain commands and environments. Writing Memoize, I went to
-great lengths to make it flexible, yet easy to use. This resulted in
-automemoization deploying two specifically developed auxiliary packages:
-package Advice, which provides a generic framework for extending the
-functionality of selected commands and environments, and package CollArgs,
-which provides a command for collection of the arguments conforming to the
-given (slightly extended) \pkg{xparse} argument specification.
-
-This section lists the considerations which went into designing the system,
-followed by short tutorials on both auxiliary packages, which include several
-examples of how Memoize uses the underlying advising framework.
-
-\paragraph{(De)activation}
-Ideally, all commands and environments where memoization makes sense would
-support Memoize (or Memoize would support them) and nothing would ever go
-wrong. In this dream world, memoization would be completely transparent to the
-author. However, things will go wrong, so at the very least, we need to
-offer the author a \emph{simple} way to \emph{selectively} switch automemoization
-on and off. This is achieved by keys \refmmz{activate} and
-\refmmz{deactivate}.
-
-\paragraph{Submission}
-Of course, there will be commands without official support by either Memoize or
-the package which defines them; clearly, at the moment when I write this, all
-commands but \cs{tikz}, \env{tikzpicture} and \env{forest} are such. Or, the
-author might want to automemoize his or her own command. Ideally, submitting a
-new command to automemoization would be as simple as
-|memoize=|{\meta{command}}, and for environments, this is in fact achievable,
-although the actual interface is
-\refmmz{auto}|=|\braces{\meta{environment}}\braces{\refmmzauto{memoize}}. But
-simply submitting the name cannot work for commands, because commands are where
-we encounter the major \hologo{TeX}nical problem with automemoization: we need
-to somehow collect the arguments of the command --- without executing the
-command itself.
-
-\paragraph{Argument collection using CollArgs}
-\hologo{TeX} being \hologo{TeX}, automatically determining the scope of a
-command in general is just plain impossible. Note that inspecting the
-|\meaning| is not enough in general, because the ``real'' and the formal
-arguments of a command can, and quite often do, differ wildly. The author (or the
-package writer) will need to tell Memoize about the argument structure of the
-command. And as there is already a nice and general argument specification on
-the market --- I'm obviously referring to the argument specification of package
-\pkg{xparse}, which was recently even integrated into the core \hologo{LaTeX}
---- why not use that? Memoize comes with an auxiliary package CollArgs, which
-(given the slightly extended \pkg{xparse}-style argument specification)
-collects the arguments of a command into a single entity. All the user needs
-to write to enable automemoization for a command is thus
-\refmmz{auto}|=|\meta{command}\braces{\refmmzauto{memoize},
- \refmmzauto{args}|=|\braces{\meta{argument specification}}}. Even simpler,
-when it comes to commands defined by \pkg{xparse}'s \refcmd{NewDocumentCommand}
-or friends, writing \refmmz{auto}|=|\meta{command}\braces{\refmmzauto{memoize}}
-will suffice, as the argument specification of these commands can be retrieved
-by \refcmd{GetDocumentCommandArgSpec}.
-
-\paragraph{Weird commands}
-Not every argument structure can be described using \pkg{xparse}'s argument
-specification, a case in point being \refcmd{tikz} with its totally
-idiosyncratic syntax --- and if Memoize won't support \cs{tikz}, why have it at
-all? The interface to automemoization must be flexible enough to cover even
-the craziest commands, and this is why Memoize allows for arbitrary argument
-collectors. These are defined by the advanced user or package writer and then
-declared to be used for parsing the argument structure of a command by writing
-\code{\refmmz{auto}=\marg{command}\bracestt{...,\refmmzauto{collector}=\meta{argument
- collector}}}.
-
-\paragraph{Over and above automemoization: handlers}
-The framework facilitating automemoization must cover more than just that. For
-one, it sometimes makes sense to automatically \emph{prevent} memoization
-during the execution of certain commands (as in the \refmmzauto{nomemoize}
-example in section~\ref{sec:tut:redefinitions}). It follows that the action
-performed to an invocation of a command should not be fixed. In the advising
-framework, implemented by the auxiliary package Advice, we assign each
-advised command a handler --- a command which does the real work of
-memoizing or whatever. Crucially, the handler and the collector are
-independent of each other, allowing a single memoization handler to handle
-commands with both standard and non-standard argument structure, and allowing a
-single collector to serve either the memoization or the no-memoization handler.
-
-\paragraph{Over and above automemoization: the outer and the inner handler}
-Second, unlike the memoization handler set by \refmmzauto{memoize}, not all
-handlers work with the entire argument list of the advised command. Some
-handlers don't care about the arguments of the advised command at all:
-\refmmzauto{abort} simply aborts memoization whenever the advised command is
-executed. Other handlers are only intended to advise a single command or a
-small family of commands, and need to inspect specific arguments of the advised
-command: for example, \refmmzauto{ref} needs to append the internal
-cross-reference macro to the context, and it of course constructs the name of
-this macro from the reference key. For all such handlers, it would be plain
-wasteful to first collect the arguments and then tear them apart to inspect
-them (or not). The advising framework therefore recognizes two kinds of
-handlers. The abortion and the cross-reference handler are examples of an
-\emph{outer} handler, which is simply placed in front of the arguments of the
-handled command as they are, without invoking the collector. The memoization
-handler, on the other hand, is an example of an \emph{inner} handler, which
-receives the entire argument list from the collector (as a single argument) ---
-or more precisely, even \refmmzauto{memoize} sets up an outer handler, but this
-outer handler doesn't do much more than invoke the collector, which in turn
-invokes the inner handler.
-
-\paragraph*{Run conditions} are another, if minor, lego piece of the
-advising framework. Using key \refmmzauto{run conditions}, the user can set the
-conditions under which the (outer) handler is executed; for example,
-cross-reference commands are only advised when memoization is underway. And
-the same goes for \refcmd{label}, and for the \refmmzauto{replicate}d
-\refcmd{index}, and for \refcmd{savepos}, upon which memoization must be
-aborted. The bottom-line is that run conditions repeat across handlers, so
-it makes sense to separate them out as an independent component of the
-framework, with the added bonus that the system can make sure that an
-invocation of a advised command which does not satisfy the run conditions
-will incur as litte overhead as possible.
-
-\paragraph{Bailout handler}
-An automemoized command applies the next-options (set by \refcmd{mmznext}), but
-what happens when the run conditions are not satisfied? If nothing happened,
-the existing next-options might apply to the next instance of
-(auto)memoization, which would not be what the author intended. This is why
-Advice introduces the bailout handler, a piece of code executed before the
-original command when the run conditions are not met. Obviously, the bailout
-handler for memoization clears out the next-options (and does not process
-them).
-
-\paragraph{The structure of advice}
-Together, the components mentioned above form a piece of \emph{advice}:
-
-\begin{center}
- \begin{forest}
- /tikz/edge node/.style={midway, font=\scriptsize, yshift=0.5ex},
- for tree={l sep=5ex, % manual, to avoid a page break
- child anchor=north, s sep+=2em},
- [activated?
- [\refmmzauto{run conditions}?,
- edge=dashed, edge label={node[edge node, left]{yes}}
- [\refmmzauto{outer handler}\meta{args},
- edge=->, edge label={node[edge node, left]{yes}},
- [default:,
- edge=dashed,
- [\refmmzauto{collector}\meta{args}, edge=->,
- [\refmmzauto{inner handler}
- \textcolor{red}{\bracestt{\textcolor{black}{\meta{args}}}},
- edge=->
- ]
- ]
- ]
- [other:\\whatever\rlap\footnotemark,
- align=center, edge=dashed]
- ]
- [\refmmzauto{bailout handler},
- edge=->, edge label={node[edge node, right]{no}},
- [\meta{original command}\meta{args}, edge=->]
- ]
- ]
- [\meta{original command}\meta{args},
- edge=dashed, edge label={node[edge node, right]{no}},
- ]
- ]
- \end{forest}%
- \footnotetext{The handler may do whatever as long as it consumes all and
- only the arguments of the original command.}
-\end{center}
-
-
-
-\paragraph{Deferred activation}
-Memoize needs to be loaded early, but activation should take place late, so
-that it can surely override the submitted commands; a case in point,
-\pkg{hyperref} redefines \refcmd{ref} very late. To address the issue, the
-advising framework implements the deferred \refmmz{activation} regime, under
-which (de)activation commands are not executed but collected in a special
-style, \refmmz{activate deferred}. Memoize deploys the deferred activation
-regime throughout the preamble, and executes \refmmz{activate deferred} at the
-latest possible \docref{hook:begindocument} hook; as a bonus, it also offers
-the author a way to avoid automatic activation completely by invoking key
-\refmmz{manual}.
-
-
-\paragraph{Install anywhere}
-Once all this machinery is developed, why not offer it to others as well?
-
-Once I decided to offer Advice (at the time, still called Auto) as a standalone
-package (and I freely admit that the framework got much cleaner once I
-separated its code out of Memoize) it became immediately clear that if it is
-to serve as a generic framework, it should be possible for multiple packages to
-use it without interfering with each other. The package thus allows any number
-of installations into different namespaces, each namespace a \pkg{pgfkeys}
-keypath. The installation is a breeze:
-\cs{pgfkeys}\bracestt{\meta{namespace}/\refkey{/handlers/.install advice}}.
-
-
-\subsubsection{Using package Advice}
-\label{sec:advice}
-
-In this section, we will provide some examples of handler declarations, mainly
-based on how Memoize deploys the advising framework.
-
-\paragraph{\refmmzauto[show keypath]{memoize}}
-
-In section~\ref{sec:tut:automemoization}, the author was instructed to submit a
-command to automemoization by writing
-\code{\refmmz{auto}=\meta{command}\bracestt{\refmmzauto{memoize},...}}. The
-auto-key \refmmzauto{memoize} is a style (defined by Memoize rather than
-Advice) which sets the appropriate components of the automemoization advice.
-Residing in keypath \refkeypath{/mmz/auto}, it is effectively defined as
-follows:
-
-\begin{tcblisting}{listing only}
-\mmzset{
- auto/memoize/.style={
- run if memoization is possible,
- bailout handler=\mmz at auto@bailout,
- outer handler=\mmz at auto@outer,
- inner handler=\mmz at auto@memoize
- }
-}
-\end{tcblisting}
-
-The heart of this advice is its inner handler, which actually triggers
-memoization by executing \refcmd{Memoize}. Remember that the first argument of
-\refcmd{Memoize} is the code which the md5sum is computed off of. This
-argument must therefore be identical to the author's invocation of the
-automemoized command or environment. Given what Advice offers, this is easy to
-construct: \refcmd{AdviceReplaced} holds the code replaced by the advice, and
-the \meta{arguments} of the automemoized command are waiting for us in |#1|.
-The second argument of \refcmd{Memoize} will be similar, but as this is the
-code which will get compiled, we have to execute the original definition of the
-command, followed by the (unbraced!) \meta{arguments} as |#1|; this is a job
-for \refcmd{AdviceOriginal}. (Note that \emph{executing}
-\refcmd{AdviceReplaced} would run the auto-handler again, resulting in an
-infinite loop! Or at least a pile of errors.)
-
-However, the overly simplistic approach shown below won't necessarily work.
-The issue is that the arguments of \refcmd{Memoize} contain
-\refcmd{AdviceReplaced} and \refcmd{AdviceOriginal} themselves, instead of their
-contents, i.e.\ (first) expansions.
-
-\begin{tcblisting}{listing only, bad}
-\long\def\mmz at auto@memoize#1{%
- \Memoize{\AdviceReplaced#1}{\AdviceOriginal#1}%
-}
-\end{tcblisting}
-
-Regarding the first argument, the problem is that the code md5sum will be
-computed off of the token list \refcmd{AdviceReplaced}\meta{arguments} ---
-exactly as you see it.\footnote{If you inspected a c-memo, you would find
- \refcmd{AdviceReplaced}\meta{arguments} in the \refcmd{mmzSource} section.}
-This implies that two commands sharing exactly the same \meta{arguments} will
-receive the same (c-)memo. For example, if you automemoized first
-|\textit{foo}| and then |\textbf{foo}|, both would come out as a bold ``foo''
-upon utilization.
-
-The second argument illustrates a general issue about the lifespan of
-\refcmd{AdviceOriginal} and other |\Advice...| commands.\footnote{The list of
- all commands available only within the handler can be found in the
- documentation of key \refmmzauto{outer handler} in
- section~\ref{sec:ref:advice}.} By executing \refcmd{Memoize}, we leave the
-advice and thereby cannot be sure that when expanded, \refcmd{AdviceOriginal}
-will mean what it means at the moment. In general, another piece of advice might be
-triggered until its expansion, or the group might be closed, etc. For example,
-the author may have issued \refcmd{mmznext}\bracestt{\refmmz{at begin
- memoization}=\cs{label}\marg{key}}, and as the pre-memoization code is
-executed before the memoization driver, the \cs{label}, which is submitted to
-Advice so that any \cs{label}s inside the memoized code ``just work'', would
-execute another piece of advice, redefining \refcmd{AdviceOriginal} and friends.
-Effectively, you'd end up memoizing an invocation of the \cs{label} command.
-
-The bottom line is that while the code following the above template
-\emph{might} sometimes work, Advice offers no guarantees that it will, so I
-advise against using it. The actual definition of the memoization inner
-handler is shown below. In this definition, we expand \refcmd{AdviceReplaced}
-and \refcmd{AdviceOriginal} --- exactly once! --- into the respective arguments
-of \refcmd{Memoize}; of course, as the entire invocation of \refcmd{Memoize} is
-expanded, we have to guard against expanding the collected arguments (|#1|) and
-\refcmd{Memoize} itself.\footnote{As the final touch, the handler also contains
- \refcmd{ignorespaces} after the invocation of \refcmd{Memoize}, if this was
- requested using \refmmz{ignore spaces}. Note that this could not be done
- without pre-expanding the \cs{ifmmz at ignorespaces} conditional, as
- \refcmd{Memoize} closes the group in which the auto- and the next-options are
- applied.} The result of the expansion is shown under the code: the first and
-the second line assume we are automemoizing command |\foo| and environment
-|bar|, respectively. Note that \refcmd{AdviceOriginal} expands into an
-invocation of \refcmd{AdviceGetOriginal}, a command which may be safely used
-outside the advice; the first argument of this command is the auto-namespace
-(in our case, \refkeypath{/mmz}), and the second argument is the advised
-command. For a \hologo{LaTeX} environment, the advised command is actually
-\refcmd{begin}, and this is why the call of \refcmd{AdviceGetOriginal} is of
-course followed by the environment name.
-
-\makeexcerpt{_auto-memoize-inner}
-\tcbinputexample[.tex][.excerpt]{%
- listing and comment, one file, no attachment,
- title={The implementation of the \refmmzauto{inner handler}
- for automemoization},
- comment={%
- $\rightarrow$ \refcmd{Memoize}\braces{\cs{foo}\Arg1}%
- \braces{\refcmd{AdviceGetOriginal}\bracestt{/mmz}\bracestt{\cs{foo}}\Arg1}\\
- $\rightarrow$ \refcmd{Memoize}\braces{\cs{begin}\bracestt{bar}\Arg1}%
- \braces{\refcmd{AdviceGetOriginal}%
- \bracestt{/mmz}\bracestt{\cs{begin}}\bracestt{bar}\Arg1}
- }
-}
-
-Let us now move backwards in time and look at the outer handler installed by
-\refmmzauto{memoize}. It is very simple, but performs an important function of
-applying the auto- and the next-options (in this order), which also
-necessitates opening a group (closed by \refcmd{Memoize}). The final line
-invokes the argument collector, which then calls the inner handler; remember
-that the invocation of \refcmd{AdviceCollector} is the sole function of the
-default outer handler.
-
-\makeexcerpt{_auto-memoize-outer}
-\tcbinputexample[.tex][.excerpt]{%
- listing only, one file, no attachment,
- title={The implementation of the \refmmzauto{outer handler}
- for automemoization},
-}
-
-Moving even further back in time, we arrive at the run conditions. The
-\refmmzauto{memoize} style invokes \refmmzauto{run if memoization is possible},
-defined as \code{run conditions=\cs{mmz at auto@rc at if@memoization at possible}}, with
-the installed macro as shown below. Indeed, memoization only makes when
-Memoize is \refmmz{enable}d (which we test using \refcmd{ifmemoize}), but we're
-not already ``inside \refcmd{Memoize}'' (which we test using
-\refcmd{ifinmemoize}). The latter condition is true when we're either
-memoizing or regularly compiling some code submitted to memoization (see the
-diagram in section~\ref{sec:Memoize}). Note that it is not necessary to invoke
-\refcmd{AdviceRunfalse} in branches where the run conditions are not satisfied.
-
-\makeexcerpt{_auto-run-if-memoization-is-possible}
-\tcbinputexample[.tex][.excerpt]{%
- listing only, one file, no attachment,
- title={The implementation of
- \refmmzauto{run if memoization is possible}},
-}
-
-While it is clear that double memoization is a no-no, why should we avoid
-memoizing inside a regular compilation? Imagine that Memoize decides not to
-memoize a Forest tree, perhaps because \refmmz{readonly} is in effect. Under
-the hood, Forest creates many \env{tikzpicture}s. Should all of them be
-(auto)memoized now? Certainly not.
-
-Finally, what happens when the run conditions are not met? Not much, but
-something important nevertheless: by consuming the next-options, the
-\refmmzauto{bailout handler} makes sure they will not erroneously apply to the
-next instance of (auto)memoization.
-
-\makeexcerpt{_auto-memoize-bailout}
-\tcbinputexample[.tex][.excerpt]{%
- listing only, one file, no attachment,
- title={The implementation of the \refmmzauto{bailout handler}
- for automemoization},
-}
-
-The only component of the automemoization advice not determined by style
-\refmmzauto{memoize} is the argument collector, which allows the user to submit
-a command with a weird argument structure to automemoization simply by setting
-key \refmmzauto{collector} in addition to executing \refmmzauto{memoize}. For
-example, Memoize submits \refcmd{tikz} to automemoization by loading
-\reffile{advice-tikz.code.tex}, which contains Advice's definition of the
-\cs{tikz} collector \refcmd{AdviceCollectTikZArguments}, and issuing the
-following declaration.
-
-\makeexcerpt{_auto-tikz-collector}
-\tcbinputexample[.tex][.excerpt]{%
- listing only, one file, no attachment,
- title={Declaring automemoization of command \cs{tikz}},
-}
-
-\paragraph{\refmmzauto[show keypath]{ref}}
-
-The cross-reference advice presents an example of an outer handler radically
-different from the default outer handler. This outer handler does not invoke
-the collector at all. As shown below, it grabs the argument of \refcmd{ref} (or
-whichever cross-referencing command) on its own --- remember that the outer
-handler receives the arguments of the handled command ``as they are,'' i.e.\
-uncollected. It then asks \refcmd{mmzNoRef} to do the real job of getting the
-reference key into the context, and finally executes the original \cs{ref}.
-
-\ExampleName{_auto-ref}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{%
- listing only, no attachment,
- title={\hypercolor{link}{white}A simplified\footnotemark{} definition
- of \refmmzauto{ref}},
- % This only works if there is a single footnote in the memoized code.
- /mmz/context/.expanded={footnote=\numexpr\thefootnote-1},
-}\footnotetext{The real outer handler allows for arbitrary optional arguments of
- the cross-referencing command, and shares code with \refmmzauto{force
- ref}.}
-
-The run conditions of this style are agonizingly simple: \refmmzauto{run if
- memoizing} sets \refmmzauto{run conditions} to a macro defined as
-\refcmd{ifmemoizing}\hyp\refcmd{AdviceRuntrue}\hyp\cs{fi}.
-
-\paragraph{\refmmzauto[show keypath]{abort}}
-
-The advice for aborting memoization is very simple --- it merely executes
-\refcmd{mmzAbort} --- but also very sneaky. Here, the \refmmzauto{run
- conditions} do the real work of aborting memoization, while the ``real,''
-i.e.\ outer handler, never even gets executed; note the absence of
-\refcmd{AdviceRuntrue}, which implies \refcmd{AdviceRunfalse}, which triggers
-the execution of the original command after the run conditions are ``checked.''
-
-\makeexcerpt{_auto-abort}
-\tcbinputexample[.tex][.excerpt]{%
- listing only, one file, no attachment,
- title={The definition of \refmmzauto{abort}},
-}
-
-The point here is that executing \refcmd{mmzAbort} (itself a single-liner
-setting an internal conditional) is cheaper than testing for the real run
-conditions (\refmmzauto{run if memoizing}) and aborting only if they are
-satisfied. Of course, the trick only works because (i) the advice doesn't
-need to inspect any arguments of advised command, and because (ii) setting the
-internal abortion conditional outside memoization does no damage.
-
-\paragraph{Advice in chains}
-
-A command may be submitted to several instances of the advising framework,
-i.e.\ instances installed under different keypaths. In the example below, we
-submit |\foo| both to the instance of Advice installed in keypath |/one| and to
-the one installed in keypath |/two|. Under |/one|, the result of |\foo{...}|
-will be boxed (|\fboxWrap|); under |/two|, in will be parenthesized
-(|\parenWrap|). The order in which this happens depends on the order in which
-|\foo| was activated under different keypaths. If we first activate it under
-|/one| with the boxing effect and then under two with the parenthesizing
-effect, the box will appear within parenthesis; if we reverse the activation
-order, the parenthesis will appear inside the box.
-
-\ExampleName{chained-advice}
-\makeexample{\examplename.tex.c1 N=2}
-\tcbinputexample{comment=\input{\examplepath.tex.c2}}
-
-First of all, looking at the code above, you have probably noticed the absence
-of key \refmmz{auto}. This is because by default, \refkey{/handlers/.install
- advice} defines the \emph{setup key} to be |advice| --- Memoize overrides
-this default by installing the framework with \refkey{/handlers/.install
- advice}|=|\bracestt{\refkey{/advice/install/setup key}=auto, ...}.
-
-Next, |advice'| is a variant of |advice| which prevents automatic activation
-upon setup (and the same holds for |auto'| vs.\ |auto| in Memoize). We have
-used the bar variant above to make it clear that it is the order of activation,
-rather than declaration by |advice|\slash \refmmz{auto}, which matters in
-determining which handler is applied first.
-
-Finally, note that the deactivation order must be the reverse of the activation
-order. So if we activate |\foo| first in |/one| and then in |/two|, we should
-deactivate it in |/two| first and in |/one| next, otherwise Advice will
-complain.
-
-
-\paragraph{A simple collector}
-
-Let us implement a collector for a command which accepts one (standard
-\hologo{LaTeX}) optional argument and one mandatory argument; in \pkg{xparse}
-terms, a command with argument specification |om|.
-
-Using \cs{NewDocumentCommand}, such a collector is very easy to implement. We
-simply define a command with signature |om| and distinguish two possibilities
-regarding the presence of the optional argument, which we test using
-\cs{IfValueTF}. If the true branch, we pass a \emph{braced} |[#1]{#2}| to the
-inner handler, which we invoke by \refcmd{AdviceInnerHandler}; in the false
-branch, we omit the optional argument, passing it an (additionally) braced
-|{#2}|.
-
-\ExampleName{om-collector-NewDocumentCommand}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{listing only}
-
-Defining a functionally equivalent collector using \cs{newcommand} would be a
-bit more involved, as \hologo{LaTeX2e} does not offer a standardized way to
-test for the \emph{presence} of the optional argument. Consider the following
-collector, whose optional argument has the same default value as the advised
-command. Is it functionally equivalent to the one above?
-
-\ExampleName{om-collector-newcommand}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{listing only}
-
-While there will be no visual difference, there is a difference under the hood.
-If you compile both documents, you will see that the first one creates three
-memos\slash externs, while the second one only creates two: |\foo{...}| does
-not have its own memo anymore, but creates and uses the same memo as
-|\foo[green]{...}|.
-
-While the second version might sometimes be preferred, perhaps even in the
-context of memoization, the initial collector, which deploys command
-\refcmd{CollectArguments}, behaves like the \cs{NewDocumentCommand}-defined
-collector above,\footnote{In fact, there is a slight difference after all.
- While the above-defined collector won't distinguish between the single-token
- mandatory argument given with or without braces, \refcmd{CollectArguments}
- will again faithfully replicate the original argument tokens.} as it attempts
-to perfectly replicate the command invocation. Furthermore, this behaviour
-makes it unnecessary for the author to provide the default values of optional
-arguments (and even allows them to replace |O{default}| in the argument
-specification by \docref{xparse:o}).
-
-We now turn to the package CollArgs, which implements the actual argument
-collection; we'll revisit the initial collector of Advice at the end of the
-following subsection.
-
-
-
-\subsubsection{Using package CollArgs}
-\label{sec:collargs}
-
-Automemoization is implemented on top of the framework offered by package
-Advice, and that package in turn couldn't really work as intended without
-package CollArgs. A regular user of Memoize shouldn't need to know anything
-about CollArgs, but a package writer wanting to support Memoize might have to.
-
-The package provides two public commands, \refcmd{CollectArguments} and
-\refcmd{CollectArgumentsRaw}; we'll focus on the former first.
-\refcmd{CollectArguments} takes three arguments: optional \meta{options} in the
-form of a \pkg{pgfkeys} keylist; a mandatory \meta{argument specification} in a
-(slightly extended) \pkg{xparse} format; and the \meta{next-code}:
-\begin{center}
- \refcmd{CollectArguments}\oarg{options}\marg{argument specification}\marg{next-code}%
- \textcolor{gray}{\meta{tokens}}
-\end{center}
-Following the three formal arguments of \refcmd{CollectArguments} are some
-\meta{tokens} --- the rest of the document, really --- and the job of
-\refcmd{CollectArguments} is to figure out the extent to which these
-\meta{tokens} conform to the given \meta{argument specification}. In other
-words, \refcmd{CollectArguments} will consume as many of the \meta{tokens} as a
-\meta{command} defined by \refcmd{NewDocumentCommand}\meta{command}\marg{argument
- specification}\bracestt{...} would. Once these \meta{argument tokens} are
-collected, \refcmd{CollectArguments} executes the \meta{next-code} with the
-\meta{argument tokens} given as a single, braced argument (clearly, the
-\meta{rest} of the \meta{tokens}, i.e.\ the non-consumed tokens, will follow):
-\begin{center}
- \meta{next-code}\marg{argument tokens}\textcolor{gray}{\meta{rest}}
-\end{center}
-
-In the example below, we define macro |\PrintAndDo|, which takes two arguments,
-a command and the collected arguments of that command, prints out which command
-we're about to execute and with what arguments, and then executes that command
-with those arguments --- |#1#2| at the end of the definition. Note that |#2|
-immediately following |#1| is not braced, so
-|\PrintAndDo\makebox{[5em][r]{text}}| executes |\makebox[5em][r]{text}|.
-
-{\tolerance 1000 Executing |\PrintAndDo\makebox{[5em][r]{text}}| directly would
- thus yield the first line of the result below --- and in fact, this is
- precisely what gets executed to yield that line, but in a roundabout fashion.
- Given the argument specification |oom| (two optional arguments followed by a
- mandatory argument), \refcmd{CollectArguments} figures out how many tokens
- following its formal arguments conform to this argument specification ---
- below, these would be |[5em][r]{text}| following
- \refcmd{CollectArguments}\hyp|{oom}{\PrintAndDo\makebox}|--- and puts them,
- braced, behind its \meta{next-code} argument, |\PrintAndDo\makebox|, yielding
- |\PrintAndDo|\hyp|\makebox{[5em][r]{text}}|.\par}
-
-\ExampleName{collargs-makebox}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{listing and compile}
-
-Seeing the arguments of |\makebox| without the immediately preceding |\makebox|
-might seem strange, but remember that \refcmd{CollectArguments} is about the
-arguments of a command, not about the command's control sequence. It doesn't
-know or care which command the argument tokens ``belong'' to, as long as they
-conform to the given specification. In the example above, it is only in |#1#2|
-of |\PrintAndDo| that |\makebox| is ``reunited'' with its arguments, but note
-that the reunion is far from obligatory.
-
-CollArgs supports all the argument types (and modifiers) that \pkg{xparse}
-does, including the environment type \docref{xparse:b}, as exemplified below.
-Again, the code below might seem strange, as it features an |\end{minipage}|
-without the matching |\begin{minipage}|, but the logic is similar as for
- commands: just as \refcmd{CollectArguments} occurs in front of the command
- arguments, without the command itself, so it occurs in front of the
- environment body, without the opening of that body. However, while
- \refcmd{CollectArguments} never needs to know the command name, we need to
- inform it of the environment name, so that it can find the end of the
- environment. This can be achieved as shown below, using key
- \refcollargs{environment} in the optional argument of the command, or by our
- extension to the \pkg{xparse} argument specification, where the environment
- argument type \docref{xparse:b} may be followed by a braced environment name.
- In the example below, we could therefore also invoke argument collection by
- \refcmd{CollectArguments}\braces
- {\docref{xparse:+}\docref{xparse:b}\braces{\env{minipage}}} (we have preceded
- \docref{xparse:b} with a \docref{xparse:+} to allow for an environment body
- containing paragraph tokens).\footnote{CollArgs automatically adapts to the
- format, i.e.\ it knows that environments are tagged by \cs{}\meta{name} and
- \cs{end}\meta{name} in \hologo{plainTeX} and by \cs{start}\meta{name} and
- \cs{stop}\meta{name} in \hologo{ConTeXt}.}
-
-\ExampleName{collargs-minipage}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{comment=\input{\examplepath.tex.c1}}
-
-You might wonder why didn't we provide \refcmd{CollectArguments} in the
-previous example with argument specification |omb| --- after all, the
-\env{minipage} environment takes an optional and a mandatory argument. While that
-would work, and produce the same result,\footnote{Unless argument processing
- was in effect; see section~\ref{sec:ref:collargs} for details.} note that
-\refcmd{CollectArguments} is only interested in finding the scope of the
-arguments, and grabbing everything until |\end{minipage}| is the same as first
-grabbing the optional argument, maybe, then the mandatory argument, and finally
-the argument body.
-
-\refcmd{CollectArguments} not only supports \pkg{xparse}'s verbatim argument
-type \docref{xparse:v}, it can grab an argument of \emph{any} type in the
-verbatim mode, triggered by option \refcollargs{verbatim}.\footnote{We refer to
- the verbatim mode triggered by \refcollargs{verbatim} as the full verbatim
- mode, where all characters are of category ``other''. There is also the
- partial verbatim mode, triggered by \refcollargs{verb}, where braces retain
- their normal category codes.} We illustrate this key below, where we also use
-option \refcollargs{tags}, which makes CollArgs automatically surround the
-grabbed environment body with the begin tag \refcmd{begin}\marg{environment
- name} and the end tag \cs{end}\marg{environment name}, and use
-\refcmd{scantokens} to execute the grabbed environment. Consult
-section~\ref{sec:ref:collargs} for the full reference on the verbatim mode and
-its limitations.
-
-% A harmless warning:
-% Missing character: There is no ^M (U+000D) in font [lmmono10-regular]:!
-% because \texttt in the example is fed a (verbatim) carriage return
-\ExampleName{collargs-verbatim}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{listing and compile}
-
-Finally, CollArgs extends the \pkg{xparse} specification by modifier
-\docref{xparse:amp}, which allows the user to specify options which apply only
-to the following argument, as opposed to the options given as the optional
-argument of \refcmd{CollectArguments}, which apply to all the arguments. A
-third way to invoke the environment body collection in the above example is
-thus \refcmd{CollectArguments}%
-\bracestt{\docref{xparse:amp}\bracestt{\refcollargs{environment}=minipage}+b}.
-
-Both the single-argument and the common options can be given not only as
-\pkg{pgfkeys} keys, but also in the raw, ``programmer's interface'' format.
-Every option key has a corresponding macro; for example, key
-\refcollargs{environment} is matched by macro \refcmd{collargsEnvironment}.
-The macros are listed alongside their corresponding keys in the reference
-section~\ref{sec:ref:collargs}; here, we merely learn how to use them.
-
-To use raw options for a single argument, double the ampersand in the argument
-specification. Therefore, the fourth way to specify the environment name is
-\code{\docref{xparse:amp}\docref{xparse:amp}\bracestt{%
- \refcmd{collargsEnvironment}\bracestt{minipage}}\docref{xparse:+}\docref{xparse:b}}.
-
-To set the raw options for all arguments, use \refcmd{CollectArgumentsRaw}, the
-second public command of the package. This command is exactly like
-\refcmd{CollectArguments}, excepts that it expects the options in the raw
-format and as a \emph{mandatory} argument:
-
-\begin{center}
- \refcmd{CollectArgumentsRaw}\marg{raw options}\marg{argument specification}%
- \marg{next-code}\textcolor{gray}{\meta{tokens}}
-\end{center}
-
-This leads us to the fifth way to set the environment name (an overkill, I
-know): \refcmd{CollectArgumentsRaw}\hyp
-\bracestt{\refcmd{collargsEnvironment}\bracestt{minipage}}\bracestt{+b}\marg{next-code}.
-Furthermore, you can use a mixture of raw and key--value options: the raw
-option commands include \refcmd{collargsSet}, which applies the given option
-keylist. The idea here (incarnated by both Auto and Memoize) is that the
-package will provide CollArgs with the raw options, for speed, while the
-author can supplement them in the friendly keylist format --- and this leads
-us to the sixth, and thankfully final way to set the environment name:
-\refcmd{CollectArgumentsRaw}\bracestt{\refcmd{collargsSet}%
- \bracestt{\refcollargs{environment}=minipage}}\bracestt{+b}\marg{next-code}.
-
-
-\paragraph{The initial collector}
-
-As the final example, let us study Advice's initial collector; this is a macro
-which is used as the collector when key \refmmzauto{collector} is not given.
-This macro is not really \refcmd{CollectArguments}, as we sometimes state to
-simplify matters, but a macro which acts as the ``bridge'' between Advice and
-CollArgs, by compiling an invocation of \refcmd{CollectArgumentsRaw} from the
-given advice setup, and executing it.
-
-The bridge macro is shown below in its full glory, but it is really less
-complicated than it might appear at first sight. In line~2, we use
-\refcmd{AdviceIfArgs} to see whether the argument structure of the handled
-command was given by the user. If it wasn't, we assume that the handled
-command was defined using \cs{NewDocumentCommand} or similar, and use
-\refcmd{GetDocumentCommandArgSpec} to retrieve it (line~3; note that
-\refcmd{AdviceName} holds the handled control sequence) and store it into
-\refcmd{AdviceArgs} (line~4), which also receives the argument specification
-when given by the user via key \refmmzauto{args}.
-
-\makeexcerpt{_advice-CollectArgumentsRaw}(../../advice.edtx)
-\tcbinputexample[.tex][.excerpt]{%
- listing only, one file, no attachment,
- listing options app={numbers=left, numberstyle=\tiny, numbersep=0.5em},
- title={The definition of the initial collector},
-}
-
-Lines~6--17 are somewhat of an expansion mess, because we have to construct the
-invocation of the CollArgs' collector from the advice setup stored in various
-macros. But once we think away all the (non-)expansion commands, we're left
-with \refcmd{CollectArgumentsRaw} plus the following three arguments:
-\begin{enumerate}
-\item The raw options (lines 8--11):
- \begin{enumerate}
- \item In line 8, the advised command's control sequence is designated as the
- \refcollargs{caller}. The effect is that if the given arguments don't
- conform to the specification, the error thrown seems to come from the
- advised command rather than some internal CollArgs macro. The author will
- be grateful for this little detail.
- \item In line 9, we add any \refmmzauto{raw collector options} set by Advice
- (plus the package deploying Advice, like Memoize); user-given options are of
- course possible, but not really expected here, because:
- \item In lines 10--11, we add the user-given \refmmzauto{collector options},
- if there are any, embedded under \refcmd{collargsSet}.
- \end{enumerate}
-\item The argument specification (line 14).
-\item The \refmmzauto{inner handler} (line 15).
-\end{enumerate}
-
-
-
-\section{Reference}
-\label{sec:reference}
-
-In this section, the extra information about keys, offered at the right of a
-key in parenthesis, may contain the initial value of a key, and also a default
-value of a key. In this context, the terms ``initial'' and ``default'' have
-the meaning employed by the \pkg{pgfkeys} utility. Initial value refers to the
-value that is set by the package. More often, we would call this the default
-or the package-default value, but in the \pkg{pgfkeys} parlance, the default value
-refers to the value that is passed to the key in the absence of the argument.
-Honestly, I only kept to this convention in the reference section; elsewhere,
-I often say ``default'' or ``package-default'' and mean the initial value.
-
-Another convention I kept to in this section is the color-coding of the keys
-and commands. Green background indicates a basic key or command, which any
-user might want to know about. Red background indicates other, more or less
-advanced keys and commands.
-
-
-\subsection{Loading}
-\label{sec:ref:loading}
-\paragraph{\hologo{LaTeX}}
-
-Load Memoize by \cs{usepackage}\braces{\docaux{package}{memoize}} or
-\cs{usepackage}\oarg{options}\bracestt{memoize}. The latter form functions
-almost identically to the former followed by
-\refcmd{mmzset}\bracestt{\meta{options}}, with two exceptions.
-
-First, the \meta{options} as package options may not contain spaces. This is a
-constraint imposed by \hologo{LaTeX} itself. To avoid it for a particular key,
-define a spaceless variant of the key in \reffile{memoize.cfg}. For example,
-to use \refmmz{memo dir} as a package option, put
-\refcmd{mmzset}|{memodir/.style={|\refmmz{memo dir}|={#1}}}| into that file.
-
-Second, key \refmmz{extract} only make senses as a package option, or within
-\reffile{memoize.cfg}. Internal extern extraction can only be performed before
-the output PDF is opened, and to maximize chances of that, Memoize is designed
-to extract as soon as it can: while it is being loaded. Consequently, changing
-the extraction method in the document preamble has no effect.
-
-The necessity of performing extraction before the output PDF is opened is also
-why Memoize prefers to be loaded early with respect to other packages. In
-particular, it must be loaded before \PGF library |fadings| (see
-section~\ref{sec:fadings-problem}) and before the \pkg{beamer} class (see
-section~\ref{sec:tut:beamer}).
-
-If you're familiar with the \TikZ externalization library, you might wonder
-whether Memoize has an equivalent of the \refcmd{tikzexternalize} command. It
-doesn't. Memoize assumes that if you loaded it, you want to use it --- but you
-can always disable it using key \refmmz{disable}, or even by loading package
-\refpkg{nomemoize} in its stead.
-
-In \hologo{LaTeX}, initialization and finalization are completely automatic.
-Memoize defines several initialization and finalization styles ---
-\docaux{key}{begindocument/before}, \docaux{key}{begindocument},
-\docaux{key}{begindocument/end} and \docaux{key}{enddocument/afterlastpage}
----and executes them at the cognominal \hologo{LaTeX} hooks.
-
-\paragraph{\hologo{plainTeX}}{}
-
-Load Memoize by |\input memoize|. As package options cannot be provided in
-\hologo{plainTeX}, the author must trigger extraction from \refcmd{mmzset}
-using key \refmmz{extract}; I recommend doing this immediately after loading
-the package. This key may be invoked with or without a value. In the latter
-case, Memoize will extract using the package default method
-\refmmz{extract=perl}, unless its has been overriden from
-\reffile{memoize.cfg}.
-
-Furthermore, as \hologo{plainTeX} has no concept of a document body, the text
-must be manually enclosed in \refcmd{mmzset}\bracestt{\docaux{key}{begin
- document}} and \refcmd{mmzset}\bracestt{\docaux{key}{end document}}; this
-is where the initialization and finalization hooks described above will be
-executed. Note that \refmmz{extract}, when used, must precede this enclosure.
-
-Finally, \hologo{plainTeX} has another reason for preferring the early loading
-of the package. In this format, Memoize must redefine \refcmd{shipout}, at a
-time the meaning of this control sequence is still primitive. In particular,
-this means that Memoize must be loaded before \pkg{atbegshi}.
-
-\begin{tcboxedraster}[raster column skip=1cm]{blankest}
- \ExampleName{plainTeX}%
- \makeexample{\examplename.tex.c1}%
- \tcbinputexample{listing only, title=A minimal \hologo{plainTeX} example}
- %
- \ExampleName{ConTeXt}%
- \makeexample{\examplename.tex.c1}%
- \tcbinputexample{listing only, title=A minimal \hologo{ConTeXt} example}
-\end{tcboxedraster}
-
-\paragraph{\hologo{ConTeXt}}{}
-
-Load Memoize by \cs{usemodule}\bracketstt{memoize} or
-\cs{usemodule}\bracketstt{memoize}\oarg{options}. Unlike in \hologo{LaTeX},
-spaces are allowed in \meta{options}; the remarks on key \refmmz{extract} and
-loading order are the same as for \hologo{LaTeX}.
-
-In \hologo{LaTeX}, Memoize automatically executes its initialization and
-finalization code when at the beginning and the end of the document body. Due
-to my very limited experience with \hologo{ConTeXt}, and its project structure
-in particular, I don't know what the appropriate place for initialization and
-finalization would be in \hologo{ConTeXt}. I therefore provisionally leave it
-to the author to execute \refcmd{mmzset}\bracestt{\refmmz{begin document}} and
-\refcmd{mmzset}\bracestt{\refmmz{end document}} manually, and hope for advice
-on how to handle this properly.
-
-
-\paragraph{Auxiliary packages}
-
-\begin{doc}{package={name=nomemoize}}
- Loading this package instead of Memoize completely disables memoization, but
- does not require removing any Memoize commands from the document (they all
- become no-ops).
-\end{doc}
-
-\begin{doc}{package={name=memoizable}}
- This package is a programmer's stub: if Memoize is loaded, it does nothing;
- otherwise, it provides the no-op variants Memoize commands. See
- section~\ref{sec:loading-memoize} for details.
-\end{doc}
-
-
-\subsection{Configuration}
-\label{sec:ref:configuration}
-
-
-\begin{doc}{easy,cmd={name=mmzset, par=\marg{options}}}
-
- Update the Memoize configuration.
-
- The \meta{options} are a comma-separated list of \meta{key}|=|\meta{value}
- pairs. They are processed using the \pkg{pgfkeys} utility of PGF/\TikZ (see
- \PGFmanual{87}), with the default path set to \docaux{key path}{mmz}.
-
- The changes are local to the current \hologo{TeX} group, except for keys
- where explicitly noted otherwise.
-\end{doc}
-
-\begin{doc}{easy,
- cmd={name=mmznext, par=\marg{options}},
- }
-
- This command accepts the same \meta{options} as \refcmd{mmzset}, but
- interprets them as \emph{next-options} --- options which will be applied to
- the next, and only the next, \emph{auto}memoized command or environment.
- (Remember that a command or environment is submitted to automemoization by
- \refmmz{auto}|=|\marg{command or
- environment}\bracestt{\refmmzauto{memoize},...}; see
- sections~\ref{sec:tut:automemoization}, \ref{sec:tut:working-on-a-picture}
- and \ref{sec:ref:advicememoization} for details.)
-
- Remarks for the author:
-
- \begin{itemize}
- \item Key \refmmz{enable} has no effect inside \refcmd{mmznext}.
- \item If \refcmd{mmznext} is used more than once preceding an automemoized
- command, only the final invocation takes effect.
- \item The next-options also apply to commands and environments for which
- memoization is autodisabled via \refmmz{auto}|=|\marg{command or
- environment}\bracestt{\refmmzauto{nomemoize},...}.
- \item It is safe to set the next-options in front of a command submitted to
- automemoization which does not actually undergo memoization in this
- particular instance. In other words, the absence of memoization will not
- cause the next-options to ``leak'' to the next automemoized command.
- \item Only the linear (execution) order of \refcmd{mmznext} and the
- automemoized command matters. Key \refcmd{mmznext} will correctly apply to
- a single following automemoized command even if it occurs outside the group
- which that command is executed from; and it will apply to the following
- automemoized command even if it is called within a group closed before that
- command is executed.
- \end{itemize}
-
- Remarks for the programmer:
- \begin{itemize}
- \item The next-options are set globally.
- \item The effect of \refcmd{mmznext} is \emph{not} cumulative. Consequently,
- \refcmd{mmznext}\braces{} clears the next-options.
- \item The next-options are applied by executing \refcmd{mmzAutoInit} within
- the advice. Any piece of advice applying the next options should also clear them
- when the \refmmzauto{run conditions} are not met. This is streamlined by
- style \refmmzauto{apply options}, intended for use within \refmmz{auto}
- declarations. Out of the box, this style is deployed by
- \refmmzauto{memoize}, \refmmzauto{nomemoize} and \refmmzauto{noop}, but it
- may be used by any piece of advice. Note that the \refmmzauto{outer handler}
- declared by this style opens a group (to apply the options in) but leaves
- it to the (undeclared) \refmmzauto{inner handler} to close that group.
- \item Key \refmmz{enable} has no effect inside \refcmd{mmznext} because when
- Memoize is disabled upon encountering an automemoized command, the advice
- bails out without ever applying the next-options. More generally, this
- applies to any advised command whose run conditions require
- \refcmd{ifmemoize} to be true. Key \refmmz{disable}, on the other hand,
- takes effect, because \refcmd{ifmemoize} is checked within \refcmd{Memoize}
- as well.
- \end{itemize}
-\end{doc}
-
-\begin{doc}{easy,file={name=memoize.cfg, desc=file}}
- The configuration file, loaded just before processing the package options.
- It will typically contain a \refcmd{mmzset} command, but it may contain any
- \hologo{TeX} code.
-
- As for any other file loaded by \hologo{TeX}, the location of the file
- determines whether it applies system-wide, user-wide or directory-wide.
-
- This file is also loaded by package \refpkg{nomemoize}, on the off chance it
- defines some commands other than \refkeypath{/mmz} keys. (It is not loaded
- by package \refpkg{memoizable}, though).
-\end{doc}
-
-\begin{doc}{cmd={name=nommzkeys, par=\marg{key}}}
- In package \refpkg{memoize}, this command is a no-op; in packages
- \refpkg{nomemoize} and \refpkg{memoizable}, it defines key
- \refkeypath{/mmz}|/|\meta{key} as a no-op.
-
- As explained in section~\ref{sec:tut:final}, use this command to declare any
- \refkeypath{/mmz} keys you have used outside \refcmd{mmzset} when switching
- to package \refpkg{nomemoize}.
-\end{doc}
-
-
-\subsection{Memoization}
-\label{sec:ref:memoization}
-
-\subsubsection{Manual memoization commands}
-\label{sec:ref:memoization:manual}
-
-\begin{doc}{easy,
- cmd={name=mmz, par=\oarg{options}\marg{code}},
- env={name=memoize, par=\oarg{options}},
- }
- Submit \meta{code} or \meta{environment body} to memoization.
-
- Prior to memoization, the configuration is locally updated by executing
- \meta{options} given as the optional argument to this command, i.e.\ the
- given options take precedence to options previously set via \refcmd{mmzset}.
- Note that next-options, set by \refcmd{mmznext}, are not applied.
-
- The effect of the macro and of the environment version of this command is the
- same, except that the command version memoizes \meta{code} exactly as-is,
- while the environment version trims away any spaces at the beginning and the
- end of the code.\footnote{What actually happens is that at the beginning of
- the environment body, all space tokens will be discarded. At the end of
- the body, no spaces are actually discarded; Memoize simply issues an
- \refcmd{unskip}. This should not matter to a regular user who simply
- writes down the environment.} The space-trimming feature of the environment
- ensures that you can write |\begin{memoize}| and |\end{memoize}| in separate
- lines (as shown above), but no extra space will creep into the extern.
-
- The space-trimming feature of the environment, which trims spaces at the
- beginning and at the end of the \meta{environment body}, should not be
- confused with the effect of \refmmz{ignore spaces}, which ignores spaces
- following the environment end-tag (in \hologo{LaTeX}, |\end{...}|) --- and
- which does not apply to manual memoization at all!
-
- The argument of \refcmd{mmz} \emph{must} be enclosed in braces.
-\end{doc}
-
-\begin{doc}{easy,
- cmd={name=nommz, par=\oarg{options}\marg{code}},
- env={name=nomemoize, par=\oarg{options}},
- }
- Disable Memoize for the span of the compilation of \meta{code} or
- \meta{environment body}.
-
- This command consumes the \meta{options} in the same way as
- \refcmd{mmz}\slash \refenv{memoize} described above. The macro and the
- environment version of the command exhibit the same space-trimming behaviour
- as their \refcmd{mmz}\slash \refenv{memoize} counterparts, and the argument
- of \refcmd{nommz} \emph{must} be enclosed in braces.
-\end{doc}
-
-
-\subsubsection{Basic configuration}
-\label{sec:ref:memoization:basic}
-
-\begin{doc}{easy,
- key={name=enable},
- key={name=disable},
- }
- Enable or disable the functionality of the package.
-
- What happens when Memoize is enabled depends on the memoization mode
- (\refmmz{normal}, \refmmz{readonly}, \refmmz{recompile}) and many other
- factors. When the package is disabled, it neither creates new memos and
- externs, nor uses the existing ones; this applies to both manual and
- automatic memoization. The effect is close to not having Memoize loaded, or
- to loading NoMemoize, but it is not completely the same; for example, the
- record file (\dmmz) \emph{is} updated while Memoize is disabled, reflecting
- the fact that nothing was memoized (or utilized) in the disabled
- state.\footnote{Cleaning the folder (\S\ref{sec:cleanup-scripts}) after
- disabling the package for the entire document is thus a bad idea.}
-
- If these keys are used in the preamble, their effect is delayed until the
- beginning of the document, to ensure that Memoize is never enabled in the
- preamble. Other than that, all these keys do is set the \hologo{TeX}
- conditional \refcmd{ifmemoize}, which you may use in your code to test
- whether Memoize is enabled. You may also use \refcmd{memoizetrue} and
- \refcmd{memoizefalse}, as long as you never enable the package in the
- preamble.
-
- Key \refmmz{enable} cannot be applied to automemoized commands via
- \refcmd{mmznext}. It will take effect for manual memoization, though, and
- key \refmmz{disable} will work for both, as expected.
-\end{doc}
-
-\begin{doc}{easy,
- key={name=normal, desc=the default mode},
- key={name=readonly},
- key={name=recompile},
- }
-
- Select the memoization mode.
-
- Each piece of code submitted to (either manual or automatic) memoization is
- associated to several files: one c-memo, one cc-memo, and some externs (zero
- or more, typically one). When Memoize encounters a piece of code submitted
- to memoization, it takes one of the following actions:
-
- \begin{description}
- \item[memoization] The code is compiled in a special way which produces the
- associated memos and externs.
- \item[utilization] The code is not compiled. Its effect is replicated by
- processing the cc-memo; typically, this includes the single extern into the
- document.
- \item[regular compilation] The code is compiled as if Memoize was absent or
- disabled (the memos and externs are neither utilized nor produced).
- \end{description}
-
- \begin{minipage}[t]{0.48\linewidth}
- The action taken depends on the memoization mode and on whether all the
- memos and externs associated with the code exist, as shown in the table on
- the right. Note that a single missing memo or extern implies the ``no''
- column of the table, and that memoization will create \emph{all} the
- associated memos and externs, even those which already exist.
- \end{minipage}
- \hfill
- \begin{tabular}[t]{@{}lll@{}}
- \toprule
- &\multicolumn{2}{c}{Do the memos and externs exist?}\\
- \cmidrule{2-3}
- mode&yes&no\\
- \midrule
- normal&utilization&memoization\\
- readonly&utilization®ular compilation\\
- recompile&memoization&memoization\\
- \bottomrule
- \end{tabular}
-
- The memoization mode only has effect when Memoize is \refmmz{enable}d. Mode
- selection is orthogonal to enabling\slash disabling the package; for example,
- if you switch to a new mode while the package is disabled, the new mode will
- be in effect once the package is enabled.
-\end{doc}
-
-
-\begin{doc}{easy,
- desc={style},
- key={name=verbatim},
- key={name=verb},
- key={name=no verbatim},
- }
- When \refmmz{verbatim} or \refmmz{verb} is in effect, the code submitted to
- memoization is read verbatim; \refmmz{no verbatim} reverts to the normal,
- non-verbatim collection of the code. This applies to both manual and
- automatic memoization.
-
- The long version, \refmmz{verbatim}, switches to the full verbatim mode,
- where all characters are assigned category code 12 (other). With the short
- version, \refmmz{verb}, the braces, |{| and |}|, retain category codes 1 and
- 2, which can be useful for verbatim collection of optional arguments. For
- details, see the documentation of CollArgs' \refcollargs{verbatim} in
- section~\ref{sec:ref:collargs}.
-
- Under the hood, these keys have two effects. First, they are passed on to the
- argument collector (typically, \refcmd{CollectArguments} of the auxiliary
- package CollArgs; for details, see section~\ref{sec:ref:advicememoization}),
- instructing it to collect the code in the specified fashion, as described
- above. Second, if the collected verbatim code is eventually compiled (either
- regularly, or memoized), Memoize first rescans it using \refcmd{scantokens}.
-\end{doc}
-
-\begin{doc}{easy,
- key={name=padding left, par=\meta{dimension},
- desc={no default, initially |1| |in|}},
- key={name=padding right, par=\meta{dimension},
- desc={no default, initially |1| |in|}},
- key={name=padding top, par=\meta{dimension},
- desc={no default, initially |1| |in|}},
- key={name=padding bottom, par=\meta{dimension},
- desc={no default, initially |1| |in|}},
- }
- Set the left/right/top/bottom padding of the extern in the extern PDF.
-
- Without padding, the (PDF) page holding the extern would tightly fit the
- bounding box of the extern. These keys enlarge the extern page by the given
- amounts, so that any parts of the extern lying outside the bounding box will
- be correctly included when using the extern. See
- section~\ref{sec:tut:padding} for details.
-
- \meta{dimension} is evaluated with \hologo{eTeX}'s |\dimexpr|, and may
- contain control sequences \docaux{cmd}{width}, \docaux{cmd}{height} and
- \docaux{cmd}{depth}, which will refer to the dimensions of the extern.
- \refcmd{width} and friends behave like dimension registers, so it is ok to write
- e.g.\ \refmmz{padding right}|=0.5\width|.
-
- The default padding is what \hologo{pdfTeX} puts around the page anyway, 1
- inch, but we use |1 in| rather than |1 true in|, which is the true default
- value of PDF registers \docref{pdfvar:horigin} and \docref{pdfvar:vorigin},
- as we want the padding to adjust with \refcmd{mag}nification.
-\end{doc}
-
-\begin{doc}{easy,key={name=padding, par=\meta{dimension}, desc={style, no default}}}
- Set all the above |padding| keys to the given value.
-\end{doc}
-
-\begin{doc}{
- easy,
- key={name=context, par=\meta{tokens}, desc={cumulative, no default,
- initially set by \refmmz{padding to context}}},
- key={name=clear context},
- }
- These keys append the given \meta{tokens} to the \emph{context expression},
- or clear this expression. Memoized code gets recompiled whenever the
- expansion of the context expression changes.
-
- The \meta{tokens} must be fully expandable (modulo protection); in
- \hologo{LaTeX}, they will be expanded by |\protected at edef| when calculating
- the md5sum of the context.
-
- The context expression is evaluated at the end of the memoization, and at
- utilization attempts after inputting the c-memo (note that the c-memo
- contains any additions to the context expression accumulated during
- memoization). At evaluation, the given context expression is fully expanded,
- yielding the \emph{context value}, whose md5 sum forms the \meta{context md5
- sum} part of the filename of the cc-memo and the extern.
-
- These keys may be used both prior to the memoization process, or during
- memoization. In the former case, their effect is local to the current group;
- in the latter case, the effect is global, so that the changes surely survive
- until the end of memoization, when the c-memo, where the context expression
- is stored, is written into a file.
-
- Under the hood, these keys manipulate token registers
- \docaux{cmd}{mmzContext} and \docaux{cmd}{mmzContextExtra}, changing the
- contents of the former while not memoizing, and the contents of the latter
- during memoization. These token registers may also be manipulated directly
- by the user, as long as one keeps to the convention of adjusting
- \refcmd{mmzContext} locally and only while not memoizing, and adjusting
- \refcmd{mmzContextExtra} globally and only during memoization.
-\end{doc}
-
-\begin{doc}{
- key={easy, name=meaning to context,
- par=\marg{comma-separated list of commands and environment names}},
- key={name=csname meaning to context, par=\marg{control sequence name}},
- key={name=key meaning to context,
- par=\marg{full path to a pgfkeys command key}},
- key={name=key value to context,
- par=\marg{full path to a pgfkeys value key}},
- }
- These keys append the definition of the given construct to the context.
-
- Essentially, the ``meaning'' keys append \cs{meaning}\meta{control sequence}
- to the context, for the appropriate \meta{control sequence}. For example,
- \refmmz{meaning to context} appends \cs{meaning}\cs{foo} when given \cs{foo}
- as in item in the list, and it appends the internal environment macros
- appropriate to the format when given an environment name. Similarly,
- \refmmz{key meaning to context} resolves \meta{control sequence} to the
- internal macro holding the key command.
-
- Key \refmmz{key value to context} should be used on keys which store values,
- e.g.\ keys initialized by \pkg{pgfkeys} handler \code{.initial}; see
- \PGFmanual{87.3.4 and \S87.4.5}.
-
- All the keys prefix the meaning\slash value by the name of the command\slash
- environment\slash key, in order to prevent ambiguous contexts, see
- section~\ref{sec:cross-referencing} for details. Furthermore, they all
- operate through \code{\cs{csname}...\cs{endcsname}} construct, allowing one
- to safely add internal commands to the context.
-\end{doc}
-
-\begin{doc}{
- keypath={/handlers},
- key={easy, name=.meaning to context},
- key={name=.value to context},
- }
- These are the handler variants of \refmmz{key meaning to context} and
- \refmmz{key value to context}.
-\end{doc}
-
-
-\begin{doc}{key={name=padding to context,desc=style}}
- This key appends the values of the \refmmz{padding} keys to the context,
- causing the memoized code to be recompiled whenever the padding values are
- changed. This key is used to initialize \refmmz{context}, so the author
- normally shouldn't have to use this key.
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzNoRef, par=\marg{reference key}, desc=\hologo{LaTeX} only},
- cmd={name=mmzForceNoRef, par=\marg{reference key}, desc=\hologo{LaTeX} only},
- }
- These macros append the current value of the \meta{reference key} to the
- context,\footnote{More precisely, it is \meta{reference key}|=|\marg{current
- value} which is appended.} causing the memoized code to be recompiled
- when the reference changes.
-
- If the reference key is undefined, \refcmd{mmzNoRef} aborts memoization,
- while \refcmd{mmzForceNoRef} uses \cs{relax} as the reference string.
-
- These commands are deployed in the implementation of \refmmzauto[show
- keypath]{ref}, \refmmzauto{force ref} and friends. If the
- cross-referencing commands you are using are advised by these keys, you most
- likely have no need of these macros.
-\end{doc}
-
-\begin{doc}{easy, key={name=per overlay,desc=style}}
- This key is only defined in the Beamer class. When in effect, the memoized
- code will produce a cc-memo (and extern) for each overlay of the frame. For
- implementation, see section~\ref{sec:per-overlay}.
-\end{doc}
-
-\begin{doc}{
- key={name=capture, par=\docAux{of=key:/mmz/capture, value={name=hbox}, text=\alt,
- value={name=vbox,of=key:/mmz/capture}},
- desc={no default, initially \refmmz{capture=hbox}}}
- }
- Select the capture mode. This setting only applies to the default
- memoization \refmmz{driver}.
-
- By default, it is assumed that the memoized code should be executed in the
- horizontal mode, so the default memoization driver captures the output of the
- memoized code in a \refcmd{hbox} --- and also issues a \refcmd{quitvmode}
- (both in the document and in the cc-memo), just in case the memoized code
- occurs at the start of the paragraph.
-
- Use \refmmz{capture}|=|\refmmz{capture=vbox} to execute the memoized code in
- the vertical mode: Memoize will capture the output of the memoized code in a
- \refcmd{vbox}, and avoid issuing \refcmd{quitvmode}. For example, this
- capture mode is necessary to memoize a \refenv{verbatim} environment.
-\end{doc}
-
-
-\subsubsection{Inside the memoization process}
-\label{sec:ref:memoization:process}
-
-\begin{doc}{easy, cmd={name=mmzAbort}}
- This command aborts the ongoing memoization.
-
- The memoization will proceed as usual (i.e.\ the extern boxes and the cc-memo
- code will be produced), but at the end of this process, no memos will be
- produced, no externs shipped out to extern pages, and no record files
- updated.
-\end{doc}
-
-\begin{doc}{easy, cmd={name=mmzUnmemoizable}}
- This command aborts the ongoing memoization, and marks the submitted code as
- unmemoizable.
-
- The ongoing memoization produces a c-memo setting conditional
- \docaux{cmd}{ifmmzUnmemoizable} to true. Upon utilizing this c-memo, the
- system switches to regular compilation.
-
- For example, if you are automemoizing |tcolorbox|es of package
- \pkg{tcolorbox}, you will want to refrain from memoizing boxes marked as
- |breakable| or |float|ing. Simply aborting the memoization cannot do the
- trick here, as memoization compiles the submitted code in a \hologo{TeX} box.
- Marking a |breakable| or |float|ing |tcolorbox| as unmemoizable (either
- manually using this macro, or automatically using \refmmz{auto key}) makes
- sure that after the first compilation when memoization is attempted, the box
- will be compiled regularly, and will have the intended ability to break
- across pages, or float.
-\end{doc}
-
-
-\begin{doc}{
- cmd={name=Memoize, par=\marg{key}\marg{code}},
- }
- Depending on various factors, this command either memoizes \meta{code} under
- key \meta{key}, utilizes the results of a previous memoization, or performs a
- regular compilation of \meta{code}.
-
- The outcome of executing this command --- memoization, utilization or regular
- compilation --- depends upon the Memoize's state (\refcmd{ifmemoize},
- \refcmd{ifmemoizing}) and mode (\refmmz{normal}, \refmmz{readonly},
- \refmmz{recompile}), and the existence of the relevant memos and externs.
- The decision process is depicted in section~\ref{sec:Memoize}.
-
- This command expects to be executed in a dedicated group, which it will close
- itself.
-
- Invoking memoization through \refcmd{Memoize} might be useful for packages
- which want to save the results of intensive computations, regardless of
- whether the author loads (and enables) memoization or not. However, this
- usage is not yet officially allowed, because there is currently no way to
- load the core memoization routines without loading the entire package,
- thereby forcing the author to use Memoize.
-\end{doc}
-
-\begin{doc}{
- key={
- name=driver, par=\marg{code},
- desc={no default, initially \refcmd{mmzSingleExternDriver}}
- },
- }
- This key sets \meta{code} as the memoization driver.
-
- Given some code submitted to memoization, the memoization driver should
- produce the memos and externs which will replicate the effect of that code
- (while retaining its regular effect). For details, see
- section~\ref{sec:memoization-drivers}.
-
- Typically, the \meta{code} argument of this key will consist of a single
- control sequence (the driver control sequence), but any amount of tokens is
- allowed. Memoize executes the driver followed by the code which it is
- supposed to memoize, in braces, and only cares that the driver consumes that
- code.
-\end{doc}
-
-
-\begin{doc}{
- key={name=at begin memoization, par=\meta{code}, desc={cumulative, initially empty}},
- key={name=at end memoization, par=\meta{code}, desc={cumulative, initially empty}},
- key={name=after memoization, par=\meta{code}, desc={cumulative, initially empty}},
- }
- Use these keys to set up memoization hooks.
-
- These keys may be used both prior to the memoization process, or during
- memoization. In the former case, their effect is local to the current group;
- in the latter case, the code given to \refmmz{at begin memoization} is
- executed immediately, while the assignment performed by the other two keys is
- global, so that the changes surely survive until the end of memoization.
-
- The code given to hook \refmmz{at begin memoization} is kept in macro
- \docaux{cmd}{mmzAtBeginMemoization}, while the content of the other two hooks
- resides in two macros per hook: \docaux{cmd}{mmzAtEndMemoization} and
- \docaux{cmd}{mmzAtEndMemoizationExtra}, and \docaux{cmd}{mmzAfterMemoization}
- and \docaux{cmd}{mmzAfterMemoizationExtra}. All these macros may be
- manipulated directly by the user,\footnote{Use \cs{appto}, \cs{eappto},
- \cs{gappto} and \cs{xappto} of package \pkg{etoolbox} (loaded by Memoize)
- to easily append code to these macros.} as long as one keeps to the
- convention of adjusting the macros without ``\texttt{Extra}'' locally and
- only while not memoizing, and adjusting the macros with ``\texttt{Extra}''
- globally and only during memoization. The ``\texttt{Extra}'' macros require
- global assignments as they might be manipulated by code residing within a
- \hologo{TeX} group of any depth.
-
- These complications explained, let us take a look at how memoization proceeds
- to learn when the hooks are used:
- \begin{enumerate}
- \item Initialize various conditionals, macros and token registers. (Here is
- where the ``\texttt{Extra}'' hooks are cleared.) Remember that at this
- point, we're inside a group opened by \refcmd{Memoize}.
- \item Execute \refmmz{at begin memoization} hook, i.e.\ the contents of
- macro \refcmd{mmzAtBeginMemoization}.
- \item Execute the memoization \refmmz{driver}.
- \item Execute \refmmz{at end memoization} hook, i.e.\ the contents of macros
- \refcmd{mmzAtEndMemoization} and \refcmd{mmzAtEndMemoizationExtra} (in this
- order).
- \item Write out the memos and ship out the externs to extern pages (unless
- memoization was aborted).
- \item Close the memoization group.
- \item Execute \refmmz{after memoization} hook, i.e.\ the contents of
- macros \refcmd{mmzAfterMemoization} and \refcmd{mmzAfterMemoizationExtra}
- (in this order).
- \end{enumerate}
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzCMemo, desc={token register, global, empty at the start of memoization}},
- }
- This token register mediates the construction of the c-memo. During
- memoization (and only during memoization), arbitrary code may be added to
- this register; at the end of memoization, Memoize writes out its contents to
- the free-form part of the c-memo.
-
- All assignments to this register should be global. Use \refcmd{gtoksapp} and
- \refcmd{xtoksapp} to easily append tokens to the register.
-\end{doc}
-
-
-\begin{doc}{
- cmd={name=mmzCCMemo, desc={token register, global, empty at the start of memoization}},
- }
- This token register mediates the construction of the cc-memo. During
- memoization, the memoization driver should append cc-memo code to
- \refcmd{mmzCCMemo}; at the end of memoization, Memoize writes out its
- contents to the cc-memo (preceded by the list of produced externs).
-
- All assignments to this register should be global. Local assignments would
- not work, because the memoized code may contain commands, like \cs{label} and
- \cs{ref}, which contribute content to cc-memo as well, but these commands may
- appear within a \hologo{TeX} group of any depth.
-
- Use \refcmd{gtoksapp} and \refcmd{xtoksapp} to easily append tokens to the
- register.
-\end{doc}
-
-\begin{doc}{
- par=\meta{token register}\marg{tokens},
- cmd={name=toksapp},
- cmd={name=gtoksapp},
- cmd={name=etoksapp},
- cmd={name=xtoksapp},
- }
- These commands append the given \meta{tokens} to the \meta{token register}.
- \refcmd{etoksapp} and \refcmd{xtoksapp} expand the \meta{tokens} before
- appending them; \refcmd{gtoksapp} and \refcmd{xtoksapp} perform a global
- assignment.
-
- These commands are actually provided by CollArgs, and they are defined only
- if they don't already exist; in particular, note that \hologo{LuaTeX}
- provides them as primitives.
-
- Unlike the \hologo{LuaTeX} primitive variant, these commands require the
- \meta{token register} to be given by a (|\toksdef|fed) control sequence; it
- cannot be given as |\toks|\meta{number}.
-\end{doc}
-
-\begin{doc}{
- cmd={name=ifmemoize},
- cmd={name=memoizetrue},
- cmd={name=memoizefalse},
- }
- Use the \hologo{TeX}-style conditional \refcmd{ifmemoize} to test whether
- Memoize is currently enabled. Within the document body, the conditional may
- be set using \refcmd{memoizetrue} and \refcmd{memoizefalse}, which are then
- functionally equivalent to \refmmz{enable} and \refmmz{disable}. Do not set
- the conditional in the preamble of the document (unless you really know what
- you are doing).
-\end{doc}
-
-\begin{doc}{
- cmd={name=ifmemoizing,desc={readonly}},
- }
- Use this \hologo{TeX}-style conditional to test whether Memoize is currently
- memoizing. It may be only inspected; you should \emph{never} set this
- conditional yourself.
-\end{doc}
-
-\begin{doc}{
- cmd={name=ifinmemoize,desc={readonly}},
- }
- Use this \hologo{TeX}-style conditional to test whether Memoize is currently
- active, in the sense of either memoizing or regularly compiling some code ---
- so inside a call to \refcmd{Memoize}. The conditional may be only inspected;
- you should \emph{never} set it yourself.
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzSingleExternDriver, par=\marg{code}},
- }
- This is the default memoization \refmmz{driver}, producing exactly one extern
- containing whatever is typeset by the submitted \meta{code}.
-
- The \meta{code} is compiled either within a horizontal or vertical box,
- depending on the value of key \refmmz{capture}. In the case of a horizontal
- capture, the driver makes sure that the horizontal mode is entered prior to
- both typesetting the resulting box in the document, or utilizing the extern.
-
- For the implementational details, see
- section~\ref{sec:memoization-driver-default}.
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzExternalizeBox, par=\marg{box}\marg{token register}},
- }
- This macro is indended to be called by memoization drivers to produce an
- extern page. The given \meta{box} is dumped into the document as a separate
- extern page, while the \meta{token register} receives the cc-memo extern
- inclusion code.
-
- The \meta{box} may be given either as a control sequence (declared via
- |\newbox|), or as box number. The resulting extern page will contain a
- \emph{copy} of the given box, padded by the \refmmz{padding} values in effect
- at the time of invocation of \refcmd{mmzExternalizeBox}.
-
- An implementation detail is that \refcmd{mmzExternalizeBox} does not ship out
- the extern page immediately. This action is delayed until the end of the
- memoization process; more precisely, it is carried out (in tandem with
- writing out the c-memo and the cc-memo) between execution of hooks \refmmz{at
- end memoization} and \refmmz{after memoization}. This delay guarantees
- that no extern pages are produced in the event of aborting memoization, even
- if the abortion is triggered after executing \refcmd{mmzExternalizeBox}.
-
- The \meta{token register} may be given either as a control sequence (declared
- via |\newtoks|) or as control sequence |\toks| followed by the register
- number. The register will receive the code, which, when executed from the
- cc-memo, includes the extern file into the main document. This code consists
- of a single invocation of \refcmd{mmzIncludeExtern}. It is the
- responsibility of the driver to include the code received by \meta{token
- register} in the register \refcmd{mmzMemo}, whose contents are, unless
- memoization is aborted, written into the cc-memo. (See
- \refcmd{ifmmzkeepexterns} and \refmmz{after memoization} to learn about
- another way to use the code received by \meta{token register}.)
-
- The invocation of \refcmd{mmzIncludeExtern} in the produced extern-inclusion
- code is adapted to the type of the box (horizontal or vertical), which is
- detected automatically --- the memoization driver does not need to inform
- \refcmd{mmzExternalizeBox} about this type explicitly.\footnote{This does not
- negate the need for key \refmmz{capture}, which applies to the default ---
- and therefore generic --- memoization driver. This driver cannot know
- whether the memoized code would prefer to be compiled in a horizontal or
- vertical box. It is precisely key \refmmz{capture} which gives the user an
- opportunity to inform Memoize about this preference. Only once the
- memoized code is compiled into a box of the appropriate type, it is trivial
- to detect the type of that box.}
-\end{doc}
-
-\begin{doc}{
- cmd={name=ifmmzkeepexterns, desc={initially \cs{iffalse}}},
- cmd={name=mmzkeepexternstrue},
- cmd={name=mmzkeepexternsfalse},
- }
- Setting this conditional to true makes Memoize keep the extern boxes in the
- global temporary storage even after shipping them out as extern pages. (The
- temporary storage is emptied at the start of the next memoization.)
-
- The extern inclusion code received by the \refcmd{mmzCCMemo} when executing
- \refcmd{mmzExternalizeBox} is primarily meant to be executed by inputting the
- cc-memo file; i.e.\ when the cc-memo is input, \refcmd{mmzIncludeExtern} is
- defined to include the extern file into the document. However, it sometimes
- makes sense to execute the cc-memo contents immediately after memoization;
- for example, if memoization produces several externs, intricately integrated
- into the surrouding environment, it might be cumbersome to replicate their
- typesetting both in the memoizing compilation and in the cc-memo code ---
- easier to build up the cc-memo code and execute it right after memoization.
- This is why Memoize, just before executing the contents of \refmmz{after
- memoization} hook, redefines \refcmd{mmzIncludeExtern} to include externs
- from the temporary storage rather than from (at that point still
- non-existing) extern files. However, as this mechanism requires Memoize to
- keep the externs around even after memoization, it is not enabled by default:
- it must be enabled by setting conditional \refcmd{ifmmzkeepexterns} to true.
-\end{doc}
-
-\begin{doc}{
- key={keypath=/mmz/auto, name=integrated driver, par=\marg{name}}
- }
- Use this key to easily setup a memoization driver which is integrated into
- the command itself.
-
- \begin{tcolorbox}[warning]
- This is an auto-key residing in keypath \refkeypath{/mmz/auto}.
- \end{tcolorbox}
-
- An integrated driver must have a way of telling whether it is memoizing or
- regularly compiling the code. This key declares a driver-specific
- conditional which may be inspected, using \refcmd{IfMemoizing}, to determine
- this. The conditional is set to true by the formal driver of the command
- (set up by the invocation of this key), executed at the start of memoization;
- it should never be set elsewhere. See
- section~\ref{sec:memoization-multiple-externs} for details and an example.
-\end{doc}
-
-\begin{doc}{
- cmd={name=IfMemoizing, par=\oarg{offset}\marg{name}\marg{true code}\marg{false code}}
- }
- This \hologo{LaTeX}-style conditional is meant to be used by the
- \refmmzauto{integrated driver} with the given \meta{name}. It tests whether
- this particular driver is currently memoizing some code.
-
- Potentially recursive commands are supported via the optional argument
- \meta{offset}. If given, the conditional will only execute the \meta{true
- code} when the current \hologo{TeX} group level matches the \hologo{TeX}
- group level at the time of the invocation of the formal driver (held in
- \refcmd{memoizinggrouplevel}), plus the \meta{offset}. In effect, the inner
- invocation of the integrated driver will perform a regular compilation. For
- details, see section~\ref{sec:memoization-complex-single-driver}.
-\end{doc}
-
-\begin{doc}{
- cmd={name=memoizinggrouplevel,desc={readonly}},
- }
- During memoization, this macro holds the \hologo{TeX} group level in effect
- at the start of the memoization.
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzRegularPages, desc=readonly counter},
- }
- This counter holds the number of pages shipped out (so far) by the format's
- regular shipout routine. \emph{Do not change its value!}
-
- In \hologo{LaTeX}, this counter is synonymous with
- \refcmd{ReadonlyShipoutCounter}, and in \hologo{ConTeXt}, it is synonymous
- with \refcmd{realpageno}. Memoize does not touch its value.
-
- In \hologo{plainTeX}, Memoize hijacks the \refcmd{shipout} control sequence
- to count (and only to count) regular shipouts. In order for its value to be
- realistic, Memoize should be loaded before other packages which hack
- \cs{shipout} --- in particular, before \pkg{atbegshi}.
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzExternPages, desc=readonly counter},
- }
- This counter holds the number of extern pages Memoize has shipped out (so
- far). \emph{Do not change its value!}
-
- A third-party tool may inspect this counter to have a realistic count of
- shipped-out pages.
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzExtraPages, desc=counter},
- }
- This counter holds the number of pages shipped out (so far) in a way not
- tracked by either \refcmd{mmzRegularPages} or \refcmd{mmzExternPages}. It
- \emph{should} be advanced by any code which performs such shipouts, or
- Memoize won't work correctly.
-\end{doc}
-
-
-\subsubsection{Tracing}
-\label{sec:ref:memoization:tracing}
-
-\begin{doc}{
- key={name=trace, conditional=false},
- }
- When tracing is on, Memoize shows information about its decision processes on
- the terminal. You can learn whether the memoized code is being memoized,
- utilized or regularly compiled; find out the md5sum of the code and which
- input line it comes from; etc.
-
- This key has the syntax of a conditional, but there is no underlying
- \hologo{TeX} conditional. The low-level interface for switching the tracing
- on and off consists of macros \docaux{cmd}{mmzTracingOn} and
- \docaux{cmd}{mmzTracingOff}.
-
- To learn about tracing the ``auto'' part of the automemoization process, also
- \refcmd{AdviceTracingOn}.
-\end{doc}
-
-\begin{doc}{
- key={name=include source in cmemo, conditional=true},
- }
-
- As a courtesy towards a curious user and a debugging aid, Memoize can include
- a copy of the memo source in the c-memo. This feature is switched on by
- default, but as the package itself never uses that information, it can be
- safely switched off at any time.
-\end{doc}
-
-\begin{doc}{
- key={name=include context in ccmemo, conditional=false},
- }
- When this key is in effect, the expanded context expression is appended to
- the cc-memo, behind the \docaux{cmd}{mmzThisContext} marker.
-
- Memoize never uses the context information from the cc-memo; this information
- is only for tracing purposes.
-\end{doc}
-
-\begin{doc}{
- key={name=direct ccmemo input, conditional=false},
- }
- When this key is set to false, a cc-memo is processed indirectly: it is first
- read into a token register, and it is the contents of this register which are
- executed. When the key is set to true, the cc-memo is simply |\input|ted.
-
- The indirect execution is implemented to facilitate inverse search. Under
- the direct cc-memo input, inverse search pointed at an included extern will
- visit the cc-memo, which is not practical; under the indirect regime, the
- inverse search will work as expected, and this is why the indirect cc-memo
- input is the default.
-
- The overhead produced by the default indirect input method seems negligible,
- but there are other factors which might make the user switch to the direct
- input. For one, a cc-memo changing some category codes will require direct
- input (no such cc-memos are ever produced out of the box). Less crucially,
- sometimes one would like to use the inverse search to figure whether a part
- of the document was produced by regular compilation or utilization, and which
- memos\slash externs were utilized if the latter. Figuring this out under the
- indirect input regime is harder: (i) reading the tracing information shown by
- \refmmz{trace} is the surest way to learn what's going on, although (ii)
- visual inspection of the externs and (iii) grepping through the |.memo.dir|
- folder for particular code often help, as well.
-
- Both input methods use the same cc-memos; there is no need to
- \refmmz{recompile} the memos when switching the cc-memo input method. Note
- that the default indirect input method crucially relies on cc-memos ending
- with \refcmd{mmzEndMemo}; this macro should not appear in the cc-memo itself.
-\end{doc}
-
-
-
-\subsubsection{Internal memo commands}
-\label{sec:ref:memoization:internal}
-
-The end-user should never have to use these commands. They are not formally
-marked as internal by a |@| in their name only because doing so would
-complicate |\input|ting the memos due to the category code changes it would
-require.
-
-\begin{doc}{
- cmd={name=mmzMemo},
- }
- This macro marks the beginning of a c-memo and a cc-memo core. Without it,
- utilization of a memo will not work.
-\end{doc}
-
-\begin{doc}{cmd={name=mmzSource}}
- This macro marks the beginning of the memoized source in the c-memo. That
- source is not used by Memoize in any way. It's inclusion into the c-memo may
- be switched off by \refmmz{include source in cmemo}|=false|.
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzResource, par=\marg{filename}},
- }
- This is an internal command, which only occurs in a cc-memo. It checks
- whether file \meta{filename} exists and is non-empty, and triggers
- recompilation of the memoized code if the check fails.
-\end{doc}
-
-\begin{doc}{
- cmd={
- name=mmzIncludeExtern,
- par=\marg{seq}\braces{\refcmd{hbox}\alt\refcmd{vbox}}%
- \marg{expected width}\marg{expected height}\marg{expected depth}%
- \marg{padding left}\marg{padding bottom}\marg{padding right}\marg{padding top},
- }
- }
-
- This is an internal command, which only occurs in a cc-memo. It includes the
- extern identified by the sequential number \meta{seq} into the document as a
- box of the specified type (horizontal or vertical). The extern is trimmed by
- the given padding values. After trimming, the command checks whether the
- size of the resulting box matches the given expectations; if it doesn't, a
- warning is yielded.
-
- Before this command is executed, the externs should be listed by a sequence
- of \refcmd{mmzResource} commands; \meta{seq} refers to the sequential number
- of an extern in this sequence.
-
- This command may also be executed by executing the entire contents of
- \refcmd{mmzCCMemo} after memoization.
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzLabel, par=\marg{label key}\marg{label value}}
- }
- This is an internal command written into the cc-memo by the auto-handler of
- \refcmd{label}. It temporarily stores \meta{label value} into
- \refcmd{@currentlabel} and then executes \cs{label}\marg{label name}.
-\end{doc}
-
-\begin{doc}{
- cmd={name=mmzEndMemo},
- }
- This macro marks the end of a cc-memo. It is used to grab the cc-memo core
- (everything between \refcmd{mmzMemo} and \refcmd{mmzEndMemo}) under the
- indirect cc-memo input regime, i.e.\ when \refmmz{direct ccmemo input} is not
- in effect.
-\end{doc}
-
-
-\subsection{Location of memos and externs}
-\label{sec:ref:dirs}
-
-\begin{doc}{easy,
- key={name=memo dir,par=\meta{name}, desc={style, default \texttt{\string\jobname}}},
- }
- A convenient way to store memos and externs in a dedicated directory (and
- create this directory in case it does not exist). Without an argument, this
- key places these files in subdirectory \meta{document name}|.memo.dir| of the
- current directory. See section~\ref{sec:tut:memodir} for the tutorial.
-
- This key sets \refmmzpath{relative} to |false|, \refmmzpath{dir} to
- \meta{name}|.memo.dir|, \refmmzpath{prefix} to empty and \refmmz{mkdir} to
- |true|. In effect, memos and externs are placed in the subdirectory
- \meta{name}|.memo.dir| of the directory containing the document; their
- filenames contain no \meta{name} prefix, as \meta{name} already occurs in the
- directory name. The latter feature also makes it easy for projects (or parts
- of a project) to share memos and externs; see section~\ref{sec:tut:multifile}
- for a typical usage case.
-\end{doc}
-
-\begin{doc}{easy,key={name=no memo dir,desc=style}}
- A convenient way to undo the effect of \refmmz{memo dir} and revert to the
- initial settings where memos and externs are located in the current
- directory. See section~\ref{sec:tut:memodir} for the tutorial.
-
- This key resets the subkeys of \refmmz{path} to their initial values, and
- sets \refmmz{mkdir} to |false|.
-\end{doc}
-
-\begin{doc}{
- key={name=path, par=\meta{keylist}, desc=style},
- }
-
- Set the \emph{path prefix} to memo and extern files by executing the given
- \meta{keylist} in keypath \docaux{key path}{mmz/path}.
-
- The path prefix contains both the location of the memo\slash extern files
- (set by \refmmzpath{relative} and \refmmzpath{dir}) and the initial,
- fixed part of their filenames (set by \refmmzpath{prefix}):
- \begin{center}
- \meta{path prefix}=[|./|]\meta{directory}|/|\meta{prefix}
- \end{center}
- where ``|./|'' is only present when \refmmzpath{relative} is in effect.
- Given the \meta{path prefix}, this is how the paths to memos and externs are
- constructed:
- \begin{center}
- \begin{tabular}{r@{ }l}
- c-memo:&\meta{path prefix}\meta{code md5 sum}|.memo|\\
- cc-memo:&\meta{path prefix}\meta{code md5 sum}|-|\meta{context md5 sum}|.memo|\\
- extern:&\meta{path prefix}\meta{code md5 sum}|-|\meta{context md5 sum}-$N$|.pdf|\\
- &(where ``-$N$'' is only present when $N\neq 0$, i.e.\ for non-first externs)\\
- \end{tabular}
- \end{center}
-
- For example, the default \refmmz{no memo dir} \meta{path prefix} equals
- \meta{jobname}|.| (note the dot), and after issuing \refmmz{memo dir}, the
- \meta{path prefix} is \meta{jobname}|.memo.dir|. Executing
- \refmmz{path}|=|\bracestt{\refmmzpath{dir}=project-memo-dir,
- \refmmzpath{prefix}=\string\jobname.} would set the memo prefix to
- |./project-memo-dir/|\meta{jobname}|.|, allowing the author to keep the memo
- files of all documents compiled from the project directory in the same place.
-
- The following keys may occur in the \meta{keylist}:
- \yadocset{keypath=/mmz/path}
-
- \begin{doc}{
- key={name=relative, conditional=true},
- }
-
- This key determines whether the location is relative to the current
- directory, i.e.\ the directory where \hologo{TeX} is executed from;
- usually, this will be the directory where the compiled |.tex| file resides.
- When set to |true| (the default), the directory set by \refmmzpath{dir}
- is prefixed by ``|./|''.
-
- You will probably need to set |openout_any=a| in \reffile{texmf.cnf}
- to allow writing to an arbitrary directory --- but remember, this might be
- dangerous!
- \end{doc}
-
- \begin{doc}{
- key={name=dir, par=\meta{name}, desc={no default, initially empty}},
- }
-
- Set the \meta{directory} where Memoize will search for memos and externs
- and\slash or create them in.
-
- Unless \refmmzpath{relative} is set to |false|, this location is
- relative to the current directory. Given the empty default, memos and
- externs are therefore created in the directory holding the source file by
- default.
-
- In principle, the given memo directory must already exist, but see
- \refmmz{mkdir}.
- \end{doc}
-
- \begin{doc}{
- key={name=prefix, par=\meta{name},
- desc={no default, initially \texttt{\string\jobname.}}},
- }
-
- Set the \meta{prefix}, i.e.\ the initial part of the memo\slash extern
- filename. Initially, it is set to the document name.
- \end{doc}
-
- After processing the \meta{keylist}, \refmmz{path} records the new path
- prefix by invoking \refmmz{record/record type/prefix}, which typically results
- in a \refcmd{mmzPrefix} entry in the \dmmz file, and attempts to create the
- \meta{directory} is \refmmz{mkdir} is in effect --- except when \refmmz{path}
- is executed in the document preamble: then, these actions are only carried
- out at the beginning of the document, for the final value of the keys.
-\end{doc}
-
-\begin{doc}{key={name=mkdir, conditional=false}}
- When this key is set to true, Memoize will attempt to create the directory
- set by \refmmz{path} if it does not yet exist. The directory creation takes
- place at the beginning of the document and at every subsequent invocation of
- key \refmmz{path}, using the system command specified by \refmmz{mkdir
- command}.
-\end{doc}
-
-\begin{doc}{
- key={name=mkdir command, par=\meta{system command invocation},
- desc={no default, initially |mkdir "\#1"|}},
- }
- Sets the command used to create \refmmzpath{dir} when \refmmz{mkdir} is in effect.
-
- This is a system command, so an appropriate shell escape mode must be in
- effect to execute it successfully.
-
- The default should work on Linux, MacOS and Windows. On Linux systems, it
- makes sense to change the default (using \reffile{memoize.cfg}) to |mkdir
- -p "#1"|, which can create parent folders as necessary.
-
- Note that extraction methods \refmmz{extract=perl} and
- \refmmz{extract=python} set \refmmz{mkdir command} to
- \refscript{memoize-extract.pl} \refscript{memoize-extract.pl--mkdir} and
- \refscript{memoize-extract.py} \refscript{memoize-extract.pl--mkdir},
- respectively. Unlike system |mkdir|, these commands are safe in the sense
- that they conform to the paranoid openout regime (|openout_any=p|).
-\end{doc}
-
-\begin{doc}{
- key={name=output-directory, par=\meta{directory},
- desc={no default, initially undefined, as package option only}},
- cmd={name=mmzOutputDirectory, par=\meta{directory}, desc={pre-defineable}}
- }
- If the \hologo{TeX} binary was invoked with option |-output-directory|, one
- should use this key or define this macro to inform Memoize this, as there is
- no way to learn about it automatically.
-
- The given \meta{directory} must end with a slash (|/|).
-
- The key is only allowed as a package option. But I imagine that defining the
- macro from the command line, prior to inputting the document source, will be
- more common:
- \begin{center}
- |pdftex -output-directory somedir '\def\mmzOutputDirectory{somedir/}\input doc'|
- \end{center}
-\end{doc}
-
-
-\subsection{Extern extraction}
-\label{sec:ref:externs}
-
-\begin{doc}{easy,
- key={name=extract,par=\marg{extraction method},
- desc={preamble-only, initially \refmmz{extract=perl}, default: see below}},
- }
- This key selects or executes the extern \meta{extraction method}, i.e. the
- method which Memoize will use to extract the extern pages out of the document
- PDF.
-
- Out of the box, Memoize recognizes the following \meta{extraction method}
- keywords: \docAux{easy, of=key:/mmz/extract, value={name=perl}, comma,
- value={name=python}, comma, value={name=tex}, and, value={name=no}}. The final
- keyword (not available in \hologo{plainTeX}) instructs Memoize to \emph{not}
- perform to extraction; it should be used when extraction is performed
- externally (for details, see section~\ref{sec:setup}). Additional methods
- may be installed by defining key \docaux{key
- path}{mmz/extract}|/|\meta{extraction method}.
-
- When invoked from \reffile{memoize.cfg} or used as a package option, this key
- \emph{selects} the extraction method. In this case, the key has no default
- value, i.e.\ it is illegal to use it without an argument. The method
- selected by the package option overrides the method selected in
- \reffile{memoize.cfg}, which in turn overrides the package-initial value
- \refmmz{extract=perl}.
-
- In \hologo{LaTeX} and \hologo{ConTeXt}, the selected method is executed at
- the end of loading the package. Afterwards, the key is disabled. If you
- really need to invoke extraction in the preamble, or again, write
- \refmmz{extract}|/|\meta{method}.
-
- When invoked from a \refcmd{mmzset} in the document preamble, this key
- immediately \emph{executes} the given extraction method. In the preamble
- incarnation, the key may be invoked without a value to execute the previously
- selected method.
-
- As we want to allow a \hologo{plainTeX} author to override the extraction
- method specified by the package (\refmmz{extract=perl}) or in
- \reffile{memoize.cfg}, Memoize does \emph{not} perform extern extraction
- while loading the package in this format. In \hologo{plainTeX}, internal
- extraction can only be triggered by an explicit invocation of
- \refmmz{extract} in the ``document preamble'' --- i.e.\ between |\input
- memoize| and \refcmd{mmzset}\braces{\refmmz{begin document}}. In this case,
- the key does not require an argument; invoking it without the argument will
- execute either the package-default, \refmmz{extract=perl}, or whatever the
- user had selected in \reffile{memoize.cfg}.
-
- Executing extraction method \refmmz{extract=perl} or \refmmz{extract=python}
- has an additional effect of setting the \refmmz{mkdir command} to the
- extraction script with option \refscript{memoize-extract.pl--mkdir}. This
- obviates the need to include |mkdir| among the restricted shell commands if
- one is using the restricted shell mode.
-\end{doc}
-
-\subsubsection{Perl- and Python-based extraction}
-\label{sec:ref:extraction-perl-python}
-
-\begin{doc}{
- key={name=perl extraction command, par=\meta{system command},
- desc={no default, initially \refscript{memoize-extract.pl}}},
- key={name=perl extraction options, par=\meta{options},
- desc={no default, initially: see below}},
- key={name=python extraction command, par=\meta{system command},
- desc={no default, initially \refscript{memoize-extract.py}}},
- key={name=python extraction options, par=\meta{options},
- desc={no default, initially: see below}},
- }
- These keys determine the system calls used for invoking the extraction
- scripts \refscript{memoize-extract.pl} and \refscript{memoize-extract.py}.
- All the details below apply both to the Perl and the Python version.
-
- Use |perl|\slash|python extraction command| to set the name of the extraction
- script. If necessary, include the full path to the scrip, or |perl|\slash
- |python| plus the path to the script. Whatever you set here must be allowed
- by the shell escape mode.
-
- Use |perl|\slash|python extraction options| to set the options that the
- script will receive; consult the documentation of
- \refscript{memoize-extract.pl} for their meaning.
-
- The initial value of \refmmz{perl extraction options}, shown below, (i) sets
- embedded mode, which prefixes each output line with the script name, (ii)
- requests a log file named \meta{jobname}|.mmz.log| (|-l|; |\input|ting the
- log after extraction informs Memoize, and the author, whether extraction was
- successful), and (iii) sets the warning template suitable for the format
- (|-w|; this way, any warning messages issued by the script can be reissued by
- the compilation of the document).
-
- These keys were initialized using \pkg{pgfkeys} handler |.initial|, so their
- values may be modified by handlers |.prefix|, |.append|, etc. During the
- execution of the system call, their values are fully expanded --- thus the
- |\string|s in the initial options value below. The guards
- (|%<latex>| etc.) make sure that
- each format asks for a warning message it understands (the code shown below
- is an excerpt from the |.dtx| file; ultimately, each format receives a single
- of the three guarded lines).
-
- \makeexcerpt{perl-extraction-options}
- \tcbinputexample[.tex][.excerpt]{listing only, one file, no attachment,
- title={The initial value of \refmmz{perl extraction
- options}}, full width, left=-2mm, right=0mm, listing options app={
- morecomment={[s]{\%<}>} }, }
-\end{doc}
-
-\begin{doc}{
- script={name=memoize-extract.pl, par=[\meta{options}] \meta{name}\texttt{.mmz}},
- script={name=memoize-extract.py, par=[\meta{options}] \meta{name}\texttt{.mmz}},
- }
- These scripts extract the new externs recorded in \meta{name}\dmmz from
- \meta{name}|.pdf|. Memoize invokes them when loaded with package option
- \refmmz{extract}|=|\refmmz{extract=perl} (the default) or
- \refmmz{extract}|=|\refmmz{extract=python}.
-
- The script inspects the given record file, \meta{name}\dmmz, for lines of
- form \refcmd{mmzNewExtern}\marg{extern filename}\marg{extern page
- number}\braces{\meta{expected width}\texttt{pt}}\braces{\meta{expected
- height}\texttt{pt}}. For each such line, page number
- \meta{extern page number} is extracted from \meta{name}|.pdf| into
- \meta{extern filename}. Other lines are ignored (and so are commented
- invocations of \refcmd{mmzNewExtern}).
-
- The \meta{extern filename} may contain a (relative or absolute) path to the
- new extern file. The relative paths are relative to the location of the
- \dmmz file, even when the script is invoked from some other directory.
-
- To guard against extracting a wrong page, the script checks whether the size
- of each extracted page matches the \meta{expected width} and \meta{expected
- height}.\footnote{\label{fn:tolerance}To avoid false positives, the match
- need not be exact, a difference up to |0.01pt| is tolerated. Some PDF
- tools, notably the |PDF::API2| library deployed by the Perl version of the
- script, round the dimensions of a PDF page, recorded in |/MediaBox|, to two
- digits.} If it does not, the script refuses to extract the page, yields a
- warning and even removes the extern file if it exist.
-
- The extraction script's paranoia extends further. It will refuse to extract
- the page, yielding a warning, if a (c)c-memo associated to the extern does
- not exist. And it will refuse (yielding a runtime error) to write to any
- file whose absolute path does not occur under the current directory or the
- directory set by \docref{TEXMFOUTPUT} (in \reffile{texmf.cnf} or as an
- environment variable); \docref{TEXMFOUTPUT} is also not allowed to point to
- the root directory, except on Windows, where it can only point to a drive
- root.
-
- \yadocset{
- of=script:memoize-extract.pl,
- override/.style={
- index annotation={option of {%
- \hypercolor{link}{gray}%
- \hyperref[script:memoize-extract.pl]{\texttt{memoize-extract}}%
- }},
- },
- }
- \begin{doc}{
- option={name=pdf, short name=P, par=\meta{pdf}}
- }
- Extract the externs from the given \meta{pdf} instead of the default
- \meta{name}\texttt{.mmz}. Note that file \meta{pdf}, despite the different
- name, should be produced by the same compilation that produced
- \meta{name}\dmmz, otherwise wrong pages might be extracted.
- \end{doc}
- \begin{doc}{
- option={name=prune, short name=p}
- }
- After extraction, remove the extracted extern pages from the document PDF.
- \end{doc}
- \begin{doc}{
- option={name=keep, short name=k}
- }
- By default, the script comments out the \refcmd{mmzNewExtern} lines in the
- \dmmz file, to prevent multiple extractions. Specifying this option
- prevents this behaviour.
- \end{doc}
- \begin{doc}{
- option={name=log, short name=l, parameters=\meta{filename}}
- }
- When given this option, the script creates a log file with the given name.
- The log will receive any warnings yielded by the script, see also
- \refscript{memoize-extract.pl--warning-template}. A well-formed log ends
- with |\endinput|, indicating that the \dmmz file was completely processed.
-
- In absence of this option, the warnings are printed to the standard error.
- \end{doc}
- \begin{doc}{
- option={name=warning-template, short name=w, parameters=\meta{template}},
- }
- When this option is given, a warning message is not logged (or printed to
- standard error) as-is; it is wrapped by the template given as the argument
- of this option. More precisely, the script will log the given
- \meta{template}, with all occurrences of \docaux{cmd}{warningtext}
- substituted by the actual warning message.
-
- The option can be used to propagate warnings to the upstream compilation,
- simply by setting this option appropriately (see the initial value of
- \refmmz{python extraction options}) and |\input|ting the log after
- extraction.
- \end{doc}
- \begin{doc}{
- option={name=force, short name=f}
- }
- Force extern extraction even if the size-check fails. The failure will
- still be logged.
- \end{doc}
- \begin{doc}{
- option={name=quiet, short name=q}
- }
- Normally, the script prints what it is doing to the standard output; in
- particular, it prints out the page number and the filename of each extern
- it is extracting. This option disables this behaviour.
- \end{doc}
- \begin{doc}{
- option={name=embedded, short name=e}
- }
- When given this option, everything printed to the standard output is
- prefixed by the script name. The idea is to use this option when calling
- the script from the \hologo{TeX} compilation, as it makes it easy to
- identify the output of the script.
- \end{doc}
- \begin{doc}{
- option={name=mkdir, short name=m}
- }
- When given this option, the extraction script transforms into a paranoid
- |mkdir| (|-p|). Argument \meta{name}|.mmz| is interpreted as a path to the
- directory to create; all other options are ignored.
-
- The ancestors of the directory are created as needed. The script will
- refuse to create any directory whose absolute path does not occur under the
- current directory or a directory listed in \docref{TEXMFOUTPUT} (set in
- \reffile{texmf.cnf} or as an environment variable).
-
- This option exists so that the author using the restricted shell mode does
- not have to list |mkdir| among the restricted shell mode commands (and it
- is also safer than a plain |mkdir|).
-
- Memoize automatically uses this script to create the memo directory when
- option \refmmz{extract} is given value \refmmz{extract=perl} or
- \refmmz{extract=python}.
- \end{doc}
- \begin{doc}{
- option={name=help, short name=h}
- }
- Show help.
- \end{doc}
- \begin{doc}{
- option={name=version, short name=V}
- }
- Show Memoize version.
- \end{doc}
-
- Functionally, the Perl (|.pl|) and the Python (|.py|) version of the script
- are almost equivalent. The minor differences, listed below, are mostly due
- to the underlying PDF-processing library:
- \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} in Perl, and
- \hreftt{https://pypi.org/project/pdfrw2/}{pdfrw2} in Python.
- \begin{itemize}
- \item The Python script is about twice as fast as the Perl script. However,
- both scripts are very fast compared to the \hologo{TeX}-based extraction.
- On my computer, extracting 160 externs out of a 400-page book takes 1.4s
- with Python, 3.7s with Perl, and 65s with \hologo{TeX}. But even when
- using \hologo{TeX}-based extraction, externalization using Memoize is
- extremely fast compared to the \TikZ's externalization library. Adding
- the regular compilation time of one minute to the above numbers, we arrive
- at the maximum externalization time of about two minues, whereas my
- estimate for the production of all 160 externs using \TikZ's
- externalization would be an \emph{hour} or more.
- \item The Python script cannot extract externs out of PDFs created without
- stream compression, i.e.\ with \refcmd{pdfvariable}
- \docref{pdfvar:compresslevel} set to |0|.
- \item Occasionally, the Perl script crashes during extraction externs; see
- section~\ref{sec:troubleshooting} for details.
- \end{itemize}
-\end{doc}
-
-\subsubsection{\texorpdfstring{\hologo{TeX}}{TeX}-based extraction}
-\label{sec:ref:extraction-tex}
-
-\begin{doc}{
- key={name=tex extraction command, par=\meta{system command},
- desc={no default, initially |pdftex|}},
- key={name=tex extraction options, par=\meta{options},
- desc={no default, initially: see below}},
- key={name=tex extraction script, par=\meta{\hologo{TeX} code},
- desc={no default, initially: see below}},
- }
- \yadocset{
- before/.style={label prefix=cmd:te},
- index annotation={defined at \hologo{TeX} extraction},
- }
-
- Together, these keys determine the system call used for invoking the
- \hologo{TeX}-based extraction script, \refscript{memoize-extract-one.tex}.
- (They were initialized using \pkg{pgfkeys} handler |.initial|, so their
- values may be modified by handlers |.prefix|, |.append|, etc.)
-
- Memoize uses the resulting system call string at the following occasions.
- First, it executes it, once for each new extern in the \dmmz file, during the
- internal extraction, i.e.\ when it is loaded with package option
- \refmmz{extract}|=|\refmmz{extract=tex}. Second, it uses it to construct
- record files of types \refmmz{record=sh}, \refmmz{record=bat} and
- \refmmz{record=makefile} when recording of these types is requested.
-
- Key \refmmz{tex extraction command} sets the \hologo{TeX} binary used for
- \hologo{TeX}-based internal extraction. By default, the \hologo{pdfTeX}
- engine |pdftex| is used; other sensible values for this key are |luatex| and
- |xetex|, or the program name including the path. Note that shell escape mode
- must be configured appropriately, and that
- \refscript{memoize-extract-one.tex} must be compiled with the
- \hologo{plainTeX} format.
-
- The value of key \refmmz{tex extraction options} is passed as options to the
- \hologo{plainTeX} binary. As shown in the initial value in the frame below,
- it may use the temporary macro \docaux{cmd}{externbasepath}, which expands to
- the path to the extern without the |.pdf| suffix. This macro is available
- during internal \hologo{TeX}-based extraction and during the execution of
- \refmmz[show keypath]{record/record type/new extern} key for and \meta{record
- type}.
-
- \makeexcerpt{tex-extraction-options}
- \tcbinputexample[.tex][.excerpt]{%
- listing only, one file, no attachment, title={The initial value of
- \refmmz{tex extraction options}},
- }
-
- Key \refmmz{tex extraction script} defines the command-line script executed
- to extract the extern. The default value, shown in the frame below, invokes
- \refscript{memoize-extract-one.tex} after setting its parameter macros.
- Temporary macros \docAux{cmd={name=pagenumber}, comma, cmd={name=expectedwidth},
- and, cmd={name={expectedheight}}} are defined at the same occasions as
- \refcmd{te:externbasepath} above, i.e.\ during internal \hologo{TeX}-based
- extraction and during the execution of a \refmmz{record/record type/new
- extern} key. The initial value requests the extern to be of the same version as
- the main document, if possible;\footnote{As far as I know, it is impossible
- to access the version of the PDF being produced in \hologo{XeTeX}, i.e.\
- there are no registers \docref{reg:pdfmajorversion} and
- \docref{reg:pdfminorversion}. To request production of a specific version of
- PDF, \hologo{XeTeX} must be invoked by with command-line option
- \code{-output-driver 'xdvipdfmx -V $N$'}.} note that Memoize defines
- \docref{reg:pdfmajorversion} and \docref{reg:pdfminorversion} in \hologo{LuaTeX}.
-
- \makeexcerpt{tex-extraction-script}
- \tcbinputexample[.tex][.excerpt]{%
- listing only, one file, title=The initial value of \refmmz{tex extraction
- script}, no attachment}
-
- As the value of \refmmz{tex extraction script} is fully expanded when used,
- the initial value shown above must prevent the expansion of much code.
- Furthermore, the initial value varies with the \hologo{TeX} format, as
- indicated by the |.dtx| guards in the definition of
- \refcmd{meo:warningtemplate}.
-\end{doc}
-
-\begin{doc}{
- script={
- head prefix={%
- (\texttt{pdf}\alt\texttt{lua}\alt\texttt{xe})\texttt{tex -jobname}
- \meta{extern filename}
- \texttt{"}\meta{parameters}
- \cs{input}\texttt{\textbraceleft}
- },
- name=memoize-extract-one.tex,
- head infix=\texttt{\textbraceright"},
- },
- }
- Compiling \refscript{memoize-extract-one.tex} with \hologo{plainTeX}
- produces an extern file containing a single (extern) page extracted from the
- document PDF.
-
- Memoize invokes this script, once for each new extern appearing in the \dmmz
- file, when loaded with package option
- \refmmz{extract}|=|\refmmz{extract=tex}.
-
- The desired \meta{extern filename} is given as the value of option |-jobname|
- of the \hologo{TeX} binary. To set the extraction \meta{parameters}, define
- the following macros before |\input|ting the file:
- \yadocset{
- index annotation=option of \texttt{memoize-extract-one.tex},
- before/.style={head prefix=\cs{def}, label prefix=cmd:meo},
- }
- \begin{doc}{cmd={name=fromdocument, parameters=\marg{document pdf filename}}}
- Defining this macro sets the filename of the PDF which the externs will be
- extracted from. The filename is relative to the working directory.
- \end{doc}
- \begin{doc}{cmd={name=pagenumber, parameters=\marg{number}}}
- Defining this macro sets the number of the page to extract. The first page
- has number 1.
- \end{doc}
- \begin{doc}{
- cmd={name=expectedwidth, parameters=\marg{dimension}, desc=optional},
- cmd={name=expectedheight, parameters=\marg{dimension}, desc=optional},
- }
- Defining these macros sets the expected width and height of the extracted
- page.
-
- To guard against extracting a wrong page, the dimensions of the extracted
- page are compared against the expected width and height. If the size check
- fails,\footnote{The match need not be exact, see
- footnote~\ref{fn:tolerance}.} the resulting extern PDF is empty (which
- counts as non-existent when Memoize checks for its presence when it
- attempts to utilize it), and a warning message (formatted via
- \refcmd{meo:warningtemplate}) is printed to the log file, if logging was
- requested via \refcmd{meo:logfile}.
-
- If any of these macros is undefined, the size check will be skipped.
- \end{doc}
- \begin{doc}{cmd={name=logfile, parameters=\marg{filename}, desc=optional}}
- Defining this macro sets the name of the log file. If not defined, no log
- file will be produced.
-
- The log file is intended to be used when the script is invoked from an
- outer \hologo{TeX} compilation. In particular, it is intended to be
- |\input| by that compilation to see whether the extraction was successful.
- Upon a failed size check, it will contain a warning (formatted by
- \refcmd{meo:warningtemplate}, if that macro is defined). The log file ends
- with |\endinput| to signal that extraction actually took place.
- \end{doc}
- \begin{doc}{cmd={name=warningtemplate, parameters=\marg{code}, desc=optional}}
- Defining this macro determines how to log the warning message in the case
- of a failed size check. The macro should expand to a \hologo{TeX}
- format-specific warning message code containing the warning text given in
- \docAux{cmd={name=warningtext, label prefix=cmd:meo, into
- index=false}}.\footnote{Macro \refcmd{meo:warningtemplate} is passed
- the warning text by a macro rather than a formal parameter to avoid
- category code problems with the parameter character when setting key
- \refmmz{tex extraction script}.}
-
- While the script formats the warning message \emph{text} on its own (``I
- refuse to extract page \dots''), the warning message is not written into
- the log unadorned. The log file is intended to be \cs{input} by the outer
- \hologo{TeX} compilation, and the idea is that inputting it should yield a
- warning in that compilation (in the case of a failed size
- check). Therefore, the content of the log file must contain an invocation
- of the command used to produce warning messages in the \hologo{TeX} format
- used by the outer compilation.
-
- For example, when this script is invoked from within a \hologo{LaTeX}
- compilation, it makes sense to define something like
- |\def\warningtemplate{\PackageWarning{memoize}{\warningtext}}|.
- \end{doc}
- \begin{doc}{cmd={name=force,
- parameters=\bracestt{\texttt{true}\Alt\texttt{false}}, desc=optional}}
- If this macro is defined to |true|, extern extraction will be carried out
- even if the size-check fails. The failure will still be logged.
- \end{doc}
- \begin{doc}{
- cmd={name=mmzpdfmajorversion, parameters=\marg{number}, desc=optional},
- cmd={name=mmzpdfminorversion, parameters=\marg{number}, desc=optional},
- }
- Defining (one or both of) these macros requests that the extern PDF be
- produced with the given major\slash minor PDF version, i.e.\ the extraction
- script will set registers \docref{reg:pdfmajorversion} and
- \docref{reg:pdfminorversion}.
- \end{doc}
-
- After extracting the extern, the script will end the compilation, i.e.\
- intentionally, only one page documents can be produced.
-\end{doc}
-
-\subsubsection{The clean-up scripts}
-\label{sec:cleanup-scripts}
-
-\begin{doc}{
- script={name=memoize-clean.pl, par=[\meta{options}] [\meta{name}\texttt{.mmz} \dots]},
- script={name=memoize-clean.py, par=[\meta{options}] [\meta{name}\texttt{.mmz} \dots]},
- }
- This script removes memo and extern files whose filenames start with
- \meta{path prefix}es mentioned in the given \dmmz files or by the
- \refscript{memoize-clean.pl--prefix} option. Unless option
- \refscript{memoize-clean.pl--all} is given, the script only deletes the
- \emph{stale} files, i.e.\ the files not mentioned in any of the given \dmmz
- files.
-
- A \meta{path prefix} of a memo or an extern is what was set by key
- \refmmz{path}, or more commonly, one of the shortcut keys \refmmz{memo dir}
- and \refmmz{no memo dir}; see section~\ref{sec:ref:dirs} for details on the
- form of a memo/extern filename.
-
- In detail, the script scans the given \dmmz files for occurrences of
- \refcmd{mmzPrefix}, and adds their \meta{path prefix} arguments to the list
- of prefixes given on the command line by option
- \refscript{memoize-clean.pl--prefix}; a \meta{path prefix} occurring in some
- \dmmz file is interpreted relatively to the location of the \dmmz file. The
- script removes all files whose full pathname (relative to the current
- directory) matches pattern \meta{path
- prefix}\meta{md5sum}(|-|\meta{md5sum})(|.memo|\alt(|-|$N$)|.pdf|\alt|.log|),%
- \footnote{The |.log| files are produced by the \hologo{TeX}-based extraction
- script.} except those which occur as the \meta{filename} argument to one
- of \refcmd{mmzUsedCMemo}, \refcmd{mmzUsedCCMemo}, \refcmd{mmzUsedExtern},
- \refcmd{mmzNewCMemo}, \refcmd{mmzNewCCMemo} and \refcmd{mmzNewExtern} in one
- of the \dmmz files.
-
- The script is fairly paranoid. It refuses to delete anything if a \dmmz file
- is malformed in any way (but not if it doesn't exist or is completely empty,
- which facilitates its usage in clean-up scripts), or if it would remove a
- file not residing under the current directory. Before removing the files, it
- lists the files to be removed and asks for confirmation.
-
- Functionally, the Perl (|.pl|) and the Python (|.py|) version are completely
- equivalent.
-
- \yadocset{
- of=script:memoize-clean.pl,
- override/.style={
- index annotation={option of {%
- \hypercolor{link}{gray}%
- \hyperref[script:memoize-clean.pl]{\texttt{memoize-clean}}%
- }},
- },
- }
- \begin{doc}{
- option={name=prefix, short name=p, par=\meta{path prefix}}
- }
- Add \meta{path prefix} to the list of prefixes; the given prefix is
- relative to the current directory. This option may be given multiple
- times.
- \end{doc}
- \begin{doc}{
- option={name=all, short name=a}
- }
- When given this option, the script removes \emph{all} memos and externs
- belonging to the document, not just the stale ones, i.e.\ it effectively
- ignores the occurences of \refcmd{mmzUsedCMemo} and friends in the \dmmz
- file.
- \end{doc}
- \begin{doc}{
- option={name=yes, short name=y}
- }
- When given this option, the script does not ask for confirmation before
- removing the files.
- \end{doc}
- \begin{doc}{
- option={name=quiet, short name=q}
- }
- Normally, the script prints what it is doing to the standard output; in
- particular, it prints out the filename of each file as it is deleting it.
- This option disables this behaviour.
- \end{doc}
- \begin{doc}{
- option={name=help, short name=h}
- }
- Show help.
- \end{doc}
- \begin{doc}{
- option={name=version, short name=V}
- }
- Show Memoize version.
- \end{doc}
-\end{doc}
-
-\subsubsection{Record files}
-\label{sec:ref:record-files}
-
-\begin{doc}{
- key={name=record, par=\marg{record type},
- desc={cumulative, initially \refmmz{record=mmz}, no default}},
- key={name=no record},
- }
- Memoize records which externs were produced and used in the compilation,
- producing a record file of every type found in the record-type list. These
- keys add \meta{record type} to the record-type list, or clear this list. See
- section~\ref{sec:record-files} for details.
-
- Note that passing an undefined \meta{record type} to this key will not yield
- an error.
-
- Out of the box, the following \meta{record type}s are recognized:
- \yadocset{of=key:/mmz/record}
- \begin{doc}{value={name=mmz}}
- This record type produces a \dmmz file recording new/used
- externs/c-memos/cc-memos and changes in the \refmmz{path} prefix to these
- files; see section~\ref{sec:.mmz} for details.
-
- The produced file is named \docAux{file={name=mmz,
- name prefix=\meta{jobname}.,
- ref prefix=.,
- index prefix=.,
- }}. This name cannot be changed.
-
- The \dmmz file is a \hologo{TeX} file, but uses only a simple subset of
- the \hologo{TeX} syntax, to be easily parsable by the external scripts such
- as \refscript{memoize-extract.pl}. Each line of the file consists of a
- (possibly commented) invocation of one of the commands listed below; the
- final line is |\endinput|. The \meta{path prefix} below consists of the
- path to memos/externs and the immutable \refmmzpath{prefix} of their filename.
- \begin{doc}{
- cmd={name=mmzUsedCMemo, par=\marg{filename}},
- cmd={name=mmzUsedCCMemo, par=\marg{filename}},
- cmd={name=mmzUsedExtern, par=\marg{filename}},
- }
- Record that the (c)c-memo or extern residing in file \meta{filename} was
- utilized.
- \end{doc}
- \begin{doc}{
- cmd={name=mmzNewCMemo, par=\marg{filename}},
- cmd={name=mmzNewCCMemo, par=\marg{filename}},
- }
- Record that a new (c)c-memo residing in file \meta{filename} was
- produced.
- \end{doc}
- \begin{doc}{cmd={name=mmzNewExtern, par=\marg{filename}\marg{page number}%
- \marg{expected width}\marg{expected height}}}
- Record that a new extern was produced and dumped as page \meta{page
- number} into the document, that it should be extracted into file
- \meta{filename}, and that it should be \meta{expected width} wide and
- \meta{expected height} high (modulo tolerance of |0.01pt|, see
- footnote~\ref{fn:tolerance}), where the height is the total height
- comprising both \hologo{TeX} height and depth.
- \end{doc}
- \begin{doc}{cmd={name=mmzPrefix, par=\marg{path prefix}}}
- Record that the \refmmz{path} prefix of memo and extern files was
- changed.
- \end{doc}
- \end{doc}
- \begin{doc}{value={name=makefile}}
- This record type produces a makefile which, when processed by the |make|
- utility, triggers \hologo{TeX}-based extraction of the new externs.
- \begin{doc}{
- key={name=makefile, par=\marg{filename}, desc={no default,
- initially \texttt{memoize-extract.\jobname.makefile}}}
- }
- Use this key to change the filename of the produced makefile.
- \end{doc}
- \end{doc}
- \begin{doc}{value={name=sh}, value={name=bat}}
- These record types produce a shell script which, when executed, triggers
- \hologo{TeX}-based extraction of the new externs.
-
- Use \refmmz{record=sh} on Unix-like systems, and \refmmz{record=bat} on
- Windows.
-
- \begin{doc}{
- key={name=sh, par=\marg{filename}, desc={no default,
- initially \texttt{memoize-extract.\jobname.sh}}},
- key={name=bat, par=\marg{filename}, desc={no default,
- initially \texttt{memoize-extract.\jobname.bat}}}
- }
- Use these keys to change the filename of the produced shell script.
- \end{doc}
- \end{doc}
-\end{doc}
-
-\begin{doc}{
- desc=definable,
- keypath=/mmz/record/\meta{record type},
- keypath label=/mmz/record/record type,
- name prefix={\meta{record type}/}, ref prefix=,
- index annotation={key in \texttt{/mmz/\meta{record type}}},
- key={name=begin},
- key={name=prefix, par=\marg{path prefix}},
- key={name=new extern, par=\marg{filename}},
- key={name=new cmemo, par=\marg{filename}},
- key={name=new ccmemo, par=\marg{filename}},
- key={name=used extern, par=\marg{filename}},
- key={name=used cmemo, par=\marg{filename}},
- key={name=used ccmemo, par=\marg{filename}},
- key={name=end},
- }
- A new record type can be implemented by defining these keys in keypath
- \docAux{key path={name=mmz/record/\meta{record type}, label=mmz/record/record
- type}} (using the standard \pkg{pgfkeys} handlers such as |.code| and
- |.style|). The keys are invoked by Memoize where appropriate if recording
- for the defined type is activated by \refmmz{record}|=|\meta{record type},
- just as for the predefined types. Only those keys which are required for
- implementing the desired functionality need to be defined.
-
- The following macros are available during the execution of key
- \refmmz{record/record type/new extern}:
- \yadocset{before/.style={label prefix=cmd:ne},
- index annotation=defined at \refmmz{record/record type/new extern}}
- \begin{doc}{cmd={name=pagenumber}}
- This macro holds the number of the extern page.
- \end{doc}
- \begin{doc}{
- cmd={name=expectedwidth},
- cmd={name=expectedheight},
- }
- These macros hold the width and the height of the extern page.
- \end{doc}
- \begin{doc}{cmd={name=externbasepath}}
- This macro holds the filename of the extern, minus the |.pdf| suffix (but
- including the path leading to the extern).
- \end{doc}
-\end{doc}
-
-
-
-\subsection{Automemoization}
-\label{sec:ref:advicememoization}
-
-
-\subsubsection{Package Advice}
-\label{sec:ref:advice}
-
-Package Advice is a namesake of \Emacs's Advice. As such, it implements a
-generic framework for extending the functionality of selected commands and
-environments. Each \emph{advised} command and environment is assigned a piece
-of \emph{advice} --- a command which is executed instead of the advised command
-and environment, and which may, or may not, invoke the original command or
-environment during its execution. The package offers an elegant way of
-declaring advice, setting up the conditions upon which the advised command will
-actually be replaced by the advice, collecting the arguments of the advised
-command and invoking it, and (de)activating the advice.
-
-Before the advising framework can be used, it must be installed into a selected
-\pkg{pgfkeys} keypath (multiple installations into different keypaths are
-allowed, even if they handle the same commands). Memoize installs the
-framework into keypath \refkeypath{/mmz} and (primarily) uses it to
-automatically memoize the results of compilation of selected commands and
-environments.
-
-\begin{doc}{
- keypath={/handlers},
- key={name=.install advice,
- sort index=install advice,
- par=\marg{configuration keylist},
- },
- }
- This key is a \pkg{pgfkeys} key handler (see \PGFmanual{87.3.5}) which
- installs the advising framework into the keypath which it was invoked from
- --- henceforth, the \meta{namespace}.
-
- For example, \code{\cs{pgfkeys}\bracestt{/my/\refkey{/handlers/.install
- advice}}} installs the framework into keypath |/my|.
-
- Argument \meta{configuration keylist} may contain the following keys:
- \yadocset{keypath=/advice/install}
- \begin{doc}{key={
- name=setup key,
- parameters=\marg{name},
- description={no default, initially \texttt{advice}}},
- }
- This key determines the names of the user-interface keys used to setup
- advice for commands and environments in \meta{namespace}.
-
- The keys whose names are determined by this key are the following:
- \meta{name}, \code{\meta{name} csname}, \code{\meta{name} key},
- \code{\meta{name}'}, \code{\meta{name} csname'} and \code{\meta{name}
- key'}. Memoize sets \refkey{/advice/install/setup key}|=auto|, and
- thereby defines \refmmz{auto}, \refmmz{auto csname}, \refmmz{auto key},
- \refmmz{auto'}, \refmmz{auto csname'} and \refmmz{auto key'}.
- \end{doc}
- \begin{doc}{key={
- name=activation,
- par=\meta{initial activation type},
- desc={no default, initially \texttt{immediate}}
- },
- }
- This key sets the \meta{initial activation type} for \meta{namespace}.
-
- At the end of the installation, the system will execute
- \meta{namespace}|/|\refmmz{activation}|=|\meta{initial activation type};
- consequently, \meta{initial activation type} must be one of
- \refmmz{activation=immediate} and \refmmz{activation=deferred}. In
- Memoize, the \meta{initial activation type} is
- \refmmz{activation=deferred}.
-
- Setting the \refmmz{activation} type during the installation only matters
- in \hologo{LaTeX}, where the installation ends by advising \refcmd{begin}
- to implement advising of environments.
- \end{doc}
-
- \begin{tcolorbox}[warning]
- Writing the documentation for Advice, I was faced with a dilemma. Should
- the documentation reflect the fact that the full names of keys defined by
- the package depend on the installed instance of the framework, in
- particular on \meta{namespace} and \meta{setup key}? For example, should
- the reference headers contain things like \meta{namespace}|/activate| and
- \meta{namespace}|/|\meta{setup key} |csname|? In my opinion, this would
- make the reference hard to read, so I decided to have the reference headers
- refer to the Advice keys of the Memoize installation, where
- \meta{namespace}=|/mmz| and \meta{setup key}=|auto|, resulting in
- friendlier headers such as \refkeypath{/mmz}|/|\refmmz{activate} and
- \refkeypath{/mmz}|/|\refmmz{auto csname}. (Consequently, it also made
- sense to document Advice within the Memoize documentation.)
-
- The bottomline: if you're reading this section with a non-Memoize
- installation in mind, you have to mentally replace any \refkeypath{/mmz}
- and \refmmz{auto} in the reference headers with the \meta{namespace} and
- the \meta{setup key} selected by that installation.
- (Section~\ref{sec:ref:advice:memoization} is another matter. Keys
- described there are only available in Memoize.)
- \end{tcolorbox}
-
- In more detail, key handler \refkey{/handlers/.install advice} performs the
- following actions (as explained in the box above, we assume that the
- advising framework was installed into keypath \refkeypath{/mmz} with the
- setup key named |auto|):
- \begin{itemize}
- \item \edef\origtolerance{\the\tolerance}\tolerance=1000
- It defines the following keys in keypath \refkeypath{/mmz}:
- \refmmz{auto},
- \refmmz{auto csname},
- \refmmz{auto key},
- \refmmz{auto'},
- \refmmz{auto csname'},
- \refmmz{auto key'},
- \refmmz{activation},
- \refmmz{activate deferred},
- \refmmz{activate},
- \refmmz{deactivate},
- \refmmz{activate csname},
- \refmmz{deactivate csname}.
- \refmmz{activate key},
- \refmmz{deactivate key}.
- \refmmz{force activate},
- \refmmz{try activate}.
- \item
- It defines the following keys in keypath \refkeypath{/mmz/auto}:
- \refmmzauto{run conditions},
- \refmmzauto{outer handler},
- \refmmzauto{bailout handler},
- \refmmzauto{collector},
- \refmmzauto{args},
- \refmmzauto{collector options},
- \refmmzauto{clear collector options},
- \refmmzauto{raw collector options},
- \refmmzauto{clear raw collector options},
- \refmmzauto{inner handler},
- \refmmzauto{options},
- \refmmzauto{clear options},
- \refmmzauto{reset}.
- \item \tolerance\origtolerance\relax
- It defines the |.unknown| key handler for \refkeypath{/mmz/auto}.
- This handler appends any unknown keys (and their values) to
- \refmmzauto{options}.
- \item It executes \refmmz[show keypath]{activation}|=|\meta{initial
- activation type}.
- \item In \hologo{LaTeX}, it submits \refcmd{begin} to advising, thereby
- enabling environment support in this format. Consequently, advising of
- environments can switched off by writing \refmmz{deactivate}|=\begin|.
- \end{itemize}
-\end{doc}
-
-\paragraph*{The keys installed into keypath \metabf{namespace}} are used to
-declare and (de)activate advice. In the documentation in this subsection,
-we assume that \meta{namespace}=\refkeypath{/mmz} and that \meta{setup
- key}=\refmmz{auto}. In particular, this also applies to the reference
-headers.
-
-\begin{doc}{
- key={name=activation, par=\docAux{of=key:/mmz/activation, value={name=immediate},
- text=\Alt, value={name=deferred}}, desc={style, no default}},
- key={name=activate deferred, desc={style}},
- }
- Key \refmmz{activation} selects the activation regime. Under the
- \refmmz{activation=immediate} regime, keys \refmmz{activate},
- \refmmz{deactivate}, \refmmz{force activate} and \refmmz{try activate} behave
- as described in their documentation below. Under the
- \refmmz{activation=deferred} regime, however, those keys are not executed;
- rather, their invocations are appended to style \refmmz{activate deferred}.
- For example, writing \refmmz{activate}|=\foo| in the deferred activation
- regime appends \refmmz{activate}|=\foo| to \refmmz{activate deferred}. It is
- up to the user if and when to execute the keys collected in \refmmz{activate
- deferred}; see the documentation of \refmmz{manual} to learn what Memoize
- does with the contents of this style.
-\end{doc}
-
-
-\begin{doc}{easy,
- par=\marg{list of commands and/or environments},
- desc={style},
- key={name=activate},
- key={name=deactivate},
- }
- These keys activate or deactivate the advice for the given commands and
- environments. When the advice is activated, it replaces the advised command;
- when it is deactivated, the command is reverted to its original definition.
-
- In Memoize, these keys are most commonly used to activate or deactivate
- automemoization for the given commands or environments. For example, write
- \refmmz{deactivate}|={\tikz,tikzpicture}| to deactivate automemoization of
- \TikZ pictures (which is declared and active by default). The curly braces
- may be omitted if the list contains a single command or environment, e.g.\
- \refmmz{deactivate}|=\tikz| or \refmmz{deactivate}|=tikzpicture|.
-
- (De)activation of a piece of advice is completely orthogonal to its declaration with
- \refmmz{auto}. For example, there is no need to deactivate a command before
- redeclaring its advice, and reactivate it afterwards. A command may be
- activated even before declaring its advice --- however, the command itself
- must be defined at the time of activation.
-
- As the advice is normally automatically activated upon declaration with
- \refmmz{auto}, explicit activation is rarely needed, but see \refmmz{auto'}.
- The effect of these keys under the deffered activation regime is described in
- \refmmz{activation}.
-
- Note that I sometimes speak of (de)activating a command, and sometimes of
- (de)activating its advice. I mean the same thing.
-\end{doc}
-
-\begin{doc}{
- par=\marg{control sequence name},
- desc={style},
- key={name=activate csname},
- key={name=deactivate csname},
- }
- These keys activate and deactivate a command given by its \meta{control
- sequence name}; for example, \refmmz{activate csname}|=foo| is equivalent
- to \refmmz{activate}|=\foo|. Note that unlike the regular \refmmz{activate}
- and \refmmz{deactivate}, their |csname| variants only accept a single command
- at a time (otherwise, including a comma in the command name would be
- impossible).
-\end{doc}
-
-\begin{doc}{
- par=\marg{list of full key names},
- desc={style},
- key={name=activate key},
- key={name=deactivate key},
- }
- These keys activate and deactivate |pgfkeys| keys. Note that \emph{full} key
- names must be given, i.e.\ the names must include the keypath.
-
- Under the hood, these keys merely execute \refmmz{activate} and
- \refmmz{deactivate} on the internal macros corresponding to the given keys.
-\end{doc}
-
-\begin{doc}{
- key={name=try activate, conditional=false},
- }
- When this conditional is set to true, \refmmz{activate} will not yield an
- error if the advice is already activated, and \refmmz{deactivate} will not
- yield an error if the advice is not yet activated.
-
- This key applies to the next, and only to the next, invocation of key
- \refmmz{activate} or \refmmz{deactivate}, i.e.\ it is reset back to |false|
- after invoking \refmmz{activate} or \refmmz{deactivate}.
-\end{doc}
-
-\begin{doc}{
- key={name=force activate, conditional=false},
- }
- When this conditional is set to true, \refmmz{activate} will activate even a
- previously activated command, provided that, additionally, the command has
- been redefined since the prior activation.
-
- In more detail, the original definition of the advised command is saved upon
- activation (to provide the possibility of both deactivation and the usage of
- the original command by the handler). Consequently, activation of an already
- activated command would result in the saved original definition being
- overwritten by the redefinition made during the first activation. However,
- if the handled command was meanwhile redefined by a third party, reactivation
- makes sense, under the assumption that the former original definition is
- obsolete and should be replaced by the (third party) redefinition. As a
- safeguard, however, \refmmz{activate} requires such reactivation to be
- explicitly requested using conditional \refmmz{force activate}.\footnote{A
- potential problem, not (yet) addressed, is that the third party might be
- another incarnation of the advising framework. In this case, forced
- reactivation will result in the loss of the original command and a circular
- dependency between the two pieces of advice.}
-
- This key applies to the next, and only to the next, invocation of key
- \refmmz{activate}, i.e.\ it is reset back to |false| after invoking
- \refmmz{activate}. This key does not apply to \refmmz{deactivate}.
-\end{doc}
-
-\begin{doc}{easy,
- key={name=auto, par=\marg{command or environment}{\marg{keylist}}, desc=style}}
-
- This key sets up the advice for the given command or environment, or updates
- the configuration of an existing piece of advice.
-
- In Memoize, this key is most commonly used to submit a command or environment
- to automemoization. For an environment (say, |bar|), it suffices to write
- \refmmz{auto}|=|\bracestt{bar}\bracestt{\refmmzauto{memoize}}; for a command,
- we usually need to include its argument specification:
- \refmmz{auto}|=\foo|\bracestt{\refmmzauto{memoize},
- \refmmzauto{args}=\bracestt{...}}. Another common usage is to prevent
- memoization during the execution of a command or environment:
- \refmmz{auto}|=|\bracestt{bar}\bracestt{\refmmzauto{nomemoize}}. For
- details, see sections~\ref{sec:tut:automemoization},
- \ref{sec:tut:working-on-a-picture} and~\ref{sec:tut:verbatim}.
-
- The advice is configured by the given \meta{keylist}, which is executed with
- the default keypath set to \docaux{key path}{mmz/auto}. Any unknown keys in
- \meta{keylist} are passed on to key \refmmzauto{options}; for example, a
- plain \refmmz{verbatim} or \refmmz{padding}|=2in| have the same effect as
- \refmmzauto{options}|=|\refmmz{verbatim} or
- \code{\refmmzauto{options}=\braces{\refmmz{padding}=2in}}.
-
- This key automatically activates the declared advice, unless it is already
- activated; under the deferred activation regime, the automatic activation is
- deferred as well. Use variant \refmmz{auto'} when you don't want to
- automatically activate the advice.
-
- When this key is used on a command or environment with an existing piece of advice,
- the advice is merely updated. This makes it easy to, for example,
- temporarily switch to verbatim collection of an environment in Memoize:
- \refmmz{auto}|=|\bracestt{tcolorbox}\braces{\refmmz{verbatim}}. Use key
- \refmmzauto{reset} to setup the advice from scratch:
- \refmmz{auto}|=|\bracestt{...}\bracestt{\refmmzauto{reset}, ...}.
-
- A piece of advice consists of several interlocked components, declared by keys
- residing in path \refkeypath{/mmz/auto}: \refmmzauto{run conditions},
- \refmmzauto{bailout handler}, \refmmzauto{outer handler},
- \refmmzauto{collector} and \refmmzauto{inner handler}. During the execution
- of the advice, these components are available through the following macros:
- \refcmd{AdviceRunConditions}, \refcmd{AdviceBailoutHandler},
- \refcmd{AdviceOuterHandler}, \refcmd{AdviceCollector} and
- \refcmd{AdviceInnerHandler}. These macros are also defined during setup, and
- it is possible to change the configuration by modifying them directly; in
- that case, it likely also makes sense to use the low-level variant of this
- key, macro \refcmd{AdviceSetup}.
-
- Control sequences used in the advice components do not need to be defined at
- the time of invoking key \refmmz{auto}, or activating the advice; they must
- only be defined at the time the advice is actually executed. It is thus
- perfectly fine to declare \refmmzauto{inner handler}|=\myinnerhandler| before
- defining |\myinnerhandler|, or to redefine |\myinnerhandler| between the
- invocations of the advised command.
-
- This key configures not only the components, but also the options of the
- advice. These options are set by keys \refmmzauto{args},
- \refmmzauto{collector options}, \refmmzauto{raw collector options} and
- \refmmzauto{options}. Whether these options are used or not depends on the
- advice components. (Same as the components, the options have their
- corresponding low-level macros: \refcmd{AdviceArgs},
- \refcmd{AdviceCollectorOptions}, \refcmd{AdviceRawCollectorOptions} and
- \refcmd{AdviceOptions}.)
-
- Parameter symbols (i.e.\ |#|) are not allowed in advice settings.
-
- A command or environment may be submitted to several instances of the
- advising framework, i.e.\ instances installed under different keypaths. The
- effect of such chained advice depends on the order of activation. If advice
- $A$ is activated before advice $B$, it will also be applied before $B$.
-
- The advice setup takes place in a group. Use key \refmmzauto{after setup} to
- execute code outside this group.
-
- In general, the name of this key equals whatever was submitted to
- \refkey{/advice/install/setup key} during the installation of the advising
- framework via \refkey{/handlers/.install advice}.
-\end{doc}
-
-\begin{doc}{
- key={name=auto csname, par=\marg{control sequence name}{\marg{keylist}}, desc=style}
- }
- This key is a variant of \refmmz{auto}, but with the command of the first
- argument given as a control sequence name, i.e.\ \refmmz{auto
- csname}|={foo}{...}| is equivalent to \refmmz{auto}|=\foo{...}|.
-\end{doc}
-
-\begin{doc}{
- key={name=auto key, par=\marg{full key}{\marg{keylist}}, desc=style},
- }
- This key is a variant of \refmmz{auto}, but it works with \pkg{pgfkeys} keys.
- The first argument should be a \meta{full key} like |/tcb/float|, i.e.\ it
- must consist of both the keypath and the keyname.
-
- This key sets up advice for the internal command corresponding to the given
- \meta{full key}, and also properly initializes the collector, so that
- \refmmzauto{inner handler} will ``just work.''
-\end{doc}
-
-\begin{doc}{
- key={name=auto', par=\marg{command or environment}{\marg{keylist}}, desc=style},
- key={name=auto csname', par=\marg{control sequence name}{\marg{keylist}}, desc=style},
- key={name=auto key', par=\marg{full key}{\marg{keylist}}, desc=style},
- }
- These keys are variants of \refmmz{auto}, \refmmz{auto csname} and
- \refmmz{auto key} which do not attempt to activate the command after setting
- it up.
-\end{doc}
-
-\begin{doc}{
- cmd={
- name=AdviceSetup,
- par=\marg{namespace}\marg{command or environment}{\marg{setup code}}
- }}
- This macro is the low-level variant of key \refmmz{auto}. The differences
- between the two are the following:
- \begin{itemize}
- \item An invocation of the macro must provide the namespace (i.e.\ the
- installation keypath) as the first argument.
- \item There is no automatic activation at the end of the setup.
- \item The final argument should not be a keylist (of keys belonging to
- \refkeypath{/mmz/auto}) but \hologo{TeX} code adjusting the contents of the
- settings macros \refcmd{AdviceRunConditions}, \refcmd{AdviceBailoutHandler},
- \refcmd{AdviceOuterHandler}, etc. For the full list of available macros, see
- the documentation of their corresponding keys below; the setting macros are
- mentioned at the end of each entry.
- \end{itemize}
-\end{doc}
-
-\begin{doc}{
- cmd={name=AdviceTracingOn},
- cmd={name=AdviceTracingOff},
- }
- Advice tracing is initially off. When it is on, Advice will show (on the
- terminal and in the |.log| file) which advice components are executed, and
- what arguments and options they have received.
-\end{doc}
-
-\paragraph*{The keys installed into keypath \metabf{namespace}\texttt{/}\metabf{setup key}}
-are used to configure advice. They may only occur within the second
-argument of the setup key. In the documentation in this subsection, we assume
-that \meta{namespace}=\refkeypath{/mmz} and that \meta{setup
- key}=\refmmz{auto}. In particular, this also applies to the reference
-headers.
-
-\begingroup
-\yadocset{keypath=/mmz/auto}
-
-\begin{doc}{%
- key={name=run conditions, par=\meta{\hologo{TeX} code},
- desc=initially and default: \refcmd{AdviceRuntrue}},
- }
-
- This key declares the \meta{control sequence} as the run conditions component
- of the advice.
-
- The run conditions macro is executed at the very start of the advice. Its
- function is to decide whether we should proceed to advise the command by
- executing the outer handler, or execute the original command (after invoking
- the bailout handler).
-
- The run conditions macro should take no arguments. If it determines that the
- run conditions are satisfied, it should set the \hologo{TeX} conditional
- \docaux{cmd}{ifAdviceRun} to true by executing \docaux{cmd}{AdviceRuntrue}.
- There is no need to execute \docaux{cmd}{AdviceRunfalse} when the run
- conditions are not satisfied.
-
- Initially, the run conditions are set to \refcmd{AdviceRuntrue}, translating to
- ``always run.'' For two non-trivial examples, see \refmmzauto{run if
- memoization is possible} and \refmmzauto{run if memoizing}. Executing this
- key without a value restores it to the initial value.
-
- During advising and advice setup, the run conditions of the advised command
- are accessible through \docaux{cmd}{AdviceRunConditions}, a parameterless
- macro expanding to the given \meta{\hologo{TeX} code}.
-\end{doc}
-
-\begin{doc}{%
- key={name=bailout handler, par=\meta{\hologo{TeX} code},
- desc=initially and default: \cs{relax}},
- }
-
- This key declares the \meta{\hologo{TeX} code} as the bailout handler
- component of the advice.
-
- The bailout handler is executed when the run conditions are not met, just prior
- to executing the original definition of the advised command. The bailout
- handler should take no arguments.
-
- The initial bailout handler, |\relax|, does nothing. Memoize defines and
- uses a bailout handler which clears the next-options. Executing this key
- without a value restores it to the initial value.
-
- During advising and advice setup, the bailout handler of the handled command is
- accessible through \docaux{cmd}{AdviceBailoutHandler}, a parameterless macro
- expanding to the given \meta{\hologo{TeX} code}.
-\end{doc}
-
-\begin{doc}{
- key={name=outer handler,par=\meta{\hologo{TeX} code},
- desc=initially and default: see below},
- }
-
- This key declares the \meta{\hologo{TeX} code} as the outer handler component
- of the advice.
-
- The outer handler can be safely imagined as the command which replaces the
- handled command. This also holds for handled environments, but with a
- caveat: for a \hologo{plainTeX} or \hologo{ConTeXt} environment |foo|, the
- outer handler replaces |\foo| and |\startfoo|, respectively; in the case of a
- \hologo{LaTeX} environment, it replaces |\begin{foo}|.%]
-
- The outer handler is the first component which has the opportunity to inspect
- the arguments given to the handled command. It is invoked just in front of
- these arguments (which are, in case \hologo{TeX} hasn't seen them yet,
- untokenized), and while it is expected that the advice will consume
- the same arguments as the advised command itself would, how precisely that
- happens may vary from situation to situation. In particular, the argument
- structure of the outer handler is not prescribed.
-
- In fact, the outer handler has complete control over the remainder of the
- advising process. In situations where advising requires knowledge of the
- advised command's arguments as a whole, the outer handler executes the
- collector, which in turn invokes the inner handler, which does the real work;
- see \refmmzauto{memoize} for the usage case which inspired this design.
- Sometimes, however, it is the outer handler which does the real work (and
- there is thus no inner handler). This is the case in situations when the
- arguments of the handled command are irrelevant for the functioning of the
- advice, or when the advice needs to inspect some individual argument of the
- handled command; for examples of such situations, see \refmmzauto{abort} and
- \refmmzauto{ref}.
-
- To reiterate the argument situation of the outer handler, it sees the
- arguments of the handled command as they were given. The arguments are
- \emph{not} collected before invoking the outer handler --- in fact, avoiding
- the argument collection is the raison d'être of the outer handler! (In the case
- of an advised environment, the environment body can be seen as an argument of
- \pkg{xparse} type \docref{xparse:+}\docref{xparse:b}.)
-
- The outer handler (and any other component of the advice it invokes) has
- access to the following auxiliary macros, defined by the framework:
- \begin{itemize}
- \item \tolerance 1000 the macros holding the configuration of the advised
- command, as set up by \refmmz{auto}: \refcmd{AdviceRunConditions},
- \refcmd{AdviceBailoutHandler} and \refcmd{AdviceOuterHandler} are probably
- useless, as they refer to components already invoked, but the remaining
- components (\refcmd{AdviceCollector} and \refcmd{AdviceInnerHandler}) and
- their options (\refcmd{AdviceArgs}, \refcmd{AdviceCollectorOptions},
- \refcmd{AdviceRawCollectorOptions} and \refcmd{AdviceOptions}) should be
- commonly used.
- \item the macros holding information about the namespace and the advised
- command or environment: \refcmd{AdviceNamespace}, \refcmd{AdviceName},
- \refcmd{AdviceReplaced} and \refcmd{AdviceOriginal}. (Command
- \refcmd{AdviceGetOriginal} might also be useful, although using
- \refcmd{AdviceOriginal} will likely be more practical.)
- \end{itemize}
-
- This key is initially set to an internal control sequence which merely
- invokes the collector by executing \refcmd{AdviceCollector}; in other words,
- the initial outer handler leaves all the work to the collector and the inner
- handler. There is no need to specifically set up the outer handler when
- using the inner handler. Executing this key without a value restores it to
- the initial value.
-
- During advising and advice setup, the outer handler of the advised command is
- accessible through \docaux{cmd}{AdviceOuterHandler}, a parameterless macro
- expanding to the given \meta{\hologo{TeX} code}.
-\end{doc}
-
-\begin{doc}{
- key={name=collector,par=\meta{\hologo{TeX} code},
- desc=initially and default: see below},
- }
-
- This key declares the \meta{\hologo{TeX} code} as the collector component of
- the advice.
-
- The collector, if used, is invoked by the outer handler. It is invoked
- immediately in front of the advised command's arguments (which are, in case
- \hologo{TeX} hasn't seen them yet, untokenized), and its function is to
- collect these arguments and pass them on, as a single argument, to the inner
- handler.
-
- While this manual occasionally states that the initial argument collector is
- \refcmd{CollectArguments} of package CollArgs, this is, if we're precise,
- incorrect on two counts. For one, the initial collector is not a CollArgs
- command, but a macro which acts as the ``bridge'' between Advice
- and CollArgs. Second, the initial collector does not really invoke
- \refcmd{CollectArguments}, but its cousin, \refcmd{CollectArgumentsRaw},
- which allows Advice (and Memoize) to fine tune its behaviour
- using the fast low-level (``programmer interface'') commands rather than the
- slower \pkg{pgfkeys} interface; clearly, the latter point also provides
- raison d'être for \refmmzauto{raw collector options}. Summing up, this key
- is initially set to an internal control sequence which compiles the settings
- provided by \refmmzauto{args}, \refmmzauto{collector options} and
- \refmmzauto{raw collector options} into an invocation of
- \refcmd{CollectArgumentsRaw} of package CollArgs.\footnote{The initial
- collector also sets the CollArgs' option \refcollargs{caller} to the
- name of the advised command or environment.} Executing this key without a
- value restores it to the initial value.
-
- The above-mentioned collector settings were clearly tailored to suit
- \refcmd{CollectArgumentsRaw}. In general, a collector might or might not use
- them, and if it does, it may interpret them in any way. For example, Advice
- ships with a \refcmd{tikz} collector, \docaux{cmd}{mmzCollectTikZArguments}, which
- ignores them completely, as it knows everything about the idiosyncrasies of
- that command anyway. Incidentally, \refcmd{mmzCollectTikZArguments} becomes
- available upon loading \reffile{advice-tikz.code.tex} (which Memoize does
- automatically in the presence of \TikZ).
-
- The collector has access to the same auxiliary macros as the outer handler.
- In particular, it will \emph{have} to use \refcmd{AdviceInnerHandler} (followed
- by the braced collected arguments) to invoke the inner handler.
-
- During advising and advice setup, the collector of the advised command is
- accessible through \docaux{cmd}{AdviceCollector}, a parameterless macro
- expanding to the given \meta{\hologo{TeX} code}.
-\end{doc}
-
-\begin{doc}{easy,
- key={name=args,par=\meta{argument specification}, desc=initially and default: unset},
- }
-
- This key describes the \meta{argument specification} of the advised command.
-
- Assuming that the initial value of \refmmzauto{collector} has not been
- modified, the given \meta{argument specification} is eventually interpreted
- by command \refcmd{CollectArguments} of package CollArgs, which expects an
- argument specification in the format specified by package \pkg{xparse}; the
- format is summarized in the frame below for convenience, for details, see the
- \pkg{xparse} manual. If the specification is not given, the initial collector
- assumes that the advised command was defined using \refcmd{NewDocumentCommand}
- (or similar) of package \pkg{xparse}, and will attempt to retrieve the argument
- specification automatically via \refcmd{GetDocumentCommandArgSpec}.
-
- \begin{tcolorbox}[float, before float=\hfill,
- title={The {\pkg[white]{xparse}} argument specification
- (as understood by \refcmd[link color=white]{CollectArguments})}]
- \begin{tabularx}{\linewidth}{>{\tt}lX}
- \multicolumn{2}{l}{\rm\textbf{Mandatory argument types}}\\
- m&standard (a single token or multiple tokens in braces)\\
- r\meta{token$_1$}\meta{token$_2$}&delimited by
- \meta{token$_1$} and \meta{token$_2$} \\
- v&verbatim, in the style of \cs{verb}\\
- b&the body of an environment\\
- [1ex]\multicolumn{2}{l}{\rm\textbf{Optional argument types}}\\
- o&in square brackets\\
- d\meta{token$_1$}\meta{token$_2$}&delimited by
- \meta{token$_1$} and \meta{token$_2$}\\
- s&an optional star\\
- t\meta{token}&an optional \meta{token}\\
- e\marg{tokens}&a set of embellishments\\
- [1ex]\multicolumn{2}{l}{\rm\textbf{Weird argument types}}\\
- l&a mandatory argument until the first begin-group token\\
- u\marg{tokens}&\hologo{TeX}'s delimited argument\\
- g&an optional argument inside braces\\
- [1ex]\multicolumn{2}{l}{\rm\textbf{Modifiers}}\\
- +&allow the next argument to be long\\
- !&disallow spaces before arguments of type \docref{xparse:d} and \docref{xparse:t}\\
- >\marg{processor}&process the next argument\\
- [1ex]\multicolumn{2}{l}{\rm\textbf{CollArgs extensions}}\\
- \docref{xparse:b}\marg{name}&set the environment name for this environment\\
- \docref{xparse:amp}\marg{options}&apply CollArgs options to the next argument\\
- \docref{xparse:amp}\docref{xparse:amp}\marg{raw options}&apply raw CollArgs options to the next argument\\
- \end{tabularx}
-
- \smallskip
-
- \refcmd{CollectArguments} can grab an argument of any type in the
- \refcollargs{verbatim} mode.
-
- As \refcmd{CollectArguments} does not use the arguments but only collects
- them, it does not care about the default values of optional arguments.
- Therefore, argument types with defaults (\docref{xparse:O},
- \docref{xparse:D} and \docref{xparse:R}) may be substituted by their
- \texttt{-NoValue-} counterparts (\docref{xparse:o}, \docref{xparse:d} and
- \docref{xparse:r}) and are therefore not included in the above table.
- \end{tcolorbox}
-
- In general, however, an argument collector may this interpret this setting it
- in any way it sees fit --- or not at all. For example, in Memoize the value
- of \refmmzauto{args} is ignored for command \refcmd{tikz}, which requires a special
- collector (\refcmd{mmzCollectTikZArguments}).
-
- When setting up advice for a \emph{command}, this key is initially
- ``unset,'' i.e.\ it holds a special value indicating that the argument
- specification is not provided. Note that this special value is not an empty
- string --- \refmmzauto{args}|={}|, or simply \refmmzauto{args}|=|, indicates
- a command which takes no arguments. During the execution of the advice, one
- may use the \hologo{LaTeX}-style conditional
- \docaux{cmd}{AdviceIfArgs}\marg{true branch}\marg{false branch} to test
- whether the argument specification was provided. Executing this key without
- a value restores it to the initial, unset value.
-
- When setting up the advice of an \emph{environment}, this key is
- initialized to \docref{xparse:+}\docref{xparse:b} (a long environment body),
- making it unnecessary to specify this value manually. Note that this holds
- even for environments with arguments other that the environment body
- ``argument'': those arguments will be caught as the start of the body even if
- not explicitly specified.
-
- During advising and advice setup, the argument specification of the advised
- command is accessible through \docaux{cmd}{AdviceArgs}, a parameterless macro
- expanding to the given \meta{argument specification}.
-\end{doc}
-
-
-\begin{doc}{
- key={name=collector options, par=\marg{keylist},
- desc={cumulative, initially empty, value required}},
- key={name=raw collector options, par=\marg{code},
- desc={cumulative, initially empty, value required}},
- }
- These keys append the given value to the list of user-friendly and raw
- collector options, respectively. A comma is prefixed to the user-friendly
- \meta{keylist} before appending it.
-
- Both kinds of collector options are intended to be used by the collector,
- which may interpret them in any way it sees fit --- or not at all. The
- initial collector, which invokes \refcmd{CollectArgumentsRaw} of package
- CollArgs, passes both lists to this command, which interprets
- \refmmzauto{collector options} as a user-friendly \pkg{pgfkeys} keylist
- (which therefore requires a bit of processing) and \refmmzauto{raw collector
- options} as plain \hologo{TeX} code (expecting it to contain only the
- allowed, ``programmer's interface'' macros).\footnote{Clearly,
- \refmmzauto{raw collector options} are why Advice deploys
- \refcmd{CollectArgumentsRaw} rather than \refcmd{CollectArguments}. But
- how does it then pass the user-friendly \refmmzauto{collector options} to
- that command? It embeds them in \refcmd{collargsSet}.} The raw variant is
- used internally by both Advice and Memoize, and may be used by a package
- deploying the advising framework which wants to save a few processing cycles.
- In CollArgs, the two kinds of options are functionally equivalent; both are
- documented in section~\ref{sec:ref:collargs}.
-
- Initially, the list of collector options is empty, and for commands, so is
- the list of raw collector options. For environments, however, the latter
- list is initialized to set (the raw equivalent of) \refcollargs{environment}
- to the environment name, and \refcollargs{end tag} to true. The rationale
- for the latter is that the environment body containing the end tag (e.g.\
- |\end{foo}|) is nicely compatible with \refcmd{AdviceReplaced} (which equals
-the begin tag, e.g.\ |\begin{foo}|) and \refcmd{AdviceOriginal} (which executes
- the original definition of e.g.\ |\begin{foo}|). For example, thanks to
- \refcollargs{end tag}, writing \refcmd{AdviceOriginal}|#1| in the inner
- handler executes the original environment. Importantly, the original
- environment can be executed without explicitly referring to the
- environment's name, and with code that works not only for environments of
- any \hologo{TeX} format, but is actually the same as the code which invokes
- an original \emph{command}. Consequently, the same inner handler works for
- both commands and environments, and in all \hologo{TeX} formats.
-
- Furthermore, the initial collector also sets option \refcollargs{caller}
- to the name of the advised command or environment (however,
- \refcollargs{caller} never appears in any of the collector options
- lists; it is simply prefixed to them while constructing the invocation of
- \refcmd{CollectArgumentsRaw}). And in Memoize, using keys
- \refmmz{verbatim}, \refmmz{verb} or \refmmz{no verbatim} triggers the
- addition of the cognominal \refcollargs{verbatim}, \refcollargs{verb}
- or \refcollargs{no verbatim} among the collector options.
-
- Precious few CollArgs' options thus remain to be set by the author. For
- memoization, the most likely candidates are \refcollargs{ignore nesting}
- and \refcollargs{ignore other tags}, which could help deal with unusual
- environments. Overriding the initial \refcollargs{end tag} by
- \refcollargs{begin tag}, \refcollargs{end tag} and/or \refcollargs{tags}
- might also be useful on occasion.
-
- During advising and advice setup, the \pkg{pgfkeys} and the raw collector
- options of the advised command are accessible through
- \docaux{cmd}{AdviceCollectorOptions} and
- \docaux{cmd}{AdviceRawCollectorOptions}, both a parameterless macro expanding
- to the given \meta{keylist} and \meta{code}, respectively.
-\end{doc}
-
-\begin{doc}{
- key={name=clear collector options},
- key={name=clear raw collector options},
- }
- These keys empty the list of user-friendly and raw collector options,
- respectively.
-\end{doc}
-
-\begin{doc}{
- key={name=inner handler, par=\meta{\hologo{TeX} code}, long description=0.5,
- desc=initially and default: see below},
- }
-
- This key declares the \meta{\hologo{TeX} code} as the inner handler component
- of the advice.
-
- The inner handler is intended to be used in situations which require
- knowledge of the advised command's arguments as a whole. In such situations,
- the outer handler will normally invoke the collector, which will in turn
- execute the inner handler and provide it with a single (braced) argument,
- containing the collected arguments of the advised command. See
- \refmmzauto{memoize} for the usage case which inspired this design.
-
- The simplest example of an inner handler is a (single-parameter) macro which
- does nothing. Surprisingly enough, such an inner handler could be useful.
- Defining |\def\Gobble#1{}| and setting
- \code{\refmmz{auto}=\cs{foo}\bracestt{\refmmzauto{inner handler}=\cs{Gobble},
- \refmmzauto{args}=\bracestt{...}}} with the argument structure
- appropriate for |\foo| could be used to eradicate all invocations of |\foo|
- from the document.
-
- The inner handler has access to all the macros available to the outer
- handler, but given that most of them have already fulfilled their function,
- only the following will likely be useful in the inner handler:
- \refcmd{AdviceNamespace}, \refcmd{AdviceName}, \refcmd{AdviceReplaced},
- \refcmd{AdviceOriginal}, and \refcmd{AdviceOptions}.
-
- Because there is clearly no reasonable default for the inner handler, this key
- is initially set to an internal control sequence producing an ``undefined
- inner handler'' error. Note that it is not necessary to define a dummy inner
- handler when handling is entirely performed by the outer handler, i.e.\ in
- cases when the inner handler is not invoked. Executing this key without a
- value restores it to the initial value.
-
- During advising and advice setup, the inner handler of the advised command is
- accessible through \docaux{cmd}{AdviceInnerHandler}, a parameterless macro
- expanding to the given \meta{\hologo{TeX} code}.
-\end{doc}
-
-\begin{doc}{easy,
- key={name=options,par=\marg{keylist},
- desc={cumulative, initially empty, value required}},
- key={name=clear options},
- }
-
- The first key appends the given \meta{keylist} to the list of advice options
- (after prefixing it by a comma), and the second one empties this list. For a
- \meta{key} undefined in keypath \refkeypath{/mmz/auto},
- \meta{key}|=|\meta{value} has the same effect as
- \refmmzauto{options}|=|\bracestt{\meta{key}=\meta{value}}.
-
- In Memoize, the options set by this key are known as
- \emph{auto-options} --- options which are applied (using \refcmd{mmzset}) at
- every invocation of the advised command or environment. For example, the
- \env{tcolorbox} environment of package \pkg{tcolorbox} is used extensively
- for typesetting this manual, and I have submitted this environment to
- automemoization. However, the \env{tcolorbox}es in this manual often include
- code listings. To memoize such environments successfully, their bodies must
- be grabbed verbatim. I have therefore submitted the \env{tcolorbox}
- environment to automemoization like this:
- \refmmz{auto}|={tcolorbox}|\bracestt{\refmmzauto{memoize},
- \refmmzauto{options}=\refmmz{verbatim}}; the simpler
- \refmmz{auto}|={tcolorbox}|\bracestt{\refmmzauto{memoize}, \refmmz{verbatim}}
- would work as well.
-
- In general, whether to use the options set by this key, and how, remains at
- the sole discretion of the advice. Note that they might be used by either
- the outer or the inner handler, or perhaps even the collector.
-
- During advising and advice setup, the options of the advised command are
- accessible through \docaux{cmd}{AdviceOptions}, a parameterless macro expanding
- to the given \meta{keylist}.
-\end{doc}
-
-\begin{doc}{key={name=reset, desc=style}}
- Executing this key restores all \refmmz{auto} keys to their initial values.
-
- Invoking \refmmz{auto} on the same command or environment again
- \emph{updates} the advice configuration. Use this key to start from scratch.
-\end{doc}
-
-\begin{doc}{key={name=after setup, desc={initially empty, cumulative}}}
- The code given to this key will be executed after exiting the group opened by
- \refmmz{auto}. The same effect may be achieved by appending to macro
- \docaux{cmd}{AdviceAfterSetup}.
-
- For example, \refmmzauto{integrated driver} uses this key to declare a new
- conditional.
-\end{doc}
-
-\paragraph{Commands available during the execution of advice}
-
-With the exception of \refcmd{AdviceGetOriginal}, the commands listed below
-only become available in the outer handler, and if that handler does nothing
-funky, they should be available in the collector and the inner handler, as
-well. However, once the advice yields control to foreign code, these macros
-are not guaranteed to hold the expected values anymore, because the foreign
-code might trigger another piece of advice. Consequently, these macros
-should be expanded, once, before integrating them into arbitrary (non-advice)
-code; in particular, this applies to \refcmd{AdviceOriginal}.
-
-\begin{doc}{cmd={name=AdviceNamespace}}
- This macro holds the \meta{namespace}, i.e.\ the keypath which this instance
- of the advising framework was installed into.
-\end{doc}
-
-\begin{doc}{cmd={name=AdviceName}}
- This macro holds the name of the advised command or environment, i.e.\ the
- name which was used as the first argument to \refmmz{auto}. For a command,
- this will be a control sequence, e.g.\ |\foo|; for environments (in any
- \hologo{TeX} format), their name, e.g.\ |foo|.
-\end{doc}
-
-\begin{doc}{cmd={name=AdviceReplaced}}
- This macro holds the code which was replaced by the outer handler. For
- commands, this will be the command itself, e.g.\ |\foo|, so
- \refcmd{AdviceReplaced} will equal \refcmd{AdviceName}. For an environment
- |foo|, \refcmd{AdviceReplaced} is set to |\begin{foo}| in \hologo{LaTeX},
- |\foo| in \hologo{plainTeX} and |\startfoo| in \hologo{ConTeXt}.
-\end{doc}
-
-\begin{doc}{cmd={name=AdviceOriginal}}
- This macro executes the original code of the advised command.
-
- This macro is defined as \refcmd{AdviceGetOriginal}\marg{namespace}\marg{name},
- and therefore acts as a shortcut for an explicit invocation of
- \refcmd{AdviceGetOriginal}. When executing the original command directly from
- the advice, one may safely write \refcmd{AdviceOriginal}. However, whenever
- \refcmd{AdviceOriginal} is embedded in code which might contain other advised
- commands, it should be pre-expanded, exactly once.
-\end{doc}
-
-\begin{doc}{cmd={name=AdviceGetOriginal, par=\marg{namespace}\marg{control sequence}}}
- This command invokes the original definition of the \meta{control sequence}
- advised by the \meta{namespace} instantiation of Advice. It may be safely
- used outside the advice, even if the advised command is not activated.
-
- For example, upon executing key |/ns/|\refmmz{auto}|=\foo{...}|,
- |\AdviceGetOriginal{/ns}{\foo}| will recall the original definition of |\foo|
- if |\foo| is activated, and simply execute |\foo| otherwise.
-
- The second argument of this command should \emph{not} be an environment name.
- To execute the original environment |foo| in \hologo{TeX} or
- \hologo{ConTeXt}, use \refcmd{AdviceGetOriginal} with the appropriate macro:
- |\AdviceGetOriginal{/ns}{\foo}| or |\AdviceGetOriginal{/ns}{\startfoo}|. In
- \hologo{LaTeX}, one should use |\AdviceGetOriginal{/ns}{\begin}{foo}|, which
- executes the original \refcmd{begin} and provides it with the environment name.
-
- Within the advice, you will probably never have to use this command directly,
- but will rather rely on the (plain or pre-expanded) \refcmd{AdviceOriginal}.
- However, outside the advice, this command provides the only means to
- access the original definition of an advised command. (Unlike the
- commands described above, this command is available throughout the
- document.)
-
- \begin{tcolorbox}[warning]
- A typo in the invocation of this command may result in an infinite loop.
- Assume that the advice for |\foo|, declared in namespace \refkeypath{/mmz},
- executes \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, which incorrectly refers
- to the non-existing namespace |/zzm|, and that command |\foo| is activated.
- Executing |\foo| will eventually execute
- \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, which won't find the original
- definition of |\foo| in the non-existing namespace |/zzm| and will thus
- execute macro |\foo| (again), which, being advised, will lead to another
- \refcmd{AdviceGetOriginal}|{/zzm}{\foo}|, etc. My advice is to define an
- abbreviation like |\def\mmzAdviceGetOriginal{\AdviceGetOriginal{/mmz}}|.
- And note that the name\-space is a full keypath, which begins with a slash
- (|/|), but has no slash at the end.
- \end{tcolorbox}
-\end{doc}
-
-\paragraph{Support for specific packages}
-At the moment, Advice only implements specific support for \TikZ, by defining a
-\refmmzauto{collector} for command \refcmd{tikz}.
-
-\begin{doc}{
- cmd={name=AdviceCollectTikZArguments},
- }
- This command collects the arguments in the format expected by \refcmd{tikz}, and
- executes macro \refcmd{AdviceInnerHandler} with the collected arguments given
- as a single braced argument. The collector supports both the group and the
- semicolor invocation of |\tikz|, i.e.\ both |\tikz{...}| and |\tikz...;|.
-
- This command is only available upon |\input|ting file
- \docaux{file}{advice-tikz.code.tex}.
-\end{doc}
-
-\endgroup % keypath = /mmz/auto
-
-\subsubsection{Memoization-related additions to the advising framework}
-\label{sec:ref:advice:memoization}
-
-In section~\ref{sec:ref:advice}, we have seen that Memoize installs the
-advising framework into keypath \refkeypath{/mmz}, with setup key name
-\refmmz{auto}. This populates keypaths \refkeypath{/mmz} and
-\refkeypath{/mmz/auto} with various generic advice keys. However, Memoize
-installs further advice\slash automemoization-related keys into these keypaths.
-It is these keys which are described in this section.\footnote{One such key,
- \refmmzauto{integrated driver}, is actually documented in
- section~\ref{sec:ref:memoization}.}
-
-Therefore, in contrast to section~\ref{sec:ref:advice}, \refkeypath{/mmz} and
-\refmmz{auto} have no secret generic meaning here, i.e.\ they should \emph{not}
-be generalized to \meta{namespace} and \meta{setup key} of
-\refkey{/handlers/.install advice}.
-
-\paragraph{Keys residing in \refkeypath{/mmz}}
-
-\begingroup
-\setlength\textfloatsep{20pt plus 2pt minus 4pt}% back to the default
-
-\begin{doc}{easy,
- key={name=manual, description={preamble-only,{ }}, conditional=false},
- }
- When this conditional is set to true, no commands are \refmmz{activate}d at
- the beginning of the document. The list of commands and environments advised
- and activated out of the box can be found in Table~\ref{tab:advised-commands}.
-
- The auto-framework allows the activation to be deferred (see
- \refmmz{activation} and \refkey{/handlers/.install advice}) but leaves it
- open to the specific instance of the framework to use the deferred activation
- commands as it sees fit. Normally, Memoize switches to immediate activation
- at the end of the preamble (hook \refmmz{begindocument/before}) and issues
- \refmmz{activate deferred} at the beginning of the document, more precisely
- in hook \refmmz{begindocument/end} (afterwards, \refmmz{activate deferred} is
- emptied). However, when \refmmz{manual} is in effect, the deferred
- activation is suppressed (though it may be still carried out by the user by
- executing \refmmz{activate deferred}).
-
- In \hologo{LaTeX}, \refmmz{manual} affects the (internal) activation of
- \refcmd{begin} as well, which effectively deactivates handling of all environments.
-\end{doc}
-
-\begin{table}
- \centering
- \begin{tabularx}{\linewidth}{llX}
- \toprule
- command/environment&handler¬es\\
- \midrule
- \refcmd{begin}&custom&Only in \hologo{LaTeX}; declared by Advice.\\
- \refcmd{errmessage}&\refmmzauto{abort}&Not available in \hologo{LuaTeX},
- where better error-detection is implemented.\\
- \refenv{forest}&\refmmzauto{memoize}\\
- \refcmd{Forest}&\refmmzauto{memoize}\\
- \refcmd{index}&\refmmzauto{replicate}&The argument is expanded prior to replication.\\
- \refcmd{label}&custom&Globally appends \refcmd{mmzLabel}\marg{label key}\marg{current label}
- to register \refcmd{mmzCCMemo}; \refmmzauto{run if memoizing}.\\
- \refcmd{pageref}&\refmmzauto{ref}\\
- \refcmd[into index=false]{pdfsavepos}&\refmmzauto{abort}&Not available in \hologo{LuaTeX}.\\
- \refcmd{pgfsys at getposition}&\refmmzauto{abort}&Available only in \TikZ is
- loaded, it aborts memoization of a picture which gets accidentally marked as
- ``remembered''.\\
- \refcmd{ref}&\refmmzauto{ref}\\
- \refcmd[short]{savepos}&\refmmzauto{abort}&Available only in \hologo{LuaTeX}.\\
- \refcmd{tikz}&\refmmzauto{memoize}\\
- \refenv{tikzpicture}&\refmmzauto{memoize}\\
- \bottomrule
- \end{tabularx}
- \caption{Commands advised by Memoize}
- \label{tab:advised-commands}
-\end{table}
-
-\begin{doc}{easy,
- key={name=ignore spaces, conditional=false},
- }
- Ignore any spaces following \emph{auto}memoized code. This key has no effect
- for manual memoization, i.e.\ command \refcmd{mmz} and environment
- \refenv{memoize}.
-
- It is common practice to conclude the definition of a command by \hologo{TeX}
- primitive \refcmd{ignorespaces}, which consumes any following spaces, to
- prevent unintended blank space after the command's invocation. Automemoizing
- such a command disrupts this behaviour.\footnote{It is clear that
- \refcmd{ignorespaces} is disrupted during utilization; in this case, the
- original command, including the concluding \cs{ignorespaces}, is never even
- executed. However, the disruption also occurs during memoization, and even
- during regular compilation. In both cases, the memoized code is embedded
- in some internal Memoize code. Therefore, the original \cs{ignorespaces}
- does not occur directly in front of the rest of the document.} The
- workaround is to use this key, normally as an option in the \refmmz{auto}
- declaration; it will work both for automemoized macros and environments.
-\end{doc}
-
-\paragraph{Keys residing in \refkeypath{/mmz/auto}}
-
-\begingroup
-\yadocset{keypath=/mmz/auto}
-
-\begin{doc}{easy,key={name=memoize, desc=style}}
- This key sets up advice which triggers memoization of the command or
- environment whenever it is encountered; we often refer to such a command as
- ``automemoized,'' or say that it was ``submitted to automemoization.''
-
- An automemoized command will consume the next-options, whether memoization
- actually occurs or not.
-
- Under the hood, this key declares both an \refmmzauto{outer handler} and an
- \refmmzauto{inner handler}. The outer handler opens the memoization group
- (so that options can be applied locally), applies the auto-options (given by
- \refmmzauto{options} within \refmmz{auto}) and the next-options (given by
- \refcmd{mmznext}) by executing \refmmzauto{apply options}, and appends the
- verbatim keys to \refmmzauto{collector options} if necessary. The inner
- handler invokes \refcmd{Memoize} (which closes the group opened by the outer
- handler): the first argument is \refcmd{AdviceReplaced}, expanded once and
- followed by the arguments of the handled command; the second argument is
- \refcmd{AdviceOriginal}, also expanded once and followed by the arguments of
- the handled command. The inner handler also makes sure that \refmmz{ignore
- spaces} is respected.
-\end{doc}
-
-\begin{doc}{easy,key={name=nomemoize,desc=style}}
- This key installs advice which disables memoization for the space of the
- command or environment; we sometimes refer to such commands as
- ``autodisabled.''
-
- This key is merely an abbreviation for \refmmzauto{noop}|,|
- \refmmzauto{options}|=|\refmmz{disable}. See the documentation of
- \refmmzauto{noop} for further details.
-\end{doc}
-
-\begin{doc}{key={name=noop,desc=style}}
- This key sets up advice which does nothing.
-
- Ok, not nothing at all. The installed handler applies the auto-options and
- the next-options by executing \refmmzauto{apply options}, and makes sure that
- \refmmz{verbatim} and \refmmz{ignore spaces} are respected.
-
- For commands and non-\hologo{LaTeX} environments, this key declares the same
- outer handler as \refmmzauto{memoize}, while the inner handler merely
- executes the original command (respecting the potential verbatim mode),
- closes the group opened by the outer handler, and makes sure that
- \refmmz{ignore spaces} is respected.
-
- For \hologo{LaTeX} environments, which open the group necessary for the local
- application of options themselves, this key declares an outer handler which
- adds the relevant code into the next hook |env/|\meta{environment
- name}|/begin|. There is no need to open a group, collect the environment
- body, or make special provisions for the verbatim mode.
-\end{doc}
-
-
-\begin{doc}{key={name=apply options,desc=style}}
- This style, used by \refmmzauto{memoize}, \refmmzauto{nomemoize} and
- \refmmzauto{noop} described above, installs two handlers:
- \begin{itemize}
- \item an outer handler which opens a group, applies auto-options and
- next-options by executing \refcmd{mmzAutoInit}, and executes the collector;
- and
- \item a bailout handler which clears the next-options.
- \end{itemize}
-\end{doc}
-
-
-\begin{doc}{cmd={name=mmzAutoInit}}
- This macro applies the auto-options and the next-options.
-
- Additionally, if \refmmz{verbatim}, \refmmz{verb} or \refmmz{no verbatim} was
- previously executed, this style appends the corresponding CollArgs key
- (\refcollargs{verbatim}, \refcollargs{verb} or \refcollargs{no
- verbatim}) to \refcmd{AdviceRawCollectorOptions}. In case several of the
- verbatim keys were executed, the final one takes effect.
-\end{doc}
-
-
-\begin{doc}{easy, key={name=abort,desc=style}}
- This key sets up advice which aborts any ongoing memoization.
-
- Under the hood, the advice merely executes \refcmd{mmzAbort} followed by
- \refcmd{AdviceOriginal}. The advised command does \emph{not} consume the
- next-options. Out of the box, we submit two control sequences to this
- handler:
- \begin{itemize}
- \item \refcmd{errmessage}: this allows us to detect and abort upon at least some
- errors.
- \item \refcmd[into index=false]{pdfsavepos} (in \hologo{LuaTeX},
- \refcmd[short]{savepos}): one common effect is that memoization of any
- \TikZ picture with \refkey{/tikz/remember picture} set is aborted.
- \end{itemize}
-\end{doc}
-
-
-\begin{doc}{key={name=unmemoizable,desc=style}}
- This key sets up advice which aborts the ongoing memoization and marks the
- automemoized code as unmemoizable, so that it will be henceforth compiled
- regularly.
-
- Under the hood, the advice merely executes \refcmd{mmzUnmemoizable} followed
- by \refcmd{AdviceOriginal}. The advised command does \emph{not} consume the
- next-options.
-
- Out of the box, we submit no control sequences to this advice, but it might
- make sense to submit \refcmd[into index=false]{pdfsavepos}\slash
- \refcmd[short]{savepos}. Keys \refmmzauto{abort}|=|\refcmd[short]{savepos}
- and \refmmzauto{unmemoizable}|=|\refcmd[short]{savepos} will most often have
- the same effect, as far as the author is concerned; the former was chosen as
- the default because it does not produce a c-memo; see
- \refcmd{mmzUnmemoizable} for a situation where \refmmzauto{unmemoizable} is
- preferred.
-\end{doc}
-
-
-\begin{doc}{easy,
- key={name=ref,desc=style},
- key={name=force ref,desc=style},
- }
-
- These keys set up advice which adds the reference key to the context
- expression. They are intended to be used with cross-referencing commands
- such as \refcmd{ref} and \refcmd{pageref}.
-
- Indeed, \refcmd{ref} and \refcmd{pageref} are submitted to this advice by
- Memoize, with the effect that standard cross-referencing inside memoized code
- ``just works.'' Note that the stabilization of the document after changing
- the reference takes three compilation cycles, i.e.\ one cycle more than
- without memoization.
-
- The advice set up by \refmmzauto{ref} aborts memoization if the reference
- key is undefined, the rationale being that the produced memo and extern would
- most often be useless, and could even obscure an undefined reference. The
- \refmmzauto{force ref} handler produces the memo and the extern even when the
- reference is undefined.
-
- The reference produced by the advised command should be fully expandable
- (because it will be expanded as a part of the context expression).
-
- Typically, a \refcmd{ref} command takes a single argument, the reference key.
- However, some packages may define a reference command which takes optional
- arguments, as well; in particular, the \pkg{hyperref}'s incarnation of \refcmd{ref}
- takes an optional star. This advice does not care: it will accept any number
- of any kind of optional arguments, as long as the reference key is the first
- braced argument following the advised command; for example, |\ref*{key}|,
- |\ref[opt]{key}|, |\ref*[opt]{key}| etc.\ will all be handled correctly,
- while |\ref{mand}{key}| will not work. Effectively, it is as if we had set
- \refmmzauto{args}|=lm| --- and with the same downside, namely that an
- unlikely unbraced single-token reference key, like |\ref k|, will not work.
-
- Under the hood, these two pieces of advice pass the reference key to macros
- \refcmd{mmzNoRef} and \refcmd{mmzForceNoRef}, and it is these commands ---
- which may also be used in user-defined advice or the document itself ---
- which actually add the reference key to the context expression.
-\end{doc}
-
-\begin{doc}{easy,
- key={name=refrange,desc=style},
- key={name=force refrange,desc=style},
- }
- These keys have the same function as \refmmzauto{ref} and \refmmzauto{force
- ref}, but they operate on reference-range commands, such as \pkg{cleveref}'s
- \refcmd{crefrange}, which take two arguments (the starting and the ending reference
- key).
-\end{doc}
-
-\begin{doc}{easy,
- key={name=multiref,desc=style},
- key={name=force multiref,desc=style},
- }
- These keys have the same function as \refmmzauto{ref} and \refmmzauto{force
- ref}, but they operate on ``multireference'' commands, such as
- \pkg{cleveref}'s \refcmd{cref}, which allow the author to list several
- comma-separated reference keys in a single argument.
-\end{doc}
-
-\begin{doc}{
- key={name=replicate, desc=style}
- }
- This key sets up advice which replicates the invocation of the command in
- the cc-memo during memoization.
-
- When using this key, it is necessary to set \refmmzauto{args} as well. For
- \refcmd{index}, Memoize executes
- \refmmz{auto}|=\index|\bracestt{\refmmzauto{args}=m, \refmmzauto{replicate}}.
-
- This key takes an auto-option, \docAux{keypath=/mmz/auto/replicate,
- key={name=expanded}}.\footnote{This option is unrelated to Memoize's options,
- settable by \refcmd{mmzset}.} If given, the collected arguments will be
- expanded before replicating them in the cc-memo; in \hologo{LaTeX}, this
- expansion is |\protect|ed.
-
- In \hologo{LaTeX}, Memoize submits \refcmd{index} to this handler (with
- expansion). Therefore, any |\index{key}| in the memoized code gets copied
- into the cc-memo. Effectively, indexing from within the memoized code ``just
- works.''
-
- Note that \refcmd{label}, despite essentially requiring replication, cannot
- use this advice, because it needs to replicate not only the label key but
- \refcmd{@currentlabel} as well.
-\end{doc}
-
-\begin{doc}{key={name=run if memoization is possible, desc=style}}
- Under the run conditions installed by this key, a command is only advised if
- Memoize is enabled but we're not already ``within Memoize,'' i.e.\ memoizing
- or normally compiling some code submitted to memoization. In code:
- \refcmd{ifmemoize}\refcmd{ifinmemoize}|\else|\refcmd{AdviceRuntrue}|\fi\fi|.
-
- Internally, this key is used by \refmmzauto{memoize} and \refmmzauto{noop}.
-\end{doc}
-
-\begin{doc}{key={name=run if memoizing, desc=style}}
- Under the run conditions installed by this key, a command is only advised
- during memoization. In code:
- \refcmd{ifmemoize}\refcmd{ifmemoizing}\refcmd{AdviceRuntrue}|\fi\fi|.
-
- Internally, this key is used by \refmmzauto{abort}, \refmmzauto{replicate},
- and \refmmzauto{ref} and friends.
-\end{doc}
-
-\endgroup % keypath = /mmz/auto
-\endgroup
-
-
-
-\subsubsection{Package CollArgs}
-\label{sec:ref:collargs}
-
-\begingroup
-\yadocset{keypath=/collargs}
-
-\begin{doc}{cmd={
- name=CollectArguments,
- par=\oarg{options}\marg{argument specification}\marg{next-code}%
- \textcolor{gray}{\meta{tokens}}}}
-
- This command determines the extent to which the \meta{tokens} following the
- the three formal arguments of the command conform to the given \meta{argument
- specification}, effectively splitting \meta{tokens} into \meta{argument
- tokens} and the \meta{rest} of the tokens, and then executes
- \meta{next-code} with the \meta{argument tokens} provided as a single, braced
- argument:
- \begin{center}
- \meta{next-code}\marg{argument tokens}\textcolor{gray}{\meta{rest}}
- \end{center}
- If the initial part of \meta{tokens} does not conform to \meta{argument
- specification}, \refcmd{CollectArguments} throws an error. (In this case,
- \meta{next-code} is not executed, and the \meta{tokens} collected until the
- error are thrown away.)
-
- The optional \meta{options} are processed using the \pkg{pgfkeys} utility of
- PGF/\TikZ (see \PGFmanual{87}), with the default path set to
- \docaux{key path}{collargs}. The given options apply to all the arguments in
- \meta{argument specification}. The recognized keys are listed in the rest of
- the section.
-
- The \meta{argument specification} should be given in the \pkg{xparse} format
- (we summarize this format in the documentation for \refmmzauto{args} in
- section~\ref{sec:ref:advice}), with several extensions:\footnote{Collargs
- internally uses a dot (|.|) to delimit the argument specification from the
- following argument tokens. Therefore, the dot really counts as an extra
- argument type, in the sense that Collargs will stop working if the dot
- becomes an argument type or a modifier in some future release of
- \pkg{xparse}.}
- \begin{itemize}
- \item We introduce modifier \docAux{xparse modifier={name=\&,label=amp,index
- annotation/.prefix={additional\ }, index annotation/.append={\
- (options)} }} taking a mandatory argument specifying the options to
- apply to the following argument in the specification. Options given here
- override the \meta{options} given as the optional argument.
- \item The environment body type \docAux{xparse type={name=b,index
- annotation/.append={\ (environment body)}}} may be followed by an optional
- \emph{braced} argument providing the name of the environment to collect.
- The name given here overrides the name given by the
- \refcollargs{environment} option.
- \item The number of collected ``arguments'' is unlimited.
- \end{itemize}
-
- Also note that the effect of \docref{xparse:O}\marg{default} is the same as
- the effect of \docref{xparse:o}, and similarly for other pairs of types with
- and without defaults (\docref{xparse:R} and \docref{xparse:r},
- \docref{xparse:D} and \docref{xparse:d}, and \docref{xparse:E} and
- \docref{xparse:e}). CollArgs is dedicated to collecting the argument tokens
- precisely as they are given: if an optional argument is missing, its default
- value is \emph{not} inserted among the collected arguments --- consequently,
- \refcmd{CollectArguments} is utterly uninterested in the default value.
-
- Collection of environments automatically adapts to the format, i.e.\ given
- environment body name |foo|, \refcmd{CollectArguments} knows to search for
- \refcmd{begin}\bracestt{foo} |...| \refcmd{end}\bracestt{foo} in
- \hologo{LaTeX}, \cs{foo} |...| \cs{endfoo} in \hologo{plainTeX}, and
- \cs{startfoo} |...| \cs{stopfoo} in \hologo{ConTeXt}. For further
- information on environment collection, see keys \refcollargs{ignore
- nesting} and \refcollargs{tags}.
-\end{doc}
-
-\begin{doc}{cmd={
- name=CollectArgumentsRaw,
- par=\marg{option-setting code}\marg{argument specification}\marg{next-code}%
- \textcolor{gray}{\meta{tokens}}}}
-
- This command is the programmer's interface to CollArgs, intended to be used
- instead of \refcmd{CollectArguments} when compilation speed is an issue. The
- two commands only differ in how they deal with options.
-
- One difference is that for \refcmd{CollectArgumentsRaw}, the options form a
- mandatory rather than an optional argument. More importantly, however, they
- do not take the form of a keylist, but should be composed out of low-level
- option-setting commands. Each key documented in this section has a
- corresponding low-level macro; these macros are listed in footnotes alongside
- the keys. The name of the macro starts with |\collargs| and continues with
- the name of the key, without spaces, each word capitalized; if the key is
- boolean, this convention applies to the base of the \hologo{TeX}
- conditional. For example,
- \begin{tcblisting}{listing only, listing options app={escapechar=|}}
-\CollectArguments[caller=\foo, tags, verbatim]|\marg{argument specification}\marg{next-code}|
- \end{tcblisting}
- is equivalent to
- \begin{tcblisting}{listing only, listing options app={escapechar=|}}
-\CollectArgumentsRaw{%
- \collargsCaller{\foo}%
- \collargsBeginTagtrue\collargsEndTagtrue
- \collargsVerbatim
-}|\marg{argument specification}\marg{next-code}|
- \end{tcblisting}
-
- Withing the option-setting code, the programmer may also deploy macro
- \docaux{cmd}{collargsSet}, which processes the \meta{options} in the keylist
- format. One idea could be to execute this macro at the end of the low-level
- options; this would set the ``defaults'' using the fast programmer's
- interface, but still allow for user customization.
-\end{doc}
-
-
-\begin{doc}[
- pi={\docaux{cmd}{collargsCaller}},
- ]{
- key={name=caller, par=\meta{control sequence (name)},
- desc={no default, initially \cs{CollectArguments}}},
- }
- Set the control sequence to refer to in error messages.
-
- If \meta{tokens} do not match the \meta{argument specification},
- \refcmd{CollectArguments} throws an error. By default, the error message
- contains a reference to \refcmd{CollectArguments} itself, for example %
- |! Argument of \CollectArguments has an extra }.| However, this might not be
-very informative to the author. When \refcollargs{caller}|=\cs| is in effect,
-the error messages will refer to the given |\cs| instead.
-
-If the value of this key is not a control sequence, it is assumed to be an
-environment name, but as the caller must be a macro, this name will be
-converted into a control sequence. Setting \refcollargs{caller}|=foo| will
-result in error messages referencing |\foo| in \hologo{plainTeX}, |\startfoo|
-in \hologo{ConTeXt} and |\begin{foo}| (a single control sequence!) in
- \hologo{LaTeX}.
-\end{doc}
-
-\begin{doc}[
- pi=\docaux{cmd}{collargsEnvironment},
- ]{key={name=environment, par=\meta{environment name},
- desc={applicable to type \docref{xparse:b}, no default, initially empty}}
- }
-
- Set the name of the environment collected by argument type
- \docref{xparse:b}.
-\end{doc}
-
-\begin{doc}[
- pi={\docaux{cmd}{ifcollargsBeginTag}, \docaux{cmd}{ifcollargsEndTag};
- \refcollargs{tags} has no corresponding low-level command}
- ]{
- key={name=begin tag, desc={applicable to type \docref{xparse:b},{ }}, conditional=false},
- key={name=end tag, desc={applicable to type \docref{xparse:b},{ }}, conditional=false},
- key={name=tags, par=\meta{boolean}, % not a conditional
- desc={applicable to type \docref{xparse:b}, style, default |true|}},
- }
-
- When \refcollargs{begin tag}\slash \refcollargs{end tag} is in effect, the
- begin\slash end tag will be will be prepended/appended to the collected
- environment body. Style \refcollargs{tags} is a shortcut for setting
- \refcollargs{begin tag} and \refcollargs{end tag} simultaneously.
-
- In \hologo{LaTeX}, using \refcollargs{tags} will thus dress up the
- collected body in a pair or \refcmd{begin}\marg{environment name} and
- \refcmd{end}\marg{environment name}. CollArgs will automatically use the tags
- appropriate to the format.
-
- In the verbatim modes, the added tags are verbatim as well, with the detail
- that in \hologo{LaTeX}, there is a slight difference between the full
- \refcollargs{verbatim} and the partial \refcollargs{verb} mode. In the
- full verbatim mode, the braces surrounding the environment name are verbatim
- (the characters used as braces are actually determined by key
- \refcollargs{braces}). In the partial verbatim, as well as the
- non-verbatim mode, the environment name is surrounded by a pair of actual
- braces of category 1 and 2, regardless of which characters are of these
- categories in the calling code.
-\end{doc}
-
-\begin{doc}[
- pi=\docaux{cmd}{ifcollargsIgnoreNesting},
- ]{
- key={name=ignore nesting, desc={applicable to type \docref{xparse:b},{ }},
- conditional=false},
- }
- When this key is \emph{not} in effect, CollArgs respects the hierarchical
- structure created by tag pairs such as \refcmd{begin}\bracestt{foo} and
- \refcmd{end}\bracestt{foo}. Given the situation below on the
- left, argument type |b{foo}| will collect everything up until the
- \emph{second} |\end{foo}|. Now this is what we usually want, because
-\hologo{LaTeX} keeps track of environment embedding as well. However, all
-\emph{verbatim} environments I know of, starting with the standard
-\hologo{LaTeX} \refenv{verbatim}, will ignore the nesting and simply scoop up
-everything up to the first \cs{end}|{verbatim}|. In CollArgs, we can replicate
-their behaviour by setting \refcollargs{ignore nesting}, as shown below on the
-right. (Of course we also need to set \refcollargs{verbatim} if we want to
-grab the environment body in the verbatim mode.)
-
- \begin{tcbraster}[raster columns=2]
- \begin{tcblisting}{listing only, mark region={1}{5},
- example title=\texttt{ignore nesting=false}}
- ...
- \begin{foo}
- ...
- \end{foo}
- ...
-\end{foo}
- \end{tcblisting}
- \ExampleName{_collargs-verbatim}
- \makeexample{\examplename.tex.c1}
- \tcbinputexample{listing only, no attachment, mark region={1}{3},
- example title=\texttt{ignore nesting=true}}
- \end{tcbraster}
-
- This key applies not only to argument type \docref{xparse:b} (in either
- normal or verbatim mode), but also to the verbatim argument type
- \docref{xparse:v} and to argument types \docref{xparse:m} and
- \docref{xparse:g} in the \refcollargs{verbatim} (but not normal, or
- \refcollargs{verb}) mode. With these keys, the relevant structure markers
- are braces, |{| and |}|.
-\end{doc}
-
-\begin{doc}[
- pi=\docaux{cmd}{ifcollargsIgnoreOtherTags},
- ]{
- key={name=ignore other tags, desc={applicable to type \docref{xparse:b},{ }},
- conditional=false},
- }
- In \hologo{LaTeX}, the environment tags, \refcmd{begin}\marg{name} and
- \refcmd{end}\marg{name}, contain braces, which retain their usual category codes
- in the non-verbatim and in the partial verbatim mode. Consequently, CollArgs
- cannot easily search for the full tags to delimit the
- environment.
-
- When this key is \emph{not} in effect, CollArgs takes the easy path, and
- determines the end of the environment only by inspection of \cs{begin}s and
- \cs{end}s, without reference to what \meta{name} they begin or end. Only
- when this key \emph{is} in effect does CollArgs inspect these \meta{name}s,
- effectively ignoring any \cs{begin}s and \cs{end}s not followed by the name of
- environment being collected. The effect of absence vs.\ presence of this key
- is shown below, where the shaded area marks the code collected into
- environment |foo|.
-
- \begin{tcbraster}
- \ExampleName{_collargs-ignore-other-tags}
- \makeexample{\examplename.tex.c1 N=2}
- \tcbinputexample{listing only, no attachment, mark region={3}{3},
- example title=\texttt{ignore other tags=false}}
- \tcbinputexample[.tex][.c2]{listing only, no attachment, mark region={3}{5},
- example title=\texttt{ignore other tags=true}}
- \end{tcbraster}
-
- This key does not have any effect the full verbatim mode, which always
- behaves as if this key was set to true, because braces are of category
- ``other'' as well. Similarly, it is as if this key was always true in
- \hologo{plainTeX} and \hologo{ConTeXt}, simply because environment tags in
- these formats don't contain braces.
-\end{doc}
-
-\begin{doc}[
- pi={\raggedright
- \docaux{cmd}{collargsAppendPreprocessor},
- \docaux{cmd}{collargsPrependPreprocessor},
- \docaux{cmd}{collargsAppendPostprocessor},
- \docaux{cmd}{collargsPrependPostprocessor}},
- ]{
- key={name=append preprocessor, par=\meta{code}, desc={no default}},
- key={name=prepend preprocessor, par=\meta{code}, desc={no default}},
- key={name=append postprocessor, par=\meta{code}, desc={no default}},
- key={name=prepend postprocessor, par=\meta{code}, desc={no default}},
- }
- These keys declare processors which will transform the collected argument
- before appending it to the argument list.
-
- A collected argument undergoes the following transformations:
- \begin{itemize}
- \item First, the argument is processed by any \emph{pre}processors, in the
- order indicated by |append| and |prepend|.
- \item Next, the processed argument is dressed up in the delimiters according
- to its type. For example, an optional argument of type \docref{xparse:o}
- will be surrounded by square brackets.
- \item Finally, the delimited argument is processed by any
- \emph{post}processors, again in the order indicated by |append| and
- |prepend|.
- \end{itemize}
-
- \meta{code} will typically consist of a single control sequence pointing to a
- one-argument macro, which will receive the collected argument (possibly
- modified by the processors already applied). In general, however, the value
- of this key may be any code; Collargs will execute \meta{code}\marg{collected
- argument}.
-
- The processed argument should be returned by storing it into token register
- \docaux{cmd}{collargsArg}.
-
- The following example illustrates how one could go about reimplementing Bruno
- Le Floch ingenious package \pkg{cprotect}.\footnote{This example is merely a
- proof of concept. For the bells and whistles which would make it useful in
- real life, see the documentation of \pkg{cprotect}.} We define processor
- |\writetofile| which dumps the argument into a file, replacing it with the
- |\input| statement. (Of course, to allow for verbatim content in the
- footnote, we also have to mark the argument as \refcollargs{verbatim}.
- And we use \refcollargs{no delimiters} to get rid of the braces around the
- footnote text.)
-
- \ExampleName{collargs-processor}
- \makeexample{\examplename.tex.c1}
- \tcbinputexample{listing and text, listing file=\examplepath.tex.c1}
-\end{doc}
-
-\begin{doc}[
- pi={\docaux{cmd}{collargsClearPreprocessors},
- \docaux{cmd}{collargsClearPostprocessors}},
- ]{
- key={name=clear preprocessors},
- key={name=clear postprocessors},
- }
- Clear the list of pre- or post-processors.
-\end{doc}
-
-\begin{doc}[
- pi={\docaux{cmd}{collargsAppendExpandablePreprocessor},
- \docaux{cmd}{collargsPrependExpandablePreprocessor},
- \docaux{cmd}{collargsAppendExpandablePostprocessor},
- \docaux{cmd}{collargsPrependExpandablePostprocessor}
- },
- ]{
- key={name=append expandable preprocessor, par=\meta{code}, desc={no default}},
- key={name=prepend expandable preprocessor, par=\meta{code}, desc={no default}},
- key={name=append expandable postprocessor, par=\meta{code}, desc={no default}},
- key={name=prepend expandable postprocessor, par=\meta{code}, desc={no default}},
- }
- These keys may be used to add fully expandable processors. A processor added
- with one of these keys will end up among the processors declared by
- \refcollargs{append preprocessor} et al.
-
- A processor declared by one of these keys will define the processed argument
- as the full expansion (|\edef|) of \meta{code}\marg{collected argument}.
- \meta{code} will typically consist of a single control sequence pointing to a
- fully expandable one-argument macro.
-
- For example, |\trim at spaces@noexp| from package \pkg{trimspaces} could be used
- as an expandable processor of environment body to remove the spaces around
- the grabbed environment body.
-\end{doc}
-
-\begin{doc}[
- pi={\docaux{cmd}{collargsAppendPrewrap},
- \docaux{cmd}{collargsPrependPrewrap},
- \docaux{cmd}{collargsAppendPostwrap},
- \docaux{cmd}{collargsPrependPostwrap}},
- ]{
- key={name=append prewrap, par=\meta{macro definition}, desc={no default}},
- key={name=prepend prewrap, par=\meta{macro definition}, desc={no default}},
- key={name=append postwrap, par=\meta{macro definition}, desc={no default}},
- key={name=prepend postwrap, par=\meta{macro definition}, desc={no default}},
- }
- These keys add processors which transform the collected argument in a single
- expansion.
-
- The declared processor will use \meta{macro definition} to define a temporary
- one-argument \meta{macro}, and then set the \meta{processed argument} to be
- the single expansion of \meta{macro}\marg{collected argument}.
-
- For example, to add quotes around the collected argument, write
- \refcollargs{append prewrap}|={``#1''}| (doubling the hash when executing
- \refcmd{CollectArguments} from a macro, of course). Or, perhaps more
- usefully, \refcollargs{append prewrap}|={\scantokens{#1}}| can be used to
- retokenize a verbatim argument (during the execution of the
- \meta{next-code}).
-\end{doc}
-
-
-\begin{doc}[
- pi=\docaux{cmd}{ifcollargsNoDelimiters}
- ]{
- key={name=no delimiters, conditional=false},
- }
- When this key is in effect, the collected argument will not be dressed up
- into delimiters that it was dressed up in \meta{argument tokens}. For
- example, an optional argument, encountered as [\meta{argument}] inside
- \meta{argument tokens}, will be spit out simply as \meta{argument}.
-
- Any user-specified pre- or post-processing will still be applied.
- \ExampleName{collargs-nodelimiters}
- \makeexample{\examplename.tex.c1}
- \tcbinputexample{comment=\input{\examplepath.tex.c1}}
-\end{doc}
-
-
-\begin{doc}[
- pi={\docaux{cmd}{collargsVerbatim},
- \docaux{cmd}{collargsVerb},
- \docaux{cmd}{collargsNoVerbatim}.
- To ensure the same effect as with the keys,
- place these macros at the end of the option code},
- ]{
- key={name=verbatim, desc=style},
- key={name=verb, desc=style},
- key={name=no verbatim, desc=style},
- }
- Select the full verbatim, the partial verbatim, or the non-verbatim mode of
- argument collection.
-
- In the full \refcollargs{verbatim} mode, the arguments are collected under
- a category code regime in which all characters are of category 12, ``other''.
- The same goes for the partial \refcollargs{verb} mode, except that in this
- case, the grouping characters --- usually the braces |{| and |}| --- retain
- their usual category codes 1 and 2. Key \refcollargs{no verbatim} selects
- the normal, non-verbatim mode.
-
- The partial \refcollargs{verb} mode can be useful for verbatim collection
- of an optional argument. To pass |]| as an optional argument to command
-|\foo|, we normally enclose it in braces: ˇ\foo[{~]~}]ˇ. However, if we try to
-collect |[{]}]| with \refcmd{CollectArguments}|[|\refmmz{verbatim}|]{o}|, we
-will get |{| (and most likely an error, as well), because in the
- \refmmz{verbatim} mode, braces do not have their grouping function. Using
- the \refmmz{verb} mode solves the problem: occuring within braces, the first
- |]| is ``invisible'' to \refcmd{CollectArguments}|[|\refmmz{verb}|]|, so the
-optional argument is correctly recognized as ending at the second |]|.
-
-The partial \refcollargs{verb} mode is also useful for collecting the bodies
-of \emph{\hologo{LaTeX}} environments. The full \refcollargs{verbatim} mode
-will only correctly collect these bodies when the relevant \refcmd{begin}
-and\slash or \refcmd{end} control sequences are followed by the grouped
-environment name without any intervening spaces. The partial
-\refcollargs{verb} mode has no such restriction.
-
-In the verbatim modes, modifier \docref{xparse:+} has no effect. The arguments
-are always collected as if they were long.
-
-To correctly collect arguments in the verbatim modes, CollArgs has to mimic the
-many details of \hologo{TeX}'s tokenization and argument delineation. These
-details depend on the category code regime, and CollArgs automatically adapts
-to the ``outside'' category code regime, i.e.\ the regime in effect at the time
-of invoking \refcmd{CollectArguments}. In particular, CollArgs remembers which
-characters were of category codes 0, 1, 2, 5, 10 and 11, and adapts the
-argument collection accordingly. For example, it will correctly pick up a
-control sequence as a single-token \docref{xparse:m}-type (\hologo{TeX}'s undelimited)
-argument even when it begins with a non-standard character of category code 0.
-The single caveat is that only a single pair of characters can function as the
-grouping characters in the full verbatim mode; to compensate for the
-deficiency, this character pair is customizable via key
-\refcollargs{braces}.
-\end{doc}
-
-\begin{doc}[
- pi={\docaux{cmd}{collargsFixFromVerbatim},
- \docaux{cmd}{collargsFixFromVerb},
- \docaux{cmd}{collargsFixFromNoVerbatim}},
- ]{
- key={name=fix from verbatim, desc=style},
- key={name=fix from verb, desc=style},
- key={name=fix from no verbatim, desc=style},
- }
- Key \refcollargs{fix from no verbatim} should be used when the first
- argument should be collected in a verbatim mode, but the outside code has
- already tokenized the first character of the subsequent input stream (most
- probably by a |\futurelet|) in the non-verbatim category code regime. Using
- this key will trigger a CollArgs' ``mode transition'' (described below) which
- will fix the situation. (This key is used in the implementation of
- \refcmd{mmz}.)
-
- The other two keys should be used in the unlikely reverse situation, where
- the outside code has tokenized the following character in a verb(atim) mode,
- while CollArgs is requested to collect the first argument in the non-verbatim
- mode.
-\end{doc}
-
-\begin{doc}[
- pi=\docaux{cmd}{collargsBraces}
- ]{
- key={name=braces, par={\meta{begin-group char}\meta{end-group char}},
- desc={no default, for details see below}},
- }% here
- This key sets the verbatim begin-group and end-group characters. The setting
- affects collection of argument types \docref{xparse:m}, \docref{xparse:g},
- \docref{xparse:v}, \docref{xparse:l}, \docref{xparse:e} and (in
- \hologo{LaTeX}) \docref{xparse:b} in the \emph{full} verbatim
- mode.\footnote{The choice of the verbatim grouping characters also affects
- the effect of \refcollargs{begin tag} and\slash or \refcollargs{end tag};
- see the documentation of these keys for details.}
-
- For example, in the non-verbatim and the partial verbatim mode, an
- \docref{xparse:m}-type argument may be delimited by any characters of
- category code 1 (``begin-group'') and 2 (``end-group''). In the full
- verbatim mode, there are of course no characters of these categories, so
- CollArgs internally assigns the grouping function to some pair of characters.
- When entering the full verbatim mode, CollArgs automatically sets the
- verbatim grouping characters to characters which were of categories 1 and 2
- in the ``outside'' category code regime, i.e.\ the regime in effect at the
- time of invoking \refcmd{CollectArguments}. However, in contrast to
- \hologo{TeX}'s internal argument parser, only one pair of characters may
- serve as the begin-group and the end-group character in CollArgs' full
- verbatim mode. In case multiple characters were of category 1 or 2 on the
- outside, CollArgs therefore has to make a choice, and it chooses the
- candidate with the lowest character code. This choice may be overridden by
- the user by invoking key \refcollargs{braces}; the user may even choose
- characters which did not belong to categories 1 and 2 in the outside regime.
-
- When \meta{begin-group char} and \meta{end-group char} are of categories 1
- and 2 in the outside category regime, they must be enclosed in a triple
- group. For example, if both |()| and |{}| have the grouping function on the
- outside, and the user wants to select |{}| as the verbatim grouping
- characters (CollArgs would go for |()|, as this pair has lower character
- codes), the correct way to invoke this key is |braces={{{{}}}}| or
- |braces=((({})))|. \footnote{This complication is due to the details of
- \pkg{pgfkeys}' keylist processing, and does not apply to
- \refcmd{collargsBraces}.}
-\end{doc}
-
-\begin{doc}[
- pi=\docaux{cmd}{collargsVerbatimRanges}
- ]{
- key={name=verbatim ranges,
- par={\bracestt{\meta{from}\texttt{-}\meta{to}(\texttt{,} \meta{from}\texttt{-}\meta{to})*}},
- desc={no default, initially \texttt{0-255}}},
- }
- If run under the \hologo{pdfTeX} or \hologo{XeTeX} engine, this key
- determines which characters will be assigned category code 12 in the verbatim
- mode. In \hologo{pdfTeX}, the range should remain at the initial |0-255|,
- but in \hologo{XeTeX}, some rare situations might require extending this
- range (don't attempt to set the full range of |0-1114111|, as this would be
- very slow and you would most likely run out of save stack).
-
- In \hologo{LuaTeX}, we switch the category code regime using category code
- tables, so this key has another meaning: it determines the range in which
- CollArgs will scan for characters of category codes 1, 2 and 14, whose
- identity it needs to know, for internal reasons.
-\end{doc}
-
-
-
-\paragraph{Mode transition limitations}
-
-\refcmd{CollectArguments} has some minor limitations regarding the transition
-from a verbatim into non-verbatim mode, or vice versa. The gist of the issue
-is best illustrated with the optional argument type \docref{xparse:o} collected
-in the verbatim mode. CollArgs determines whether an argument of this type is
-present by peeking ahead (using \hologo{TeX}'s |\futurelet| primitive) into the
-input stream. If the argument is present (i.e.\ if the input stream continues
-with an open bracket, |[|%]
-), all is well. But when the optional argument is absent, the peek-ahead will
-tokenize the following character, which presents a problem when no more
-arguments are present in the input stream, like in the example below, where the
-verbatim \docref{xparse:o} is the (only and) final type in the argument
-specification. In this case, the peek-ahead ``incorrectly'' assigns category
-code 12 (``other'') to the first |$|. %$
-This character was intended to be tokenized as the math shift character of
-category 3, to start the math mode after \refcmd{CollectArguments} is finished,
-but having been assigned category code 12, it cannot perform this function,
-resulting in error |! Missing $ inserted| %$
-once \hologo{TeX} encounters the superscript character |^|.
-
-\ExampleName{collargs-transition-ok}
-\makeexample{\examplename.tex.c1}
-\tcbinputexample{sidebyside, lefthand ratio=0.6,
- comment=\input{\examplepath.tex.c1}}
-
-Well --- this is what \emph{would} happen if CollArgs didn't address the
-transition issue described above. In fact, the above example compiles just
-fine, because CollArgs \emph{does} address this issue, but unfortunately,
-certain transition problems simply cannot be resolved --- read on to learn what
-can go wrong.
-
-For example, you can typeset the name of the document author via
-\hologo{LaTeX}'s internal command |\@author|, but to use this command in the
-document, you have to precede it by |\makeatletter|. As shown by the first
-line of the example below, this works rather nicely: |\makeatletter| sets the
-category code of |@| to 11 (``letter''), so |@| may help form the control word
-|\@author| --- importantly, |\makeatletter| sets the category code of |@|
-\emph{before} control sequence |\@author| is constructed, even if it precedes
-it immediately.
-
-\ExampleName{collargs-transition-cs}
-\makeexample{\examplename.tex.c1}
-\long\def\ShowArgs#1{Collected: ``{\color{red}\tt\detokenize{#1}}''}
-\tcbinputexample{comment=\input{\examplepath.tex.c1}}
-
-In the second line of the example, our clever invocation of |\@author| is
-immediately preceded by a call to \refcmd{CollectArguments}, which tries to
-collect a verbatim argument of type \docref{xparse:o}. It doesn't find it, which results in
-the wrong, verbatim tokenization of the escape character of |\makeletter|.
-CollArgs realizes the problem and tries to fix it. But while it is searching
-for the end of control sequence |\makeletter| (which it successfully
-constructs), it triggers the tokenization of what follows --- which, as |@| is
-at that point of category 12 (``other''), yields the control symbol |\@| (later
-followed by word ``author'', typeset in the example).
-
-In short, the solution has created another, delayed instance of the problem ---
-an instance which cannot be addressed any further. But we're nevertheless
-better off, as this particular issue will bite only in the case when the
-``corrupted'' control sequence immediately following the invocation
-\refcmd{CollectArguments} changes category codes in a way that affects the
-tokenization of what immediately follows it.
-
-This was an example of what can go wrong in the transition from the
-\meta{argument tokens} to the \meta{rest} of the tokens following an invocation
-of \refcmd{CollectArguments}. As the ``outside world'' is non-verbatim, this
-transition can only be problematic if the argument which corrupted the first of
-the \meta{rest} tokens was verbatim, so if the transition was from the verbatim
-to the non-verbatim mode. Such a transition can also occur within the
-\meta{argument tokens}, but the good news here is that CollArgs successfully
-solves any problems that occur there, so you should only worry about the
-end-of-arguments situation.
-
-The other direction of the transition, from the non-verbatim to the verbatim
-mode, however, can affect both the internal and the external transitions. Let
-us illustrate the problem with the internal transition. Say you want to
-collect an optional argument (\docref{xparse:o}) in the non-verbatim mode, and
-then a mandatory argument (\docref{xparse:m}) in (either full or partial)
-verbatim mode. In the first invocation of \refcmd{CollectArguments} below, the
-optional argument is present, and we get what we expect: the percent sign is
-collected, verbatim, into the mandatory argument. In the second invocation,
-however, the percent character retains its usual commenting function ---
-despite the fact that we have requested verbatim mode for the mandatory
-argument --- which results in the group in the second line being picked up as
-the mandatory argument. Again, this happens because CollArgs has to peek ahead
-in the input stream when determining whether the optional argument is present.
-Having requested non-verbatim mode for the optional argument, the peeking is
-performed in the non-verbatim mode, and as the optional argument is not
-present, it finds the comment character, which fulfills its regular function of
-disappearing along with the rest of the line. Once CollArgs sets to find the
-(verbatim) mandatory argument, the rest of the line is already gone, so it
-searches for, and finds, this argument in the next line.
-
-\ExampleName{collargs-transition-comment}
-\makeexample{\examplename.tex.c1}
-\long\def\ShowArgs#1{Collected: ``{\color{red}\tt\detokenize{#1}}''}
-\tcbinputexample{listing and text}
-
-Nothing can be done here --- commenting deletes information, irrevocably ---
-and in a similar fashion, nothing can be done to catch a verbatim end-of-line
-character when preceded by an absent optional argument in the non-verbatim mode
-(because it was already tokenized into a space token).
-
-Finally, the transition issues are not limited to transits from argument type
-\docref{xparse:o}. The full list of argument types which give rise to
-transition problems (when transiting \emph{from} arguments of these types) is
-as follows: \docref{xparse:o}, \docref{xparse:d}, \docref{xparse:s},
-\docref{xparse:t}, \docref{xparse:g}, \docref{xparse:e}.
-
-
-\endgroup % /collargs
-
-
-\section{Varia}
-
-\subsectionclearpagefalse
-
-\subsection{Known issues}
-\label{sec:known-issues}
-
-\paragraph*{Bitmap graphics export} is coming up in the next release of the
-package.
-
-
-\paragraph{Disappearing errors}
-If a non-fatal internal \hologo{TeX} error occurs during memoization, the memos
-and externs may be nevertheless produced and utilized in subsequent
-compilations. In such a case, the erroneous code won't be compiled again, and
-therefore won't yield any errors, giving the mistaken impression that the code
-is error-free.
-
-This problem does not apply to errors which trigger \refcmd{errmessage},
-because that control sequence is advised by \refmmzauto{abort}.
-Note that internal \hologo{TeX} errors like |Undefined control sequence| are
-\emph{not} reported through \refcmd{errmessage}, and will therefore cause the
-issue.
-
-This problem does not affect \hologo{LuaTeX}, because this engine allows
-Memoize to detect errors and abort memoization if it encounters any.
-
-
-\paragraph{A minimal issue with \hologo{XeLaTeX}}
-If the very first page of a document of class |minimal|, compiled by
-\hologo{XeLaTeX}, happens to be an extern page, we have a problem: all the
-regular pages of the document will be of the same size as that extern page.
-\hologo{pdfLaTeX} and \hologo{LuaLaTeX} do not exhibit this behaviour, nor do
-\hologo{LaTeX} classes other than |minimal|, even when compiled with
-\hologo{XeLaTeX}.
-
-
-\paragraph{CollArgs}
-
-Due to an unfortunate design decision, CollArgs does not accept a dot |.| as
-the \meta{token} argument of types, \docref{xparse:r}, \docref{xparse:R},
-\docref{xparse:d}, \docref{xparse:D}, and \docref{xparse:t}. Furthermore,
-\refcollargs{append prewrap} and friends (and their macro counterparts) do not
-accept a parameter symbol consistently. These issues will be fixed in the next
-release.
-
-
-\subsection{Troubleshooting}
-\label{sec:troubleshooting}
-
-\paragraph{Extern extraction does not work}
-\label{sec:fadings-problem}
-\label{sec:loading-order}
-
-If the internal extern extraction does not work, you should have gotten one of
-the following warnings:
-\begin{enumerate}
-\item \code{Package memoize Warning: Extraction of externs from document
- "\meta{jobname}.pdf" using method "perl" was unsuccessful. Have you set the
- shell escape mode as suggested in chapter 1 of the manual?}
-
- This warning is thrown when the extraction log \meta{jobname}|.mmz.log|,
- which should be produced by the extraction script, is incomplete, i.e.\ it
- does not finish by |\endinput|.\footnote{The extraction log will never be
- absent, because Memoize clears it before executing the extraction script
- --- and clearing it creates it if it doesn't exist.} The most likely
- reason is that the script was not executed at all, so the question asked in
- the warning message is pertinent. However, if all is well with the
- shell-mode, you have a mystery at your hands. To gather information, I
- suggest inspecting:
- \begin{itemize}
- \item \meta{jobname}|.log| --- search for \code{runsystem(memoize-extract.pl
- ...)}, it will tell you whether the script was executed;
- \item the terminal output --- if the script was executed, it should've
- announced itself by \code{Extracting externs from \meta{jobname}.pdf}; are
- there any further messages between this header and the warning message?
- \item \meta{the path to the extern}|.log|, if you are using the
- \hologo{TeX}-based extraction (\code{\refmmz{extract}=\refmmz{extract=tex}}).
- \end{itemize}
- \hologo{TeX}-based extraction will yield this error even in the case of a
- missing or corrupted PDF, i.e.\ in
- case~\ref{item:warning:extract:cannot-read} below.
-
-\item\label{item:warning:extract:cannot-read}
- \code{Package memoize (perl-based extraction) Warning: Cannot read file
- "\meta{jobname}.pdf". Perhaps you have to load Memoize earlier in the
- preamble?}
-
- Embedded extern extraction requires an intact document PDF from the previous
- compilation, so Memoize must be loaded before the document PDF is opened for
- writing the results of the ongoing compilation. In particular, the PDF is
- opened by \PGF library |fadings|, included by \TikZ's libraries |fadings|
- and |shadows|, so Memoize must be loaded before any of these libraries.
-
- With \pkg{beamer}, the problem is particularly acute because the PDF is
- opened while loading the class. In this case, simply moving
- |\usepackage{memoize}| up the preamble, as suggested, won't help: you have to
- write |\RequirePackage{memoize}| \emph{before} |\documentclass{beamer}|!
-
-\item\label{item:warning:extract:unexpected-size} \code{Package memoize
- (perl-based extraction) Warning: I refuse to extract page $n$ from
- "\meta{jobname}.pdf", because its size ($\meta{width}\times\meta{height}$)
- is not what I expected\\ ($\meta{expected width}\times\meta{expected
- height}$)}
-
- If the compilation which produced the offending extern pages yielded any
- errors, you should probably disregard this warning, fix the errors, and
- compile again. Otherwise, you have somehow winded up with mismatched
- \meta{jobname}|.pdf| and \meta{jobname}\dmmz (the latter file contains
- instructions on which pages to extract, complete with the expected
- dimensions). Are you sure that they were produced by the same compilation,
- and have remained untouched since? Are you perhaps trying to perform the
- extraction the second time, after the first extraction
- \refscript{memoize-extract.pl--prune}d the PDF?
-
- If the warning stubbornly persists, but you are sure that the page the script
- is refusing to extract is correct, you can force the extraction by adding
- option \refscript{memoize-extract.pl--force} to the script invocation, which
- can be set by \refmmz{perl extraction options}. However, as such a situation
- probably indicates a bug in Memoize, please let me know about it.
-\item\code{\myfontexpansionon Invalid dictionary key at
- /.../perl5/site\_perl/PDF/API2/Basic/PDF/File.pm line $N$\myfontexpansionoff}
-
- This can sometimes happen when extracting with the Perl-based script. The
- issue might be related to PDF version --- by default, \hologo{TeX} produces
- PDFs of version 1.5, while the PDF library
- \hreftt{https://metacpan.org/pod/PDF::API2}{PDF::API2} officially only
- supports versions up to 1.4 --- but I'm afraid I haven't identified the exact
- circumstances yet. Possibly, the externalizing a picture containing an
- embedded PDF file might be the culprit. The workaround is to use Python- or
- \hologo{TeX}-based extraction, i.e.\ load Memoize with package option
- \refmmz{extract}|=|\refmmz{extract=python} or
- \refmmz{extract}|=|\refmmz{extract=tex}.
-\end{enumerate}
-
-The failure to extract externs intentionally doesn't result in an error,
-because what if you deleted the document PDF on purpose? At any rate, the
-compilation will succeed, it's just that as Memoize cannot extract the externs,
-they will be produced, and dumped into your document, at each and every
-compilation.
-
-\paragraph{An extern won't be included}
-
-Did you receive a warning or error message?
-\begin{enumerate}
-\item \code{Package memoize Warning: Unexpected size of extern "\meta{extern
- path}.pdf"; expected\\ $\meta{expected width}\times\meta{expected
- height}$, got $\meta{width}\times\meta{height}$}
-
- This warning is related to warning~\ref{item:warning:extract:unexpected-size}
- above, only that it occurs once the extern is extracted. The same
- investigative methods apply.
-\item \code{!pdfTeX error: pdflatex (file \meta{extern path}.pdf): reading
- image file failed}
-
- This is a fatal error. The extern file got corrupted, somehow ---
- inexistent, or even empty, extern files merely trigger recompilation.
-\end{enumerate}
-
-If there was no warning or error --- are you certain that Memoize is
-\refmmz{enable}d, and that it is not in the \refmmz{recompile} mode? Remember
-that these settings can also apply only to a part of the document; search for
-any stray \refcmd{mmzset} or \refcmd{mmznext} commands.
-
-
-\paragraph*{Warnings about duplicate labels, indices, etc.} may be safely
-disregarded.
-
-Externalization causes any (non-immediate) |\write| commands in the extern to
-be executed twice, once upon the shipout of the regular page, and once upon the
-shipout of the extern page. This results in warnings about doubly defined
-labels, hyperreferences, indices, etc. For example, you might get
-\texttt{LaTeX Warning: Label `<name>' multiply defined} or \texttt{warning (pdf
- backend): ignoring duplicate destination with the name '<name>'}. You can
-safely disregard these warnings; they will disappear once the extern is
-utilized.
-
-
-\paragraph{Memoization was aborted}
-
-This warning means that either:
-\begin{itemize}
-\item you are trying to (auto)memoize a \refenv{tikzpicture} with
- \refkey{/tikz/remember picture} set, or more generally, some code which
- contains \refcmd{savepos} --- this can't be done, see
- section~\ref{sec:tut:good-to-know}; or
-\item an error occurred during memoization --- in this case, Memoize cowardly
- refuses to proceed with memoization, see section~\ref{sec:known-issues} for
- details.
-\end{itemize}
-
-
-\subsection{License}
-
-Copyright \textcopyright\ 2020- Sašo Živanović.
-
-This work may be distributed and/or modified under the conditions of the
-\hologo{LaTeX} Project Public License, either version 1.3c of this license or
-(at your option) any later version. The latest version of this license is
-in \url{https://www.latex-project.org/lppl.txt} and version 1.3 or later is
-part of all distributions of \hologo{LaTeX} version 2008 or later.
-
-This work comprises of the sources, generated files, accompanying scripts,
-auxiliary files and scripts, documentation, documentation sources, examples and
-example sources of packages |memoize|, |nomemoize|, |memoizable|, |auto| and
-|collargs|. The files belonging to this work and covered by LPPL are listed in
-\meta{texmf}|/doc/generic/memoize/FILES|.
-
-This work has the LPPL maintenance status `maintained.' The Current Maintainer
-of this work is Sašo Živanović. The work is available on CTAN
-at \url{https://ctan.org/pkg/memoize} and on GitHub
-at \url{https://github.com/sasozivanovic/memoize}.
-
-
-\subsection{Changelog}
-
-\newcommand\githubrelease[2]{%
- \href{https://github.com/sasozivanovic/memoize/releases/tag/#2}{#1 #2}%
-}
-\begin{description}
-\item[\githubrelease{2020/07/17}{v0.1}]
- The proof-of-concept, available only at GitHub.
-\item[\githubrelease{2023/10/10}{v1.0.0}]
- A complete, fully documented reimplementation.
-\end{description}
-
-\subsection{Acknowledgments}
-
-First and foremost, my gratitude goes to Stefan Müller of Language Science
-Press for his encouragement and patience --- without him, this package might've
-remained an idea forever!
-
-But there were others as well. Shunsaku Hirata of the \hologo{XeTeX} team
-promptly resolved an issue with forward references in |xdvipdfmx|, and thereby
-made Memoize work with \hologo{XeTeX}. Joseph Wright kindly didn't object to
-me misappropriating \pkg{etoolbox} for other formats. Petra Rübe-Pugliese
-(CTAN) and Karl Berry (\hologo{TeX}Live) offered good advice on packaging the
-package; I would never get to call Advice Advice without their advice. Karl
-also performed a security review of the extraction scripts, providing some very
-useful comments along the way, and agreed to include the scripts among the
-shell escape commands allowed in \hologo{TeX}Live.
-
-Thank you all!
-
-
-\printindex
-
-\end{document}
-
-
-% Local Variables:
-% TeX-engine: luatex
-% TeX-master: t
-% End:
Modified: trunk/Master/texmf-dist/doc/man/man1/memoize-clean.1
===================================================================
--- trunk/Master/texmf-dist/doc/man/man1/memoize-clean.1 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/man/man1/memoize-clean.1 2024-01-03 21:18:51 UTC (rev 69285)
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pandoc 3.0.1
+.\" Automatically generated by Pandoc 3.1.5
.\"
.\" Define V font for inline verbatim, using C font in formats
.\" that render this, and otherwise B font.
@@ -14,7 +14,7 @@
. ftr VB CB
. ftr VBI CBI
.\}
-.TH "memoize-clean" "1" "October 10, 2023" "memoize-clean of Memoize v1.0.0" "User Manual"
+.TH "memoize-clean" "1" "January 02, 2024" "memoize-clean of Memoize v1.1.0" "User Manual"
.nh
.SH NAME
.PP
Modified: trunk/Master/texmf-dist/doc/man/man1/memoize-clean.man1.pdf
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/doc/man/man1/memoize-clean.pl.man1.pdf
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/doc/man/man1/memoize-clean.py.man1.pdf
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/doc/man/man1/memoize-extract.1
===================================================================
--- trunk/Master/texmf-dist/doc/man/man1/memoize-extract.1 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/doc/man/man1/memoize-extract.1 2024-01-03 21:18:51 UTC (rev 69285)
@@ -1,4 +1,4 @@
-.\" Automatically generated by Pandoc 3.0.1
+.\" Automatically generated by Pandoc 3.1.5
.\"
.\" Define V font for inline verbatim, using C font in formats
.\" that render this, and otherwise B font.
@@ -14,7 +14,7 @@
. ftr VB CB
. ftr VBI CBI
.\}
-.TH "memoize-extract" "1" "October 10, 2023" "memoize-extract of Memoize v1.0.0" "User Manual"
+.TH "memoize-extract" "1" "January 02, 2024" "memoize-extract of Memoize v1.1.0" "User Manual"
.nh
.SH NAME
.PP
@@ -47,18 +47,19 @@
Therefore, after compiling \f[I]document.tex\f[R], the externs should be
extracted by \f[B]memoize-extract\f[R] \f[I]document.mmz\f[R].
.PP
+\f[I]document.mmz\f[R] may also be given as document* or
+\f[I]document.tex\f[R].
+When environment variable \f[I]TEXMF_OUTPUT_DIRECTORY\f[R] is set, this
+filename is relative to the output directory specified by this variable.
+.PP
\f[I]document.mmz\f[R] also records the expected width and height of
each extern.
In case of a mismatch, \f[B]memoize-extract\f[R] refuses to extract the
page and removes the extern file if it already exist, and prints a
warning message to the standard error.
-The script furthermore refuses to extract the page if a (c)c-memo
-associated to the extern does not exist, and to write to any file whose
-absolute path does not occur under the current directory or the
-directory set by TEXMFOUTPUT (in \f[I]texmf.cnf\f[R] or as an
-environment variable); TEXMFOUTPUT is also not allowed to point to the
-root directory, except on Windows, where it can only point to a drive
-root.
+The script also refuses to extract the page if a (c)c-memo associated to
+the extern does not exist.
+See also section SECURITY.
.PP
The Perl (.pl) and the Python (.py) version of the script are
functionally equivalent.
@@ -79,33 +80,78 @@
extracted.
By default, they are commented out to prevent double extraction.
.TP
+\f[B]-F, --format\f[R] \f[I]latex\f[R]|\f[I]plain\f[R]|\f[I]context\f[R]
+When this option is given, the script assumes that it was called from
+within a TeX compilation of a document in the given format: it prefixes
+all output by the script name, and creates a log file
+\f[I]document.mmz.log\f[R], which receives any extraction-related
+warnings and errors.
+.TP
\f[B]-f, --force\f[R]
Extract the extern even if the size-check fails.
.TP
-\f[B]-l, --log\f[R] \f[I]filename\f[R]
-Log size mismatches to \f[I]filename\f[R] rather than to the standard
-error.
-.TP
-\f[B]-w, --warning-template\f[R] \f[I]string\f[R]
-Wrap the size mismatch warning text into the given template:
-\[rs]warningtext in the template will be replaced by the warning text.
-.TP
\f[B]-q, --quiet\f[R]
Don\[cq]t describe what\[cq]s happening.
.TP
-\f[B]-e, --embedded\f[R]
-Prefix all messages to the standard output with the script name.
-.TP
\f[B]-m, --mkdir\f[R]
A paranoid \f[I]mkdir -p\f[R].
(No extraction occurs, \f[I]document.mmz\f[R] is interpreted as a
-directory name.)
+directory name, which may end in any suffix; no suffix mangling is
+performed.)
.TP
+\f[B]-V, --version\f[R]
+Show the Memoize version number and exit.
+.TP
\f[B]-h, --help\f[R]
Show help and exit.
+.SH SECURITY
+.PP
+This script respects the restrictions on file input and output imposed
+by the TeX configuration, more precisely, the variables
+\f[I]openin_any\f[R] and \f[I]openout_any\f[R] of the \f[B]kpathsea\f[R]
+library (https://tug.org/kpathsea).
+You can inspect the values of these variables by executing
+`\f[B]kpsewhich\f[R] -var-value=openin_any' and `\f[B]kpsewhich\f[R]
+-var-value=openout_any'.
+The interpretation is as follows:
.TP
-\f[B]-V, --version\f[R]
-Show the Memoize version number and exit.
+\f[B]a\f[R] (or \f[B]y\f[R] or \f[B]1\f[R]) any
+Allows any file to be opened.
+.TP
+\f[B]r\f[R] (or \f[B]n\f[R] or \f[B]0\f[R]) restricted
+Means disallowing special file names.
+.TP
+\f[B]p\f[R] (or any other value) paranoid
+Means being really paranoid: disallowing special file names and
+restricting input/output files to be in or below the working directory
+or the directory specified by \f[I]TEXMFOUTPUT\f[R] or
+\f[I]TEXMF_OUTPUT_DIRECTORY\f[R].
+\f[I]TEXMFOUTPUT\f[R] may be set either in \f[I]texmf.cnf\f[R] (e.g.\ by
+\f[B]tlmgr\f[R]) or as an environment variable.
+\f[I]TEXMF_OUTPUT_DIRECTORY\f[R] may only be set as an environment
+variables; it is automatically set by TeX when called by
+\f[I]-output-directory\f[R] option (starting in TeXLive 2024).
+.SH EXIT STATUS
+.TP
+\f[B]0\f[R]
+The externs were successfully extracted.
+This exit code is returned even if no externs need to be extracted, or
+if \f[I]document.mmz\f[R] does not exist.
+.TP
+\f[B]10\f[R]
+A warning also reported back to the compilation when given option
+\f[B]--format\f[R].
+Currently, either: (i) size-mismatch; (ii) a non-existing associated
+(c)c-memo file; or (iii) unavailable \f[I]kpsewhich\f[R].
+.TP
+\f[B]11\f[R]
+An error also reported back to the compilation when given option
+\f[B]--format\f[R].
+Currently, either: (i) a currupted document PDF, or (ii) a kpathsea
+permission error.
+.PP
+Other exit codes are as produced by the underlying scripting language
+(Perl of Python).
.SH SEE ALSO
.PP
Memoize manual (https://ctan.org/pkg/memoize), section 6.6.1.
Modified: trunk/Master/texmf-dist/doc/man/man1/memoize-extract.man1.pdf
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/doc/man/man1/memoize-extract.pl.man1.pdf
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/doc/man/man1/memoize-extract.py.man1.pdf
===================================================================
(Binary files differ)
Modified: trunk/Master/texmf-dist/scripts/memoize/memoize-clean.pl
===================================================================
--- trunk/Master/texmf-dist/scripts/memoize/memoize-clean.pl 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/scripts/memoize/memoize-clean.pl 2024-01-03 21:18:51 UTC (rev 69285)
@@ -18,18 +18,19 @@
# The files belonging to this work and covered by LPPL are listed in
# <texmf>/doc/generic/memoize/FILES.
+my $PROG = 'memoize-clean.pl';
+my $VERSION = '2024/01/02 v1.1.0';
+
use strict;
use Getopt::Long;
-# I intend to rewrite this script using Path::Class.
use Cwd 'realpath';
use File::Spec;
use File::Basename;
-my $VERSION = '2023/10/10 v1.0.0';
-
-my $usage = "usage: memoize-clean.py [-h] [--yes] [--all] [--quiet] [--prefix PREFIX] [mmz ...]\n";
+my $usage = "usage: $PROG [-h] [--yes] [--all] [--quiet] [--prefix PREFIX] " .
+ "[mmz ...]\n";
my $Help = <<END;
-Remove (stale) memo and extern files.
+Remove (stale) memo and extern files produced by package Memoize.
positional arguments:
mmz .mmz record files
@@ -41,7 +42,8 @@
--all, -a Remove *all* memos and externs.
--quiet, -q
--prefix PREFIX, -p PREFIX
- A path prefix to clean; this option can be specified multiple times.
+ A path prefix to clean;
+ this option can be specified multiple times.
For details, see the man page or the Memoize documentation.
END
@@ -111,8 +113,11 @@
opendir(MD, $dir) or die "Cannot open directory '$dir'";
while( (my $fn = readdir(MD)) ) {
my $path = File::Spec->catfile(($dir),$fn);
- if ($fn =~ /\Q$basename_prefix\E[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(\.memo|(?:-[0-9]+)?\.pdf|\.log)/ and ($all || !exists($keep{$path}))) {
- push @tbdeleted, $path;
+ if ($fn =~
+ /\Q$basename_prefix\E[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?#
+ (\.memo|(?:-[0-9]+)?\.pdf|\.log)/x
+ and ($all || !exists($keep{$path}))) {
+ push @tbdeleted, $path;
}
}
closedir(MD);
@@ -161,3 +166,7 @@
} elsif (!$quiet) {
print "Nothing to do, the directory seems clean.\n";
}
+
+# Local Variables:
+# after-save-hook: pl2dtx
+# End:
Modified: trunk/Master/texmf-dist/scripts/memoize/memoize-clean.py
===================================================================
--- trunk/Master/texmf-dist/scripts/memoize/memoize-clean.py 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/scripts/memoize/memoize-clean.py 2024-01-03 21:18:51 UTC (rev 69285)
@@ -18,13 +18,14 @@
# The files belonging to this work and covered by LPPL are listed in
# <texmf>/doc/generic/memoize/FILES.
-__version__ = '2023/10/10 v1.0.0'
+__version__ = '2024/01/02 v1.1.0'
import argparse, re, sys, pathlib, os
parser = argparse.ArgumentParser(
description="Remove (stale) memo and extern files.",
- epilog = "For details, see the man page or the Memoize documentation."
+ epilog = "For details, see the man page or the Memoize documentation "
+ "(https://ctan.org/pkg/memoize)."
)
parser.add_argument('--yes', '-y', action = 'store_true',
help = 'Do not ask for confirmation.')
@@ -63,7 +64,8 @@
elif endinput:
raise RuntimeError(
- r'Bailing out, \endinput is not the last line of file $mmz_fn.')
+ rf'Bailing out, '
+ rf'\endinput is not the last line of file {mmz_fn}.')
elif m := re_prefix.match(line):
prefix = m[1]
@@ -109,7 +111,8 @@
def populate_tbdeleted(folder, basename_prefix):
re_aux = re.compile(
re.escape(basename_prefix) +
- '[0-9A-F]{32}(?:-[0-9A-F]{32})?(?:-[0-9]+)?(?:\.memo|(?:-[0-9]+)?\.pdf|\.log)$')
+ '[0-9A-F]{32}(?:-[0-9A-F]{32})?'
+ '(?:-[0-9]+)?(?:\.memo|(?:-[0-9]+)?\.pdf|\.log)$')
try:
for f in folder.iterdir():
if re_aux.match(f.name) and (args.all or f not in keep):
@@ -157,3 +160,8 @@
print("Bailing out.")
elif not args.quiet:
print('Nothing to do, the directory seems clean.')
+
+# Local Variables:
+# fill-column: 79
+# after-save-hook: py2dtx
+# End:
Modified: trunk/Master/texmf-dist/scripts/memoize/memoize-extract.pl
===================================================================
--- trunk/Master/texmf-dist/scripts/memoize/memoize-extract.pl 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/scripts/memoize/memoize-extract.pl 2024-01-03 21:18:51 UTC (rev 69285)
@@ -18,188 +18,757 @@
# The files belonging to this work and covered by LPPL are listed in
# <texmf>/doc/generic/memoize/FILES.
+my $PROG = 'memoize-extract.pl';
+my $VERSION = '2024/01/02 v1.1.0';
+
use strict;
+use File::Basename qw/basename/;
use Getopt::Long;
-use Path::Class;
-use File::Spec;
-use File::Basename;
-use PDF::API2;
+use File::Spec::Functions
+ qw/splitpath catpath splitdir rootdir file_name_is_absolute/;
+use File::Path qw(make_path);
+# We will only try to import the PDF processing library once we set up the error
+# log.
-my $VERSION = '2023/10/10 v1.0.0';
+# Declare variables for command-line arguments and for |kpathsea|
+# variables. They are defined here so that they are global in the subs which use
+# them.
+our ($pdf_file, $prune, $keep, $format, $force, $quiet,
+ $pdf_library, $print_version, $mkdir, $help,
+ $openin_any, $openout_any, $texmfoutput, $texmf_output_directory);
-my $usage = "usage: memoize-extract.pl [-h] [--pdf PDF] [--prune] [--keep] [--force] [--log LOG] [--warning-template WARNING_TEMPLATE] [--quiet] [--mkdir] mmz\n";
+# \paragraph{Messages}
+
+# The messages are written both to the extraction log and the terminal (we
+# output to stdout rather than stderr so that messages on the TeX terminal and
+# document |.log| appear in chronological order). Messages are automatically
+# adapted to the TeX |--format|.
+
+# The format of the messages. It depends on the given |--format|; the last
+# entry is for t the terminal output.
+
+my %ERROR = (
+ latex => '\PackageError{memoize (perl-based extraction)}{$short}{$long}',
+ plain => '\errhelp{$long}\errmessage{memoize (perl-based extraction): $short}',
+ context => '\errhelp{$long}\errmessage{memoize (perl-based extraction): $short}',
+ '' => '$header$short. $long');
+
+my %WARNING = (
+ latex => '\PackageWarning{memoize (perl-based extraction)}{$texindent$text}',
+ plain => '\message{memoize (perl-based extraction) Warning: $texindent$text}',
+ context => '\message{memoize (perl-based extraction) Warning: $texindent$text}',
+ '' => '$header$indent$text.');
+
+my %INFO = (
+ latex => '\PackageInfo{memoize (perl-based extraction)}{$texindent$text}',
+ plain => '\message{memoize (perl-based extraction): $texindent$text}',
+ context => '\message{memoize (perl-based extraction): $texindent$text}',
+ '' => '$header$indent$text.');
+
+# Some variables used in the message routines; note that |header| will be
+# redefined once we parse the arguments.
+
+my $exit_code = 0;
+my $log;
+my $header = '';
+my $indent = '';
+my $texindent = '';
+
+# The message routines.
+
+sub error {
+ my ($short, $long) = @_;
+ if (! $quiet) {
+ $_ = $ERROR{''};
+ s/\$header/$header/;
+ s/\$short/$short/;
+ s/\$long/$long/;
+ print(STDOUT "$_\n");
+ }
+ if ($log) {
+ $long =~ s/\\/\\string\\/g;
+ $_ = $ERROR{$format};
+ s/\$short/$short/;
+ s/\$long/$long/;
+ print(LOG "$_\n");
+ }
+ $exit_code = 11;
+ endinput();
+}
+
+sub warning {
+ my $text = shift;
+ if ($log) {
+ $_ = $WARNING{$format};
+ s/\$texindent/$texindent/;
+ s/\$text/$text/;
+ print(LOG "$_\n");
+ }
+ if (! $quiet) {
+ $_ = $WARNING{''};
+ s/\$header/$header/;
+ s/\$indent/$indent/;
+ s/\$text/$text/;
+ print(STDOUT "$_\n");
+ }
+ $exit_code = 10;
+}
+
+sub info {
+ my $text = shift;
+ if ($text && ! $quiet) {
+ $_ = $INFO{''};
+ s/\$header/$header/;
+ s/\$indent/$indent/;
+ s/\$text/$text/;
+ print(STDOUT "$_\n");
+ if ($log) {
+ $_ = $INFO{$format};
+ s/\$texindent/$texindent/;
+ s/\$text/$text/;
+ print(LOG "$_\n");
+ }
+ }
+}
+
+# Mark the log as complete and exit.
+sub endinput {
+ if ($log) {
+ print(LOG "\\endinput\n");
+ close(LOG);
+ }
+ exit $exit_code;
+}
+
+sub die_handler {
+ stderr_to_warning();
+ my $text = shift;
+ chomp($text);
+ error("Perl error: $text", '');
+}
+
+sub warn_handler {
+ my $text = shift;
+ chomp($text);
+ warning("Perl warning: $text");
+}
+
+# This is used to print warning messages from PDF::Builder, which are output to
+# STDERR.
+my $stderr;
+sub stderr_to_warning {
+ if ($stderr) {
+ my $w = ' Perl info: ';
+ my $nl = '';
+ for (split(/\n/, $stderr)) {
+ /(^\s*)(.*?)(\s*)$/;
+ $w .= ($1 ? ' ' : $nl) . $2;
+ $nl = "\n";
+ }
+ warning("$w");
+ $stderr = '';
+ }
+}
+
+# \paragraph{Permission-related functions}
+
+# We will need these variables below. Note that we only support Unix and
+# Windows.
+my $on_windows = $^O eq 'MSWin32';
+my $dirsep = $on_windows ? '\\' : '/';
+
+# |paranoia_in|/|out| should work exactly as
+# |kpsewhich -safe-in-name|/|-safe-out-name|.
+sub paranoia_in {
+ my ($f, $remark) = @_;
+ error("I'm not allowed to read from '$f' (openin_any = $openin_any)",
+ $remark) unless _paranoia($f, $openin_any);
+}
+
+sub paranoia_out {
+ my ($f, $remark) = @_;
+ error("I'm not allowed to write to '$f' (openin_any = $openout_any)",
+ $remark) unless _paranoia($f, $openout_any);
+}
+
+sub _paranoia {
+ # |f| is the path to the file (it should not be empty), and |mode| is the
+ # value of |openin_any| or |openout_any|.
+ my ($f, $mode) = @_;
+ return if (! $f);
+ # We split the filename into the directory and the basename part, and the
+ # directory into components.
+ my ($volume, $dir, $basename) = splitpath($f);
+ my @dir = splitdir($dir);
+ return (
+ # In mode `any' (|a|, |y| or |1|), we may access any file.
+ $mode =~ /^[ay1]$/
+ || (
+ # Otherwise, we are at least in the restricted mode, so we should
+ # not open dot files on Unix-like systems (except file called
+ # |.tex|).
+ ! (!$on_windows && $basename =~ /^\./ && !($basename =~ /^\.tex$/))
+ && (
+ # If we are precisely in the restricted mode (|r|, |n|, |0|),
+ # then there are no further restrictions.
+ $mode =~ /^[rn0]$/
+ # Otherwise, we are in the paranoid mode (officially |p|, but
+ # any other value is interpreted as |p| as well). There are
+ # two further restrictions in the paranoid mode.
+ || (
+ # We're not allowed to go to a parent directory.
+ ! grep(/^\.\.$/, @dir) && $basename ne '..'
+ &&
+ # If the given path is absolute, is should be a descendant
+ # of either |TEXMF_OUTPUT_DIRECTORY| or |TEXMFOUTPUT|.
+ (!file_name_is_absolute($f)
+ ||
+ is_ancestor($texmf_output_directory, $f)
+ ||
+ is_ancestor($texmfoutput, $f)
+ )))));
+}
+
+# Only removes final "/"s. This is unlike |File::Spec|'s |canonpath|, which also
+# removes |.| components, collapses multiple |/| --- and unfortunately also goes
+# up for |..| on Windows.
+sub normalize_path {
+ my $path = shift;
+ my ($v, $d, $n) = splitpath($path);
+ if ($n eq '' && $d =~ /[^\Q$dirsep\E]\Q$dirsep\E+$/) {
+ $path =~ s/\Q$dirsep\E+$//;
+ }
+ return $path;
+}
+
+# On Windows, we disallow ``semi-absolute'' paths, i.e.\ paths starting with the
+# |\| but lacking the drive. |File::Spec|'s function |file_name_is_absolute|
+# returns 2 if the path is absolute with a volume, 1 if it's absolute with no
+# volume, and 0 otherwise. After a path was sanitized using this function,
+# |file_name_is_absolute| will work as we want it to.
+sub sanitize_path {
+ my $f = normalize_path(shift);
+ my ($v, $d, $n) = splitpath($f);
+ if ($on_windows) {
+ my $a = file_name_is_absolute($f);
+ if ($a == 1 || ($a == 0 && $v) ) {
+ error("\"Semi-absolute\" paths are disallowed: " . $f,
+ "The path must either both contain the drive letter and " .
+ "start with '\\', or none of these; paths like 'C:foo\\bar' " .
+ "and '\\foo\\bar' are disallowed");
+ }
+ }
+}
+
+sub access_in {
+ return -r shift;
+}
+
+sub access_out {
+ my $f = shift;
+ my $exists;
+ eval { $exists = -e $f };
+ # Presumably, we get this error when the parent directory is not executable.
+ return if ($@);
+ if ($exists) {
+ # An existing file should be writable, and if it's a directory, it
+ # should also be executable.
+ my $rw = -w $f; my $rd = -d $f; my $rx = -x $f;
+ return -w $f && (! -d $f || -x $f);
+ } else {
+ # For a non-existing file, the parent directory should be writable.
+ # (This is the only place where function |parent| is used, so it's ok
+ # that it returns the logical parent.)
+ my $p = parent($f);
+ return -w $p;
+ }
+}
+
+# This function finds the location for an input file, respecting
+# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the
+# filesystem. It returns an absolute file as-is. For a relative file, it tries
+# |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (always), and
+# |TEXMFOUTPUT| directory (if defined), in this order. The first readable file
+# found is returned; if no readable file is found, the file in the current
+# directory is returned.
+sub find_in {
+ my $f = shift;
+ sanitize_path($f);
+ return $f if file_name_is_absolute($f);
+ for my $df (
+ $texmf_output_directory ? join_paths($texmf_output_directory, $f) : undef,
+ $f,
+ $texmfoutput ? join_paths($texmfoutput, $f) : undef) {
+ return $df if $df && -r $df;
+ }
+ return $f;
+}
+
+# This function finds the location for an output file, respecting
+# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the
+# filesystem. It returns an absolute file as-is. For a relative file, it tries
+# |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (unless
+# |TEXMF_OUTPUT_DIRECTORY| is defined), and |TEXMFOUTPUT| directory (if
+# defined), in this order. The first writable file found is returned; if no
+# writable file is found, the file in either the current or the output directory
+# is returned.
+sub find_out {
+ my $f = shift;
+ sanitize_path($f);
+ return $f if file_name_is_absolute($f);
+ for my $df (
+ $texmf_output_directory ? join_paths($texmf_output_directory, $f) : undef,
+ $texmf_output_directory ? undef : $f,
+ $texmfoutput ? join_paths($texmfoutput, $f) : undef) {
+ return $df if $df && access_out($df);
+ }
+ return $texmf_output_directory ? join_paths($texmf_output_directory, $f) : $f;
+}
+
+# We next define some filename-related utilities matching what Python offers out
+# of the box. We avoid using |File::Spec|'s |canonpath|, because on Windows,
+# which has no concept of symlinks, this function resolves |..| to the parent.
+
+sub name {
+ my $path = shift;
+ my ($volume, $dir, $filename) = splitpath($path);
+ return $filename;
+}
+
+sub suffix {
+ my $path = shift;
+ my ($volume, $dir, $filename) = splitpath($path);
+ $filename =~ /\.[^.]*$/;
+ return $&;
+}
+
+sub with_suffix {
+ my ($path, $suffix) = @_;
+ my ($volume, $dir, $filename) = splitpath($path);
+ if ($filename =~ s/\.[^.]*$/$suffix/) {
+ return catpath($volume, $dir, $filename);
+ } else {
+ return catpath($volume, $dir, $filename . $suffix);
+ }
+}
+
+sub with_name {
+ my ($path, $name) = @_;
+ my ($volume, $dir, $filename) = splitpath($path);
+ my ($v,$d,$f) = splitpath($name);
+ die "Runtime error in with_name: " .
+ "'$name' should not contain the directory component"
+ unless $v eq '' && $d eq '' && $f eq $name;
+ return catpath($volume, $dir, $name);
+}
+
+sub join_paths {
+ my $path1 = normalize_path(shift);
+ my $path2 = normalize_path(shift);
+ return $path2 if !$path1 || file_name_is_absolute($path2);
+ my ($volume1, $dir1, $filename1) = splitpath($path1, 'no_file');
+ my ($volume2, $dir2, $filename2) = splitpath($path2);
+ die if $volume2;
+ return catpath($volume1,
+ join($dirsep, ($dir1 eq $dirsep ? '' : $dir1, $dir2)),
+ $filename2);
+}
+
+# The logical parent. The same as |pathlib.parent| in Python.
+sub parent {
+ my $f = normalize_path(shift);
+ my ($v, $dn, $_dummy) = splitpath($f, 1);
+ my $p_dn = $dn =~ s/[^\Q$dirsep\E]+$//r;
+ if ($p_dn eq '') {
+ $p_dn = $dn =~ /^\Q$dirsep\E/ ? $dirsep : '.';
+ }
+ my $p = catpath($v, $p_dn, '');
+ $p = normalize_path($p);
+ return $p;
+}
+
+# This function assumes that both paths are absolute; ancestor may be '',
+# signaling a non-path.
+sub is_ancestor {
+ my $ancestor = normalize_path(shift);
+ my $descendant = normalize_path(shift);
+ return if ! $ancestor;
+ $ancestor .= $dirsep unless $ancestor =~ /\Q$dirsep\E$/;
+ return $descendant =~ /^\Q$ancestor/;
+}
+
+# A paranoid |Path.mkdir|. The given folder is preprocessed by |find_out|.
+sub make_directory {
+ my $folder = find_out(shift);
+ if (! -d $folder) {
+ paranoia_out($folder);
+ # Using |make_path| is fine because we know that
+ # |TEXMF_OUTPUT_DIRECTORY|/|TEXMFOUTPUT|, if given, exists, and that
+ # ``folder'' contains no |..|.
+ make_path($folder);
+ # This does not get logged when the function is invoked via |--mkdir|,
+ # as it is not clear what the log name should be.
+ info("Created directory $folder");
+ }
+}
+
+sub unquote {
+ shift =~ s/"(.*?)"/\1/rg;
+}
+
+# \paragraph{Kpathsea}
+
+# Get the values of |openin_any|, |openout_any|, |TEXMFOUTPUT| and
+# |TEXMF_OUTPUT_DIRECTORY|.
+
+my $maybe_backslash = $on_windows ? '' : '\\';
+my $query = 'kpsewhich -expand-var=' .
+ "openin_any=$maybe_backslash\$openin_any," .
+ "openout_any=$maybe_backslash\$openout_any," .
+ "TEXMFOUTPUT=$maybe_backslash\$TEXMFOUTPUT";
+my $kpsewhich_output = `$query`;
+if (! $kpsewhich_output) {
+ # No TeX? (Note that |kpsewhich| should exist in MiKTeX as well.) In
+ # absence of |kpathsea| information, we get very paranoid.
+ ($openin_any, $openout_any) = ('p', 'p');
+ ($texmfoutput, $texmf_output_directory) = ('', '');
+ # Unfortunately, this warning can't make it into the log. But then again,
+ # the chances of a missing |kpsewhich| are very slim, and its absence would
+ # show all over the place anyway.
+ warning('I failed to execute "kpsewhich", is there no TeX system installed? ' .
+ 'Assuming openin_any = openout_any = "p" ' .
+ '(i.e. restricting all file operations to non-hidden files ' .
+ 'in the current directory of its subdirectories).');
+} else {
+ $kpsewhich_output =~ /^openin_any=(.*),openout_any=(.*),TEXMFOUTPUT=(.*)/;
+ ($openin_any, $openout_any, $texmfoutput) = @{^CAPTURE};
+ $texmf_output_directory = $ENV{'TEXMF_OUTPUT_DIRECTORY'};
+ if ($openin_any =~ '^\$openin_any') {
+ # When the |open*_any| variables are not expanded, we assume we're
+ # running MiKTeX. The two config settings below correspond to TeXLive's
+ # |openin_any| and |openout_any|; afaik, there is no analogue to
+ # |TEXMFOUTPUT|.
+ $query = 'initexmf --show-config-value=[Core]AllowUnsafeInputFiles ' .
+ '--show-config-value=[Core]AllowUnsafeOutputFiles';
+ my $initexmf_output = `$query`;
+ $initexmf_output =~ /^(.*)\n(.*)\n/m;
+ $openin_any = $1 eq 'true' ? 'a' : 'p';
+ $openout_any = $2 eq 'true' ? 'a' : 'p';
+ $texmfoutput = '';
+ $texmf_output_directory = '';
+ }
+}
+
+# An output directory should exist, and may not point to the root on Linux. On
+# Windows, it may point to the root, because being absolute also implies
+# containing the drive; see |sanitize_filename|.
+sub sanitize_output_dir {
+ return unless my $d = shift;
+ sanitize_path($d);
+ # On Windows, |rootdir| returns |\|, so it cannot possibly match |$d|.
+ return $d if -d $d && $d ne rootdir();
+}
+
+$texmfoutput = sanitize_output_dir($texmfoutput);
+$texmf_output_directory = sanitize_output_dir($texmf_output_directory);
+
+# We don't delve into the real script when loaded from the testing code.
+return 1 if caller;
+
+# \paragraph{Arguments}
+
+my $usage = "usage: $PROG [-h] [-P PDF] [-p] [-k] [-F {latex,plain,context}] [-f] " .
+ "[-L {PDF::API2,PDF::Builder}] [-q] [-m] [-V] mmz\n";
my $Help = <<END;
-Extract extern pages out of the document PDF.
+Extract extern pages produced by package Memoize out of the document PDF.
positional arguments:
- mmz the record file produced by Memoize: doc.mmz when compiling doc.tex
+ mmz the record file produced by Memoize:
+ doc.mmz when compiling doc.tex
+ (doc and doc.tex are accepted as well)
options:
- --help, -h show this help message and exit
- --version, -V show version and exit
- --pdf PDF, -P PDF extract from file PDF
- --prune, -p remove the extern pages after extraction
- --keep, -k do not mark externs as extracted
- --force, -f extract even if the size-check fails
- --log LOG, -l LOG the log file
- --warning-template WARNING_TEMPLATE, -w WARNING_TEMPLATE
- \warningtext in the template will be replaced by the warning message
- --quiet, -q don't describe what's happening
- --embedded, -e prefix all messages to the standard output with the script name
- --mkdir, -m create a directory (and exit)
+ -h, --help show this help message and exit
+ -P PDF, --pdf PDF extract from file PDF
+ -p, --prune remove the extern pages after extraction
+ -k, --keep do not mark externs as extracted
+ -F, --format {latex,plain,context}
+ the format of the TeX document invoking extraction
+ -f, --force extract even if the size-check fails
+ -q, --quiet describe what's happening
+ -L, --library {PDF::API2, PDF::Builder}
+ which PDF library to use for extraction (default: PDF::API2)
+ -m, --mkdir create a directory (and exit);
+ mmz argument is interpreted as directory name
+ -V, --version show program's version number and exit
For details, see the man page or the Memoize documentation.
END
-my ($pdf_arg, $prune, $keep, $log, $quiet, $embedded, $force, $mkdir, $help, $print_version);
-my $warning_template = '\warningtext';
+my @valid_libraries = ('PDF::API2', 'PDF::Builder');
Getopt::Long::Configure ("bundling");
GetOptions(
- "pdf|P=s" => \$pdf_arg,
+ "pdf|P=s" => \$pdf_file,
+ "prune|p" => \$prune,
"keep|k" => \$keep,
- "prune|p" => \$prune,
- "log|l=s" => \$log,
+ "format|F=s" => \$format,
+ "force|f" => \$force,
"quiet|q" => \$quiet,
- "embedded|e" => \$embedded,
- "force|f" => \$force,
- "warning-template|w=s" => \$warning_template,
+ "library|L=s" => \$pdf_library,
"mkdir|m" => \$mkdir,
+ "version|V" => \$print_version,
"help|h|?" => \$help,
- "version|V" => \$print_version,
) or die $usage;
+
if ($help) {print("$usage\n$Help"); exit 0}
-if ($print_version) { print("memoize-extract.pl of Memoize $VERSION\n"); exit 0 }
-die $usage unless @ARGV == 1;
-my $message_prefix = $embedded ? basename($0) . ': ' : '';
-print("\n") if ($embedded);
+if ($print_version) { print("$PROG of Memoize $VERSION\n"); exit 0 }
-my @output_paths = (dir()->absolute->resolve);
-my $texmfoutput = `kpsewhich --var-value=TEXMFOUTPUT`;
-$texmfoutput =~ s/^\s+|\s+$//g;
-if ($texmfoutput) {
- my $texmfoutput_dir = dir($texmfoutput)->absolute->resolve;
- push(@output_paths, $texmfoutput_dir) unless $texmfoutput_dir->dir_list == 1 && ! $texmfoutput_dir->volume;
-}
+die "${usage}$PROG: error: the following arguments are required: mmz\n"
+ unless @ARGV == 1;
-sub paranoia {
- my $file = $_[0];
- die "${message_prefix}Cannot create a hidden file or dir: $file"
- if $file->basename =~ /^\./;
- my $parent = $file->parent->absolute->resolve;
- die "${message_prefix}Cannot write outside the current working or output directory tree: $file"
- unless grep($_->contains($parent), @output_paths);
-}
+die "${usage}$PROG: error: argument -F/--format: invalid choice: '$format' " .
+ "(choose from 'latex', 'plain', 'context')\n"
+ unless grep $_ eq $format, ('', 'latex', 'plain', 'context');
-my $mmz_arg = $ARGV[0];
-my $mmz_file = file($mmz_arg);
-my $mmz_dir = $mmz_file->parent;
+die "${usage}$PROG: error: argument -L/--library: invalid choice: '$pdf_library' " .
+ "(choose from " . join(", ", @valid_libraries) . ")\n"
+ if $pdf_library && ! grep $_ eq $pdf_library, @valid_libraries;
+$header = $format ? basename($0) . ': ' : '';
+
+# start a new line in the TeX terminal output
+print("\n") if $format;
+
+# \paragraph{Initialization}
+
+# With |--mkdir|, argument |mmz| is interpreted as the directory to create.
if ($mkdir) {
- my $dir = dir($mmz_arg)->cleanup;
- my $current = dir(File::Spec->catpath($dir->volume,
- $dir->is_absolute ? File::Spec->rootdir : File::Spec->curdir,
- ''))->absolute;
- for my $c ($dir->components) {
- $current = $current->subdir($c);
- if (-d $current) {
- $current = $current->resolve;
- } else {
- paranoia($current);
- mkdir($current);
- print("${message_prefix}Created directory $current\n") unless $quiet;
- }
- }
- exit;
-} else {
- die "${message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '$mmz_file'\n" unless $mmz_file->basename =~ /\.mmz$/;
+ make_directory($ARGV[0]);
+ exit 0;
}
-# Enable in-place editing (of the .mmz file).
-paranoia($mmz_file) unless $keep;
-$^I = "" unless $keep;
+# Normalize the |mmz| argument into a |.mmz| filename.
+my $mmz_file = $ARGV[0];
+$mmz_file = with_suffix($mmz_file, '.mmz')
+ if suffix($mmz_file) eq '.tex';
+$mmz_file = with_name($mmz_file, name($mmz_file) . '.mmz')
+ if suffix($mmz_file) ne '.mmz';
-my $pdf_file = $pdf_arg ? file($pdf_arg) : $mmz_dir->file($mmz_file->basename =~ s/\.mmz$/\.pdf/r)->cleanup;
-paranoia($pdf_file) if $prune;
+# Once we have the |.mmz| filename, we can open the log.
+if ($format) {
+ my $_log = find_out(with_suffix($mmz_file, '.mmz.log'));
+ paranoia_out($_log);
+ info("Logging to '$_log'");
+ $log = $_log;
+ open LOG, ">$log";
+}
-if ($log) {
- paranoia(file($log));
- open LOG, ">$log";
+# Now that we have opened the log file, we can try loading the PDF processing
+# library.
+if ($pdf_library) {
+ eval "use $pdf_library";
+ error("Perl module '$pdf_library' was not found",
+ 'Have you followed the instructions is section 1.1 of the manual?')
+ if ($@);
} else {
- *LOG = *STDERR;
+ for (@valid_libraries) {
+ eval "use $_";
+ if (!$@) {
+ $pdf_library = $_;
+ last;
+ }
+ }
+ if (!$pdf_library) {
+ error("No suitable Perl module for PDF processing was found, options are " .
+ join(", ", @valid_libraries),
+ 'Have you followed the instructions is section 1.1 of the manual?');
+ }
}
-my ($pdf, %extern_pages);
-print "${message_prefix}Extracting externs from $pdf_file:\n" unless $quiet;
+# Catch any errors in the script and output them to the log.
+$SIG{__DIE__} = \&die_handler;
+$SIG{__WARN__} = \&warn_handler;
+close(STDERR);
+open(STDERR, ">", \$stderr);
+# Find the |.mmz| file we will read, but retain the original filename in
+# |$given_mmz_file|, as we will still need it.
+my $given_mmz_file = $mmz_file;
+$mmz_file = find_in($mmz_file, 1);
+if (! -e $mmz_file) {
+ info("File '$given_mmz_file' does not exist, assuming there's nothing to do");
+ endinput();
+}
+paranoia_in($mmz_file);
+paranoia_out($mmz_file,
+ 'I would have to rewrite this file unless option --keep is given.')
+ unless $keep;
+
+# Determine the PDF filename: it is either given via |--pdf|, or constructed
+# from the |.mmz| filename.
+$pdf_file = with_suffix($given_mmz_file, '.pdf') if !$pdf_file;
+$pdf_file = find_in($pdf_file);
+paranoia_in($pdf_file);
+paranoia_out($pdf_file,
+ 'I would have to rewrite this file because option --prune was given.')
+ if $prune;
+
+# Various initializations.
+my $pdf;
+my %extern_pages;
+my $new_mmz;
my $tolerance = 0.01;
-my $done_message = "${message_prefix}Done (there was nothing to extract).\n";
-
-while (<>) {
- if (/^\\mmzNewExtern *{(?P<extern_path>(?P<prefix>.*?)(?P<code_md5sum>[0-9A-F]{32})-(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P<page_n>[0-9]+)}{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}/) {
- my $extern_file = file($+{extern_path});
- if (! $extern_file->is_absolute) {
- $extern_file = $mmz_dir->file($+{extern_path});
- }
- paranoia($extern_file);
- my $page = $+{page_n};
- my $expected_width_pt = $+{expected_width};
- my $expected_height_pt = $+{expected_height};
- my $c_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '.memo');
- my $cc_memo_file = $mmz_dir->file($+{prefix} . $+{code_md5sum} . '-' . $+{context_md5sum} . '.memo');
- if (!(-e $c_memo_file && -e $cc_memo_file)) {
- print LOG ($warning_template =~ s/\\warningtext/Not extracting page $page into extern $extern_file, because the associated (c)c-memo does not exist/gr), "\n\\endinput\n";
- last;
- }
- eval { $pdf = PDF::API2->open($pdf_file->stringify) unless $pdf; };
- if ($@) {
- print LOG ($warning_template =~ s/\\warningtext/Cannot read file "$pdf_file". Perhaps you have to load Memoize earlier in the preamble?/gr), "\n\\endinput\n";
- close LOG;
- die "${message_prefix}File '$pdf_file' cannot be read, bailing out.\n";
- }
- my $extern = PDF::API2->new();
- $extern->version($pdf->version);
- $extern->import_page($pdf, $page);
- my $extern_page = $extern->open_page(1);
- my ($x0, $y0, $x1, $y1) = $extern_page->get_mediabox();
- my $width_pt = ($x1 - $x0) / 72 * 72.27;
- my $height_pt = ($y1 - $y0) / 72 * 72.27;
- my $warning = '';
- if (abs($width_pt - $expected_width_pt) > $tolerance
- || abs($height_pt - $expected_height_pt) > $tolerance)
- {
- $warning = "I refuse to extract page $page from $pdf_file, because its size (${width_pt}pt x ${height_pt}pt) is not what I expected (${expected_width_pt}pt x ${expected_height_pt}pt)";
- print LOG ($warning_template =~ s/\\warningtext/$warning/gr), "\n";
- }
- if ($warning && !$force) {
- unlink $extern_file;
- } else {
- $extern->saveas($extern_file->stringify);
- $done_message = "${message_prefix}Done.\n";
- print STDOUT "${message_prefix} Page $page --> $extern_file\n" unless $quiet;
- $extern_pages{$page} = 1 if $prune;
- print("%") unless $keep;
- }
+info("Extracting new externs listed in '$mmz_file' " .
+ "from '$pdf_file' using Perl module $pdf_library");
+my $done_message = "Done (there was nothing to extract)";
+$indent = ' ';
+$texindent = '\space\space ';
+my $dir_to_make;
+
+# \paragraph{Process \texttt{.mmz}}
+#
+# We cannot process the .mmz file using in-place editing. It would fail when
+# the file is writable but its parent directory is not.
+
+open (MMZ, $mmz_file);
+while (<MMZ>) {
+ my $mmz_line = $_;
+ if (/^\\mmzPrefix *{(?P<prefix>)}/) {
+ # Found |\mmzPrefix|: create the extern directory, but only later, if an
+ # extern file is actually produced. We parse the prefix in two steps
+ # because we have to unquote the entire prefix.
+ my $prefix = unquote($+{prefix});
+ warning("Cannot parse line '$mmz_line'") unless
+ $prefix =~ /(?P<dir_prefix>.*\/)?(?P<name_prefix>.*?)/;
+ $dir_to_make = $+{dir_prefix};
+ } elsif (/^\\mmzNewExtern\ *{(?P<extern_path>.*?)}{(?P<page_n>[0-9]+)}#
+ {(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}/x) {
+ # Found |\mmzNewExtern|: extract the extern page into an extern file.
+ $done_message = "Done";
+ my $ok = 1;
+ my %m_ne = %+;
+ # The extern filename, as specified in |.mmz|:
+ my $extern_file = unquote($m_ne{extern_path});
+ # We parse the extern filename in a separate step because we have to
+ # unquote the entire path.
+ warning("Cannot parse line '$mmz_line'") unless
+ $extern_file =~ /(?P<dir_prefix>.*\/)?(?P<name_prefix>.*?)#
+ (?P<code_md5sum>[0-9A-F]{32})-#
+ (?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf/x;
+ # The actual extern filename:
+ my $extern_file_out = find_out($extern_file);
+ paranoia_out($extern_file_out);
+ my $page = $m_ne{page_n};
+ # Check whether c-memo and cc-memo exist (in any input directory).
+ my $c_memo = with_name($extern_file,
+ $+{name_prefix} . $+{code_md5sum} . '.memo');
+ my $cc_memo = with_name($extern_file,
+ $+{name_prefix} . $+{code_md5sum} .
+ '-' . $+{context_md5sum} . '.memo');
+ my $c_memo_in = find_in($c_memo);
+ my $cc_memo_in = find_in($cc_memo);
+ if ((! access_in($c_memo_in) || ! access_in($cc_memo_in)) && !$force) {
+ warning("I refuse to extract page $page into extern '$extern_file', " .
+ "because the associated c-memo '$c_memo' and/or " .
+ "cc-memo '$cc_memo' does not exist");
+ $ok = '';
+ }
+ # Load the PDF. We only do this now so that we don't load it if there
+ # is nothing to extract.
+ if ($ok && ! $pdf) {
+ if (!access_in($pdf_file)) {
+ warning("Cannot open '$pdf_file'", '');
+ endinput();
+ }
+ # Temporarily disable error handling, so that we can catch the error
+ # ourselves.
+ $SIG{__DIE__} = undef; $SIG{__WARN__} = undef;
+ # All safe, |paranoia_in| was already called above.
+ eval { $pdf = $pdf_library->open($pdf_file, msgver => 0) };
+ $SIG{__DIE__} = \&die_handler; $SIG{__WARN__} = \&warn_handler;
+ error("File '$pdf_file' seems corrupted. " .
+ "Perhaps you have to load Memoize earlier in the preamble",
+ "In particular, Memoize must be loaded before TikZ library " .
+ "'fadings' and any package deploying it, and in Beamer, " .
+ "load Memoize by writing \\RequirePackage{memoize} before " .
+ "\\documentclass{beamer}. " .
+ "This was the error thrown by Perl:" . "\n$@") if $@;
+ }
+ # Does the page exist?
+ if ($ok && $page > (my $n_pages = $pdf->page_count())) {
+ error("I cannot extract page $page from '$pdf_file', " .
+ "as it contains only $n_pages page" .
+ ($n_pages > 1 ? 's' : ''), '');
+ }
+ if ($ok) {
+ # Import the page into the extern PDF (no disk access yet).
+ my $extern = $pdf_library->new(outver => $pdf->version);
+ $extern->import_page($pdf, $page);
+ my $extern_page = $extern->open_page(1);
+ # Check whether the page size matches the |.mmz| expectations.
+ my ($x0, $y0, $x1, $y1) = $extern_page->get_mediabox();
+ my $width_pt = ($x1 - $x0) / 72 * 72.27;
+ my $height_pt = ($y1 - $y0) / 72 * 72.27;
+ my $expected_width_pt = $m_ne{expected_width};
+ my $expected_height_pt = $m_ne{expected_height};
+ if ((abs($width_pt - $expected_width_pt) > $tolerance
+ || abs($height_pt - $expected_height_pt) > $tolerance) && !$force) {
+ warning("I refuse to extract page $page from $pdf_file, " .
+ "because its size (${width_pt}pt x ${height_pt}pt) " .
+ "is not what I expected " .
+ "(${expected_width_pt}pt x ${expected_height_pt}pt)");
+ } else {
+ # All tests were successful, let's create the extern file.
+ # First, the containing directory, if necessary.
+ if ($dir_to_make) {
+ make_directory($dir_to_make);
+ $dir_to_make = undef;
+ }
+ # Now the extern file. Note that |paranoia_out| was already
+ # called above.
+ info("Page $page --> $extern_file_out");
+ $extern->saveas($extern_file_out);
+ # This page will get pruned.
+ $extern_pages{$page} = 1 if $prune;
+ # Comment out this |\mmzNewExtern|.
+ $new_mmz .= '%' unless $keep;
+ }
+ }
}
- print unless $keep;
+ $new_mmz .= $mmz_line unless $keep;
+ stderr_to_warning();
}
+close(MMZ);
+$indent = '';
+$texindent = '';
+info($done_message);
-print $done_message unless $quiet;
+# Write out the |.mmz| file with |\mmzNewExtern| lines commented out. (All safe,
+# |paranoia_out| was already called above.)
+if (!$keep) {
+ open(MMZ, ">", $mmz_file);
+ print MMZ $new_mmz;
+ close(MMZ);
+}
-if ($pdf and $prune) {
- paranoia($pdf_file);
- my $pruned_pdf = PDF::API2->new();
+# Remove the extracted pages from the original PDF. (All safe, |paranoia_out|
+# was already called above.)
+if ($prune and keys(%extern_pages) != 0) {
+ my $pruned_pdf = $pdf_library->new();
for (my $n = 1; $n <= $pdf->page_count(); $n++) {
- if (! $extern_pages{$n}) {
- $pruned_pdf->import_page($pdf, $n);
- }
+ if (! $extern_pages{$n}) {
+ $pruned_pdf->import_page($pdf, $n);
+ }
}
- $pruned_pdf->save($pdf_file->stringify);
- print("${message_prefix}The following extern pages were pruned out of the PDF: ",
- join(",", sort(keys(%extern_pages))) . "\n") unless $quiet;
+ $pruned_pdf->save($pdf_file);
+ info("The following extern pages were pruned out of the PDF: " .
+ join(",", sort(keys(%extern_pages))));
}
-if ($log) {
- print LOG "\\endinput\n";
- close LOG;
-}
+endinput();
+
+# Local Variables:
+# fill-column: 79
+# after-save-hook: pl2dtx
+# End:
Modified: trunk/Master/texmf-dist/scripts/memoize/memoize-extract.py
===================================================================
--- trunk/Master/texmf-dist/scripts/memoize/memoize-extract.py 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/scripts/memoize/memoize-extract.py 2024-01-03 21:18:51 UTC (rev 69285)
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-
+#
# This file is a part of Memoize, a TeX package for externalization of
# graphics and memoization of compilation results in general, available at
# https://ctan.org/pkg/memoize and https://github.com/sasozivanovic/memoize.
@@ -18,166 +18,552 @@
# The files belonging to this work and covered by LPPL are listed in
# <texmf>/doc/generic/memoize/FILES.
-__version__ = '2023/10/10 v1.0.0'
+__version__ = '2024/01/02 v1.1.0'
-import argparse, re, sys, os, pdfrw, subprocess, itertools
-from pathlib import Path
+import argparse, re, sys, os, subprocess, itertools, traceback, platform
+from pathlib import Path, PurePath
+# We will only try to import the PDF processing library once we set up the
+# error log.
-parser = argparse.ArgumentParser(
- description = "Extract extern pages out of the document PDF.",
- epilog = "For details, see the man page or the Memoize documentation.",
- prog = 'memoize-extract.py',
-)
-parser.add_argument('--pdf', '-P', help = 'extract from file PDF')
-parser.add_argument('--prune', '-p', action = 'store_true',
- help = 'remove the extern pages after extraction')
-parser.add_argument('--keep', '-k', action = 'store_true',
- help = 'do not mark externs as extracted')
-parser.add_argument('--force', '-f', action = 'store_true',
- help = 'extract even if the size-check fails')
-parser.add_argument('--log', '-l', default = os.devnull, help = 'the log file')
-parser.add_argument('--warning-template', '-w', default = '\warningtext',
- help = '\warningtext in the template will be replaced by the warning message')
-parser.add_argument('--quiet', '-q', action = 'store_true',
- help = "describe what's happening")
-parser.add_argument('--embedded', '-e', action = 'store_true',
- help = "prefix all messages to the standard output with the script name")
-parser.add_argument('--mkdir', '-m', action = 'store_true',
- help = 'create a directory (and exit)')
-parser.add_argument('mmz',
- help = 'the record file produced by Memoize: doc.mmz when compiling doc.tex')
-parser.add_argument('--version', '-V', action = 'version',
- version = f"%(prog)s of Memoize " + __version__)
+# \paragraph{Messages}
-args = parser.parse_args()
+# The messages are written both to the extraction log and the terminal (we
+# output to stdout rather than stderr so that messages on the TeX terminal and
+# document |.log| appear in chronological order). Messages are automatically
+# adapted to the TeX |--format|.
-message_prefix = parser.prog + ': ' if args.embedded else ''
-if args.embedded:
- print()
+# The format of the messages. It depends on the given |--format|; the last
+# entry is for t the terminal output.
-# Even a bit more paranoid than required:
-# (a) disallowing TEXMFOUTPUT=/ (while allowing C:\ on Windows)
-# (b) waiting for access to '-output-directory'.
-output_paths = [Path.cwd()]
-if texmfoutput := subprocess.run(
- ['kpsewhich', '--var-value=TEXMFOUTPUT'],
- capture_output = True).stdout.decode().strip():
- texmfoutput_dir = Path(texmfoutput).resolve()
- if len(texmfoutput_dir.parts) != 1 or texmfoutput_dir.drive:
- output_paths.append(texmfoutput_dir)
+ERROR = {
+ 'latex': r'\PackageError{{{package_name}}}{{{short}}}{{{long}}}',
+ 'plain': r'\errhelp{{{long}}}\errmessage{{{package_name}: {short}}}',
+ 'context': r'\errhelp{{{long}}}\errmessage{{{package_name}: {short}}}',
+ None: '{header}{short}.\n{long}',
+}
-def paranoia(f):
- p = Path(f).resolve()
- if p.stem.startswith('.'):
- raise RuntimeError(f"{message_prefix}Cannot create a hidden file or dir: {f}")
- if not any(p.is_relative_to(op) for op in output_paths):
- raise RuntimeError(f"{message_prefix}Cannot write outside the current working or output directory tree: {f}")
+WARNING = {
+ 'latex': r'\PackageWarning{{{package_name}}}{{{texindent}{text}}}',
+ 'plain': r'\message{{{package_name}: {texindent}{text}}}',
+ 'context': r'\message{{{package_name}: {texindent}{text}}}',
+ None: r'{header}{indent}{text}.',
+}
-mmz_file = Path(args.mmz)
+INFO = {
+ 'latex': r'\PackageInfo{{{package_name}}}{{{texindent}{text}}}',
+ 'plain': r'\message{{{package_name}: {texindent}{text}}}',
+ 'context': r'\message{{{package_name}: {texindent}{text}}}',
+ None: r'{header}{indent}{text}.',
+}
+
+# Some variables used in the message routines; note that |header| will be
+# redefined once we parse the arguments.
+
+package_name = 'memoize (python-based extraction)'
+exit_code = 0
+log = None
+header = ''
+indent = ''
+texindent = ''
+
+# The message routines.
+
+def error(short, long):
+ if not args.quiet:
+ print(ERROR[None].format(short = short, long = long, header = header))
+ if log:
+ long = long.replace('\\', '\\string\\')
+ print(
+ ERROR[args.format].format(
+ short = short, long = long, package_name = package_name),
+ file = log)
+ global exit_code
+ exit_code = 11
+ endinput()
-if args.mkdir: # Here, argument "mmz" is interpreted as the directory to create.
- # We cannot simply say
- # paranoia(mmz_file)
- # mmz_file.mkdir(parents = True, exist_ok = True)
- # because have be paranoid about the intermediate directories as well:
- # memoize-extract.py -m a/../../BAD/../<cwd-name>/b
- # Note that paranoia might kick in only after a part of the given path was
- # already created. This is in accord to how "mkdir" behaves wrt existing
- # files.
- for folder in itertools.chain(reversed(mmz_file.parents), (mmz_file,)):
- if not folder.is_dir():
- paranoia(folder)
- folder.mkdir(exist_ok = True)
- if not args.quiet:
- print(f"{message_prefix}Created directory {folder}")
- sys.exit()
-elif mmz_file.suffix != '.mmz':
- raise RuntimeError(f"{message_prefix}The 'mmz' argument should be a file with suffix '.mmz', not '{mmz_file}'")
+def warning(text):
+ if log:
+ print(
+ WARNING[args.format].format(
+ text = text, texindent = texindent, package_name = package_name),
+ file = log)
+ if not args.quiet:
+ print(WARNING[None].format(text = text, header = header, indent = indent))
+ global exit_code
+ exit_code = 10
+
+def info(text):
+ if text and not args.quiet:
+ print(INFO[None].format(text = text, header = header, indent = indent))
+ if log:
+ print(
+ INFO[args.format].format(
+ text = text, texindent = texindent, package_name = package_name),
+ file = log)
+
+# Mark the log as complete and exit.
+def endinput():
+ if log:
+ print(r'\endinput', file = log)
+ log.close()
+ sys.exit(exit_code)
-mmz_dir = mmz_file.parent
-pdf_file = Path(args.pdf) if args.pdf else mmz_file.with_suffix('.pdf')
-paranoia(pdf_file)
-pdf = None
-extern_pages = []
-new_mmz = []
-args.log is os.devnull or paranoia(Path(args.log))
-re_newextern = re.compile(r'\\mmzNewExtern *{(?P<extern_fn>(?P<prefix>.*?)(?P<code_md5sum>[0-9A-F]{32})-(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf)}{(?P<page_n>[0-9]+)}{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}')
-tolerance = 0.01
-done_message = f"{message_prefix}Done (there was nothing to extract)."
+# \paragraph{Permission-related functions}
-# Complication: I want to use 'with', but don't have to open stderr.
-with open(args.log, 'w') as log:
- log = sys.stderr if args.log is os.devnull else log
+# |paranoia_in|/|out| should work exactly as
+# |kpsewhich -safe-in-name|/|-safe-out-name|.
+def paranoia_in(f, remark = ''):
+ if f and not _paranoia(f, openin_any):
+ error(f"I'm not allowed to read from '{f}' (openin_any = {openin_any})",
+ remark)
+
+def paranoia_out(f, remark = ''):
+ if f and not _paranoia(f, openout_any):
+ error(f"I'm not allowed to write to '{f}' (openout_any = {openout_any})",
+ remark)
+
+def _paranoia(f, mode):
+ # |mode| is the value of |openin_any| or |openout_any|. |f| is a
+ # |pathlib.Path| object.
+ return (
+ # In mode `any' (|a|, |y| or |1|), we may access any file.
+ mode in 'ay1'
+ or (
+ # Otherwise, we are at least in the restricted mode, so we should not
+ # open dot files on Unix-like systems (except file called |.tex|).
+ not (os.name == 'posix' and f.stem.startswith('.') and f.stem != '.tex')
+ and (
+ # If we are precisely in the restricted mode (|r|, |n|, |0|),
+ # then there are no further restrictions.
+ mode in 'rn0'
+ # Otherwise, we are in the paranoid mode (officially |p|, but
+ # any other value is interpreted as |p| as well). There are
+ # two further restrictions in the paranoid mode.
+ or (
+ # We're not allowed to go to a parent directory.
+ '..' not in f.parts
+ and
+ # If the given path is absolute, is should be a descendant
+ # of either |TEXMF_OUTPUT_DIRECTORY| or |TEXMFOUTPUT|.
+ (not f.is_absolute()
+ or
+ is_ancestor(texmf_output_directory, f)
+ or
+ is_ancestor(texmfoutput, f)
+ )))))
+
+# On Windows, we disallow ``semi-absolute'' paths, i.e.\ paths starting with
+# the |\| but lacking the drive. On Windows, |pathlib|'s |is_absolute| returns
+# |True| only for paths starting with |\| and containing the drive.
+def sanitize_filename(f):
+ if f and platform.system() == 'Windows' and not (f.is_absolute() or not f.drive):
+ error(f"\"Semi-absolute\" paths are disallowed: '{f}'", r"The path must "
+ r"either contain both the drive letter and start with '\', "
+ r"or none of these; paths like 'C:foo' and '\foo' are disallowed")
+
+def access_in(f):
+ return os.access(f, os.R_OK)
+
+# This function can fail on Windows, reporting a non-writable file or dir as
+# writable, because |os.access| does not work with Windows' |icacls|
+# permissions. Consequence: we might try to write to a read-only current or
+# output directory instead of switching to the temporary directory. Paranoia
+# is unaffected, as it doesn't use |access_*| functions.
+def access_out(f):
try:
- mmz = mmz_file.open()
- except FileNotFoundError:
- print(f'''{message_prefix}File "{mmz_file}" does not exist, assuming there's nothing to do.''',
- file = sys.stderr)
+ exists = f.exists()
+ # Presumably, we get this error when the parent directory is not
+ # executable.
+ except PermissionError:
+ return
+ if exists:
+ # An existing file should be writable, and if it's a directory, it
+ # should also be executable.
+ return os.access(f, os.W_OK) and (not f.is_dir() or os.access(f, os.X_OK))
else:
- if not args.quiet:
- print(f"{message_prefix}Extracting externs from {pdf_file}")
+ # For a non-existing file, the parent directory should be writable.
+ # (This is the only place where function |pathlib.parent| is used, so
+ # it's ok that it returns the logical parent.)
+ return os.access(f.parent, os.W_OK)
+
+# This function finds the location for an input file, respecting
+# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the
+# filesystem. It returns an absolute file as-is. For a relative file, it
+# tries |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (always),
+# and |TEXMFOUTPUT| directory (if defined), in this order. The first readable
+# file found is returned; if no readable file is found, the file in the current
+# directory is returned.
+def find_in(f):
+ sanitize_filename(f)
+ if f.is_absolute():
+ return f
+ for df in (texmf_output_directory / f if texmf_output_directory else None,
+ f,
+ texmfoutput / f if texmfoutput else None):
+ if df and access_in(df):
+ return df
+ return f
+
+# This function finds the location for an output file, respecting
+# |TEXMF_OUTPUT_DIRECTORY| and |TEXMFOUTPUT|, and the permissions in the
+# filesystem. It returns an absolute file as-is. For a relative file, it
+# tries |TEXMF_OUTPUT_DIRECTORY| (if defined), the current directory (unless
+# |TEXMF_OUTPUT_DIRECTORY| is defined), and |TEXMFOUTPUT| directory (if
+# defined), in this order. The first writable file found is returned; if no
+# writable file is found, the file in either the current or the output
+# directory is returned.
+def find_out(f):
+ sanitize_filename(f)
+ if f.is_absolute():
+ return f
+ for df in (texmf_output_directory / f if texmf_output_directory else None,
+ f if not texmf_output_directory else None,
+ texmfoutput / f if texmfoutput else None):
+ if df and access_out(df):
+ return df
+ return texmf_output_directory / f if texmf_output_directory else f
+
+# This function assumes that both paths are absolute; ancestor may be |None|,
+# signaling a non-path.
+def is_ancestor(ancestor, descendant):
+ if not ancestor:
+ return
+ a = ancestor.parts
+ d = descendant.parts
+ return len(a) < len(d) and a == d[0:len(a)]
+
+# A paranoid |Path.mkdir|. The given folder is preprocessed by |find_out|.
+def mkdir(folder):
+ folder = find_out(Path(folder))
+ if not folder.exists():
+ paranoia_out(folder)
+ # Using |folder.mkdir| is fine because we know that
+ # |TEXMF_OUTPUT_DIRECTORY|/|TEXMFOUTPUT|, if given, exists, and that
+ # ``folder'' contains no |..|.
+ folder.mkdir(parents = True, exist_ok = True)
+ # This does not get logged when the function is invoked via |--mkdir|,
+ # as it is not clear what the log name should be.
+ info(f"Created directory {folder}")
+
+_re_unquote = re.compile(r'"(.*?)"')
+def unquote(fn):
+ return _re_unquote.sub(r'\1', fn)
+
+# \paragraph{Kpathsea}
+
+# Get the values of |openin_any|, |openout_any|, |TEXMFOUTPUT| and
+# |TEXMF_OUTPUT_DIRECTORY|.
+
+kpsewhich_output = subprocess.run(['kpsewhich',
+ f'-expand-var='
+ f'openin_any=$openin_any,'
+ f'openout_any=$openout_any,'
+ f'TEXMFOUTPUT=$TEXMFOUTPUT'],
+ capture_output = True
+ ).stdout.decode().strip()
+if not kpsewhich_output:
+ # No TeX? (Note that |kpsewhich| should exist in MiKTeX as well.) In
+ # absence of |kpathsea| information, we get very paranoid, but still try to
+ # get |TEXMFOUTPUT| from an environment variable.
+ openin_any, openout_any = 'p', 'p'
+ texmfoutput, texmf_output_directory = None, None
+ # Unfortunately, this warning can't make it into the log. But then again,
+ # the chances of a missing |kpsewhich| are very slim, and its absence would
+ # show all over the place anyway.
+ warning('I failed to execute "kpsewhich"; , is there no TeX system installed? '
+ 'Assuming openin_any = openout_any = "p" '
+ '(i.e. restricting all file operations to non-hidden files '
+ 'in the current directory of its subdirectories).')
+else:
+ m = re.fullmatch(r'openin_any=(.*),openout_any=(.*),TEXMFOUTPUT=(.*)',
+ kpsewhich_output)
+ openin_any, openout_any, texmfoutput = m.groups()
+ texmf_output_directory = os.environ.get('TEXMF_OUTPUT_DIRECTORY', None)
+ if openin_any == '$openin_any':
+ # When the |open*_any| variables are not expanded, we assume we're
+ # running MiKTeX. The two config settings below correspond to TeXLive's
+ # |openin_any| and |openout_any|; afaik, there is no analogue to
+ # |TEXMFOUTPUT|.
+ initexmf_output = subprocess.run(
+ ['initexmf', '--show-config-value=[Core]AllowUnsafeInputFiles',
+ '--show-config-value=[Core]AllowUnsafeOutputFiles'],
+ capture_output = True).stdout.decode().strip()
+ openin_any, openout_any = initexmf_output.split()
+ openin_any = 'a' if openin_any == 'true' else 'p'
+ openout_any = 'a' if openout_any == 'true' else 'p'
+ texmfoutput = None
+ texmf_output_directory = None
+
+# An output directory should exist, and may not point to the root on Linux. On
+# Windows, it may point to the root, because we only allow absolute filenames
+# containing the drive, e.g.\ |F:\|; see |is_absolute|.
+def sanitize_output_dir(d_str):
+ d = Path(d_str) if d_str else None
+ sanitize_filename(d)
+ return d if d and d.is_dir() and \
+ (not d.is_absolute() or len(d.parts) != 1 or d.drive) else None
+
+texmfoutput = sanitize_output_dir(texmfoutput)
+texmf_output_directory = sanitize_output_dir(texmf_output_directory)
+
+class NotExtracted(UserWarning):
+ pass
+
+# We don't delve into the real script when loaded from the testing code.
+if __name__ == '__main__':
+
+ # \paragraph{Arguments}
+
+ parser = argparse.ArgumentParser(
+ description = "Extract extern pages produced by package Memoize "
+ "out of the document PDF.",
+ epilog = "For details, see the man page or the Memoize documentation.",
+ prog = 'memoize-extract.py',
+ )
+ parser.add_argument('-P', '--pdf', help = 'extract from file PDF')
+ parser.add_argument('-p', '--prune', action = 'store_true',
+ help = 'remove the extern pages after extraction')
+ parser.add_argument('-k', '--keep', action = 'store_true',
+ help = 'do not mark externs as extracted')
+ parser.add_argument('-F', '--format', choices = ['latex', 'plain', 'context'],
+ help = 'the format of the TeX document invoking extraction')
+ parser.add_argument('-f', '--force', action = 'store_true',
+ help = 'extract even if the size-check fails')
+ parser.add_argument('-q', '--quiet', action = 'store_true',
+ help = "describe what's happening")
+ parser.add_argument('-m', '--mkdir', action = 'store_true',
+ help = 'create a directory (and exit); '
+ 'mmz argument is interpreted as directory name')
+ parser.add_argument('-V', '--version', action = 'version',
+ version = f"%(prog)s of Memoize " + __version__)
+ parser.add_argument('mmz', help = 'the record file produced by Memoize: '
+ 'doc.mmz when compiling doc.tex '
+ '(doc and doc.tex are accepted as well)')
+
+ args = parser.parse_args()
+
+ header = parser.prog + ': ' if args.format else ''
+
+ # Start a new line in the TeX terminal output.
+ if args.format:
+ print()
+
+ # \paragraph{Initialization}
+
+ # With |--mkdir|, argument |mmz| is interpreted as the directory to create.
+ if args.mkdir:
+ mkdir(args.mmz)
+ sys.exit()
+
+ # Normalize the |mmz| argument into a |.mmz| filename.
+ mmz_file = Path(args.mmz)
+ if mmz_file.suffix == '.tex':
+ mmz_file = mmz_file.with_suffix('.mmz')
+ elif mmz_file.suffix != '.mmz':
+ mmz_file = mmz_file.with_name(mmz_file.name + '.mmz')
+
+ # Once we have the |.mmz| filename, we can open the log.
+ if args.format:
+ log_file = find_out(mmz_file.with_suffix('.mmz.log'))
+ paranoia_out(log_file)
+ info(f"Logging to '{log_file}'");
+ log = open(log_file, 'w')
+
+ # Now that we have opened the log file, we can try loading the PDF
+ # processing library.
+ try:
+ import pdfrw
+ except ModuleNotFoundError:
+ error("Python module 'pdfrw' was not found",
+ 'Have you followed the instructions is section 1.1 of the manual?')
+
+ # Catch any errors in the script and output them to the log.
+ try:
+
+ # Find the |.mmz| file we will read, but retain the original filename
+ # in |given_mmz_file|, as we will still need it.
+ given_mmz_file = mmz_file
+ mmz_file = find_in(mmz_file)
+ paranoia_in(mmz_file)
+ if not args.keep:
+ paranoia_out(mmz_file,
+ remark = 'This file is rewritten unless option --keep is given.')
+ try:
+ mmz = open(mmz_file)
+ except FileNotFoundError:
+ info(f"File '{given_mmz_file}' does not exist, "
+ f"assuming there's nothing to do")
+ endinput()
+
+ # Determine the PDF filename: it is either given via |--pdf|, or
+ # constructed from the |.mmz| filename.
+ pdf_file = find_in(Path(args.pdf)
+ if args.pdf else given_mmz_file.with_suffix('.pdf'))
+ paranoia_in(pdf_file)
+ if args.prune:
+ paranoia_out(pdf_file,
+ remark = 'I would have to rewrite this file '
+ 'because option --prune was given.')
+
+ # Various initializations.
+
+ re_prefix = re.compile(r'\\mmzPrefix *{(?P<prefix>.*?)}')
+ re_split_prefix = re.compile(r'(?P<dir_prefix>.*/)?(?P<name_prefix>.*?)')
+ re_newextern = re.compile(
+ r'\\mmzNewExtern *{(?P<extern_path>.*?)}{(?P<page_n>[0-9]+)}'
+ r'{(?P<expected_width>[0-9.]*)pt}{(?P<expected_height>[0-9.]*)pt}')
+ re_extern_path = re.compile(
+ r'(?P<dir_prefix>.*/)?(?P<name_prefix>.*?)'
+ r'(?P<code_md5sum>[0-9A-F]{32})-'
+ r'(?P<context_md5sum>[0-9A-F]{32})(?:-[0-9]+)?.pdf')
+ pdf = None
+ extern_pages = []
+ new_mmz = []
+ tolerance = 0.01
+ dir_to_make = None
+ info(f"Extracting new externs listed in '{mmz_file}' from '{pdf_file}'")
+ done_message = "Done (there was nothing to extract)"
+ indent = ' '
+ texindent = '\space\space '
+
+ # \paragraph{Process \texttt{.mmz}}
+
for line in mmz:
- if m := re_newextern.match(line):
- extern_file = mmz_dir / m['extern_fn']
- paranoia(extern_file)
- page_n = int(m['page_n'])-1
- c_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '.memo')
- cc_memo = mmz_dir / (m['prefix'] + m['code_md5sum'] + '-' + m['context_md5sum'] + '.memo')
- if not (c_memo.exists() and cc_memo.exists()):
- print(args.warning_template.replace('\warningtext', f'Not extracting page {page_n} into extern {extern_file}, because the associated (c)c-memo does not exist'), file = log)
- continue
- if not pdf:
- try:
- pdf = pdfrw.PdfReader(pdf_file)
- except pdfrw.errors.PdfParseError:
- print(f'{message_prefix}File "{pdf_file}" cannot be read, bailing out.', file = sys.stderr)
- print(args.warning_template.replace('\warningtext', f'Cannot read file "{pdf_file}". Perhaps you have to load Memoize earlier in the preamble?'), file = log)
- args.keep = True
- break
- extern = pdfrw.PdfWriter(extern_file)
- page = pdf.pages[page_n]
- expected_width_pt, expected_height_pt = float(m['expected_width']), float(m['expected_height'])
- mb = page['/MediaBox']
- width_bp, height_bp = float(mb[2]) - float(mb[0]), float(mb[3]) - float(mb[1])
- width_pt = width_bp / 72 * 72.27
- height_pt = height_bp / 72 * 72.27
- warning = None
- if abs(width_pt - expected_width_pt) > tolerance \
- or abs(height_pt - expected_height_pt) > tolerance:
- warning = (
- f'I refuse to extract page {page_n+1} from "{pdf_file}", '
- f'because its size ({width_pt}pt x {height_pt}pt) is not '
- f'what I expected ({expected_width_pt}pt x {expected_height_pt}pt)')
- print(args.warning_template.replace('\warningtext', warning), file = log)
- if warning and not args.force:
- extern_file.unlink(missing_ok = True)
- else:
+ try:
+ if m_p := re_prefix.match(line):
+ # Found |\mmzPrefix|: create the extern directory, but only
+ # later, if an extern file is actually produced. We parse
+ # the prefix in two steps because we have to unquote the
+ # entire prefix.
+ prefix = unquote(m_p['prefix'])
+ if not (m_sp := re_split_prefix.match(prefix)):
+ warning(f"Cannot parse line {line.strip()}")
+ dir_to_make = m_sp['dir_prefix']
+ elif m_ne := re_newextern.match(line):
+ # Found |\mmzNewExtern|: extract the extern page into an
+ # extern file.
+ done_message = "Done"
+ # The extern filename, as specified in |.mmz|:
+ unquoted_extern_path = unquote(m_ne['extern_path'])
+ extern_file = Path(unquoted_extern_path)
+ # We parse the extern filename in a separate step because
+ # we have to unquote the entire path.
+ if not (m_ep := re_extern_path.match(unquoted_extern_path)):
+ warning(f"Cannot parse line {line.strip()}")
+ # The actual extern filename:
+ extern_file_out = find_out(extern_file)
+ paranoia_out(extern_file_out)
+ page_n = int(m_ne['page_n'])-1
+ # Check whether c-memo and cc-memo exist (in any input
+ # directory).
+ c_memo = extern_file.with_name(
+ m_ep['name_prefix'] + m_ep['code_md5sum'] + '.memo')
+ cc_memo = extern_file.with_name(
+ m_ep['name_prefix'] + m_ep['code_md5sum']
+ + '-' + m_ep['context_md5sum'] + '.memo')
+ c_memo_in = find_in(c_memo)
+ cc_memo_in = find_in(cc_memo)
+ if not (access_in(c_memo_in) and access_in(cc_memo_in)) \
+ and not args.force:
+ warning(f"I refuse to extract page {page_n+1} into extern "
+ f"'{extern_file}', because the associated c-memo "
+ f"'{c_memo}' and/or cc-memo '{cc_memo}' "
+ f"does not exist")
+ raise NotExtracted()
+ # Load the PDF. We only do this now so that we don't load
+ # it if there is nothing to extract.
+ if not pdf:
+ if not access_in(pdf_file):
+ warning(f"Cannot open '{pdf_file}'")
+ endinput()
+ try:
+ # All safe, |paranoia_in| was already called above.
+ pdf = pdfrw.PdfReader(pdf_file)
+ except pdfrw.errors.PdfParseError as err:
+ error(rf"File '{pdf_file}' seems corrupted. Perhaps you "
+ rf"have to load Memoize earlier in the preamble",
+ f"In particular, Memoize must be loaded before "
+ f"TikZ library 'fadings' and any package "
+ f"deploying it, and in Beamer, load Memoize "
+ f"by writing \RequirePackage{{memoize}} before "
+ f"\documentclass{{beamer}}. "
+ f"This was the error thrown by Python: \n{err}")
+ # Does the page exist?
+ if page_n >= len(pdf.pages):
+ error(rf"I cannot extract page {page_n} from '{pdf_file}', "
+ rf"as it contains only {len(pdf.pages)} page" +
+ ('s' if len(pdf.pages) > 1 else ''), '')
+ # Check whether the page size matches the |.mmz|
+ # expectations.
+ page = pdf.pages[page_n]
+ expected_width_pt = float(m_ne['expected_width'])
+ expected_height_pt = float(m_ne['expected_height'])
+ mb = page['/MediaBox']
+ width_bp = float(mb[2]) - float(mb[0])
+ height_bp = float(mb[3]) - float(mb[1])
+ width_pt = width_bp / 72 * 72.27
+ height_pt = height_bp / 72 * 72.27
+ if (abs(width_pt - expected_width_pt) > tolerance
+ or abs(height_pt - expected_height_pt) > tolerance) \
+ and not args.force:
+ warning(
+ f"I refuse to extract page {page_n+1} from '{pdf_file}' "
+ f"because its size ({width_pt}pt x {height_pt}pt) "
+ f"is not what I expected "
+ f"({expected_width_pt}pt x {expected_height_pt}pt)")
+ raise NotExtracted()
+ # All tests were successful, let's create the extern file.
+ # First, the containing directory, if necessary.
+ if dir_to_make:
+ mkdir(dir_to_make)
+ dir_to_make = None
+ # Now the extern file. Note that |paranoia_out| was
+ # already called above.
+ info(f"Page {page_n+1} --> {extern_file_out}")
+ extern = pdfrw.PdfWriter(extern_file_out)
extern.addpage(page)
- if not args.quiet:
- print(f"{message_prefix} Page {page_n+1} --> {extern_file}", file = sys.__stdout__)
extern.write()
- done_message = f"{message_prefix}Done."
+ # This page will get pruned.
if args.prune:
extern_pages.append(page_n)
+ # Comment out this |\mmzNewExtern|.
if not args.keep:
line = '%' + line
- if not args.keep:
- new_mmz.append(line)
+ except NotExtracted:
+ pass
+ finally:
+ if not args.keep:
+ new_mmz.append(line)
mmz.close()
- if not args.quiet:
- print(done_message)
+ indent = ''
+ texindent = ''
+ info(done_message)
+
+ # Write out the |.mmz| file with |\mmzNewExtern| lines commented
+ # out. (All safe, |paranoia_out| was already called above.)
if not args.keep:
- paranoia(mmz_file)
with open(mmz_file, 'w') as mmz:
for line in new_mmz:
print(line, file = mmz, end = '')
+
+ # Remove the extracted pages from the original PDF. (All safe,
+ # |paranoia_out| was already called above.)
if args.prune and extern_pages:
pruned_pdf = pdfrw.PdfWriter(pdf_file)
pruned_pdf.addpages(
page for n, page in enumerate(pdf.pages) if n not in extern_pages)
pruned_pdf.write()
- if not args.quiet:
- print(f"{message_prefix}The following extern pages were pruned out of the PDF:",
- ",".join(str(page+1) for page in extern_pages))
- if args.log is not os.devnull:
- print(r'\endinput', file = log)
+ info(f"The following extern pages were pruned out of the PDF: " +
+ ",".join(str(page+1) for page in extern_pages))
+
+ # Report that extraction was successful.
+ endinput()
+
+ # Catch any errors in the script and output them to the log.
+ except Exception as err:
+ error(f'Python error: {err}', traceback.format_exc())
+
+# Local Variables:
+# fill-column: 79
+# after-save-hook: py2dtx
+# End:
Modified: trunk/Master/texmf-dist/source/generic/memoize/Makefile
===================================================================
--- trunk/Master/texmf-dist/source/generic/memoize/Makefile 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/source/generic/memoize/Makefile 2024-01-03 21:18:51 UTC (rev 69285)
@@ -9,15 +9,16 @@
# Prepare the CTAN submission.
PACKAGE = memoize
-VERSION = 1.0.0
-YEAR = 2023
-MONTH = 10
-DAY = 10
+VERSION = 1.1.0
+YEAR = 2024
+MONTH = 01
+DAY = 02
FORMAT = generic
COMMON = memoize nomemoize memoizable
PLAIN = memoize-extract-one.tex
+GENERIC = memoizable.code.tex
SOURCE = memoize.edtx memoize.ins # $(makefiles)
SCRIPTS := memoize-extract memoize-clean
@@ -31,11 +32,9 @@
%.py.1: %.1
echo .so man1/$*.1 > $@ # link to .1 man page
-.PHONY: runtime
-runtime: $(RUNTIME)
-
README = doc/README.memoize.md
INSTALL = INSTALL.md
+CHANGELOG = CHANGELOG.md
MAKEFILE = Makefile
LICENCE = LICENCE
@@ -44,11 +43,11 @@
codedoc-source = memoize-code.tex \
memoize-code.sty memoize-doc-common.sty
-manual-source = memoize.tex \
+manual-source = memoize-doc.tex \
memoize-doc.sty memoize-doc-common.sty yadoc.sty \
- memoize.mst
+ memoize-doc.mst
-PDF = memoize.pdf memoize-code.pdf
+PDF = memoize-doc.pdf memoize-code.pdf
codedoc-source := $(codedoc-source:%=doc/%)
manual-source := $(manual-source:%=doc/%)
@@ -80,11 +79,19 @@
ln -sr $(TDS-DOC-DIR)/examples.zip $(CTAN-DIR)/doc
$(CTAN-END)
+%.py.dtx: %.py
+ edtx2dtx -s -c '#' -B '^__version__' -E '^# Local Variables:' $< \
+ | sed -e '/^% Local Variables:/Q' > $@
+%.pl.dtx: %.pl
+ edtx2dtx -s -c '#' -B '^my \$$PROG' -E '^# Local Variables:' $< \
+ | sed -e '/^% Local Variables:/Q' > $@
-doc/memoize-code.pdf: $(SOURCE) $(codedoc-source)
+doc/memoize-code.pdf: $(SOURCE) $(codedoc-source) \
+ advice.edtx advice.ins collargs.edtx collargs.ins \
+ $(SCRIPTS:%=%.dtx)
-doc/memoize.pdf: $(manual-source) $(examples-src)
+doc/memoize.pdf: $(manual-source) $(examples-src) memoize.edtx advice.edtx collargs.edtx
%.pdf: %.tex
latexmk -cd -lualatex -bibtex- $< && touch $@
@@ -119,11 +126,9 @@
$(call EDIT-VERSION-PYTHON,memoize-clean.py)
$(call EDIT-VERSION-MAN,doc/memoize-extract.1.md)
$(call EDIT-VERSION-MAN,doc/memoize-clean.1.md)
-# Change the date of the latest release (identified by the version).
- sed -Ei 's!^\\item\[\\githubrelease\{[0-9]{4}/[0-9]{2}/[0-9]{2}\}\{v$(VERSION)\}\] *$$!\\item\[\\githubrelease\{$(YEAR)/$(MONTH)/$(DAY)\}\{v$(VERSION)\}\]!' doc/memoize.tex
-
+ $(call EDIT-DATE-CHANGELOG,CHANGELOG.md)
define COLOR_VERSION
-grep -E --color '[0-9]{4}[/-][0-9]{2}[/-][0-9]{2}|v?[0-9]\.[0-9]\.[0-9]|(January|February|March|April|May|June|July|August|September|October|November|December) [0-9]+, [0-9]{4}'
+grep -E --color '[0-9]{4}[/-][0-9]{2}[/-][0-9]{2}|v?[0-9]\.[0-9]\.[0-9]([-a-z]*)|(January|February|March|April|May|June|July|August|September|October|November|December) [0-9]+, [0-9]{4}'
endef
versions-show:
@@ -131,9 +136,26 @@
@grep __version__ *.py | ${COLOR_VERSION}
@grep VERSION *.pl | ${COLOR_VERSION}
@grep -E '^(footer|date):' doc/memoize-*.md | ${COLOR_VERSION}
- @grep -E 'githubrelease' doc/memoize.tex | ${COLOR_VERSION}
+ @${COLOR_VERSION} CHANGELOG.md doc/CHANGELOG.advice.md doc/CHANGELOG.collargs.md
include Makefile.package
include Makefile.runtimes
VERSION-MAN = of Memoize v$(VERSION)
+
+.PHONY: all-runtimes link-all-runtimes unlink-all-runtimes test
+
+all-runtimes: runtimes
+ $(MAKE) -f Makefile.advice runtimes
+ $(MAKE) -f Makefile.collargs runtimes
+
+link-all-runtimes: link-runtimes
+ $(MAKE) -f Makefile.advice link-runtimes
+ $(MAKE) -f Makefile.collargs link-runtimes
+
+unlink-all-runtimes: unlink-runtimes
+ $(MAKE) -f Makefile.advice unlink-runtimes
+ $(MAKE) -f Makefile.collargs unlink-runtimes
+
+test:
+ cd testing && ./MakeTests.py
Modified: trunk/Master/texmf-dist/source/generic/memoize/memoize.edtx
===================================================================
--- trunk/Master/texmf-dist/source/generic/memoize/memoize.edtx 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/source/generic/memoize/memoize.edtx 2024-01-03 21:18:51 UTC (rev 69285)
@@ -1,5 +1,3 @@
-% \iffalse
-%
% memoize.edtx (this is not a .dtx file; to produce a .dtx, process it with edtx2dtx)
%
%% This file is a part of Memoize, a TeX package for externalization of
@@ -21,8 +19,6 @@
%% The files belonging to this work and covered by LPPL are listed in
%% (<texmf>/doc/generic/memoize/)FILES.
%
-% \fi
-%
% \begin{macrocode}
%
% \relax
@@ -68,7 +64,7 @@
% listing options app={basicstyle=\ttfamily\scriptsize}}}
%
% \begin{tcbraster}[raster columns=100]
-% \tcbinputlisting{raster multicolumn=55, ins listing, top=1mm, bottom=1mm, title=memoize.ins,listing file=../memoize.ins, linerange={27-38}, leftupper=1mm}
+% \tcbinputlisting{raster multicolumn=55, ins listing, top=1mm, bottom=1mm, title=memoize.ins,listing file=../memoize.ins, linerange={27-39}, leftupper=1mm}
% \begin{tcboxedraster}[raster columns=1]{blankest, raster multicolumn=45}
% \tcbinputlisting{ins listing, title=advice.ins, listing file=../advice.ins, linerange=28-31}
% \tcbinputlisting{ins listing, title=collar\raisebox{0pt}[\height][0pt]{g}s.ins, listing file=../collargs.ins, linerange=29-31}
@@ -77,8 +73,7 @@
%
% Memoize also contains two scripts, |memoize-extract| and |memoize-clean|.
% Both come in two functionally equivalent implementations: Perl (|.pl|) and a
-% Python (|.py|). The code is listed in \S\ref{sec:code:scripts}, I believe it
-% is self-explanatory enough to lack more than a occasional comment.
+% Python (|.py|). Their code is listed in \S\ref{sec:code:scripts}.
%
%
% \thispagestyle{empty}
@@ -95,14 +90,14 @@
%
% \paragraph{Identification} of |memoize|, |memoizable| and |nomemoize|.
%<*mmz>
-%<latex>\ProvidesPackage{memoize}[2023/10/10 v1.0.0 Fast and flexible externalization]
+%<latex>\ProvidesPackage{memoize}[2024/1/02 v1.1.0 Fast and flexible externalization]
%<context>%D \module[
%<context>%D file=t-memoize.tex,
-%<context>%D version=1.0.0,
+%<context>%D version=1.1.0,
%<context>%D title=Memoize,
%<context>%D subtitle=Fast and flexible externalization,
%<context>%D author=Saso Zivanovic,
-%<context>%D date=2023-10-10,
+%<context>%D date=2024-01-02,
%<context>%D copyright=Saso Zivanovic,
%<context>%D license=LPPL,
%<context>%D ]
@@ -109,17 +104,17 @@
%<context>\writestatus{loading}{ConTeXt User Module / memoize}
%<context>\unprotect
%<context>\startmodule[memoize]
-%<plain>% Package memoize 2023/10/10 v1.0.0
+%<plain>% Package memoize 2024/01/02 v1.1.0
%</mmz>
%<*mmzable>
-%<latex>\ProvidesPackage{memoizable}[2023/10/10 v1.0.0 A programmer's stub for Memoize]
+%<latex>\ProvidesPackage{memoizable}[2024/1/02 v1.1.0 A programmer's stub for Memoize]
%<context>%D \module[
%<context>%D file=t-memoizable.tex,
-%<context>%D version=1.0.0,
+%<context>%D version=1.1.0,
%<context>%D title=Memoizable,
%<context>%D subtitle=A programmer's stub for Memoize,
%<context>%D author=Saso Zivanovic,
-%<context>%D date=2023-10-10,
+%<context>%D date=2024-01-02,
%<context>%D copyright=Saso Zivanovic,
%<context>%D license=LPPL,
%<context>%D ]
@@ -126,17 +121,17 @@
%<context>\writestatus{loading}{ConTeXt User Module / memoizable}
%<context>\unprotect
%<context>\startmodule[memoizable]
-%<plain>% Package memoizable 2023/10/10 v1.0.0
+%<plain>% Package memoizable 2024/01/02 v1.1.0
%</mmzable>
%<*nommz>
-%<latex>\ProvidesPackage{nomemoize}[2023/10/10 v1.0.0 A no-op stub for Memoize]
+%<latex>\ProvidesPackage{nomemoize}[2024/1/02 v1.1.0 A no-op stub for Memoize]
%<context>%D \module[
%<context>%D file=t-nomemoize.tex,
-%<context>%D version=1.0.0,
+%<context>%D version=1.1.0,
%<context>%D title=Memoize,
%<context>%D subtitle=A no-op stub for Memoize,
%<context>%D author=Saso Zivanovic,
-%<context>%D date=2023-10-10,
+%<context>%D date=2024-01-02,
%<context>%D copyright=Saso Zivanovic,
%<context>%D license=LPPL,
%<context>%D ]
@@ -143,7 +138,7 @@
%<context>\writestatus{loading}{ConTeXt User Module / nomemoize}
%<context>\unprotect
%<context>\startmodule[nomemoize]
-%<mmz>% Package nomemoize 2023/10/10 v1.0.0
+%<mmz>% Package nomemoize 2024/01/02 v1.1.0
%</nommz>
%
% \paragraph{Required packages} and \hologo{LaTeX}ization of \hologo{plainTeX}
@@ -163,13 +158,6 @@
\def\newtoks{\alloc at 5\toks\toksdef\@cclvi}
\def\newwrite{\alloc at 7\write\chardef\sixt@@n}
%</mmz&plain>
-% |Nomemoize| has to load |pgfopts| as well, and process package options (right
-% away, why not), otherwise \hologo{LaTeX} will complain.
-%<*latex>
-%<mmz,nommz>\RequirePackage{pgfopts} % pgfkeys-based package options
-%<nommz>\pgfkeys{/memoize/package options/.unknown/.code={}}
-%<nommz>\ProcessPgfPackageOptions{/memoize/package options}
-%</latex>
% I can't really write any code without |etoolbox| \dots
%<*mmz>
%<latex>\RequirePackage{etoolbox}
@@ -238,6 +226,7 @@
}%
}%
}
+%</mmz>
%
% In \hologo{ConTeXt}, |\unexpanded| means |\protected|, and the usual
% |\unexpanded| is available as |\normalunexpanded|. Option one: use dtx
@@ -249,7 +238,19 @@
% |\unexpanded| in the |.dtx|, and |sed| through the generated \hologo{ConTeXt}
% files to replace all its occurrences by |\normalunexpanded|. Oh yeah!
%
+% Load |pgfkeys| in |nomemoize| and |memoizable|. Not necessary in |memoize|,
+% as it is already loaded by CollArgs.
+%<*nommz,mmzable>
+%<latex>\RequirePackage{pgfkeys}
+%<plain>\input pgfkeys
+%<context>\input t-pgfkey
+%</nommz,mmzable>
%
+% Different formats of |memoizable| merely load |memoizable.code.tex|, which
+% exists so that |memoizable| can be easily loaded by generic code, like a
+% |tikz| library.
+%<mmzable&!generic>\input memoizable.code.tex
+%
% \paragraph.{Shipout}
% We will next load our own auxiliary package, CollArgs, but before we do
% that, we need to grab |\shipout| in \hologo{plainTeX}. The problem is,
@@ -260,6 +261,7 @@
% section~\ref{sec:code:extern}. Below, we first check that the current
% meaning of |\shipout| is primitive, and then redefine it.
%
+%<*mmz>
%<*plain>
\def\mmz at regular@shipout{%
\global\advance\mmzRegularPages1\relax
@@ -284,7 +286,7 @@
% \paragraph.{Loading order} |memoize| and |nomemoize| are mutually exclusive,
% and |memoizable| must be loaded before either of them. |\mmz at loadstatus|: 1
% = memoize, 2 = memoizable, 3 = nomemoize.
-%<*(plain,context)&(mmz,nommz,mmzable)>
+%<*mmz,nommz>
\def\ifmmz at loadstatus#1{%
\ifnum#1=0\csname mmz at loadstatus\endcsname\relax
\expandafter\@firstoftwo
@@ -292,11 +294,9 @@
\expandafter\@secondoftwo
\fi
}
-%</(plain,context)&(mmz,nommz,mmzable)>
+%</mmz,nommz>
%<*mmz>
-%<latex>\@ifpackageloaded{nomemoize}%
-%<plain,context>\ifmmz at loadstatus{3}%
-{%
+\ifmmz at loadstatus{3}{%
\PackageError{memoize}{Cannot load the package, as "nomemoize" is already
loaded. Memoization will NOT be in effect}{Packages "memoize" and
"nomemoize" are mutually exclusive, please load either one or the other.}%
@@ -304,9 +304,7 @@
%<latex>\ProcessPgfPackageOptions{/memoize/package options}
\endinput
}{}%
-%<latex>\@ifpackageloaded{memoizable}%
-%<plain,context>\ifmmz at loadstatus{2}%
-{%
+\ifmmz at loadstatus{2}{%
\PackageError{memoize}{Cannot load the package, as "memoizable" is already
loaded}{Package "memoizable" is loaded by packages which support
memoization. Memoize must be loaded before all such packages. The
@@ -324,30 +322,20 @@
%<latex>\ProcessPgfPackageOptions{/memoize/package options}
\endinput
}{}%
-%<plain,context>\ifmmz at loadstatus{1}{\endinput}{}%
-%<plain,context>\def\mmz at loadstatus{1}%
+\ifmmz at loadstatus{1}{\endinput}{}%
+\def\mmz at loadstatus{1}%
%</mmz>
-%<*mmzable>
-%<latex>\@ifpackageloaded{memoize}%
-%<plain,context>\ifmmz at loadstatus{1}%
-{\endinput}{}
-%<latex>\@ifpackageloaded{nomemoize}%
-%<plain,context>\ifmmz at loadstatus{3}%
-{\endinput}{}%
-%<plain,context>\ifmmz at loadstatus{2}{\endinput}{}%
-%<plain,context>\def\mmz at loadstatus{2}%
-%</mmzable>
+%<*mmzable&generic>
+\ifcsname mmz at loadstatus\endcsname\endinput\fi
+\def\mmz at loadstatus{2}%
+%</mmzable&generic>
%<*nommz>
-%<latex>\@ifpackageloaded{memoize}
-%<plain,context>\ifmmz at loadstatus{1}%
-{%
+\ifmmz at loadstatus{1}{%
\PackageError{nomemoize}{Cannot load the package, as "memoize" is already
loaded; memoization will remain in effect}{Packages "memoize" and
"nomemoize" are mutually exclusive, please load either one or the other.}%
\endinput }{}%
-%<latex>\@ifpackageloaded{memoizable}%
-%<plain,context>\ifmmz at loadstatus{2}%
-{%
+\ifmmz at loadstatus{2}{%
\PackageError{nomemoize}{Cannot load the package, as "memoizable" is already
loaded}{Package "memoizable" is loaded by packages which support
memoization. (No)Memoize must be loaded before all such packages. The
@@ -363,8 +351,8 @@
of that package.}%
\endinput
}{}%
-%<plain,context>\ifmmz at loadstatus{3}{\endinput}{}%
-%<plain,context>\def\mmz at loadstatus{3}%
+\ifmmz at loadstatus{3}{\endinput}{}%
+\def\mmz at loadstatus{3}%
%</nommz>
%
%<*mmz>
@@ -373,7 +361,7 @@
% Read \hologo{TeX} file |#2| into token register |#1| (under the current
% category code regime); |\toksapp| is defined in CollArgs.
\def\filetotoks#1#2{%
- \immediate\openin0{#2}%
+ \immediate\mmz at openin0{#2}%
#1={}%
\loop
\unless\ifeof0
@@ -385,6 +373,12 @@
}
% \end{macro}
%
+% \begin{macro}{\mmz at openin,\mmz at openout}
+% A workaround for |morewrites|.
+\let\mmz at openin\openin
+\let\mmz at openout\openout
+% \end{macro}
+%
% \paragraph{Other} little things.
\newif\ifmmz at temp
\newtoks\mmz at temptoks
@@ -395,22 +389,24 @@
% \label{sec:code:configuration}
%
% \begin{macro}{\mmzset}
-% The user primarily interacts with |Memoize| through the |pgfkeys|-based
+% The user primarily interacts with Memoize through the |pgfkeys|-based
% configuration macro |\mmzset|, which executes keys in path |/mmz|. In
% |nomemoize| and |memoizable|, is exists as a no-op.
\def\mmzset#1{\pgfqkeys{/mmz}{#1}\ignorespaces}
%</mmz>
-%<nommz,mmzable>\def\mmzset#1{\ignorespaces}
+%<*nommz,mmzable&generic>
+\def\mmzset#1{\ignorespaces}
+%</nommz,mmzable&generic>
% \end{macro}
-%
-% \begin{macro}{\mmzset}
+%
+% \begin{macro}{\nommzkeys}
% Any |/mmz| keys used outside of |\mmzset| must be declared by this macro
% for |nomemoize| package to work.
%<mmz>\def\nommzkeys#1{}
-%<*nommz,mmzable>
+%<*nommz,mmzable&generic>
\def\nommzkeys{\pgfqkeys{/mmz}}
\pgfqkeys{/mmz}{.unknown/.code={\pgfkeysdef{\pgfkeyscurrentkey}{}}}
-%</nommz,mmzable>
+%</nommz,mmzable&generic>
% \end{macro}
%
% \begin{key}{enable, disable}
@@ -427,7 +423,9 @@
%
% |Nomemoize| does not need the keys themselves, but it does need the
% underlying conditional --- which will be always false.
-%<mmz,nommz,mmzable>\newif\ifmemoize
+%<*mmz,nommz,mmzable&generic>
+\newif\ifmemoize
+%</mmz,nommz,mmzable&generic>
%<*mmz>
\mmzset{%
enable/.style={begindocument/.append code=\memoizetrue},
@@ -439,10 +437,15 @@
% Memoize is enabled at the beginning of the document, unless explicitly
% disabled by the user in the preamble.
enable,
-}
% \end{macro}
% \end{key}
%
+% \begin{key}{options}
+% Execute the given value as a keylist of Memoize settings.
+ options/.style={#1},
+}
+% \end{key}
+%
% \begin{key}{normal,readonly,recompile}
% When Memoize is enabled, it can be in one of three modes
% (\MS\ref{sec:tut:working-on-a-picture}): normal, readonly, and recompile.
@@ -461,7 +464,7 @@
}
% \end{key}
%
-% \begin{key}{path, path/relative, path/dir, path/prefix}
+% \begin{key}{prefix}
% Key |path| executes the given keylist in path |/mmz/path|, to determine the
% full \emph{path prefix} to memo and extern files
% (\MS\ref{sec:tut:memodir},\ref{sec:memos}): |relative|, true by default,
@@ -471,56 +474,77 @@
% under user control, and neither is the suffix. These subkeys will be
% initialized a bit later, via |no memo dir|.
\mmzset{%
- path/.code={\pgfqkeys{/mmz/path}{#1}},
- path/.cd,
- relative/.is if=mmz at relativepath,
- dir/.store in=\mmz at dir,
- dir/.value required,
- prefix/.store in=\mmz at prefix,
- prefix/.value required,
+ prefix/.code={\mmz at parse@prefix{#1}},
}
-\newif\ifmmz at relativepath
-% Key |path| concludes by performing two post-path-setting actions: creating
-% the given directory if |mkdir| is in effect, and noting the new path prefix
-% in record files (by eventually executing |record/prefix|, which typically
-% puts a |\mmzPrefix| line in the |.mmz| file). These actions are the reason
-% for having the path setting keys grouped under |path| --- we don't want them
-% to be triggered by changes of the individual components of the path.
-% Similarly, we don't want them triggered by multiple invocations of |path| in
-% the preamble; only the final setting matters, so |path| is only equipped with
-% the action-triggering code at the beginning of the document.
+% \begin{macro}{\mmz at split@prefix}
+% This macro stores the detokenized expansion of |#1| into |\mmz at prefix|,
+% which it then splits into |\mmz at prefix@dir| and |\mmz at prefix@name| at the
+% final |/|. The slash goes into |\mmz at prefix@dir|. If there is no slash,
+% |\mmz at prefix@dir| is empty.
+\begingroup
+\catcode`\/=12
+\gdef\mmz at parse@prefix#1{%
+ \edef\mmz at prefix{\detokenize\expandafter{\expanded{#1}}}%
+ \def\mmz at prefix@dir{}%
+ \def\mmz at prefix@name{}%
+ \expandafter\mmz at parse@prefix at i\mmz at prefix/\mmz at eov
+}
+\gdef\mmz at parse@prefix at i#1/#2{%
+ \ifx\mmzeov#2%
+ \def\mmz at prefix@name{#1}%
+ \else
+ \appto\mmz at prefix@dir{#1/}%
+ \expandafter\mmz at parse@prefix at i\expandafter#2%
+ \fi
+}
+\endgroup
+% \end{macro}
+%
+% Key |prefix| concludes by performing two actions: it creates the given
+% directory if |mkdir| is in effect, and notes the new prefix in record files
+% (by eventually executing |record/prefix|, which typically puts a |\mmzPrefix|
+% line in the |.mmz| file). In the preamble, only the final setting of
+% |prefix| matters, so this key is only equipped with the action-triggering
+% code at the beginning of the document.
\mmzset{%
begindocument/.append style={
- path/.append code=\mmz at maybe@mkmemodir\mmz at record@prefix,
+ prefix/.append code=\mmz at maybe@mkmemodir\mmz at record@prefix,
},
- % Consequently, the post-path-setting actions must be triggered manually at
+ % Consequently, the post-prefix-setting actions must be triggered manually at
% the beginning of the document. Below, we trigger directory creation;
% |record/prefix| will be called from |record/begin|, which is executed at
% the beginning of the document, so it shouldn't be mentioned here.
begindocument/.append code=\mmz at maybe@mkmemodir,
}
-% Define the paths to the memo directory and the prefix.
-\def\mmz at dir@path{\ifmmz at relativepath.\fi/\mmz at dir}
-\def\mmz at prefix@path{\mmz at dir@path/\mmz at prefix}
% \end{key}
%
% \begin{key}{mkdir, mkdir command}
% Should we create the memo/extern directory if it doesn't exist? And which
-% command should we use to create it? Of course, shell escape must be
-% properly configured for this to work (\MS\ref{sec:shell-escape}).
-% ^^A todo: What about quoting the paths containing spaces on Windows? Will this work?
+% command should we use to create it? There is no initial value for the
+% latter, because |mkdir| cannot be executed out of the box, but note that
+% |extract=perl| and |extract=python| will set the extraction script with
+% option |--mkdir| as the value of |mkdir command|.
\mmzset{
mkdir/.is if=mmz at mkdir,
- mkdir command/.code={\def\mmz at mkdir@command##1{#1}},
- mkdir command={mkdir "#1"},
+ mkdir command/.store in=\mmz at mkdir@command,
+ mkdir command={},
}
% The underlying conditional \cs{ifmmz at mkdir} is only ever used in
% |\mmz at maybe@mkmemodir| below, which is itself only executed at the end of
-% |path| and in |begindocument|.
+% |prefix| and in |begindocument|.
\newif\ifmmz at mkdir
+\mmz at mkdirtrue
+% We only attempt to create the memo directory if |\ifmmz at mkdir| is in effect
+% and if both |\mmz at mkdir@command| and |\mmz at prefix@dir| are specified (i.e.\
+% non-empty).
\def\mmz at maybe@mkmemodir{%
\ifmmz at mkdir
- \pdf at system{\mmz at mkdir@command{\mmzOutputDirectory\mmz at dir@path}}%
+ \ifdefempty\mmz at mkdir@command{}{%
+ \ifdefempty\mmz at prefix@dir{}{%
+ \mmz at remove@quotes{\mmz at prefix@dir}\mmz at temp
+ \pdf at system{\mmz at mkdir@command\space"\mmz at temp"}%
+ }%
+ }%
\fi
}
% \end{key}
@@ -533,51 +557,49 @@
% filenames themselves have no prefix. Furthermore, |memo dir| triggers the
% creation of the directory.
\mmzset{%
- memo dir/.style={
- mkdir,
- path={
- relative,
- dir={#1.memo.dir},
- prefix={},
- },
- },
- memo dir/.default=\mmzUnquote\jobname,
- no memo dir/.style={
- mkdir=false,
- path={
- relative,
- dir={},
- prefix={#1.},
- },
- },
- no memo dir/.default=\mmzUnquote\jobname,
+ memo dir/.style={prefix={#1.memo.dir/}},
+ memo dir/.default=\jobname,
+ no memo dir/.style={prefix={#1.}},
+ no memo dir/.default=\jobname,
no memo dir,
}
% \end{key}
%
-% \begin{macro}{\mmzUnquote}
-% If the expanded argument is surrounded by double quotes, remove them. This
-% relies on |#1| containing no quotes other than the potential surrounding
-% quotes, which should be the case when applying the macro to |\jobname|. An
-% empty |#1| is dealt with correctly, even if |\jobname| can hardly ever be
-% empty (needs |openout_any=a|).
+% \begin{macro}{\mmz at remove@quotes}
+% This macro removes fully expands |#1|, detokenizes the expansion and then
+% removes all double quotes the string. The result is stored in the control
+% sequence given in |#2|.
%
% We use this macro when we are passing a filename constructed from
% |\jobname| to external programs.
-\def\mmzUnquote#1{\expanded{\noexpand\mmz at unquote#1}\mmz at unquote@end}
-\def\mmz at unquote#1{%
- \ifx\mmz at unquote@end#1%
+\def\mmz at remove@quotes#1#2{%
+ \def\mmz at remove@quotes at end{\let#2\mmz at temp}%
+ \def\mmz at temp{}%
+ \expanded{%
+ \noexpand\mmz at remove@quotes at i
+ \detokenize\expandafter{\expanded{#1}}%
+ "\noexpand\mmz at eov
+ }%
+}
+\def\mmz at remove@quotes at i{%
+ \CollectArgumentsRaw
+ {\collargsBraceCollectedfalse
+ \collargsNoDelimiterstrue
+ \collargsAppendPostwrap{{##1}}%
+ }%
+ {u"u\mmz at eov}%
+ \mmz at remove@quotes at ii
+}
+\def\mmz at remove@quotes at ii#1#2{%
+ \appto\mmz at temp{#1}%
+ \ifx&%
+ \mmz at remove@quotes at end
+ \expandafter\@gobble
\else
- \ifx"#1%
- \expandafter\expandafter\expandafter\mmz at unquote@quotes
- \else
- \expandafter\expandafter\expandafter\mmz at unquote@noquotes
- \expandafter\expandafter\expandafter#1%
- \fi
+ \expandafter\@firstofone
\fi
+ {\mmz at remove@quotes at i#2\mmz at eov}%
}
-\def\mmz at unquote@quotes#1"\mmz at unquote@end{#1}
-\def\mmz at unquote@noquotes#1\mmz at unquote@end{#1}
% \end{macro}
%
% \begin{key}{ignore spaces}
@@ -832,6 +854,7 @@
%<context>\let\startmemoize\startnomemoize
%<context>\let\stopmemoize\stopnomemoize
%</nommz>
+%</mmz,nommz>
% \end{environment}
%
% \subsection{The memoization process}
@@ -843,6 +866,7 @@
% anywhere else. It is checked by |\Memoize| to prevent nested
% memoizations, deployed in advice run conditions set by |run only if
% memoizing|, etc.
+%<*mmz,nommz,mmzable&generic>
\newif\ifmemoizing
% \end{macro}
%
@@ -859,7 +883,7 @@
% An auxiliary macro which rescans the given
% code using |\scantokens| if the verbatim mode is active. We also need it
% in NoMemoize, to properly grab verbatim manually memoized code.
-%</mmz,nommz>
+%</mmz,nommz,mmzable&generic>
%<*mmz>
\def\mmz at maybe@scantokens{%
\ifmmz at verbatim
@@ -1091,7 +1115,7 @@
\gtoksapp\mmzCMemo{\global\mmzUnmemoizabletrue}%
\mmz at write@cmemo
\mmz at trace@endmemoize at unmemoizable
- \PackageWarning{memoize}{Marking this code as unmemoizable}%
+ \PackageInfo{memoize}{Marking this code as unmemoizable}%
\else
\ifmmz at abort
% If memoization was aborted, we create an empty c-memo, to make sure that
@@ -1098,8 +1122,9 @@
% no leftover c-memo tricks Memoize into thinking that the code was
% successfully memoized.
\mmz at trace@endmemoize at aborted
- \PackageWarning{memoize}{Memoization was aborted}%
- \mmz at write@empty at cmemo
+ \PackageInfo{memoize}{Memoization was aborted}%
+ \mmz at compute@context at mdfivesum
+ \mmz at write@cmemo
\else
% If memoization was not aborted, we compute the \meta{context md5sum},
% open and write out the memos, and shipout the externs (as pages into the
@@ -1320,7 +1345,7 @@
%
% \paragraph.{Tracing} We populate the hooks which send the tracing info to the
% terminal.
-\def\mmz at trace#1{\immediate\write16{[tracing memoize] #1}}
+\def\mmz at trace#1{\advice at typeout{[tracing memoize] #1}}
\def\mmz at trace@context{\mmz at trace{\space\space
Context: "\expandonce{\mmz at context@key}" --> \mmz at context@mdfivesum}}
\def\mmz at trace@Memoize at on{%
@@ -1424,9 +1449,6 @@
% \end{macro}
% \end{key}
%
-%</mmz>
-%<nommz,mmzable>\newcommand\IfMemoizing[2][]{\@secondoftwo}
-%<*mmz>
%
% \subsection{Context}
% \label{sec:code:context}
@@ -1465,24 +1487,27 @@
% context, key value to context, /handlers/.meaning to context,
% /handlers/.value to context}
% Utilities to put the meaning of various stuff into |context|.
+ % \indentmacrocode
meaning to context/.code={\forcsvlist\mmz at mtoc{#1}},
- csname meaning to context/.code={\mmz at mtoc@cs{#1}},
- key meaning to context/.code={\forcsvlist\mmz at mtoc\mmz at mtoc@keycmd{#1}},
+ csname meaning to context/.code={\mmz at mtoc@csname{#1}},
+ key meaning to context/.code={%
+ \forcsvlist\mmz at mtoc\mmz at mtoc@keycmd{#1}},
key value to context/.code={\forcsvlist\mmz at mtoc@key{#1}},
- /handlers/.meaning to context/.code={%
- \expanded{\noexpand\mmz at mtoc@cs{pgfk@\pgfkeyscurrentpath/. at cmd}}},
+ /handlers/.meaning to context/.code={\expanded{%
+ \noexpand\mmz at mtoc@csname{pgfk@\pgfkeyscurrentpath/. at cmd}}},
/handlers/.value to context/.code={%
- \expanded{\noexpand\mmz at mtoc@cs{pgfk@\pgfkeyscurrentpath}}},
+ \expanded{\noexpand\mmz at mtoc@csname{pgfk@\pgfkeyscurrentpath}}},
}
+% \noindentmacrocode
\def\mmz at mtoc#1{%
\collargs at cs@cases{#1}%
{\mmz at mtoc@cmd{#1}}%
{\mmz at mtoc@error at notcsorenv{#1}}%
{%
- \mmz at mtoc@cs{%
+ \mmz at mtoc@csname{%
%<context>start%
#1}%
- \mmz at mtoc@cs{%
+ \mmz at mtoc@csname{%
%<latex,plain>end%
%<context>stop%
#1}%
@@ -1492,17 +1517,18 @@
\begingroup
\escapechar=-1
\expandafter\endgroup
- \expandafter\mmz at mtoc@cs\expandafter{\string#1}%
+ \expandafter\mmz at mtoc@csname\expandafter{\string#1}%
}
-\def\mmz at mtoc@cs#1{%
+\def\mmz at mtoc@csname#1{%
\pgfkeysvalueof{/mmz/context/. at cmd}%
- \expandafter\string\csname#1\endcsname={\expandafter\meaning\csname#1\endcsname}%
- \pgfeov
+ \detokenize{#1}={\expandafter\meaning\csname#1\endcsname}%
+ \pgfeov
}
-\def\mmz at mtoc@key#1{\mmz at mtoc@cs{pgfk@#1}}
-\def\mmz at mtoc@key#1{\mmz at mtoc@cs{pgfk@#1/. at cmd}}
+\def\mmz at mtoc@key#1{\mmz at mtoc@csname{pgfk@#1}}
+\def\mmz at mtoc@keycmd#1{\mmz at mtoc@csname{pgfk@#1/. at cmd}}
\def\mmz at mtoc@error at notcsorenv#1{%
- \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context' is neither a command nor an environment}{}%
+ \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context'
+ is neither a command nor an environment}{}%
}
% \end{key}
%
@@ -1512,7 +1538,7 @@
%
% \paragraph{The path} to a c-memo consists of the path prefix, the MD5 sum of
% the memoized code, and suffix |.memo|.
-\def\mmz at cmemo@path{\mmz at prefix@path\mmz at code@mdfivesum.memo}
+\def\mmz at cmemo@path{\mmz at prefix\mmz at code@mdfivesum.memo}
% \begin{macro}{\mmzCMemo}
% The additional, free-form content of the c-memo is collected in this token
% register.
@@ -1536,7 +1562,7 @@
% |\mmzCMemo|.
\def\mmz at write@cmemo{%
% Open the file for writing.
- \immediate\openout\mmz at out{\mmz at cmemo@path}%
+ \immediate\mmz at openout\mmz at out{\mmz at cmemo@path}%
% The memo starts with the |\mmzMemo| marker (a signal that the memo is valid).
\immediate\write\mmz at out{\noexpand\mmzMemo}%
% We store the content of |\mmzContextExtra| by writing out a command that
@@ -1559,17 +1585,6 @@
}
% \end{macro}
%
-% \begin{macro}{\mmz at write@empty at cmemo}
-% This macro is used to create an empty c-memo on aborted memoization, to
-% make sure that no leftover c-memo tricks Memoize into thinking that the
-% code was successfully memoized.
-\def\mmz at write@empty at cmemo{%
- \immediate\openout\mmz at out{\mmz at cmemo@path}%
- \immediate\closeout\mmz at out
-}
-% \end{macro}
-%
-%
% \begin{macro}{\mmzSource}
% The c-memo memoized code marker. This macro is synonymous with |\endinput|,
% so the source following it is ignored when inputting the c-memo.
@@ -1644,7 +1659,7 @@
% hyphen-separated MD5 sums of the memoized code and the (evaluated) context,
% and suffix |.memo|.
\def\mmz at ccmemo@path{%
- \mmz at prefix@path\mmz at code@mdfivesum-\mmz at context@mdfivesum.memo}
+ \mmz at prefix\mmz at code@mdfivesum-\mmz at context@mdfivesum.memo}
%
% \paragraph{The structure} of a cc-memo:
% \begin{itemize}
@@ -1699,7 +1714,7 @@
% memoized code can update the context. This is one of the two reasons why
% we couldn't write the cc-memo directly into the file, but had to collect
% its contents into token register |\mmzCCMemo|.
- \immediate\openout\mmz at out{\mmz at ccmemo@path}%
+ \immediate\mmz at openout\mmz at out{\mmz at ccmemo@path}%
% Token register |\mmz at ccmemo@resources| consists of calls to
% |\mmz at ccmemo@append at resource|, so the following code writes down the list
% of created externs into the cc-memo. Wanting to have this list at the top
@@ -1751,7 +1766,7 @@
% might be produced by a failed \hologo{TeX}-based extraction, should count
% as no file. The |0| behind |\ifnum| is there because |\pdffilesize|
% returns an empty string when the file does not exist.
- \ifnum0\pdf at filesize{\mmz at dir@path/#1}=0
+ \ifnum0\pdf at filesize{\mmz at prefix@dir/#1}=0
\ifmmz at direct@ccmemo at input
\let\mmzMemo\endinput
\else
@@ -1768,8 +1783,10 @@
%
% \begin{macro}{\mmz at process@ccmemo,\mmzThisContext,\mmzEndMemo}
% This macro processes the cc-memo.
+% \indentmacrocode
\def\mmz at process@ccmemo{%
\mmz at trace@process at ccmemo
+ % \noindentmacrocode
% The following conditional signals whether cc-memo was successfully
% utilized. If the cc-memo file does not exist, |\ifmmz at abort| will remain
% true. If it exists, it is headed by the list of resources. If a
@@ -1865,11 +1882,11 @@
% full path.
\newcount\mmz at seq
\def\mmz at extern@basename{%
- \mmz at prefix\mmz at code@mdfivesum-\mmz at context@mdfivesum
+ \mmz at prefix@name\mmz at code@mdfivesum-\mmz at context@mdfivesum
\ifnum\mmz at seq>0 -\the\mmz at seq\fi
}
\def\mmz at extern@name{\mmz at extern@basename.pdf}
-\def\mmz at extern@basepath{\mmz at dir@path/\mmz at extern@basename}
+\def\mmz at extern@basepath{\mmz at prefix@dir\mmz at extern@basename}
\def\mmz at extern@path{\mmz at extern@basepath.pdf}
%
% \begin{key}{padding left, padding right, padding top, padding bottom}
@@ -2124,12 +2141,14 @@
% |\mmzExternPages| holds the number of shipped-out extern pages; and
% |\mmzExtraPages| holds, or at least should hold, the number of pages
% shipped out using any other means.
- \edef\pagenumber{\the\numexpr\mmzRegularPages
+ \edef\pagenumber{%
+ \the\numexpr\mmzRegularPages
% In \hologo{LaTeX}, the |\mmzRegularPages| holds to number of pages
% already shipped out. In \hologo{ConTeXt}, the counter is already
% increased while processing the page, so we need to subtract $1$.
%<context> -1%
- +\mmzExternPages+\mmzExtraPages}%
+ +\mmzExternPages+\mmzExtraPages
+ }%
% Record the creation of the new extern. We do this after shipping out the
% extern page, so that the recording mechanism can serve as an after-shipout
% hook, for the unlikely situation that some package really needs to do
@@ -2212,7 +2231,7 @@
enddocument/afterlastpage/.append code={%
\ifnum\mmzExternPages>0
\PackageWarning{memoize}{The compilation produced \the\mmzExternPages\space
- new extern\ifnum\mmzExternPages>1 s\fi.}%
+ new extern\ifnum\mmzExternPages>1 s\fi}%
\fi
},
}
@@ -2304,13 +2323,13 @@
%
% \begin{macro}{\mmz at include@extern at from@tbe at box}
% Include the extern number |#1| residing in |\mmz at tbe@box| into the
-% document. This helper macro makes it possible for a complex memoization
-% driver to process the cc-memo right after memoization --- by using the
-% |\mmzkeepexternstrue\xtoksapp\mmzAfterMemoizationExtra{\the\mmzCCMemo}|
-% trick --- to ensure that the result of the memoizing compilation matches
-% the result of inputting the cc-memo. The rest of the arguments are
-% gobbled, as we don't have to do any size adjustment or checking here, and
-% the box is of the correct type.
+% document. It may be called as |\mmzIncludeExtern| from |after memoization|
+% hook if |\ifmmzkeepexterns| was set to true during memoization. The macro
+% takes the same arguments as |\mmzIncludeExtern| but disregards all but
+% the first one, the extern sequential number. Using this macro, a complex
+% memoization driver can process the cc-memo right after memoization, by
+% issuing
+% |\global\mmzkeepexternstrue\xtoksapp\mmzAfterMemoizationExtra{\the\mmzCCMemo}|.
\def\mmz at include@extern at from@tbe at box#1#2#3#4#5#6#7#8#9{%
\setbox0\vbox{%
\@tempcnta#1\relax
@@ -2344,7 +2363,7 @@
\mmzset{
extract/.estore in=\mmz at extraction@method,
extract/.value required,
- begindocument/.append style={extract/.code=\mmz at preamble@only at warning},
+ begindocument/.append style={extract/.code=\mmz at preamble@only at error},
% \end{key}
%
% \begin{key}{extract/perl, extract/python}
@@ -2352,10 +2371,10 @@
% Memoize ships with two extraction scripts, a Perl script and a Python
% script, which are selected by |extract=perl| (the default) and
% |extract=python|, respectively. We run the scripts in verbose mode
- % (without |-q|), and keep the |.mmz| file as is, i.e.\ we're not commenting
- % out the |\mmzNewExtern| lines, because we're about to overwrite it
- % anyway. We also request the log file, which will contain
- % |\mmzExtractionSuccessful| if extraction was successful.
+ % (without |-q|), and keep the |.mmz| file as is (without |-k|), i.e.\
+ % we're not commenting out the |\mmzNewExtern| lines, because we're about
+ % to overwrite it anyway. We inform the script about the format of the
+ % document (|-F|).
extract/perl/.code={%
\mmz at clear@extraction at log
\pdf at system{%
@@ -2363,17 +2382,14 @@
\mmzvalueof{perl extraction options}%
}%
\mmz at check@extraction at log{perl}%
- \def\mmz at mkdir@command##1{\mmzvalueof{perl extraction command}\space --mkdir "##1"}%
+ \def\mmz at mkdir@command{\mmzvalueof{perl extraction command} --mkdir}%
},
perl extraction command/.initial=memoize-extract.pl,
- perl extraction options/.initial={%
-% \begin{listingregion}{perl-extraction-options.tex}
- -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w
- %<latex>"\string\PackageWarning{memoize (perl-based extraction)}{\string\warningtext}"
- %<plain>"\string\warning{memoize (perl-based extraction): \string\warningtext}"
- %<context>"\string\warning{memoize (perl-based extraction): \string\warningtext}"
- "\mmzOutputDirectory\mmzUnquote\jobname.mmz"
-% \end{listingregion}
+ perl extraction options/.initial={\space
+ %<latex>-F latex
+ %<plain>-F plain
+ %<context>-F context
+ \jobname\space
},
extract=perl,
extract/python/.code={%
@@ -2383,21 +2399,20 @@
\mmzvalueof{python extraction options}%
}%
\mmz at check@extraction at log{python}%
- \def\mmz at mkdir@command##1{\mmzvalueof{python extraction command}\space --mkdir "##1"}%
+ \def\mmz at mkdir@command{\mmzvalueof{python extraction command} --mkdir}%
},
python extraction command/.initial=memoize-extract.py,
- python extraction options/.initial={%
- -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w
- %<latex>"\string\PackageWarning{memoize (python-based extraction)}{\string\warningtext}"
- %<plain>"\string\warning{memoize (python-based extraction): \string\warningtext}"
- %<context>"\string\warning{memoize (python-based extraction): \string\warningtext}"
- "\mmzOutputDirectory\mmzUnquote\jobname.mmz"
+ python extraction options/.initial={\space
+ %<latex>-F latex
+ %<plain>-F plain
+ %<context>-F context
+ \jobname\space
},
}
-\def\mmz at preamble@only at warning{%
- \PackageWarning{memoize}{%
+\def\mmz at preamble@only at error{%
+ \PackageError{memoize}{%
Ignoring the invocation of "\pgfkeyscurrentkey".
- This key may only be executed in the preamble}%
+ This key may only be executed in the preamble}{}%
}
% \end{key}
%
@@ -2406,15 +2421,14 @@
% communicate with the system command via the ``extraction log file,'' produced
% by both \hologo{TeX}-based extraction and the Perl and Python extraction
% script. This file signals whether the embedded extraction was successful ---
-% if it is, the file contains |\mmzExtractionSuccessful| --- and also contains
-% any size-mismatch warnings (these are currently only thrown by the
-% \hologo{TeX}-based extraction). As the log is really a \hologo{TeX} file,
-% the idea is to simply input it after extracting each extern (for
+% if it is, the file ends if |\endinput| --- and also contains any warnings and
+% errors thrown by the script. As the log is really a \hologo{TeX} file, the
+% idea is to simply input it after extracting each extern (for
% \hologo{TeX}-based extraction) or after the extraction of all externs (for
% the external scripts).
\def\mmz at clear@extraction at log{%
\begingroup
- \immediate\openout0{\mmzUnquote\jobname.mmz.log"}%
+ \immediate\mmz at openout0{\jobname.mmz.log}%
\immediate\closeout0
\endgroup
}
@@ -2426,9 +2440,11 @@
\@input{\jobname.mmz.log}%
\ifmmz at temp \else \mmz at extraction@error \fi \endgroup }
\def\mmz at extraction@error{%
- \PackageWarning{memoize}{Extraction of externs from document "\mmzUnquote\jobname.pdf"
- using method "\extractionmethod" was unsuccessful. Have you set the
- shell escape mode as suggested in chapter 1 of the manual?}{}}
+ \PackageError{memoize}{Extraction of externs from document
+ "\jobname.pdf" using method "\extractionmethod" was
+ unsuccessful}{The extraction script "\mmzvalueof{\extractionmethod\space
+ extraction command}" wasn't executed or didn't finish execution
+ properly.}}
%
% \subsection{The record files}
% \label{sec:code:record}
@@ -2451,7 +2467,7 @@
% surely occurs at the top of the |.mmz| file. Listing each prefix type
% separately in this hook ensures that |prefix| of a certain type is
% executed after that type's |begin|.
- /mmz/record/#1/prefix/.try/.expanded=\mmz at prefix@path,
+ /mmz/record/#1/prefix/.try/.expanded=\mmz at prefix,
},
record/prefix/.append style={/mmz/record/#1/prefix/.try={##1}},
record/new extern/.append style={/mmz/record/#1/new extern/.try={##1}},
@@ -2495,7 +2511,7 @@
% We define this macro because |\aftergroup|, used in |record/prefix|, only
% accepts a token.
\def\mmz at record@prefix{%
- \mmzset{/mmz/record/prefix/.expanded=\mmz at prefix@path}%
+ \mmzset{/mmz/record/prefix/.expanded=\mmz at prefix}%
}
% \paragraph{Initialize} the hook keys, preactivate |mmz| record type, and
% execute hooks |begin| and |end| at the edges of the document.
@@ -2516,12 +2532,11 @@
% contains are intentionally as simple as possible (just a macro plus braced
% arguments), to facilitate parsing by the external scripts.
%
-% These hooks simply put the calls of the corresponding macros into the file.
-% All but hooks but |begin| and |end| receive the full path to the relevant
-% file as the only argument (ok, |prefix| receives the full path prefix, as set
-% by key |path|).
-%
% \begin{key}{record/mmz/...}
+% These hooks simply put the calls of the corresponding macros into the file.
+% All but hooks but |begin| and |end| receive the full path to the relevant
+% file as the only argument (ok, |prefix| receives the full path prefix, as
+% set by key |path|).
\mmzset{
record/mmz/begin/.code={%
\newwrite\mmz at mmzout
@@ -2529,7 +2544,7 @@
% location (the current directory, i.e.\ the directory where \hologo{TeX}
% is executed from; usually, this will be the directory containing the
% \hologo{TeX} source).
- \immediate\openout\mmz at mmzout{\jobname.mmz}%
+ \immediate\mmz at openout\mmz at mmzout{\jobname.mmz}%
},
% The |\mmzPrefix| is used by the clean-up script, which will remove all
% files with the given path prefix but (unless called with |--all|) those
@@ -2592,7 +2607,7 @@
% \begin{key}{record/sh/...} Define the Linux shell script record type.
record/sh/begin/.code={%
\newwrite\mmz at shout
- \immediate\openout\mmz at shout{\mmz at shname}%
+ \immediate\mmz at openout\mmz at shout{\mmz at shname}%
},
record/sh/new extern/.code={%
\begingroup
@@ -2611,7 +2626,7 @@
% Rinse and repeat for Windows.
record/bat/begin/.code={%
\newwrite\mmz at batout
- \immediate\openout\mmz at batout{\mmz at batname}%
+ \immediate\mmz at openout\mmz at batout{\mmz at batname}%
},
record/bat/new extern/.code={%
\begingroup
@@ -2649,7 +2664,7 @@
% variables |.DEFAULT_GOAL| and |.PHONY|.
\newwrite\mmz at makefileout
\newtoks\mmz at makefile@externs
- \immediate\openout\mmz at makefileout{\mmz at makefilename}%
+ \immediate\mmz at openout\mmz at makefileout{\mmz at makefilename}%
\immediate\write\mmz at makefileout{.DEFAULT_GOAL = externs}%
\immediate\write\mmz at makefileout{.PHONY: externs}%
},
@@ -2693,11 +2708,10 @@
}
% \end{key}
%
-% \begingroup
-% \setlength\MacroIndent{2em}
% \begin{macro}{\mmzUsedCMemo,\mmzUsedCCMemo,\mmzUsedExtern,\mmzNewCMemo,\mmzNewCCMemo,\mmzPrefix}
% We can ignore everything but |\mmzNewExtern|s. All these macros receive a
% single argument.
+% \indentmacrocode
\def\mmzUsedCMemo#1{}
\def\mmzUsedCCMemo#1{}
\def\mmzUsedExtern#1{}
@@ -2705,7 +2719,6 @@
\def\mmzNewCCMemo#1{}
\def\mmzPrefix#1{}
% \end{macro}
-% \endgroup
%
% \begin{macro}{\mmzNewExtern}
% Command |\mmzNewExtern| takes four arguments. It instructs us to extract
@@ -2744,11 +2757,11 @@
% \end{macro}
\def\mmz at pageextraction@error{%
\PackageError{memoize}{Extraction of extern page \pagenumber\space from
- document "\mmzUnquote\jobname.pdf" using method "\extractionmethod" was unsuccessful.
- Have you set the shell escape mode as suggested in chapter 1 of the
- manual?}{If "\mmzvalueof{tex extraction command}" was executed,
- shell escape mode is not the problem, and inspecting "\externbasepath.log"
- might give you a clue what's wrong}}
+ document "jobname.pdf" using method "\extractionmethod" was
+ unsuccessful.}{Check the log file to see if the extraction script was
+ executed at all, and if it finished successfully. You might also want to
+ inspect "\externbasepath.log", the log file of the embedded TeX compilation
+ which ran the extraction script}}
%
% \begin{key}{tex extraction command, tex extraction options, tex extraction script}
% Using these keys, we set the system call
@@ -2786,12 +2799,11 @@
-halt-on-error
-interaction=batchmode
-jobname "\externbasepath"
- \ifdefempty\mmzOutputDirectory{}{-output-directory "\mmzOutputDirectory"}
% \end{listingregion}
},
tex extraction script/.initial={%
% \begin{listingregion}{tex-extraction-script.tex} ^^A todo: context
- \def\noexpand\fromdocument{"\mmzOutputDirectory"\jobname.pdf}%
+ \def\noexpand\fromdocument{\jobname.pdf}%
\def\noexpand\pagenumber{\pagenumber}%
\def\noexpand\expectedwidth{\expectedwidth}%
\def\noexpand\expectedheight{\expectedheight}%
@@ -3008,6 +3020,7 @@
%<plain,context>noop/.style={inner handler=\mmz at auto@noop},
nomemoize/.style={noop, options=disable},
replicate/.style={run if memoizing, inner handler=\mmz at auto@replicate},
+ to context/.style={run if memoizing, outer handler=\mmz at auto@tocontext},
}
%
% \paragraph{Abortion}
@@ -3221,6 +3234,27 @@
}
% \end{macro}
% \end{mmzautokey}
+%
+% \begin{mmzautokey}{to context}
+% \begin{macro}{\mmz at auto@tocontext}
+% This outer handler appends the original definition of the handled command
+% to the context. The |\expandafter| are there to expand |\AdviceName|
+% once before fully expanding |\AdviceGetOriginalCsname|.
+\def\mmz at auto@tocontext{%
+ \expanded{%
+ \noexpand\pgfkeysvalueof{/mmz/context/. at cmd}%
+ original "\AdviceNamespace" csname "\AdviceCsname"={%
+ \noexpand\expanded{%
+ \noexpand\noexpand\noexpand\meaning
+ \noexpand\AdviceCsnameGetOriginal{\AdviceNamespace}{\AdviceCsname}%
+ }%
+ }%
+ }%
+ \pgfeov
+ \AdviceOriginal
+}
+% \end{macro}
+% \end{mmzautokey}
%
% \subsection{\hologo{LaTeX}-specific handlers}
%
@@ -3277,12 +3311,7 @@
% appends the reference key to the context. |\mmzNoRef| only does that if
% the reference is defined, otherwise it aborts the memoization.
\def\mmzForceNoRef#1{%
- \ifmemoizing
- \expandafter\gtoksapp\expandafter\mmzContextExtra
- \else
- \expandafter\toksapp\expandafter\mmzContext
- \fi
- {r@#1={\csname r@#1\endcsname}}%
+ \mmz at mtoc@csname{r@#1}%
\ignorespaces
}
\def\mmzNoRef#1{%
@@ -3360,11 +3389,17 @@
\endgroup
}
%</latex>
-%</mmz>
%
% \end{macro}
%
% \section{Support for various classes and packages}
+%
+%<*latex>
+\AddToHook{shipout/before}[memoize]{\global\advance\mmzExtraPages-1\relax}
+\AddToHook{shipout/after}[memoize]{\global\advance\mmzExtraPages1\relax}
+\mmzset{auto=\DiscardShipoutBox{
+ outer handler=\global\advance\mmzExtraPages1\relax\AdviceOriginal}}
+%</latex>
%
% \subsection{\TikZ;}
% \label{sec:code:mmz:tikz}
@@ -3373,7 +3408,6 @@
% Advice). All the action happens at the end of the preamble, so that we can
% detect whether \TikZ; was loaded (regardless of whether Memoize was loaded
% before \TikZ;, or vice versa), but still input the definitions.
-%<*mmz>
\mmzset{
begindocument/before/.append code={%
%<latex>\@ifpackageloaded{tikz}{%
@@ -3384,32 +3418,23 @@
% We define and activate the automemoization handlers for the \TikZ; command
% and environment.
\mmzset{%
- % \begin{listingregion}{_auto-tikz-collector.tex}
- auto=\tikz{memoize, collector=\AdviceCollectTikZArguments},
- % \end{listingregion}
- %/utils/exec={\tracingall},
- auto={tikzpicture}{memoize},
- % A hack to prevent memoizing pictures which are accidentally marked as
- % remembered --- accidentally in the sense that because the document
- % changed, the |.aux| file contains a |\pgfsyspdfmark| command which
- % erroneously refers to the picture being memoized. We know that
- % memoizing a remembered picture can't be right, as we always abort on
- % |\pdfsavepos|. This is implemented by hacking into PGF's
- % |\pgfsys at getposition|, and aborting memoization if the mark does not
- % equal |\relax|. (We have to duplicate |#| because of |.append code|.)
- auto=\pgfsys at getposition{
- run if memoizing, outer handler=\mmz at pgfsys@getposition},
- }%
- \def\mmz at pgfsys@getposition##1{%
- \expandafter\ifx\csname pgf at sys@pdf at mark@pos@##1\endcsname\relax
- \else
- \mmzAbort
- \fi
- \AdviceOriginal{##1}%
+ auto/memoize tikz/.style={
+ memoize,
+ at begin memoization=\edef\mmz at pgfpictureid{%
+ \the\pgf at picture@serial at count
+ },
+ at end memoization=\xtoksapp\mmzCCMemo{%
+ \unexpanded{%
+ \global\expandafter\advance\csname pgf at picture@serial at count\endcsname
+ }%
+ \the\numexpr\pgf at picture@serial at count-\mmz at pgfpictureid\relax\relax
+ },
+ },
+ auto=\tikz{memoize tikz, collector=\AdviceCollectTikZArguments},
+ auto={tikzpicture}{memoize tikz},
}%
},
}
-%</mmz>
%
%
% \subsection{Forest}
@@ -3417,7 +3442,6 @@
%
% Forest will soon feature extensive memoization support, but for now, let's
% just enable the basic, single extern externalization.
-%<*mmz>
%<*latex>
\mmzset{
begindocument/before/.append code={%
@@ -3478,7 +3502,99 @@
}{}}
%</latex>
%
+% \subsection{Morewrites}
+% \label{sec:code:morewrites}
+%
+% Use the old grammar for |\openin| and |\openout| as a temporary workaround.
+% |prefix|es containing spaces must be quoted manually.
+%<*latex>
+\AddToHook{begindocument/before}{%
+ \@ifpackageloaded{morewrites}{%
+ \def\mmz at openin#1#2{\openin#1=#2\relax}%
+ \def\mmz at openout#1#2{\openout#1=#2\relax}%
+ }{}%
+}
+%</latex>
%
+% \subsection{Biblatex}
+% \label{sec:biblatex}
+%
+%<*latex>
+\mmzset{
+ begindocument/before/.append style={%
+ auto=\blx at bbl@entry{outer handler=\mmz at biblatex@entry},
+ auto/cite/.style={run if memoizing, outer handler=\mmz at biblatex@cite},
+ auto/cites/.style={run if memoizing, outer handler=\mmz at biblatex@cites},
+ auto=\cite{cite},
+ auto=\cites{cites},
+ }%
+}
+% \begin{macro}{\mmz at biblatex@entry}
+% This macro stores the MD5 sum of the |\entry| when reading the |.bbl| file.
+\def\mmz at biblatex@entry#1#2\endentry{%
+ \csxdef{mmz at bbl@#1}{\pdf at mdfivesum{#2}}%
+ \AdviceOriginal{#1}#2\endentry
+}
+% \end{macro}
+% \begin{macro}{\mmz at biblatex@cite}
+% This macro puts the cites reference keys into the context, and adds the
+% handled |\cite| command to the cc-memo.
+\def\mmz at biblatex@cite#1#{\mmz at biblatex@cite at i{#1}}
+\def\mmz at biblatex@cite at i#1#2{%
+ \forcsvlist\mmz at biblatex@cite at do@key{#2}%
+ \xtoksapp\mmzCCMemo{%
+ \noexpand\setbox0\noexpand\hbox{%
+ \expandonce\AdviceOriginal\unexpanded{#1}{#2}%
+ }}%
+ \AdviceOriginal#1{#2}%
+}
+\def\mmz at biblatex@cite at do@key#1{%
+ \mmz at mtoc@csname{mmz at bbl@#1}%
+ \ifcsdef{mmz at bbl@#1}{}{\mmzAbort}%
+}
+% \end{macro}
+% \begin{macro}{\mmz at biblatex@cites}
+% This macro puts the cites reference keys into the context, and adds the
+% handled |\cites| command to the cc-memo.
+\def\mmz at biblatex@cites{%
+ \mmz at temptoks{}%
+ \mmz at biblatex@cites at i
+}
+\def\mmz at biblatex@cites at i{%
+ \futurelet\mmz at temp\mmz at biblatex@cites at ii
+}
+\def\mmz at biblatex@cites at ii{%
+ \mmz at tempfalse
+ \ifx\mmz at temp\bgroup
+ \mmz at temptrue
+ \else
+ \ifx\mmz at temp[%]
+ \mmz at temptrue
+ \fi
+ \fi
+ \ifmmz at temp
+ \expandafter\mmz at biblatex@cites at iii
+ \else
+ \expandafter\mmz at biblatex@cites at z
+ \fi
+}
+\def\mmz at biblatex@cites at iii#1#{\mmz at biblatex@cites at iv{#1}}
+\def\mmz at biblatex@cites at iv#1#2{%
+ \forcsvlist\mmz at biblatex@cite at do@key{#2}%
+ \toksapp\mmz at temptoks{#1{#2}}%
+ \mmz at biblatex@cites at i
+}
+\def\mmz at biblatex@cites at z{%
+ \xtoksapp\mmzCCMemo{%
+ \noexpand\setbox0\noexpand\hbox{%
+ \expandonce\AdviceOriginal\the\mmz at temptoks
+ }}%
+ \expandafter\AdviceOriginal\the\mmz at temptoks
+}
+%</latex>
+% \end{macro}
+%
+%
% \section{Initialization}
% \label{sec:code:initialization}
%
@@ -3512,6 +3628,10 @@
%</plain,context>
% \end{key}
%
+% Formats other than \hologo{plainTeX} need a way to prevent extraction during
+% package-loading.
+%<!plain>\mmzset{extract/no/.code={}}
+%
% \paragraph{\texttt{memoize.cfg}} Load the configuration file. Note that
% |nomemoize| must input this file as well, because any special
% memoization-related macros defined by the user should be available; for
@@ -3521,63 +3641,91 @@
%<mmz,nommz>\InputIfFileExists{memoize.cfg}{}{}
%<*mmz>
%
-% Formats other than \hologo{plainTeX} need a way to prevent extraction during
-% package-loading.
-\mmzset{
- %<!plain>extract/no/.code={},
-% \begin{key}{output directory}
-% \begin{macro}{mmzOutputDirectory}
-% Set the |-output-directory| --- manually, as there is no other way.
-% \indentmacrocode
- output-directory/.store in=\mmzOutputDirectory,
-}
-% \end{macro}
-% \end{key}
+% For formats other than \hologo{plainTeX}, we also save the current (initial
+% or |memoize.cfg|-set) value of |extract|, so that we can restore it when
+% package options include |extract=no|. Then, |extract| can be called without
+% an argument in the preamble, triggering extraction using this method; this is
+% useful e.g.\ if Memoize is compiled into a format.
+%<!plain>\let\mmz at initial@extraction at method\mmz at extraction@method
%
% \paragraph{Process} the package options (except in \hologo{plainTeX}).
-%<latex>\ProcessPgfPackageOptions{/mmz}
+%<*latex>
+\DeclareUnknownKeyHandler[mmz]{%
+ \expanded{\noexpand\pgfqkeys{/mmz}{#1\IfBlankF{#2}{={#2}}}}}
+\ProcessKeyOptions[mmz]
+%</latex>
%<context>\expandafter\mmzset\expandafter{\currentmoduleparameters}
-%
-% Define |\mmzOutputDirectory| if |output-directory| was not given.
-\ifdefined\mmzOutputDirectory
-\else
- \def\mmzOutputDirectory{}%
-\fi
-\mmzset{output directory/.code={\PackageError{memoize}{Key "output-directory"
- may only be used as a package option}{}}}
+%
+% In \hologo{LaTeX}, |nomemoize| has to process package options as well,
+% otherwise \hologo{LaTeX} will complain.
+%</mmz>
+%<*latex&nommz>
+\DeclareUnknownKeyHandler[mmz]{}
+\ProcessKeyOptions[mmz]
+%</latex&nommz>
+% \paragraph{Extern extraction}
+% We redefine |extract| to immediately trigger extraction. This is crucial in
+% \hologo{plainTeX}, where extraction must be invoked after loading the
+% package, but also potentially useful in other formats when package options
+% include |extract=no|.
+%<*mmz>
\mmzset{
- % \paragraph{Extract} the externs using the method given by |memoize.cfg| or
- % the package options --- unless we're running \hologo{plainTeX}.
- %<*!plain>
- extract/\mmz at extraction@method,
- % In non-\hologo{plainTeX} formats, also disable |extract| in the preamble.
- extract/.code={\PackageError{memoize}{Key "extract" is only allowed as a
- package option.}{If you really want to extract in the preamble, execute
- "extract/<method>".}},
- %</!plain>
- %<*plain>
- % In \hologo{plainTeX}, where extraction must be invoked after
- % loading the package, we now have to redefine |extract|, so that it will
- % immediately trigger extraction.
extract/.is choice,
extract/.default=\mmz at extraction@method,
% But only once:
extract/.append style={
- extract/.code={\PackageError{memoize}{Key "extract" is only allowed to
- be used once.}{If you really want to extract again, execute
- "extract/<method>".}},
+ extract/.code={\PackageError{memoize}{Key "extract" was invoked twice.}{In
+ principle, externs should be extracted only once. If you really want
+ to extract again, execute "extract/<method>".}},
},
- %</plain>
+ % In formats other than \hologo{plainTeX}, we remember the current |extract|
+ % code and then trigger the extraction.
+ %<!plain>/utils/exec={\pgfkeysgetvalue{/mmz/extract/. at cmd}\mmz at temp@extract},
+ %<!plain>extract=\mmz at extraction@method,
}
+% Option |extract=no| (which only exists in formats other than
+% \hologo{plainTeX}) should allow for an explicit invocation of |extract| in
+% the preamble.
+%<*!plain>
+\def\mmz at temp{no}
+\ifx\mmz at extraction@method\mmz at temp
+ \pgfkeyslet{/mmz/extract/. at cmd}\mmz at temp@extract
+ \let\mmz at extraction@method\mmz at initial@extraction at method
+\fi
+\let\mmz at temp@extract\relax
+%</!plain>
%
% Memoize was not really born for the draft mode, as it cannot produce new
-% externs there. But we don't want to disable the package, as utilization is
-% still perfectly valid in this mode, so let's just warn the user.
+% externs there. But we don't want to disable the package, as utilization and
+% pure memoization are still perfectly valid in this mode, so let's just warn
+% the user.
\ifnum\pdf at draftmode=1
- \PackageWarning{memoize}{No memoization will be performed in the draft mode}%
+ \PackageWarning{memoize}{No externalization will be performed in the draft mode}%
\fi
%</mmz>
%
+% Several further things which need to be defined as dummies in
+% |nomemoize|\slash|memoizable|.
+%<*nommz,mmzable&generic>
+\pgfkeys{%
+ /handlers/.meaning to context/.code={},
+ /handlers/.value to context/.code={},
+}
+\let\mmzAbort\relax
+\let\mmzUnmemoizable\relax
+\newcommand\IfMemoizing[2][]{\@secondoftwo}
+\let\mmzNoRef\@gobble
+\let\mmzForceNoRef\@gobble
+\newtoks\mmzContext
+\newtoks\mmzContextExtra
+\newtoks\mmzCMemo
+\newtoks\mmzCCMemo
+\newcount\mmzExternPages
+\newcount\mmzExtraPages
+\let\mmzTracingOn\relax
+\let\mmzTracingOff\relax
+%</nommz,mmzable&generic>
+%
% \paragraph{The end} of |memoize|, |nomemoize| and |memoizable|.
%<*mmz,nommz,mmzable>
%<plain>\resetatcatcode
Modified: trunk/Master/texmf-dist/source/generic/memoize/memoize.ins
===================================================================
--- trunk/Master/texmf-dist/source/generic/memoize/memoize.ins 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/source/generic/memoize/memoize.ins 2024-01-03 21:18:51 UTC (rev 69285)
@@ -34,6 +34,7 @@
\file{memoizable.sty}{\from{memoize.dtx}{mmzable,latex}}%
\file{memoizable.tex}{\from{memoize.dtx}{mmzable,plain}}%
\file{t-memoizable.tex}{\from{memoize.dtx}{mmzable,context}}%
+ \file{memoizable.code.tex}{\from{memoize.dtx}{mmzable,generic}}%
\file{memoize-extract-one.tex}{\from{memoize.dtx}{extract-one}}%
}
\endbatchfile
Modified: trunk/Master/texmf-dist/tex/context/third/memoize/t-memoizable.tex
===================================================================
--- trunk/Master/texmf-dist/tex/context/third/memoize/t-memoizable.tex 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/tex/context/third/memoize/t-memoizable.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -26,11 +26,11 @@
%% (<texmf>/doc/generic/memoize/)FILES.
%D \module[
%D file=t-memoizable.tex,
-%D version=1.0.0,
+%D version=1.1.0,
%D title=Memoizable,
%D subtitle=A programmer's stub for Memoize,
%D author=Saso Zivanovic,
-%D date=2023-10-10,
+%D date=2024-01-02,
%D copyright=Saso Zivanovic,
%D license=LPPL,
%D ]
@@ -38,24 +38,8 @@
\unprotect
\startmodule[memoizable]
\input miniltx
-\def\ifmmz at loadstatus#1{%
- \ifnum#1=0\csname mmz at loadstatus\endcsname\relax
- \expandafter\@firstoftwo
- \else
- \expandafter\@secondoftwo
- \fi
-}
-\ifmmz at loadstatus{1}%
-{\endinput}{}
-\ifmmz at loadstatus{3}%
-{\endinput}{}%
-\ifmmz at loadstatus{2}{\endinput}{}%
-\def\mmz at loadstatus{2}%
-\def\mmzset#1{\ignorespaces}
-\def\nommzkeys{\pgfqkeys{/mmz}}
-\pgfqkeys{/mmz}{.unknown/.code={\pgfkeysdef{\pgfkeyscurrentkey}{}}}
-\newif\ifmemoize
-\newcommand\IfMemoizing[2][]{\@secondoftwo}
+\input t-pgfkey
+\input memoizable.code.tex
\stopmodule
\protect
\endinput
Modified: trunk/Master/texmf-dist/tex/context/third/memoize/t-memoize.tex
===================================================================
--- trunk/Master/texmf-dist/tex/context/third/memoize/t-memoize.tex 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/tex/context/third/memoize/t-memoize.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -26,11 +26,11 @@
%% (<texmf>/doc/generic/memoize/)FILES.
%D \module[
%D file=t-memoize.tex,
-%D version=1.0.0,
+%D version=1.1.0,
%D title=Memoize,
%D subtitle=Fast and flexible externalization,
%D author=Saso Zivanovic,
-%D date=2023-10-10,
+%D date=2024-01-02,
%D copyright=Saso Zivanovic,
%D license=LPPL,
%D ]
@@ -98,15 +98,13 @@
\expandafter\@secondoftwo
\fi
}
-\ifmmz at loadstatus{3}%
-{%
+\ifmmz at loadstatus{3}{%
\PackageError{memoize}{Cannot load the package, as "nomemoize" is already
loaded. Memoization will NOT be in effect}{Packages "memoize" and
"nomemoize" are mutually exclusive, please load either one or the other.}%
\endinput
}{}%
-\ifmmz at loadstatus{2}%
-{%
+\ifmmz at loadstatus{2}{%
\PackageError{memoize}{Cannot load the package, as "memoizable" is already
loaded}{Package "memoizable" is loaded by packages which support
memoization. Memoize must be loaded before all such packages. The
@@ -121,7 +119,7 @@
\ifmmz at loadstatus{1}{\endinput}{}%
\def\mmz at loadstatus{1}%
\def\filetotoks#1#2{%
- \immediate\openin0{#2}%
+ \immediate\mmz at openin0{#2}%
#1={}%
\loop
\unless\ifeof0
@@ -130,6 +128,8 @@
\repeat
\immediate\closein0
}
+\let\mmz at openin\openin
+\let\mmz at openout\openout
\newif\ifmmz at temp
\newtoks\mmz at temptoks
\newbox\mmz at box
@@ -145,6 +145,7 @@
disable/.code=\memoizefalse,
},
enable,
+ options/.style={#1},
}
\def\mmz at mode@normal{0}
\def\mmz at mode@readonly{1}
@@ -156,69 +157,83 @@
recompile/.code={\let\mmz at mode\mmz at mode@recompile},
}
\mmzset{%
- path/.code={\pgfqkeys{/mmz/path}{#1}},
- path/.cd,
- relative/.is if=mmz at relativepath,
- dir/.store in=\mmz at dir,
- dir/.value required,
- prefix/.store in=\mmz at prefix,
- prefix/.value required,
+ prefix/.code={\mmz at parse@prefix{#1}},
}
-\newif\ifmmz at relativepath
+\begingroup
+\catcode`\/=12
+\gdef\mmz at parse@prefix#1{%
+ \edef\mmz at prefix{\detokenize\expandafter{\normalexpanded{#1}}}%
+ \def\mmz at prefix@dir{}%
+ \def\mmz at prefix@name{}%
+ \expandafter\mmz at parse@prefix at i\mmz at prefix/\mmz at eov
+}
+\gdef\mmz at parse@prefix at i#1/#2{%
+ \ifx\mmzeov#2%
+ \def\mmz at prefix@name{#1}%
+ \else
+ \appto\mmz at prefix@dir{#1/}%
+ \expandafter\mmz at parse@prefix at i\expandafter#2%
+ \fi
+}
+\endgroup
\mmzset{%
begindocument/.append style={
- path/.append code=\mmz at maybe@mkmemodir\mmz at record@prefix,
+ prefix/.append code=\mmz at maybe@mkmemodir\mmz at record@prefix,
},
begindocument/.append code=\mmz at maybe@mkmemodir,
}
-\def\mmz at dir@path{\ifmmz at relativepath.\fi/\mmz at dir}
-\def\mmz at prefix@path{\mmz at dir@path/\mmz at prefix}
\mmzset{
mkdir/.is if=mmz at mkdir,
- mkdir command/.code={\def\mmz at mkdir@command##1{#1}},
- mkdir command={mkdir "#1"},
+ mkdir command/.store in=\mmz at mkdir@command,
+ mkdir command={},
}
\newif\ifmmz at mkdir
+\mmz at mkdirtrue
\def\mmz at maybe@mkmemodir{%
\ifmmz at mkdir
- \pdf at system{\mmz at mkdir@command{\mmzOutputDirectory\mmz at dir@path}}%
+ \ifdefempty\mmz at mkdir@command{}{%
+ \ifdefempty\mmz at prefix@dir{}{%
+ \mmz at remove@quotes{\mmz at prefix@dir}\mmz at temp
+ \pdf at system{\mmz at mkdir@command\space"\mmz at temp"}%
+ }%
+ }%
\fi
}
\mmzset{%
- memo dir/.style={
- mkdir,
- path={
- relative,
- dir={#1.memo.dir},
- prefix={},
- },
- },
- memo dir/.default=\mmzUnquote\jobname,
- no memo dir/.style={
- mkdir=false,
- path={
- relative,
- dir={},
- prefix={#1.},
- },
- },
- no memo dir/.default=\mmzUnquote\jobname,
+ memo dir/.style={prefix={#1.memo.dir/}},
+ memo dir/.default=\jobname,
+ no memo dir/.style={prefix={#1.}},
+ no memo dir/.default=\jobname,
no memo dir,
}
-\def\mmzUnquote#1{\normalexpanded{\noexpand\mmz at unquote#1}\mmz at unquote@end}
-\def\mmz at unquote#1{%
- \ifx\mmz at unquote@end#1%
+\def\mmz at remove@quotes#1#2{%
+ \def\mmz at remove@quotes at end{\let#2\mmz at temp}%
+ \def\mmz at temp{}%
+ \normalexpanded{%
+ \noexpand\mmz at remove@quotes at i
+ \detokenize\expandafter{\normalexpanded{#1}}%
+ "\noexpand\mmz at eov
+ }%
+}
+\def\mmz at remove@quotes at i{%
+ \CollectArgumentsRaw
+ {\collargsBraceCollectedfalse
+ \collargsNoDelimiterstrue
+ \collargsAppendPostwrap{{##1}}%
+ }%
+ {u"u\mmz at eov}%
+ \mmz at remove@quotes at ii
+}
+\def\mmz at remove@quotes at ii#1#2{%
+ \appto\mmz at temp{#1}%
+ \ifx&%
+ \mmz at remove@quotes at end
+ \expandafter\@gobble
\else
- \ifx"#1%
- \expandafter\expandafter\expandafter\mmz at unquote@quotes
- \else
- \expandafter\expandafter\expandafter\mmz at unquote@noquotes
- \expandafter\expandafter\expandafter#1%
- \fi
+ \expandafter\@firstofone
\fi
+ {\mmz at remove@quotes at i#2\mmz at eov}%
}
-\def\mmz at unquote@quotes#1"\mmz at unquote@end{#1}
-\def\mmz at unquote@noquotes#1\mmz at unquote@end{#1}
\newif\ifmmz at ignorespaces
\mmzset{
ignore spaces/.is if=mmz at ignorespaces,
@@ -427,12 +442,13 @@
\gtoksapp\mmzCMemo{\global\mmzUnmemoizabletrue}%
\mmz at write@cmemo
\mmz at trace@endmemoize at unmemoizable
- \PackageWarning{memoize}{Marking this code as unmemoizable}%
+ \PackageInfo{memoize}{Marking this code as unmemoizable}%
\else
\ifmmz at abort
\mmz at trace@endmemoize at aborted
- \PackageWarning{memoize}{Memoization was aborted}%
- \mmz at write@empty at cmemo
+ \PackageInfo{memoize}{Memoization was aborted}%
+ \mmz at compute@context at mdfivesum
+ \mmz at write@cmemo
\else
\mmz at compute@context at mdfivesum
\mmz at write@cmemo
@@ -526,7 +542,7 @@
\fi
}
\def\mmz at Ifmemoizing@nogrouplevel{0\relax\relax\fi\iftrue}
-\def\mmz at trace#1{\immediate\write16{[tracing memoize] #1}}
+\def\mmz at trace#1{\advice at typeout{[tracing memoize] #1}}
\def\mmz at trace@context{\mmz at trace{\space\space
Context: "\expandonce{\mmz at context@key}" --> \mmz at context@mdfivesum}}
\def\mmz at trace@Memoize at on{%
@@ -639,13 +655,14 @@
},
clear context/.value forbidden,
meaning to context/.code={\forcsvlist\mmz at mtoc{#1}},
- csname meaning to context/.code={\mmz at mtoc@cs{#1}},
- key meaning to context/.code={\forcsvlist\mmz at mtoc\mmz at mtoc@keycmd{#1}},
+ csname meaning to context/.code={\mmz at mtoc@csname{#1}},
+ key meaning to context/.code={%
+ \forcsvlist\mmz at mtoc\mmz at mtoc@keycmd{#1}},
key value to context/.code={\forcsvlist\mmz at mtoc@key{#1}},
- /handlers/.meaning to context/.code={%
- \normalexpanded{\noexpand\mmz at mtoc@cs{pgfk@\pgfkeyscurrentpath/. at cmd}}},
+ /handlers/.meaning to context/.code={\normalexpanded{%
+ \noexpand\mmz at mtoc@csname{pgfk@\pgfkeyscurrentpath/. at cmd}}},
/handlers/.value to context/.code={%
- \normalexpanded{\noexpand\mmz at mtoc@cs{pgfk@\pgfkeyscurrentpath}}},
+ \normalexpanded{\noexpand\mmz at mtoc@csname{pgfk@\pgfkeyscurrentpath}}},
}
\def\mmz at mtoc#1{%
\collargs at cs@cases{#1}%
@@ -652,10 +669,10 @@
{\mmz at mtoc@cmd{#1}}%
{\mmz at mtoc@error at notcsorenv{#1}}%
{%
- \mmz at mtoc@cs{%
+ \mmz at mtoc@csname{%
start%
#1}%
- \mmz at mtoc@cs{%
+ \mmz at mtoc@csname{%
stop%
#1}%
}%
@@ -664,19 +681,20 @@
\begingroup
\escapechar=-1
\expandafter\endgroup
- \expandafter\mmz at mtoc@cs\expandafter{\string#1}%
+ \expandafter\mmz at mtoc@csname\expandafter{\string#1}%
}
-\def\mmz at mtoc@cs#1{%
+\def\mmz at mtoc@csname#1{%
\pgfkeysvalueof{/mmz/context/. at cmd}%
- \expandafter\string\csname#1\endcsname={\expandafter\meaning\csname#1\endcsname}%
+ \detokenize{#1}={\expandafter\meaning\csname#1\endcsname}%
\pgfeov
}
-\def\mmz at mtoc@key#1{\mmz at mtoc@cs{pgfk@#1}}
-\def\mmz at mtoc@key#1{\mmz at mtoc@cs{pgfk@#1/. at cmd}}
+\def\mmz at mtoc@key#1{\mmz at mtoc@csname{pgfk@#1}}
+\def\mmz at mtoc@keycmd#1{\mmz at mtoc@csname{pgfk@#1/. at cmd}}
\def\mmz at mtoc@error at notcsorenv#1{%
- \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context' is neither a command nor an environment}{}%
+ \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context'
+ is neither a command nor an environment}{}%
}
-\def\mmz at cmemo@path{\mmz at prefix@path\mmz at code@mdfivesum.memo}
+\def\mmz at cmemo@path{\mmz at prefix\mmz at code@mdfivesum.memo}
\newtoks\mmzCMemo
\mmzset{%
include source in cmemo/.is if=mmz at include@source,
@@ -684,7 +702,7 @@
\newif\ifmmz at include@source
\mmz at include@sourcetrue
\def\mmz at write@cmemo{%
- \immediate\openout\mmz at out{\mmz at cmemo@path}%
+ \immediate\mmz at openout\mmz at out{\mmz at cmemo@path}%
\immediate\write\mmz at out{\noexpand\mmzMemo}%
\immediate\write\mmz at out{%
\global\mmzContextExtra{\the\mmzContextExtra}\collargs at percentchar
@@ -697,10 +715,6 @@
\immediate\closeout\mmz at out
\pgfkeysalso{/mmz/record/new cmemo={\mmz at cmemo@path}}%
}
-\def\mmz at write@empty at cmemo{%
- \immediate\openout\mmz at out{\mmz at cmemo@path}%
- \immediate\closeout\mmz at out
-}
\let\mmzSource\endinput
\def\mmz at process@cmemo{%
\mmz at trace@process at cmemo
@@ -732,7 +746,7 @@
\xdef\mmz at context@mdfivesum{\pdf at mdfivesum{\expandonce\mmz at context@key}}%
}
\def\mmz at ccmemo@path{%
- \mmz at prefix@path\mmz at code@mdfivesum-\mmz at context@mdfivesum.memo}
+ \mmz at prefix\mmz at code@mdfivesum-\mmz at context@mdfivesum.memo}
\newtoks\mmzCCMemo
\newif\ifmmz at include@context
\mmzset{%
@@ -743,7 +757,7 @@
direct ccmemo input/.is if=mmz at direct@ccmemo at input,
}
\def\mmz at write@ccmemo{%
- \immediate\openout\mmz at out{\mmz at ccmemo@path}%
+ \immediate\mmz at openout\mmz at out{\mmz at ccmemo@path}%
\begingroup
\the\mmz at ccmemo@resources
\endgroup
@@ -763,7 +777,7 @@
\string\mmzResource{\mmz at extern@name}\collargs at percentchar}%
}
\def\mmzResource#1{%
- \ifnum0\pdf at filesize{\mmz at dir@path/#1}=0
+ \ifnum0\pdf at filesize{\mmz at prefix@dir/#1}=0
\ifmmz at direct@ccmemo at input
\let\mmzMemo\endinput
\else
@@ -824,11 +838,11 @@
}
\newcount\mmz at seq
\def\mmz at extern@basename{%
- \mmz at prefix\mmz at code@mdfivesum-\mmz at context@mdfivesum
+ \mmz at prefix@name\mmz at code@mdfivesum-\mmz at context@mdfivesum
\ifnum\mmz at seq>0 -\the\mmz at seq\fi
}
\def\mmz at extern@name{\mmz at extern@basename.pdf}
-\def\mmz at extern@basepath{\mmz at dir@path/\mmz at extern@basename}
+\def\mmz at extern@basepath{\mmz at prefix@dir\mmz at extern@basename}
\def\mmz at extern@path{\mmz at extern@basepath.pdf}
\mmzset{
padding left/.store in=\mmz at padding@left,
@@ -924,9 +938,11 @@
\endgroup
\global\advance\mmzExternPages1
\edef\externbasepath{\mmz at extern@basepath}%
- \edef\pagenumber{\the\numexpr\mmzRegularPages
+ \edef\pagenumber{%
+ \the\numexpr\mmzRegularPages
-1%
- +\mmzExternPages+\mmzExtraPages}%
+ +\mmzExternPages+\mmzExtraPages
+ }%
\mmzset{record/new extern/.expanded=\mmz at extern@path}%
\global\advance\mmz at seq1
}
@@ -958,7 +974,7 @@
enddocument/afterlastpage/.append code={%
\ifnum\mmzExternPages>0
\PackageWarning{memoize}{The compilation produced \the\mmzExternPages\space
- new extern\ifnum\mmzExternPages>1 s\fi.}%
+ new extern\ifnum\mmzExternPages>1 s\fi}%
\fi
},
}
@@ -1024,7 +1040,7 @@
\mmzset{
extract/.estore in=\mmz at extraction@method,
extract/.value required,
- begindocument/.append style={extract/.code=\mmz at preamble@only at warning},
+ begindocument/.append style={extract/.code=\mmz at preamble@only at error},
extract/perl/.code={%
\mmz at clear@extraction at log
\pdf at system{%
@@ -1032,13 +1048,12 @@
\mmzvalueof{perl extraction options}%
}%
\mmz at check@extraction at log{perl}%
- \def\mmz at mkdir@command##1{\mmzvalueof{perl extraction command}\space --mkdir "##1"}%
+ \def\mmz at mkdir@command{\mmzvalueof{perl extraction command} --mkdir}%
},
perl extraction command/.initial=memoize-extract.pl,
- perl extraction options/.initial={%
- -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w
- "\string\warning{memoize (perl-based extraction): \string\warningtext}"
- "\mmzOutputDirectory\mmzUnquote\jobname.mmz"
+ perl extraction options/.initial={\space
+ -F context
+ \jobname\space
},
extract=perl,
extract/python/.code={%
@@ -1048,23 +1063,22 @@
\mmzvalueof{python extraction options}%
}%
\mmz at check@extraction at log{python}%
- \def\mmz at mkdir@command##1{\mmzvalueof{python extraction command}\space --mkdir "##1"}%
+ \def\mmz at mkdir@command{\mmzvalueof{python extraction command} --mkdir}%
},
python extraction command/.initial=memoize-extract.py,
- python extraction options/.initial={%
- -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w
- "\string\warning{memoize (python-based extraction): \string\warningtext}"
- "\mmzOutputDirectory\mmzUnquote\jobname.mmz"
+ python extraction options/.initial={\space
+ -F context
+ \jobname\space
},
}
-\def\mmz at preamble@only at warning{%
- \PackageWarning{memoize}{%
+\def\mmz at preamble@only at error{%
+ \PackageError{memoize}{%
Ignoring the invocation of "\pgfkeyscurrentkey".
- This key may only be executed in the preamble}%
+ This key may only be executed in the preamble}{}%
}
\def\mmz at clear@extraction at log{%
\begingroup
- \immediate\openout0{\mmzUnquote\jobname.mmz.log"}%
+ \immediate\mmz at openout0{\jobname.mmz.log}%
\immediate\closeout0
\endgroup
}
@@ -1075,14 +1089,16 @@
\@input{\jobname.mmz.log}%
\ifmmz at temp \else \mmz at extraction@error \fi \endgroup }
\def\mmz at extraction@error{%
- \PackageWarning{memoize}{Extraction of externs from document "\mmzUnquote\jobname.pdf"
- using method "\extractionmethod" was unsuccessful. Have you set the
- shell escape mode as suggested in chapter 1 of the manual?}{}}
+ \PackageError{memoize}{Extraction of externs from document
+ "\jobname.pdf" using method "\extractionmethod" was
+ unsuccessful}{The extraction script "\mmzvalueof{\extractionmethod\space
+ extraction command}" wasn't executed or didn't finish execution
+ properly.}}
\mmzset{
record/.style={%
record/begin/.append style={
/mmz/record/#1/begin/.try,
- /mmz/record/#1/prefix/.try/.expanded=\mmz at prefix@path,
+ /mmz/record/#1/prefix/.try/.expanded=\mmz at prefix,
},
record/prefix/.append style={/mmz/record/#1/prefix/.try={##1}},
record/new extern/.append style={/mmz/record/#1/new extern/.try={##1}},
@@ -1108,7 +1124,7 @@
}
}
\def\mmz at record@prefix{%
- \mmzset{/mmz/record/prefix/.expanded=\mmz at prefix@path}%
+ \mmzset{/mmz/record/prefix/.expanded=\mmz at prefix}%
}
\mmzset{
no record,
@@ -1119,7 +1135,7 @@
\mmzset{
record/mmz/begin/.code={%
\newwrite\mmz at mmzout
- \immediate\openout\mmz at mmzout{\jobname.mmz}%
+ \immediate\mmz at openout\mmz at mmzout{\jobname.mmz}%
},
record/mmz/prefix/.code={%
\immediate\write\mmz at mmzout{\noexpand\mmzPrefix{#1}}%
@@ -1154,7 +1170,7 @@
bat=memoize-extract.\jobname.bat,
record/sh/begin/.code={%
\newwrite\mmz at shout
- \immediate\openout\mmz at shout{\mmz at shname}%
+ \immediate\mmz at openout\mmz at shout{\mmz at shname}%
},
record/sh/new extern/.code={%
\begingroup
@@ -1166,7 +1182,7 @@
},
record/bat/begin/.code={%
\newwrite\mmz at batout
- \immediate\openout\mmz at batout{\mmz at batname}%
+ \immediate\mmz at openout\mmz at batout{\mmz at batname}%
},
record/bat/new extern/.code={%
\begingroup
@@ -1187,7 +1203,7 @@
record/makefile/begin/.code={%
\newwrite\mmz at makefileout
\newtoks\mmz at makefile@externs
- \immediate\openout\mmz at makefileout{\mmz at makefilename}%
+ \immediate\mmz at openout\mmz at makefileout{\mmz at makefilename}%
\immediate\write\mmz at makefileout{.DEFAULT_GOAL = externs}%
\immediate\write\mmz at makefileout{.PHONY: externs}%
},
@@ -1234,11 +1250,11 @@
}
\def\mmz at pageextraction@error{%
\PackageError{memoize}{Extraction of extern page \pagenumber\space from
- document "\mmzUnquote\jobname.pdf" using method "\extractionmethod" was unsuccessful.
- Have you set the shell escape mode as suggested in chapter 1 of the
- manual?}{If "\mmzvalueof{tex extraction command}" was executed,
- shell escape mode is not the problem, and inspecting "\externbasepath.log"
- might give you a clue what's wrong}}
+ document "jobname.pdf" using method "\extractionmethod" was
+ unsuccessful.}{Check the log file to see if the extraction script was
+ executed at all, and if it finished successfully. You might also want to
+ inspect "\externbasepath.log", the log file of the embedded TeX compilation
+ which ran the extraction script}}
\def\mmz at tex@extraction at systemcall{%
\mmzvalueof{tex extraction command}\space
\mmzvalueof{tex extraction options}\space
@@ -1250,10 +1266,9 @@
-halt-on-error
-interaction=batchmode
-jobname "\externbasepath"
- \ifdefempty\mmzOutputDirectory{}{-output-directory "\mmzOutputDirectory"}
},
tex extraction script/.initial={%
- \def\noexpand\fromdocument{"\mmzOutputDirectory"\jobname.pdf}%
+ \def\noexpand\fromdocument{\jobname.pdf}%
\def\noexpand\pagenumber{\pagenumber}%
\def\noexpand\expectedwidth{\expectedwidth}%
\def\noexpand\expectedheight{\expectedheight}%
@@ -1311,6 +1326,7 @@
noop/.style={inner handler=\mmz at auto@noop},
nomemoize/.style={noop, options=disable},
replicate/.style={run if memoizing, inner handler=\mmz at auto@replicate},
+ to context/.style={run if memoizing, outer handler=\mmz at auto@tocontext},
}
\mmzset{
auto/abort/.style={run conditions=\mmzAbort},
@@ -1379,6 +1395,19 @@
\pgfqkeys{/mmz/auto/replicate}{
expanded/.code={\let\mmz at auto@replicate at expansion\@firstofone},
}
+\def\mmz at auto@tocontext{%
+ \normalexpanded{%
+ \noexpand\pgfkeysvalueof{/mmz/context/. at cmd}%
+ original "\AdviceNamespace" csname "\AdviceCsname"={%
+ \noexpand\normalexpanded{%
+ \noexpand\noexpand\noexpand\meaning
+ \noexpand\AdviceCsnameGetOriginal{\AdviceNamespace}{\AdviceCsname}%
+ }%
+ }%
+ }%
+ \pgfeov
+ \AdviceOriginal
+}
\mmzset{
begindocument/before/.append code={%
\ifdefined\tikz
@@ -1385,18 +1414,21 @@
\input advice-tikz.code.tex
\fi
\mmzset{%
- auto=\tikz{memoize, collector=\AdviceCollectTikZArguments},
- auto={tikzpicture}{memoize},
- auto=\pgfsys at getposition{
- run if memoizing, outer handler=\mmz at pgfsys@getposition},
+ auto/memoize tikz/.style={
+ memoize,
+ at begin memoization=\edef\mmz at pgfpictureid{%
+ \the\pgf at picture@serial at count
+ },
+ at end memoization=\xtoksapp\mmzCCMemo{%
+ \normalunexpanded{%
+ \global\expandafter\advance\csname pgf at picture@serial at count\endcsname
+ }%
+ \the\numexpr\pgf at picture@serial at count-\mmz at pgfpictureid\relax\relax
+ },
+ },
+ auto=\tikz{memoize tikz, collector=\AdviceCollectTikZArguments},
+ auto={tikzpicture}{memoize tikz},
}%
- \def\mmz at pgfsys@getposition##1{%
- \expandafter\ifx\csname pgf at sys@pdf at mark@pos@##1\endcsname\relax
- \else
- \mmzAbort
- \fi
- \AdviceOriginal{##1}%
- }%
},
}
\mmzset{
@@ -1403,26 +1435,29 @@
begin document/.style={begindocument/before, begindocument, begindocument/end},
end document/.style={enddocument/afterlastpage},
}
+\mmzset{extract/no/.code={}}
\InputIfFileExists{memoize.cfg}{}{}
+\let\mmz at initial@extraction at method\mmz at extraction@method
+\expandafter\mmzset\expandafter{\currentmoduleparameters}
\mmzset{
- extract/no/.code={},
- output-directory/.store in=\mmzOutputDirectory,
+ extract/.is choice,
+ extract/.default=\mmz at extraction@method,
+ extract/.append style={
+ extract/.code={\PackageError{memoize}{Key "extract" was invoked twice.}{In
+ principle, externs should be extracted only once. If you really want
+ to extract again, execute "extract/<method>".}},
+ },
+ /utils/exec={\pgfkeysgetvalue{/mmz/extract/. at cmd}\mmz at temp@extract},
+ extract=\mmz at extraction@method,
}
-\expandafter\mmzset\expandafter{\currentmoduleparameters}
-\ifdefined\mmzOutputDirectory
-\else
- \def\mmzOutputDirectory{}%
+\def\mmz at temp{no}
+\ifx\mmz at extraction@method\mmz at temp
+ \pgfkeyslet{/mmz/extract/. at cmd}\mmz at temp@extract
+ \let\mmz at extraction@method\mmz at initial@extraction at method
\fi
-\mmzset{output directory/.code={\PackageError{memoize}{Key "output-directory"
- may only be used as a package option}{}}}
-\mmzset{
- extract/\mmz at extraction@method,
- extract/.code={\PackageError{memoize}{Key "extract" is only allowed as a
- package option.}{If you really want to extract in the preamble, execute
- "extract/<method>".}},
-}
+\let\mmz at temp@extract\relax
\ifnum\pdf at draftmode=1
- \PackageWarning{memoize}{No memoization will be performed in the draft mode}%
+ \PackageWarning{memoize}{No externalization will be performed in the draft mode}%
\fi
\stopmodule
\protect
Modified: trunk/Master/texmf-dist/tex/context/third/memoize/t-nomemoize.tex
===================================================================
--- trunk/Master/texmf-dist/tex/context/third/memoize/t-nomemoize.tex 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/tex/context/third/memoize/t-nomemoize.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -26,11 +26,11 @@
%% (<texmf>/doc/generic/memoize/)FILES.
%D \module[
%D file=t-nomemoize.tex,
-%D version=1.0.0,
+%D version=1.1.0,
%D title=Memoize,
%D subtitle=A no-op stub for Memoize,
%D author=Saso Zivanovic,
-%D date=2023-10-10,
+%D date=2024-01-02,
%D copyright=Saso Zivanovic,
%D license=LPPL,
%D ]
@@ -38,6 +38,7 @@
\unprotect
\startmodule[nomemoize]
\input miniltx
+\input t-pgfkey
\def\ifmmz at loadstatus#1{%
\ifnum#1=0\csname mmz at loadstatus\endcsname\relax
\expandafter\@firstoftwo
@@ -45,14 +46,12 @@
\expandafter\@secondoftwo
\fi
}
-\ifmmz at loadstatus{1}%
-{%
+\ifmmz at loadstatus{1}{%
\PackageError{nomemoize}{Cannot load the package, as "memoize" is already
loaded; memoization will remain in effect}{Packages "memoize" and
"nomemoize" are mutually exclusive, please load either one or the other.}%
\endinput }{}%
-\ifmmz at loadstatus{2}%
-{%
+\ifmmz at loadstatus{2}{%
\PackageError{nomemoize}{Cannot load the package, as "memoizable" is already
loaded}{Package "memoizable" is loaded by packages which support
memoization. (No)Memoize must be loaded before all such packages. The
@@ -100,9 +99,25 @@
\let\stopmemoize\stopnomemoize
\newif\ifmemoizing
\newif\ifinmemoize
-\newcommand\IfMemoizing[2][]{\@secondoftwo}
\def\mmznext#1{\ignorespaces}
\InputIfFileExists{memoize.cfg}{}{}
+\pgfkeys{%
+ /handlers/.meaning to context/.code={},
+ /handlers/.value to context/.code={},
+}
+\let\mmzAbort\relax
+\let\mmzUnmemoizable\relax
+\newcommand\IfMemoizing[2][]{\@secondoftwo}
+\let\mmzNoRef\@gobble
+\let\mmzForceNoRef\@gobble
+\newtoks\mmzContext
+\newtoks\mmzContextExtra
+\newtoks\mmzCMemo
+\newtoks\mmzCCMemo
+\newcount\mmzExternPages
+\newcount\mmzExtraPages
+\let\mmzTracingOn\relax
+\let\mmzTracingOff\relax
\stopmodule
\protect
\endinput
Added: trunk/Master/texmf-dist/tex/generic/memoize/memoizable.code.tex
===================================================================
--- trunk/Master/texmf-dist/tex/generic/memoize/memoizable.code.tex (rev 0)
+++ trunk/Master/texmf-dist/tex/generic/memoize/memoizable.code.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -0,0 +1,54 @@
+%%
+%% This is file `memoizable.code.tex',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% memoize.dtx (with options: `mmzable,generic')
+%%
+%% This file is a part of Memoize, a TeX package for externalization of
+%% graphics and memoization of compilation results in general, available at
+%% https://ctan.org/pkg/memoize and https://github.com/sasozivanovic/memoize.
+%%
+%% Copyright (c) 2020- Saso Zivanovic <saso.zivanovic at guest.arnes.si>
+%% (Sa\v{s}o \v{Z}ivanovi\'{c})
+%%
+%% This work may be distributed and/or modified under the conditions of the
+%% LaTeX Project Public License, either version 1.3c of this license or (at
+%% your option) any later version. The latest version of this license is in
+%% https://www.latex-project.org/lppl.txt and version 1.3c or later is part of
+%% all distributions of LaTeX version 2008 or later.
+%%
+%% This work has the LPPL maintenance status `maintained'.
+%% The Current Maintainer of this work is Saso Zivanovic.
+%%
+%% The files belonging to this work and covered by LPPL are listed in
+%% (<texmf>/doc/generic/memoize/)FILES.
+\ifcsname mmz at loadstatus\endcsname\endinput\fi
+\def\mmz at loadstatus{2}%
+\def\mmzset#1{\ignorespaces}
+\def\nommzkeys{\pgfqkeys{/mmz}}
+\pgfqkeys{/mmz}{.unknown/.code={\pgfkeysdef{\pgfkeyscurrentkey}{}}}
+\newif\ifmemoize
+\newif\ifmemoizing
+\newif\ifinmemoize
+\pgfkeys{%
+ /handlers/.meaning to context/.code={},
+ /handlers/.value to context/.code={},
+}
+\let\mmzAbort\relax
+\let\mmzUnmemoizable\relax
+\newcommand\IfMemoizing[2][]{\@secondoftwo}
+\let\mmzNoRef\@gobble
+\let\mmzForceNoRef\@gobble
+\newtoks\mmzContext
+\newtoks\mmzContextExtra
+\newtoks\mmzCMemo
+\newtoks\mmzCCMemo
+\newcount\mmzExternPages
+\newcount\mmzExtraPages
+\let\mmzTracingOn\relax
+\let\mmzTracingOff\relax
+\endinput
+%%
+%% End of file `memoizable.code.tex'.
Property changes on: trunk/Master/texmf-dist/tex/generic/memoize/memoizable.code.tex
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Modified: trunk/Master/texmf-dist/tex/latex/memoize/memoizable.sty
===================================================================
--- trunk/Master/texmf-dist/tex/latex/memoize/memoizable.sty 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/tex/latex/memoize/memoizable.sty 2024-01-03 21:18:51 UTC (rev 69285)
@@ -24,16 +24,9 @@
%%
%% The files belonging to this work and covered by LPPL are listed in
%% (<texmf>/doc/generic/memoize/)FILES.
-\ProvidesPackage{memoizable}[2023/10/10 v1.0.0 A programmer's stub for Memoize]
-\@ifpackageloaded{memoize}%
-{\endinput}{}
-\@ifpackageloaded{nomemoize}%
-{\endinput}{}%
-\def\mmzset#1{\ignorespaces}
-\def\nommzkeys{\pgfqkeys{/mmz}}
-\pgfqkeys{/mmz}{.unknown/.code={\pgfkeysdef{\pgfkeyscurrentkey}{}}}
-\newif\ifmemoize
-\newcommand\IfMemoizing[2][]{\@secondoftwo}
+\ProvidesPackage{memoizable}[2024/1/02 v1.1.0 A programmer's stub for Memoize]
+\RequirePackage{pgfkeys}
+\input memoizable.code.tex
\endinput
%%
%% End of file `memoizable.sty'.
Modified: trunk/Master/texmf-dist/tex/latex/memoize/memoize.sty
===================================================================
--- trunk/Master/texmf-dist/tex/latex/memoize/memoize.sty 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/tex/latex/memoize/memoize.sty 2024-01-03 21:18:51 UTC (rev 69285)
@@ -24,8 +24,7 @@
%%
%% The files belonging to this work and covered by LPPL are listed in
%% (<texmf>/doc/generic/memoize/)FILES.
-\ProvidesPackage{memoize}[2023/10/10 v1.0.0 Fast and flexible externalization]
-\RequirePackage{pgfopts} % pgfkeys-based package options
+\ProvidesPackage{memoize}[2024/1/02 v1.1.0 Fast and flexible externalization]
\RequirePackage{etoolbox}
\ifdefined\luatexversion
\directlua{memoize = {}}
@@ -49,8 +48,14 @@
}%
}
\RequirePackage{advice}
-\@ifpackageloaded{nomemoize}%
-{%
+\def\ifmmz at loadstatus#1{%
+ \ifnum#1=0\csname mmz at loadstatus\endcsname\relax
+ \expandafter\@firstoftwo
+ \else
+ \expandafter\@secondoftwo
+ \fi
+}
+\ifmmz at loadstatus{3}{%
\PackageError{memoize}{Cannot load the package, as "nomemoize" is already
loaded. Memoization will NOT be in effect}{Packages "memoize" and
"nomemoize" are mutually exclusive, please load either one or the other.}%
@@ -58,8 +63,7 @@
\ProcessPgfPackageOptions{/memoize/package options}
\endinput
}{}%
-\@ifpackageloaded{memoizable}%
-{%
+\ifmmz at loadstatus{2}{%
\PackageError{memoize}{Cannot load the package, as "memoizable" is already
loaded}{Package "memoizable" is loaded by packages which support
memoization. Memoize must be loaded before all such packages. The
@@ -73,8 +77,10 @@
\ProcessPgfPackageOptions{/memoize/package options}
\endinput
}{}%
+\ifmmz at loadstatus{1}{\endinput}{}%
+\def\mmz at loadstatus{1}%
\def\filetotoks#1#2{%
- \immediate\openin0{#2}%
+ \immediate\mmz at openin0{#2}%
#1={}%
\loop
\unless\ifeof0
@@ -83,6 +89,8 @@
\repeat
\immediate\closein0
}
+\let\mmz at openin\openin
+\let\mmz at openout\openout
\newif\ifmmz at temp
\newtoks\mmz at temptoks
\newbox\mmz at box
@@ -98,6 +106,7 @@
disable/.code=\memoizefalse,
},
enable,
+ options/.style={#1},
}
\def\mmz at mode@normal{0}
\def\mmz at mode@readonly{1}
@@ -109,69 +118,83 @@
recompile/.code={\let\mmz at mode\mmz at mode@recompile},
}
\mmzset{%
- path/.code={\pgfqkeys{/mmz/path}{#1}},
- path/.cd,
- relative/.is if=mmz at relativepath,
- dir/.store in=\mmz at dir,
- dir/.value required,
- prefix/.store in=\mmz at prefix,
- prefix/.value required,
+ prefix/.code={\mmz at parse@prefix{#1}},
}
-\newif\ifmmz at relativepath
+\begingroup
+\catcode`\/=12
+\gdef\mmz at parse@prefix#1{%
+ \edef\mmz at prefix{\detokenize\expandafter{\expanded{#1}}}%
+ \def\mmz at prefix@dir{}%
+ \def\mmz at prefix@name{}%
+ \expandafter\mmz at parse@prefix at i\mmz at prefix/\mmz at eov
+}
+\gdef\mmz at parse@prefix at i#1/#2{%
+ \ifx\mmzeov#2%
+ \def\mmz at prefix@name{#1}%
+ \else
+ \appto\mmz at prefix@dir{#1/}%
+ \expandafter\mmz at parse@prefix at i\expandafter#2%
+ \fi
+}
+\endgroup
\mmzset{%
begindocument/.append style={
- path/.append code=\mmz at maybe@mkmemodir\mmz at record@prefix,
+ prefix/.append code=\mmz at maybe@mkmemodir\mmz at record@prefix,
},
begindocument/.append code=\mmz at maybe@mkmemodir,
}
-\def\mmz at dir@path{\ifmmz at relativepath.\fi/\mmz at dir}
-\def\mmz at prefix@path{\mmz at dir@path/\mmz at prefix}
\mmzset{
mkdir/.is if=mmz at mkdir,
- mkdir command/.code={\def\mmz at mkdir@command##1{#1}},
- mkdir command={mkdir "#1"},
+ mkdir command/.store in=\mmz at mkdir@command,
+ mkdir command={},
}
\newif\ifmmz at mkdir
+\mmz at mkdirtrue
\def\mmz at maybe@mkmemodir{%
\ifmmz at mkdir
- \pdf at system{\mmz at mkdir@command{\mmzOutputDirectory\mmz at dir@path}}%
+ \ifdefempty\mmz at mkdir@command{}{%
+ \ifdefempty\mmz at prefix@dir{}{%
+ \mmz at remove@quotes{\mmz at prefix@dir}\mmz at temp
+ \pdf at system{\mmz at mkdir@command\space"\mmz at temp"}%
+ }%
+ }%
\fi
}
\mmzset{%
- memo dir/.style={
- mkdir,
- path={
- relative,
- dir={#1.memo.dir},
- prefix={},
- },
- },
- memo dir/.default=\mmzUnquote\jobname,
- no memo dir/.style={
- mkdir=false,
- path={
- relative,
- dir={},
- prefix={#1.},
- },
- },
- no memo dir/.default=\mmzUnquote\jobname,
+ memo dir/.style={prefix={#1.memo.dir/}},
+ memo dir/.default=\jobname,
+ no memo dir/.style={prefix={#1.}},
+ no memo dir/.default=\jobname,
no memo dir,
}
-\def\mmzUnquote#1{\expanded{\noexpand\mmz at unquote#1}\mmz at unquote@end}
-\def\mmz at unquote#1{%
- \ifx\mmz at unquote@end#1%
+\def\mmz at remove@quotes#1#2{%
+ \def\mmz at remove@quotes at end{\let#2\mmz at temp}%
+ \def\mmz at temp{}%
+ \expanded{%
+ \noexpand\mmz at remove@quotes at i
+ \detokenize\expandafter{\expanded{#1}}%
+ "\noexpand\mmz at eov
+ }%
+}
+\def\mmz at remove@quotes at i{%
+ \CollectArgumentsRaw
+ {\collargsBraceCollectedfalse
+ \collargsNoDelimiterstrue
+ \collargsAppendPostwrap{{##1}}%
+ }%
+ {u"u\mmz at eov}%
+ \mmz at remove@quotes at ii
+}
+\def\mmz at remove@quotes at ii#1#2{%
+ \appto\mmz at temp{#1}%
+ \ifx&%
+ \mmz at remove@quotes at end
+ \expandafter\@gobble
\else
- \ifx"#1%
- \expandafter\expandafter\expandafter\mmz at unquote@quotes
- \else
- \expandafter\expandafter\expandafter\mmz at unquote@noquotes
- \expandafter\expandafter\expandafter#1%
- \fi
+ \expandafter\@firstofone
\fi
+ {\mmz at remove@quotes at i#2\mmz at eov}%
}
-\def\mmz at unquote@quotes#1"\mmz at unquote@end{#1}
-\def\mmz at unquote@noquotes#1\mmz at unquote@end{#1}
\newif\ifmmz at ignorespaces
\mmzset{
ignore spaces/.is if=mmz at ignorespaces,
@@ -388,12 +411,13 @@
\gtoksapp\mmzCMemo{\global\mmzUnmemoizabletrue}%
\mmz at write@cmemo
\mmz at trace@endmemoize at unmemoizable
- \PackageWarning{memoize}{Marking this code as unmemoizable}%
+ \PackageInfo{memoize}{Marking this code as unmemoizable}%
\else
\ifmmz at abort
\mmz at trace@endmemoize at aborted
- \PackageWarning{memoize}{Memoization was aborted}%
- \mmz at write@empty at cmemo
+ \PackageInfo{memoize}{Memoization was aborted}%
+ \mmz at compute@context at mdfivesum
+ \mmz at write@cmemo
\else
\mmz at compute@context at mdfivesum
\mmz at write@cmemo
@@ -487,7 +511,7 @@
\fi
}
\def\mmz at Ifmemoizing@nogrouplevel{0\relax\relax\fi\iftrue}
-\def\mmz at trace#1{\immediate\write16{[tracing memoize] #1}}
+\def\mmz at trace#1{\advice at typeout{[tracing memoize] #1}}
\def\mmz at trace@context{\mmz at trace{\space\space
Context: "\expandonce{\mmz at context@key}" --> \mmz at context@mdfivesum}}
\def\mmz at trace@Memoize at on{%
@@ -600,13 +624,14 @@
},
clear context/.value forbidden,
meaning to context/.code={\forcsvlist\mmz at mtoc{#1}},
- csname meaning to context/.code={\mmz at mtoc@cs{#1}},
- key meaning to context/.code={\forcsvlist\mmz at mtoc\mmz at mtoc@keycmd{#1}},
+ csname meaning to context/.code={\mmz at mtoc@csname{#1}},
+ key meaning to context/.code={%
+ \forcsvlist\mmz at mtoc\mmz at mtoc@keycmd{#1}},
key value to context/.code={\forcsvlist\mmz at mtoc@key{#1}},
- /handlers/.meaning to context/.code={%
- \expanded{\noexpand\mmz at mtoc@cs{pgfk@\pgfkeyscurrentpath/. at cmd}}},
+ /handlers/.meaning to context/.code={\expanded{%
+ \noexpand\mmz at mtoc@csname{pgfk@\pgfkeyscurrentpath/. at cmd}}},
/handlers/.value to context/.code={%
- \expanded{\noexpand\mmz at mtoc@cs{pgfk@\pgfkeyscurrentpath}}},
+ \expanded{\noexpand\mmz at mtoc@csname{pgfk@\pgfkeyscurrentpath}}},
}
\def\mmz at mtoc#1{%
\collargs at cs@cases{#1}%
@@ -613,9 +638,9 @@
{\mmz at mtoc@cmd{#1}}%
{\mmz at mtoc@error at notcsorenv{#1}}%
{%
- \mmz at mtoc@cs{%
+ \mmz at mtoc@csname{%
#1}%
- \mmz at mtoc@cs{%
+ \mmz at mtoc@csname{%
end%
#1}%
}%
@@ -624,19 +649,20 @@
\begingroup
\escapechar=-1
\expandafter\endgroup
- \expandafter\mmz at mtoc@cs\expandafter{\string#1}%
+ \expandafter\mmz at mtoc@csname\expandafter{\string#1}%
}
-\def\mmz at mtoc@cs#1{%
+\def\mmz at mtoc@csname#1{%
\pgfkeysvalueof{/mmz/context/. at cmd}%
- \expandafter\string\csname#1\endcsname={\expandafter\meaning\csname#1\endcsname}%
+ \detokenize{#1}={\expandafter\meaning\csname#1\endcsname}%
\pgfeov
}
-\def\mmz at mtoc@key#1{\mmz at mtoc@cs{pgfk@#1}}
-\def\mmz at mtoc@key#1{\mmz at mtoc@cs{pgfk@#1/. at cmd}}
+\def\mmz at mtoc@key#1{\mmz at mtoc@csname{pgfk@#1}}
+\def\mmz at mtoc@keycmd#1{\mmz at mtoc@csname{pgfk@#1/. at cmd}}
\def\mmz at mtoc@error at notcsorenv#1{%
- \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context' is neither a command nor an environment}{}%
+ \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context'
+ is neither a command nor an environment}{}%
}
-\def\mmz at cmemo@path{\mmz at prefix@path\mmz at code@mdfivesum.memo}
+\def\mmz at cmemo@path{\mmz at prefix\mmz at code@mdfivesum.memo}
\newtoks\mmzCMemo
\mmzset{%
include source in cmemo/.is if=mmz at include@source,
@@ -644,7 +670,7 @@
\newif\ifmmz at include@source
\mmz at include@sourcetrue
\def\mmz at write@cmemo{%
- \immediate\openout\mmz at out{\mmz at cmemo@path}%
+ \immediate\mmz at openout\mmz at out{\mmz at cmemo@path}%
\immediate\write\mmz at out{\noexpand\mmzMemo}%
\immediate\write\mmz at out{%
\global\mmzContextExtra{\the\mmzContextExtra}\collargs at percentchar
@@ -657,10 +683,6 @@
\immediate\closeout\mmz at out
\pgfkeysalso{/mmz/record/new cmemo={\mmz at cmemo@path}}%
}
-\def\mmz at write@empty at cmemo{%
- \immediate\openout\mmz at out{\mmz at cmemo@path}%
- \immediate\closeout\mmz at out
-}
\let\mmzSource\endinput
\def\mmz at process@cmemo{%
\mmz at trace@process at cmemo
@@ -692,7 +714,7 @@
\xdef\mmz at context@mdfivesum{\pdf at mdfivesum{\expandonce\mmz at context@key}}%
}
\def\mmz at ccmemo@path{%
- \mmz at prefix@path\mmz at code@mdfivesum-\mmz at context@mdfivesum.memo}
+ \mmz at prefix\mmz at code@mdfivesum-\mmz at context@mdfivesum.memo}
\newtoks\mmzCCMemo
\newif\ifmmz at include@context
\mmzset{%
@@ -703,7 +725,7 @@
direct ccmemo input/.is if=mmz at direct@ccmemo at input,
}
\def\mmz at write@ccmemo{%
- \immediate\openout\mmz at out{\mmz at ccmemo@path}%
+ \immediate\mmz at openout\mmz at out{\mmz at ccmemo@path}%
\begingroup
\the\mmz at ccmemo@resources
\endgroup
@@ -723,7 +745,7 @@
\string\mmzResource{\mmz at extern@name}\collargs at percentchar}%
}
\def\mmzResource#1{%
- \ifnum0\pdf at filesize{\mmz at dir@path/#1}=0
+ \ifnum0\pdf at filesize{\mmz at prefix@dir/#1}=0
\ifmmz at direct@ccmemo at input
\let\mmzMemo\endinput
\else
@@ -784,11 +806,11 @@
}
\newcount\mmz at seq
\def\mmz at extern@basename{%
- \mmz at prefix\mmz at code@mdfivesum-\mmz at context@mdfivesum
+ \mmz at prefix@name\mmz at code@mdfivesum-\mmz at context@mdfivesum
\ifnum\mmz at seq>0 -\the\mmz at seq\fi
}
\def\mmz at extern@name{\mmz at extern@basename.pdf}
-\def\mmz at extern@basepath{\mmz at dir@path/\mmz at extern@basename}
+\def\mmz at extern@basepath{\mmz at prefix@dir\mmz at extern@basename}
\def\mmz at extern@path{\mmz at extern@basepath.pdf}
\mmzset{
padding left/.store in=\mmz at padding@left,
@@ -870,8 +892,10 @@
\endgroup
\global\advance\mmzExternPages1
\edef\externbasepath{\mmz at extern@basepath}%
- \edef\pagenumber{\the\numexpr\mmzRegularPages
- +\mmzExternPages+\mmzExtraPages}%
+ \edef\pagenumber{%
+ \the\numexpr\mmzRegularPages
+ +\mmzExternPages+\mmzExtraPages
+ }%
\mmzset{record/new extern/.expanded=\mmz at extern@path}%
\global\advance\mmz at seq1
}
@@ -903,7 +927,7 @@
enddocument/afterlastpage/.append code={%
\ifnum\mmzExternPages>0
\PackageWarning{memoize}{The compilation produced \the\mmzExternPages\space
- new extern\ifnum\mmzExternPages>1 s\fi.}%
+ new extern\ifnum\mmzExternPages>1 s\fi}%
\fi
},
}
@@ -982,7 +1006,7 @@
\mmzset{
extract/.estore in=\mmz at extraction@method,
extract/.value required,
- begindocument/.append style={extract/.code=\mmz at preamble@only at warning},
+ begindocument/.append style={extract/.code=\mmz at preamble@only at error},
extract/perl/.code={%
\mmz at clear@extraction at log
\pdf at system{%
@@ -990,13 +1014,12 @@
\mmzvalueof{perl extraction options}%
}%
\mmz at check@extraction at log{perl}%
- \def\mmz at mkdir@command##1{\mmzvalueof{perl extraction command}\space --mkdir "##1"}%
+ \def\mmz at mkdir@command{\mmzvalueof{perl extraction command} --mkdir}%
},
perl extraction command/.initial=memoize-extract.pl,
- perl extraction options/.initial={%
- -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w
- "\string\PackageWarning{memoize (perl-based extraction)}{\string\warningtext}"
- "\mmzOutputDirectory\mmzUnquote\jobname.mmz"
+ perl extraction options/.initial={\space
+ -F latex
+ \jobname\space
},
extract=perl,
extract/python/.code={%
@@ -1006,23 +1029,22 @@
\mmzvalueof{python extraction options}%
}%
\mmz at check@extraction at log{python}%
- \def\mmz at mkdir@command##1{\mmzvalueof{python extraction command}\space --mkdir "##1"}%
+ \def\mmz at mkdir@command{\mmzvalueof{python extraction command} --mkdir}%
},
python extraction command/.initial=memoize-extract.py,
- python extraction options/.initial={%
- -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w
- "\string\PackageWarning{memoize (python-based extraction)}{\string\warningtext}"
- "\mmzOutputDirectory\mmzUnquote\jobname.mmz"
+ python extraction options/.initial={\space
+ -F latex
+ \jobname\space
},
}
-\def\mmz at preamble@only at warning{%
- \PackageWarning{memoize}{%
+\def\mmz at preamble@only at error{%
+ \PackageError{memoize}{%
Ignoring the invocation of "\pgfkeyscurrentkey".
- This key may only be executed in the preamble}%
+ This key may only be executed in the preamble}{}%
}
\def\mmz at clear@extraction at log{%
\begingroup
- \immediate\openout0{\mmzUnquote\jobname.mmz.log"}%
+ \immediate\mmz at openout0{\jobname.mmz.log}%
\immediate\closeout0
\endgroup
}
@@ -1033,14 +1055,16 @@
\@input{\jobname.mmz.log}%
\ifmmz at temp \else \mmz at extraction@error \fi \endgroup }
\def\mmz at extraction@error{%
- \PackageWarning{memoize}{Extraction of externs from document "\mmzUnquote\jobname.pdf"
- using method "\extractionmethod" was unsuccessful. Have you set the
- shell escape mode as suggested in chapter 1 of the manual?}{}}
+ \PackageError{memoize}{Extraction of externs from document
+ "\jobname.pdf" using method "\extractionmethod" was
+ unsuccessful}{The extraction script "\mmzvalueof{\extractionmethod\space
+ extraction command}" wasn't executed or didn't finish execution
+ properly.}}
\mmzset{
record/.style={%
record/begin/.append style={
/mmz/record/#1/begin/.try,
- /mmz/record/#1/prefix/.try/.expanded=\mmz at prefix@path,
+ /mmz/record/#1/prefix/.try/.expanded=\mmz at prefix,
},
record/prefix/.append style={/mmz/record/#1/prefix/.try={##1}},
record/new extern/.append style={/mmz/record/#1/new extern/.try={##1}},
@@ -1066,7 +1090,7 @@
}
}
\def\mmz at record@prefix{%
- \mmzset{/mmz/record/prefix/.expanded=\mmz at prefix@path}%
+ \mmzset{/mmz/record/prefix/.expanded=\mmz at prefix}%
}
\mmzset{
no record,
@@ -1077,7 +1101,7 @@
\mmzset{
record/mmz/begin/.code={%
\newwrite\mmz at mmzout
- \immediate\openout\mmz at mmzout{\jobname.mmz}%
+ \immediate\mmz at openout\mmz at mmzout{\jobname.mmz}%
},
record/mmz/prefix/.code={%
\immediate\write\mmz at mmzout{\noexpand\mmzPrefix{#1}}%
@@ -1112,7 +1136,7 @@
bat=memoize-extract.\jobname.bat,
record/sh/begin/.code={%
\newwrite\mmz at shout
- \immediate\openout\mmz at shout{\mmz at shname}%
+ \immediate\mmz at openout\mmz at shout{\mmz at shname}%
},
record/sh/new extern/.code={%
\begingroup
@@ -1124,7 +1148,7 @@
},
record/bat/begin/.code={%
\newwrite\mmz at batout
- \immediate\openout\mmz at batout{\mmz at batname}%
+ \immediate\mmz at openout\mmz at batout{\mmz at batname}%
},
record/bat/new extern/.code={%
\begingroup
@@ -1145,7 +1169,7 @@
record/makefile/begin/.code={%
\newwrite\mmz at makefileout
\newtoks\mmz at makefile@externs
- \immediate\openout\mmz at makefileout{\mmz at makefilename}%
+ \immediate\mmz at openout\mmz at makefileout{\mmz at makefilename}%
\immediate\write\mmz at makefileout{.DEFAULT_GOAL = externs}%
\immediate\write\mmz at makefileout{.PHONY: externs}%
},
@@ -1192,11 +1216,11 @@
}
\def\mmz at pageextraction@error{%
\PackageError{memoize}{Extraction of extern page \pagenumber\space from
- document "\mmzUnquote\jobname.pdf" using method "\extractionmethod" was unsuccessful.
- Have you set the shell escape mode as suggested in chapter 1 of the
- manual?}{If "\mmzvalueof{tex extraction command}" was executed,
- shell escape mode is not the problem, and inspecting "\externbasepath.log"
- might give you a clue what's wrong}}
+ document "jobname.pdf" using method "\extractionmethod" was
+ unsuccessful.}{Check the log file to see if the extraction script was
+ executed at all, and if it finished successfully. You might also want to
+ inspect "\externbasepath.log", the log file of the embedded TeX compilation
+ which ran the extraction script}}
\def\mmz at tex@extraction at systemcall{%
\mmzvalueof{tex extraction command}\space
\mmzvalueof{tex extraction options}\space
@@ -1208,10 +1232,9 @@
-halt-on-error
-interaction=batchmode
-jobname "\externbasepath"
- \ifdefempty\mmzOutputDirectory{}{-output-directory "\mmzOutputDirectory"}
},
tex extraction script/.initial={%
- \def\noexpand\fromdocument{"\mmzOutputDirectory"\jobname.pdf}%
+ \def\noexpand\fromdocument{\jobname.pdf}%
\def\noexpand\pagenumber{\pagenumber}%
\def\noexpand\expectedwidth{\expectedwidth}%
\def\noexpand\expectedheight{\expectedheight}%
@@ -1272,6 +1295,7 @@
outer handler=\mmz at auto@noop at env, bailout handler=\mmz at auto@bailout},
nomemoize/.style={noop, options=disable},
replicate/.style={run if memoizing, inner handler=\mmz at auto@replicate},
+ to context/.style={run if memoizing, outer handler=\mmz at auto@tocontext},
}
\mmzset{
auto/abort/.style={run conditions=\mmzAbort},
@@ -1348,6 +1372,19 @@
\pgfqkeys{/mmz/auto/replicate}{
expanded/.code={\let\mmz at auto@replicate at expansion\@firstofone},
}
+\def\mmz at auto@tocontext{%
+ \expanded{%
+ \noexpand\pgfkeysvalueof{/mmz/context/. at cmd}%
+ original "\AdviceNamespace" csname "\AdviceCsname"={%
+ \noexpand\expanded{%
+ \noexpand\noexpand\noexpand\meaning
+ \noexpand\AdviceCsnameGetOriginal{\AdviceNamespace}{\AdviceCsname}%
+ }%
+ }%
+ }%
+ \pgfeov
+ \AdviceOriginal
+}
\mmzset{
auto/.cd,
ref/.style={outer handler=\mmz at auto@ref\mmzNoRef, run if memoizing},
@@ -1365,12 +1402,7 @@
\AdviceOriginal#2{#3}%
}
\def\mmzForceNoRef#1{%
- \ifmemoizing
- \expandafter\gtoksapp\expandafter\mmzContextExtra
- \else
- \expandafter\toksapp\expandafter\mmzContext
- \fi
- {r@#1={\csname r@#1\endcsname}}%
+ \mmz at mtoc@csname{r@#1}%
\ignorespaces
}
\def\mmzNoRef#1{%
@@ -1414,6 +1446,10 @@
\label{#1}%
\endgroup
}
+\AddToHook{shipout/before}[memoize]{\global\advance\mmzExtraPages-1\relax}
+\AddToHook{shipout/after}[memoize]{\global\advance\mmzExtraPages1\relax}
+\mmzset{auto=\DiscardShipoutBox{
+ outer handler=\global\advance\mmzExtraPages1\relax\AdviceOriginal}}
\mmzset{
begindocument/before/.append code={%
\@ifpackageloaded{tikz}{%
@@ -1420,18 +1456,21 @@
\input advice-tikz.code.tex
}{}%
\mmzset{%
- auto=\tikz{memoize, collector=\AdviceCollectTikZArguments},
- auto={tikzpicture}{memoize},
- auto=\pgfsys at getposition{
- run if memoizing, outer handler=\mmz at pgfsys@getposition},
+ auto/memoize tikz/.style={
+ memoize,
+ at begin memoization=\edef\mmz at pgfpictureid{%
+ \the\pgf at picture@serial at count
+ },
+ at end memoization=\xtoksapp\mmzCCMemo{%
+ \unexpanded{%
+ \global\expandafter\advance\csname pgf at picture@serial at count\endcsname
+ }%
+ \the\numexpr\pgf at picture@serial at count-\mmz at pgfpictureid\relax\relax
+ },
+ },
+ auto=\tikz{memoize tikz, collector=\AdviceCollectTikZArguments},
+ auto={tikzpicture}{memoize tikz},
}%
- \def\mmz at pgfsys@getposition##1{%
- \expandafter\ifx\csname pgf at sys@pdf at mark@pos@##1\endcsname\relax
- \else
- \mmzAbort
- \fi
- \AdviceOriginal{##1}%
- }%
},
}
\mmzset{
@@ -1480,30 +1519,102 @@
\fi
}%
}{}}
+\AddToHook{begindocument/before}{%
+ \@ifpackageloaded{morewrites}{%
+ \def\mmz at openin#1#2{\openin#1=#2\relax}%
+ \def\mmz at openout#1#2{\openout#1=#2\relax}%
+ }{}%
+}
+\mmzset{
+ begindocument/before/.append style={%
+ auto=\blx at bbl@entry{outer handler=\mmz at biblatex@entry},
+ auto/cite/.style={run if memoizing, outer handler=\mmz at biblatex@cite},
+ auto/cites/.style={run if memoizing, outer handler=\mmz at biblatex@cites},
+ auto=\cite{cite},
+ auto=\cites{cites},
+ }%
+}
+\def\mmz at biblatex@entry#1#2\endentry{%
+ \csxdef{mmz at bbl@#1}{\pdf at mdfivesum{#2}}%
+ \AdviceOriginal{#1}#2\endentry
+}
+\def\mmz at biblatex@cite#1#{\mmz at biblatex@cite at i{#1}}
+\def\mmz at biblatex@cite at i#1#2{%
+ \forcsvlist\mmz at biblatex@cite at do@key{#2}%
+ \xtoksapp\mmzCCMemo{%
+ \noexpand\setbox0\noexpand\hbox{%
+ \expandonce\AdviceOriginal\unexpanded{#1}{#2}%
+ }}%
+ \AdviceOriginal#1{#2}%
+}
+\def\mmz at biblatex@cite at do@key#1{%
+ \mmz at mtoc@csname{mmz at bbl@#1}%
+ \ifcsdef{mmz at bbl@#1}{}{\mmzAbort}%
+}
+\def\mmz at biblatex@cites{%
+ \mmz at temptoks{}%
+ \mmz at biblatex@cites at i
+}
+\def\mmz at biblatex@cites at i{%
+ \futurelet\mmz at temp\mmz at biblatex@cites at ii
+}
+\def\mmz at biblatex@cites at ii{%
+ \mmz at tempfalse
+ \ifx\mmz at temp\bgroup
+ \mmz at temptrue
+ \else
+ \ifx\mmz at temp[%]
+ \mmz at temptrue
+ \fi
+ \fi
+ \ifmmz at temp
+ \expandafter\mmz at biblatex@cites at iii
+ \else
+ \expandafter\mmz at biblatex@cites at z
+ \fi
+}
+\def\mmz at biblatex@cites at iii#1#{\mmz at biblatex@cites at iv{#1}}
+\def\mmz at biblatex@cites at iv#1#2{%
+ \forcsvlist\mmz at biblatex@cite at do@key{#2}%
+ \toksapp\mmz at temptoks{#1{#2}}%
+ \mmz at biblatex@cites at i
+}
+\def\mmz at biblatex@cites at z{%
+ \xtoksapp\mmzCCMemo{%
+ \noexpand\setbox0\noexpand\hbox{%
+ \expandonce\AdviceOriginal\the\mmz at temptoks
+ }}%
+ \expandafter\AdviceOriginal\the\mmz at temptoks
+}
\AddToHook{begindocument/before}{\mmzset{begindocument/before}}
\AddToHook{begindocument}{\mmzset{begindocument}}
\AddToHook{begindocument/end}{\mmzset{begindocument/end}}
\AddToHook{enddocument/afterlastpage}{\mmzset{enddocument/afterlastpage}}
+\mmzset{extract/no/.code={}}
\InputIfFileExists{memoize.cfg}{}{}
+\let\mmz at initial@extraction at method\mmz at extraction@method
+\DeclareUnknownKeyHandler[mmz]{%
+ \expanded{\noexpand\pgfqkeys{/mmz}{#1\IfBlankF{#2}{={#2}}}}}
+\ProcessKeyOptions[mmz]
\mmzset{
- extract/no/.code={},
- output-directory/.store in=\mmzOutputDirectory,
+ extract/.is choice,
+ extract/.default=\mmz at extraction@method,
+ extract/.append style={
+ extract/.code={\PackageError{memoize}{Key "extract" was invoked twice.}{In
+ principle, externs should be extracted only once. If you really want
+ to extract again, execute "extract/<method>".}},
+ },
+ /utils/exec={\pgfkeysgetvalue{/mmz/extract/. at cmd}\mmz at temp@extract},
+ extract=\mmz at extraction@method,
}
-\ProcessPgfPackageOptions{/mmz}
-\ifdefined\mmzOutputDirectory
-\else
- \def\mmzOutputDirectory{}%
+\def\mmz at temp{no}
+\ifx\mmz at extraction@method\mmz at temp
+ \pgfkeyslet{/mmz/extract/. at cmd}\mmz at temp@extract
+ \let\mmz at extraction@method\mmz at initial@extraction at method
\fi
-\mmzset{output directory/.code={\PackageError{memoize}{Key "output-directory"
- may only be used as a package option}{}}}
-\mmzset{
- extract/\mmz at extraction@method,
- extract/.code={\PackageError{memoize}{Key "extract" is only allowed as a
- package option.}{If you really want to extract in the preamble, execute
- "extract/<method>".}},
-}
+\let\mmz at temp@extract\relax
\ifnum\pdf at draftmode=1
- \PackageWarning{memoize}{No memoization will be performed in the draft mode}%
+ \PackageWarning{memoize}{No externalization will be performed in the draft mode}%
\fi
\endinput
%%
Modified: trunk/Master/texmf-dist/tex/latex/memoize/nomemoize.sty
===================================================================
--- trunk/Master/texmf-dist/tex/latex/memoize/nomemoize.sty 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/tex/latex/memoize/nomemoize.sty 2024-01-03 21:18:51 UTC (rev 69285)
@@ -24,18 +24,21 @@
%%
%% The files belonging to this work and covered by LPPL are listed in
%% (<texmf>/doc/generic/memoize/)FILES.
-\ProvidesPackage{nomemoize}[2023/10/10 v1.0.0 A no-op stub for Memoize]
-\RequirePackage{pgfopts} % pgfkeys-based package options
-\pgfkeys{/memoize/package options/.unknown/.code={}}
-\ProcessPgfPackageOptions{/memoize/package options}
-\@ifpackageloaded{memoize}
-{%
+\ProvidesPackage{nomemoize}[2024/1/02 v1.1.0 A no-op stub for Memoize]
+\RequirePackage{pgfkeys}
+\def\ifmmz at loadstatus#1{%
+ \ifnum#1=0\csname mmz at loadstatus\endcsname\relax
+ \expandafter\@firstoftwo
+ \else
+ \expandafter\@secondoftwo
+ \fi
+}
+\ifmmz at loadstatus{1}{%
\PackageError{nomemoize}{Cannot load the package, as "memoize" is already
loaded; memoization will remain in effect}{Packages "memoize" and
"nomemoize" are mutually exclusive, please load either one or the other.}%
\endinput }{}%
-\@ifpackageloaded{memoizable}%
-{%
+\ifmmz at loadstatus{2}{%
\PackageError{nomemoize}{Cannot load the package, as "memoizable" is already
loaded}{Package "memoizable" is loaded by packages which support
memoization. (No)Memoize must be loaded before all such packages. The
@@ -47,6 +50,8 @@
of that package.}%
\endinput
}{}%
+\ifmmz at loadstatus{3}{\endinput}{}%
+\def\mmz at loadstatus{3}%
\def\mmzset#1{\ignorespaces}
\def\nommzkeys{\pgfqkeys{/mmz}}
\pgfqkeys{/mmz}{.unknown/.code={\pgfkeysdef{\pgfkeyscurrentkey}{}}}
@@ -70,9 +75,27 @@
\let\endmemoize\endnomemoize
\newif\ifmemoizing
\newif\ifinmemoize
-\newcommand\IfMemoizing[2][]{\@secondoftwo}
\def\mmznext#1{\ignorespaces}
\InputIfFileExists{memoize.cfg}{}{}
+\DeclareUnknownKeyHandler[mmz]{}
+\ProcessKeyOptions[mmz]
+\pgfkeys{%
+ /handlers/.meaning to context/.code={},
+ /handlers/.value to context/.code={},
+}
+\let\mmzAbort\relax
+\let\mmzUnmemoizable\relax
+\newcommand\IfMemoizing[2][]{\@secondoftwo}
+\let\mmzNoRef\@gobble
+\let\mmzForceNoRef\@gobble
+\newtoks\mmzContext
+\newtoks\mmzContextExtra
+\newtoks\mmzCMemo
+\newtoks\mmzCCMemo
+\newcount\mmzExternPages
+\newcount\mmzExtraPages
+\let\mmzTracingOn\relax
+\let\mmzTracingOff\relax
\endinput
%%
%% End of file `nomemoize.sty'.
Modified: trunk/Master/texmf-dist/tex/plain/memoize/memoizable.tex
===================================================================
--- trunk/Master/texmf-dist/tex/plain/memoize/memoizable.tex 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/tex/plain/memoize/memoizable.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -24,26 +24,10 @@
%%
%% The files belonging to this work and covered by LPPL are listed in
%% (<texmf>/doc/generic/memoize/)FILES.
-% Package memoizable 2023/10/10 v1.0.0
+% Package memoizable 2024/01/02 v1.1.0
\input miniltx
-\def\ifmmz at loadstatus#1{%
- \ifnum#1=0\csname mmz at loadstatus\endcsname\relax
- \expandafter\@firstoftwo
- \else
- \expandafter\@secondoftwo
- \fi
-}
-\ifmmz at loadstatus{1}%
-{\endinput}{}
-\ifmmz at loadstatus{3}%
-{\endinput}{}%
-\ifmmz at loadstatus{2}{\endinput}{}%
-\def\mmz at loadstatus{2}%
-\def\mmzset#1{\ignorespaces}
-\def\nommzkeys{\pgfqkeys{/mmz}}
-\pgfqkeys{/mmz}{.unknown/.code={\pgfkeysdef{\pgfkeyscurrentkey}{}}}
-\newif\ifmemoize
-\newcommand\IfMemoizing[2][]{\@secondoftwo}
+\input pgfkeys
+\input memoizable.code.tex
\resetatcatcode
\endinput
%%
Modified: trunk/Master/texmf-dist/tex/plain/memoize/memoize.tex
===================================================================
--- trunk/Master/texmf-dist/tex/plain/memoize/memoize.tex 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/tex/plain/memoize/memoize.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -24,7 +24,7 @@
%%
%% The files belonging to this work and covered by LPPL are listed in
%% (<texmf>/doc/generic/memoize/)FILES.
-% Package memoize 2023/10/10 v1.0.0
+% Package memoize 2024/01/02 v1.1.0
\input miniltx
\def\PackageWarning#1#2{{%
\newlinechar`\^^J\def\MessageBreak{^^J\space\space#1: }%
@@ -73,15 +73,13 @@
\expandafter\@secondoftwo
\fi
}
-\ifmmz at loadstatus{3}%
-{%
+\ifmmz at loadstatus{3}{%
\PackageError{memoize}{Cannot load the package, as "nomemoize" is already
loaded. Memoization will NOT be in effect}{Packages "memoize" and
"nomemoize" are mutually exclusive, please load either one or the other.}%
\endinput
}{}%
-\ifmmz at loadstatus{2}%
-{%
+\ifmmz at loadstatus{2}{%
\PackageError{memoize}{Cannot load the package, as "memoizable" is already
loaded}{Package "memoizable" is loaded by packages which support
memoization. Memoize must be loaded before all such packages. The
@@ -96,7 +94,7 @@
\ifmmz at loadstatus{1}{\endinput}{}%
\def\mmz at loadstatus{1}%
\def\filetotoks#1#2{%
- \immediate\openin0{#2}%
+ \immediate\mmz at openin0{#2}%
#1={}%
\loop
\unless\ifeof0
@@ -105,6 +103,8 @@
\repeat
\immediate\closein0
}
+\let\mmz at openin\openin
+\let\mmz at openout\openout
\newif\ifmmz at temp
\newtoks\mmz at temptoks
\newbox\mmz at box
@@ -120,6 +120,7 @@
disable/.code=\memoizefalse,
},
enable,
+ options/.style={#1},
}
\def\mmz at mode@normal{0}
\def\mmz at mode@readonly{1}
@@ -131,69 +132,83 @@
recompile/.code={\let\mmz at mode\mmz at mode@recompile},
}
\mmzset{%
- path/.code={\pgfqkeys{/mmz/path}{#1}},
- path/.cd,
- relative/.is if=mmz at relativepath,
- dir/.store in=\mmz at dir,
- dir/.value required,
- prefix/.store in=\mmz at prefix,
- prefix/.value required,
+ prefix/.code={\mmz at parse@prefix{#1}},
}
-\newif\ifmmz at relativepath
+\begingroup
+\catcode`\/=12
+\gdef\mmz at parse@prefix#1{%
+ \edef\mmz at prefix{\detokenize\expandafter{\expanded{#1}}}%
+ \def\mmz at prefix@dir{}%
+ \def\mmz at prefix@name{}%
+ \expandafter\mmz at parse@prefix at i\mmz at prefix/\mmz at eov
+}
+\gdef\mmz at parse@prefix at i#1/#2{%
+ \ifx\mmzeov#2%
+ \def\mmz at prefix@name{#1}%
+ \else
+ \appto\mmz at prefix@dir{#1/}%
+ \expandafter\mmz at parse@prefix at i\expandafter#2%
+ \fi
+}
+\endgroup
\mmzset{%
begindocument/.append style={
- path/.append code=\mmz at maybe@mkmemodir\mmz at record@prefix,
+ prefix/.append code=\mmz at maybe@mkmemodir\mmz at record@prefix,
},
begindocument/.append code=\mmz at maybe@mkmemodir,
}
-\def\mmz at dir@path{\ifmmz at relativepath.\fi/\mmz at dir}
-\def\mmz at prefix@path{\mmz at dir@path/\mmz at prefix}
\mmzset{
mkdir/.is if=mmz at mkdir,
- mkdir command/.code={\def\mmz at mkdir@command##1{#1}},
- mkdir command={mkdir "#1"},
+ mkdir command/.store in=\mmz at mkdir@command,
+ mkdir command={},
}
\newif\ifmmz at mkdir
+\mmz at mkdirtrue
\def\mmz at maybe@mkmemodir{%
\ifmmz at mkdir
- \pdf at system{\mmz at mkdir@command{\mmzOutputDirectory\mmz at dir@path}}%
+ \ifdefempty\mmz at mkdir@command{}{%
+ \ifdefempty\mmz at prefix@dir{}{%
+ \mmz at remove@quotes{\mmz at prefix@dir}\mmz at temp
+ \pdf at system{\mmz at mkdir@command\space"\mmz at temp"}%
+ }%
+ }%
\fi
}
\mmzset{%
- memo dir/.style={
- mkdir,
- path={
- relative,
- dir={#1.memo.dir},
- prefix={},
- },
- },
- memo dir/.default=\mmzUnquote\jobname,
- no memo dir/.style={
- mkdir=false,
- path={
- relative,
- dir={},
- prefix={#1.},
- },
- },
- no memo dir/.default=\mmzUnquote\jobname,
+ memo dir/.style={prefix={#1.memo.dir/}},
+ memo dir/.default=\jobname,
+ no memo dir/.style={prefix={#1.}},
+ no memo dir/.default=\jobname,
no memo dir,
}
-\def\mmzUnquote#1{\expanded{\noexpand\mmz at unquote#1}\mmz at unquote@end}
-\def\mmz at unquote#1{%
- \ifx\mmz at unquote@end#1%
+\def\mmz at remove@quotes#1#2{%
+ \def\mmz at remove@quotes at end{\let#2\mmz at temp}%
+ \def\mmz at temp{}%
+ \expanded{%
+ \noexpand\mmz at remove@quotes at i
+ \detokenize\expandafter{\expanded{#1}}%
+ "\noexpand\mmz at eov
+ }%
+}
+\def\mmz at remove@quotes at i{%
+ \CollectArgumentsRaw
+ {\collargsBraceCollectedfalse
+ \collargsNoDelimiterstrue
+ \collargsAppendPostwrap{{##1}}%
+ }%
+ {u"u\mmz at eov}%
+ \mmz at remove@quotes at ii
+}
+\def\mmz at remove@quotes at ii#1#2{%
+ \appto\mmz at temp{#1}%
+ \ifx&%
+ \mmz at remove@quotes at end
+ \expandafter\@gobble
\else
- \ifx"#1%
- \expandafter\expandafter\expandafter\mmz at unquote@quotes
- \else
- \expandafter\expandafter\expandafter\mmz at unquote@noquotes
- \expandafter\expandafter\expandafter#1%
- \fi
+ \expandafter\@firstofone
\fi
+ {\mmz at remove@quotes at i#2\mmz at eov}%
}
-\def\mmz at unquote@quotes#1"\mmz at unquote@end{#1}
-\def\mmz at unquote@noquotes#1\mmz at unquote@end{#1}
\newif\ifmmz at ignorespaces
\mmzset{
ignore spaces/.is if=mmz at ignorespaces,
@@ -425,12 +440,13 @@
\gtoksapp\mmzCMemo{\global\mmzUnmemoizabletrue}%
\mmz at write@cmemo
\mmz at trace@endmemoize at unmemoizable
- \PackageWarning{memoize}{Marking this code as unmemoizable}%
+ \PackageInfo{memoize}{Marking this code as unmemoizable}%
\else
\ifmmz at abort
\mmz at trace@endmemoize at aborted
- \PackageWarning{memoize}{Memoization was aborted}%
- \mmz at write@empty at cmemo
+ \PackageInfo{memoize}{Memoization was aborted}%
+ \mmz at compute@context at mdfivesum
+ \mmz at write@cmemo
\else
\mmz at compute@context at mdfivesum
\mmz at write@cmemo
@@ -524,7 +540,7 @@
\fi
}
\def\mmz at Ifmemoizing@nogrouplevel{0\relax\relax\fi\iftrue}
-\def\mmz at trace#1{\immediate\write16{[tracing memoize] #1}}
+\def\mmz at trace#1{\advice at typeout{[tracing memoize] #1}}
\def\mmz at trace@context{\mmz at trace{\space\space
Context: "\expandonce{\mmz at context@key}" --> \mmz at context@mdfivesum}}
\def\mmz at trace@Memoize at on{%
@@ -637,13 +653,14 @@
},
clear context/.value forbidden,
meaning to context/.code={\forcsvlist\mmz at mtoc{#1}},
- csname meaning to context/.code={\mmz at mtoc@cs{#1}},
- key meaning to context/.code={\forcsvlist\mmz at mtoc\mmz at mtoc@keycmd{#1}},
+ csname meaning to context/.code={\mmz at mtoc@csname{#1}},
+ key meaning to context/.code={%
+ \forcsvlist\mmz at mtoc\mmz at mtoc@keycmd{#1}},
key value to context/.code={\forcsvlist\mmz at mtoc@key{#1}},
- /handlers/.meaning to context/.code={%
- \expanded{\noexpand\mmz at mtoc@cs{pgfk@\pgfkeyscurrentpath/. at cmd}}},
+ /handlers/.meaning to context/.code={\expanded{%
+ \noexpand\mmz at mtoc@csname{pgfk@\pgfkeyscurrentpath/. at cmd}}},
/handlers/.value to context/.code={%
- \expanded{\noexpand\mmz at mtoc@cs{pgfk@\pgfkeyscurrentpath}}},
+ \expanded{\noexpand\mmz at mtoc@csname{pgfk@\pgfkeyscurrentpath}}},
}
\def\mmz at mtoc#1{%
\collargs at cs@cases{#1}%
@@ -650,9 +667,9 @@
{\mmz at mtoc@cmd{#1}}%
{\mmz at mtoc@error at notcsorenv{#1}}%
{%
- \mmz at mtoc@cs{%
+ \mmz at mtoc@csname{%
#1}%
- \mmz at mtoc@cs{%
+ \mmz at mtoc@csname{%
end%
#1}%
}%
@@ -661,19 +678,20 @@
\begingroup
\escapechar=-1
\expandafter\endgroup
- \expandafter\mmz at mtoc@cs\expandafter{\string#1}%
+ \expandafter\mmz at mtoc@csname\expandafter{\string#1}%
}
-\def\mmz at mtoc@cs#1{%
+\def\mmz at mtoc@csname#1{%
\pgfkeysvalueof{/mmz/context/. at cmd}%
- \expandafter\string\csname#1\endcsname={\expandafter\meaning\csname#1\endcsname}%
+ \detokenize{#1}={\expandafter\meaning\csname#1\endcsname}%
\pgfeov
}
-\def\mmz at mtoc@key#1{\mmz at mtoc@cs{pgfk@#1}}
-\def\mmz at mtoc@key#1{\mmz at mtoc@cs{pgfk@#1/. at cmd}}
+\def\mmz at mtoc@key#1{\mmz at mtoc@csname{pgfk@#1}}
+\def\mmz at mtoc@keycmd#1{\mmz at mtoc@csname{pgfk@#1/. at cmd}}
\def\mmz at mtoc@error at notcsorenv#1{%
- \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context' is neither a command nor an environment}{}%
+ \PackageError{memoize}{'\detokenize{#1}' passed to key 'meaning to context'
+ is neither a command nor an environment}{}%
}
-\def\mmz at cmemo@path{\mmz at prefix@path\mmz at code@mdfivesum.memo}
+\def\mmz at cmemo@path{\mmz at prefix\mmz at code@mdfivesum.memo}
\newtoks\mmzCMemo
\mmzset{%
include source in cmemo/.is if=mmz at include@source,
@@ -681,7 +699,7 @@
\newif\ifmmz at include@source
\mmz at include@sourcetrue
\def\mmz at write@cmemo{%
- \immediate\openout\mmz at out{\mmz at cmemo@path}%
+ \immediate\mmz at openout\mmz at out{\mmz at cmemo@path}%
\immediate\write\mmz at out{\noexpand\mmzMemo}%
\immediate\write\mmz at out{%
\global\mmzContextExtra{\the\mmzContextExtra}\collargs at percentchar
@@ -694,10 +712,6 @@
\immediate\closeout\mmz at out
\pgfkeysalso{/mmz/record/new cmemo={\mmz at cmemo@path}}%
}
-\def\mmz at write@empty at cmemo{%
- \immediate\openout\mmz at out{\mmz at cmemo@path}%
- \immediate\closeout\mmz at out
-}
\let\mmzSource\endinput
\def\mmz at process@cmemo{%
\mmz at trace@process at cmemo
@@ -729,7 +743,7 @@
\xdef\mmz at context@mdfivesum{\pdf at mdfivesum{\expandonce\mmz at context@key}}%
}
\def\mmz at ccmemo@path{%
- \mmz at prefix@path\mmz at code@mdfivesum-\mmz at context@mdfivesum.memo}
+ \mmz at prefix\mmz at code@mdfivesum-\mmz at context@mdfivesum.memo}
\newtoks\mmzCCMemo
\newif\ifmmz at include@context
\mmzset{%
@@ -740,7 +754,7 @@
direct ccmemo input/.is if=mmz at direct@ccmemo at input,
}
\def\mmz at write@ccmemo{%
- \immediate\openout\mmz at out{\mmz at ccmemo@path}%
+ \immediate\mmz at openout\mmz at out{\mmz at ccmemo@path}%
\begingroup
\the\mmz at ccmemo@resources
\endgroup
@@ -760,7 +774,7 @@
\string\mmzResource{\mmz at extern@name}\collargs at percentchar}%
}
\def\mmzResource#1{%
- \ifnum0\pdf at filesize{\mmz at dir@path/#1}=0
+ \ifnum0\pdf at filesize{\mmz at prefix@dir/#1}=0
\ifmmz at direct@ccmemo at input
\let\mmzMemo\endinput
\else
@@ -821,11 +835,11 @@
}
\newcount\mmz at seq
\def\mmz at extern@basename{%
- \mmz at prefix\mmz at code@mdfivesum-\mmz at context@mdfivesum
+ \mmz at prefix@name\mmz at code@mdfivesum-\mmz at context@mdfivesum
\ifnum\mmz at seq>0 -\the\mmz at seq\fi
}
\def\mmz at extern@name{\mmz at extern@basename.pdf}
-\def\mmz at extern@basepath{\mmz at dir@path/\mmz at extern@basename}
+\def\mmz at extern@basepath{\mmz at prefix@dir\mmz at extern@basename}
\def\mmz at extern@path{\mmz at extern@basepath.pdf}
\mmzset{
padding left/.store in=\mmz at padding@left,
@@ -906,8 +920,10 @@
\endgroup
\global\advance\mmzExternPages1
\edef\externbasepath{\mmz at extern@basepath}%
- \edef\pagenumber{\the\numexpr\mmzRegularPages
- +\mmzExternPages+\mmzExtraPages}%
+ \edef\pagenumber{%
+ \the\numexpr\mmzRegularPages
+ +\mmzExternPages+\mmzExtraPages
+ }%
\mmzset{record/new extern/.expanded=\mmz at extern@path}%
\global\advance\mmz at seq1
}
@@ -939,7 +955,7 @@
enddocument/afterlastpage/.append code={%
\ifnum\mmzExternPages>0
\PackageWarning{memoize}{The compilation produced \the\mmzExternPages\space
- new extern\ifnum\mmzExternPages>1 s\fi.}%
+ new extern\ifnum\mmzExternPages>1 s\fi}%
\fi
},
}
@@ -1018,7 +1034,7 @@
\mmzset{
extract/.estore in=\mmz at extraction@method,
extract/.value required,
- begindocument/.append style={extract/.code=\mmz at preamble@only at warning},
+ begindocument/.append style={extract/.code=\mmz at preamble@only at error},
extract/perl/.code={%
\mmz at clear@extraction at log
\pdf at system{%
@@ -1026,13 +1042,12 @@
\mmzvalueof{perl extraction options}%
}%
\mmz at check@extraction at log{perl}%
- \def\mmz at mkdir@command##1{\mmzvalueof{perl extraction command}\space --mkdir "##1"}%
+ \def\mmz at mkdir@command{\mmzvalueof{perl extraction command} --mkdir}%
},
perl extraction command/.initial=memoize-extract.pl,
- perl extraction options/.initial={%
- -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w
- "\string\warning{memoize (perl-based extraction): \string\warningtext}"
- "\mmzOutputDirectory\mmzUnquote\jobname.mmz"
+ perl extraction options/.initial={\space
+ -F plain
+ \jobname\space
},
extract=perl,
extract/python/.code={%
@@ -1042,23 +1057,22 @@
\mmzvalueof{python extraction options}%
}%
\mmz at check@extraction at log{python}%
- \def\mmz at mkdir@command##1{\mmzvalueof{python extraction command}\space --mkdir "##1"}%
+ \def\mmz at mkdir@command{\mmzvalueof{python extraction command} --mkdir}%
},
python extraction command/.initial=memoize-extract.py,
- python extraction options/.initial={%
- -e -l "\mmzOutputDirectory\mmzUnquote\jobname.mmz.log" -w
- "\string\warning{memoize (python-based extraction): \string\warningtext}"
- "\mmzOutputDirectory\mmzUnquote\jobname.mmz"
+ python extraction options/.initial={\space
+ -F plain
+ \jobname\space
},
}
-\def\mmz at preamble@only at warning{%
- \PackageWarning{memoize}{%
+\def\mmz at preamble@only at error{%
+ \PackageError{memoize}{%
Ignoring the invocation of "\pgfkeyscurrentkey".
- This key may only be executed in the preamble}%
+ This key may only be executed in the preamble}{}%
}
\def\mmz at clear@extraction at log{%
\begingroup
- \immediate\openout0{\mmzUnquote\jobname.mmz.log"}%
+ \immediate\mmz at openout0{\jobname.mmz.log}%
\immediate\closeout0
\endgroup
}
@@ -1069,14 +1083,16 @@
\@input{\jobname.mmz.log}%
\ifmmz at temp \else \mmz at extraction@error \fi \endgroup }
\def\mmz at extraction@error{%
- \PackageWarning{memoize}{Extraction of externs from document "\mmzUnquote\jobname.pdf"
- using method "\extractionmethod" was unsuccessful. Have you set the
- shell escape mode as suggested in chapter 1 of the manual?}{}}
+ \PackageError{memoize}{Extraction of externs from document
+ "\jobname.pdf" using method "\extractionmethod" was
+ unsuccessful}{The extraction script "\mmzvalueof{\extractionmethod\space
+ extraction command}" wasn't executed or didn't finish execution
+ properly.}}
\mmzset{
record/.style={%
record/begin/.append style={
/mmz/record/#1/begin/.try,
- /mmz/record/#1/prefix/.try/.expanded=\mmz at prefix@path,
+ /mmz/record/#1/prefix/.try/.expanded=\mmz at prefix,
},
record/prefix/.append style={/mmz/record/#1/prefix/.try={##1}},
record/new extern/.append style={/mmz/record/#1/new extern/.try={##1}},
@@ -1102,7 +1118,7 @@
}
}
\def\mmz at record@prefix{%
- \mmzset{/mmz/record/prefix/.expanded=\mmz at prefix@path}%
+ \mmzset{/mmz/record/prefix/.expanded=\mmz at prefix}%
}
\mmzset{
no record,
@@ -1113,7 +1129,7 @@
\mmzset{
record/mmz/begin/.code={%
\newwrite\mmz at mmzout
- \immediate\openout\mmz at mmzout{\jobname.mmz}%
+ \immediate\mmz at openout\mmz at mmzout{\jobname.mmz}%
},
record/mmz/prefix/.code={%
\immediate\write\mmz at mmzout{\noexpand\mmzPrefix{#1}}%
@@ -1148,7 +1164,7 @@
bat=memoize-extract.\jobname.bat,
record/sh/begin/.code={%
\newwrite\mmz at shout
- \immediate\openout\mmz at shout{\mmz at shname}%
+ \immediate\mmz at openout\mmz at shout{\mmz at shname}%
},
record/sh/new extern/.code={%
\begingroup
@@ -1160,7 +1176,7 @@
},
record/bat/begin/.code={%
\newwrite\mmz at batout
- \immediate\openout\mmz at batout{\mmz at batname}%
+ \immediate\mmz at openout\mmz at batout{\mmz at batname}%
},
record/bat/new extern/.code={%
\begingroup
@@ -1181,7 +1197,7 @@
record/makefile/begin/.code={%
\newwrite\mmz at makefileout
\newtoks\mmz at makefile@externs
- \immediate\openout\mmz at makefileout{\mmz at makefilename}%
+ \immediate\mmz at openout\mmz at makefileout{\mmz at makefilename}%
\immediate\write\mmz at makefileout{.DEFAULT_GOAL = externs}%
\immediate\write\mmz at makefileout{.PHONY: externs}%
},
@@ -1228,11 +1244,11 @@
}
\def\mmz at pageextraction@error{%
\PackageError{memoize}{Extraction of extern page \pagenumber\space from
- document "\mmzUnquote\jobname.pdf" using method "\extractionmethod" was unsuccessful.
- Have you set the shell escape mode as suggested in chapter 1 of the
- manual?}{If "\mmzvalueof{tex extraction command}" was executed,
- shell escape mode is not the problem, and inspecting "\externbasepath.log"
- might give you a clue what's wrong}}
+ document "jobname.pdf" using method "\extractionmethod" was
+ unsuccessful.}{Check the log file to see if the extraction script was
+ executed at all, and if it finished successfully. You might also want to
+ inspect "\externbasepath.log", the log file of the embedded TeX compilation
+ which ran the extraction script}}
\def\mmz at tex@extraction at systemcall{%
\mmzvalueof{tex extraction command}\space
\mmzvalueof{tex extraction options}\space
@@ -1244,10 +1260,9 @@
-halt-on-error
-interaction=batchmode
-jobname "\externbasepath"
- \ifdefempty\mmzOutputDirectory{}{-output-directory "\mmzOutputDirectory"}
},
tex extraction script/.initial={%
- \def\noexpand\fromdocument{"\mmzOutputDirectory"\jobname.pdf}%
+ \def\noexpand\fromdocument{\jobname.pdf}%
\def\noexpand\pagenumber{\pagenumber}%
\def\noexpand\expectedwidth{\expectedwidth}%
\def\noexpand\expectedheight{\expectedheight}%
@@ -1305,6 +1320,7 @@
noop/.style={inner handler=\mmz at auto@noop},
nomemoize/.style={noop, options=disable},
replicate/.style={run if memoizing, inner handler=\mmz at auto@replicate},
+ to context/.style={run if memoizing, outer handler=\mmz at auto@tocontext},
}
\mmzset{
auto/abort/.style={run conditions=\mmzAbort},
@@ -1373,6 +1389,19 @@
\pgfqkeys{/mmz/auto/replicate}{
expanded/.code={\let\mmz at auto@replicate at expansion\@firstofone},
}
+\def\mmz at auto@tocontext{%
+ \expanded{%
+ \noexpand\pgfkeysvalueof{/mmz/context/. at cmd}%
+ original "\AdviceNamespace" csname "\AdviceCsname"={%
+ \noexpand\expanded{%
+ \noexpand\noexpand\noexpand\meaning
+ \noexpand\AdviceCsnameGetOriginal{\AdviceNamespace}{\AdviceCsname}%
+ }%
+ }%
+ }%
+ \pgfeov
+ \AdviceOriginal
+}
\mmzset{
begindocument/before/.append code={%
\ifdefined\tikz
@@ -1379,18 +1408,21 @@
\input advice-tikz.code.tex
\fi
\mmzset{%
- auto=\tikz{memoize, collector=\AdviceCollectTikZArguments},
- auto={tikzpicture}{memoize},
- auto=\pgfsys at getposition{
- run if memoizing, outer handler=\mmz at pgfsys@getposition},
+ auto/memoize tikz/.style={
+ memoize,
+ at begin memoization=\edef\mmz at pgfpictureid{%
+ \the\pgf at picture@serial at count
+ },
+ at end memoization=\xtoksapp\mmzCCMemo{%
+ \unexpanded{%
+ \global\expandafter\advance\csname pgf at picture@serial at count\endcsname
+ }%
+ \the\numexpr\pgf at picture@serial at count-\mmz at pgfpictureid\relax\relax
+ },
+ },
+ auto=\tikz{memoize tikz, collector=\AdviceCollectTikZArguments},
+ auto={tikzpicture}{memoize tikz},
}%
- \def\mmz at pgfsys@getposition##1{%
- \expandafter\ifx\csname pgf at sys@pdf at mark@pos@##1\endcsname\relax
- \else
- \mmzAbort
- \fi
- \AdviceOriginal{##1}%
- }%
},
}
\mmzset{
@@ -1399,25 +1431,16 @@
}
\InputIfFileExists{memoize.cfg}{}{}
\mmzset{
- output-directory/.store in=\mmzOutputDirectory,
-}
-\ifdefined\mmzOutputDirectory
-\else
- \def\mmzOutputDirectory{}%
-\fi
-\mmzset{output directory/.code={\PackageError{memoize}{Key "output-directory"
- may only be used as a package option}{}}}
-\mmzset{
extract/.is choice,
extract/.default=\mmz at extraction@method,
extract/.append style={
- extract/.code={\PackageError{memoize}{Key "extract" is only allowed to
- be used once.}{If you really want to extract again, execute
- "extract/<method>".}},
+ extract/.code={\PackageError{memoize}{Key "extract" was invoked twice.}{In
+ principle, externs should be extracted only once. If you really want
+ to extract again, execute "extract/<method>".}},
},
}
\ifnum\pdf at draftmode=1
- \PackageWarning{memoize}{No memoization will be performed in the draft mode}%
+ \PackageWarning{memoize}{No externalization will be performed in the draft mode}%
\fi
\resetatcatcode
\endinput
Modified: trunk/Master/texmf-dist/tex/plain/memoize/nomemoize.tex
===================================================================
--- trunk/Master/texmf-dist/tex/plain/memoize/nomemoize.tex 2024-01-03 21:18:30 UTC (rev 69284)
+++ trunk/Master/texmf-dist/tex/plain/memoize/nomemoize.tex 2024-01-03 21:18:51 UTC (rev 69285)
@@ -25,6 +25,7 @@
%% The files belonging to this work and covered by LPPL are listed in
%% (<texmf>/doc/generic/memoize/)FILES.
\input miniltx
+\input pgfkeys
\def\ifmmz at loadstatus#1{%
\ifnum#1=0\csname mmz at loadstatus\endcsname\relax
\expandafter\@firstoftwo
@@ -32,14 +33,12 @@
\expandafter\@secondoftwo
\fi
}
-\ifmmz at loadstatus{1}%
-{%
+\ifmmz at loadstatus{1}{%
\PackageError{nomemoize}{Cannot load the package, as "memoize" is already
loaded; memoization will remain in effect}{Packages "memoize" and
"nomemoize" are mutually exclusive, please load either one or the other.}%
\endinput }{}%
-\ifmmz at loadstatus{2}%
-{%
+\ifmmz at loadstatus{2}{%
\PackageError{nomemoize}{Cannot load the package, as "memoizable" is already
loaded}{Package "memoizable" is loaded by packages which support
memoization. (No)Memoize must be loaded before all such packages. The
@@ -87,9 +86,25 @@
\let\endmemoize\endnomemoize
\newif\ifmemoizing
\newif\ifinmemoize
-\newcommand\IfMemoizing[2][]{\@secondoftwo}
\def\mmznext#1{\ignorespaces}
\InputIfFileExists{memoize.cfg}{}{}
+\pgfkeys{%
+ /handlers/.meaning to context/.code={},
+ /handlers/.value to context/.code={},
+}
+\let\mmzAbort\relax
+\let\mmzUnmemoizable\relax
+\newcommand\IfMemoizing[2][]{\@secondoftwo}
+\let\mmzNoRef\@gobble
+\let\mmzForceNoRef\@gobble
+\newtoks\mmzContext
+\newtoks\mmzContextExtra
+\newtoks\mmzCMemo
+\newtoks\mmzCCMemo
+\newcount\mmzExternPages
+\newcount\mmzExtraPages
+\let\mmzTracingOn\relax
+\let\mmzTracingOff\relax
\resetatcatcode
\endinput
%%
More information about the tex-live-commits
mailing list.