texlive[52931] Master: versioned .tar.xz containers for the tlnet
commits+karl at tug.org
commits+karl at tug.org
Wed Nov 27 00:04:18 CET 2019
Revision: 52931
http://tug.org/svn/texlive?view=revision&revision=52931
Author: karl
Date: 2019-11-27 00:04:18 +0100 (Wed, 27 Nov 2019)
Log Message:
-----------
versioned .tar.xz containers for the tlnet repository
Modified Paths:
--------------
trunk/Master/texmf-dist/scripts/texlive/tlmgr.pl
trunk/Master/tlpkg/TeXLive/TLPOBJ.pm
trunk/Master/tlpkg/bin/tl-update-containers
trunk/Master/tlpkg/doc/packages.txt
Modified: trunk/Master/texmf-dist/scripts/texlive/tlmgr.pl
===================================================================
--- trunk/Master/texmf-dist/scripts/texlive/tlmgr.pl 2019-11-26 22:24:38 UTC (rev 52930)
+++ trunk/Master/texmf-dist/scripts/texlive/tlmgr.pl 2019-11-26 23:04:18 UTC (rev 52931)
@@ -1095,9 +1095,10 @@
}
if ($opts{"backup"}) {
$tlp->make_container($::progs{'compressor'}, $localtlpdb->root,
- $opts{"backupdir"},
- "${pkg}.r" . $tlp->revision,
- $tlp->relocated);
+ destdir => $opts{"backupdir"},
+ containername => "${pkg}.r" . $tlp->revision,
+ relative => $tlp->relocated,
+ user => 1);
if ($autobackup) {
# in case we do auto backups we remove older backups
clear_old_backups($pkg, $opts{"backupdir"}, $autobackup);
@@ -2166,7 +2167,9 @@
. $tlp->revision . ".tar.$compressorextension\n");
if (!$opts{"dry-run"}) {
$tlp->make_container($::progs{'compressor'}, $localtlpdb->root,
- $opts{"backupdir"}, "${pkg}.r" . $tlp->revision);
+ destdir => $opts{"backupdir"},
+ containername => "${pkg}.r" . $tlp->revision,
+ user => 1);
}
}
}
@@ -2281,8 +2284,12 @@
push (@rst_tlpobj, "tlpkg\\tlpobj\\$pkg.tlpobj");
push (@rst_info, "$pkg ^($oldrev^)");
next if ($opts{"dry-run"});
- # create backup; make_container expects file name in a format: some-name.r[0-9]+
- my ($size, undef, $fullname) = $localtlp->make_container("tar", $root, $temp, "__BACKUP_$pkg.r$oldrev");
+ # create backup; make_container expects filename in format:
+ # some-name.r[0-9]+
+ my ($size, undef, $fullname) = $localtlp->make_container("tar", $root,
+ destdir => $temp,
+ containername => "__BACKUP_$pkg.r$oldrev",
+ user => 1);
if ($size <= 0) {
tlwarn("$prg: creation of backup container failed for: $pkg\n");
return 1; # backup failed? abort
@@ -3202,8 +3209,10 @@
if ($opts{"backup"} && !$opts{"dry-run"}) {
my $compressorextension = $Compressors{$::progs{'compressor'}}{'extension'};
$tlp->make_container($::progs{'compressor'}, $root,
- $opts{"backupdir"}, "${pkg}.r" . $tlp->revision,
- $tlp->relocated);
+ destdir => $opts{"backupdir"},
+ containername => "${pkg}.r" . $tlp->revision,
+ relative => $tlp->relocated,
+ user => 1);
$unwind_package =
"$opts{'backupdir'}/${pkg}.r" . $tlp->revision . ".tar.$compressorextension";
@@ -3241,9 +3250,11 @@
# no backup was made, so let us create a temporary .tar file
# of the package
my $tlp = $localtlpdb->get_package($pkg);
- my ($s, undef, $fullname) = $tlp->make_container("tar", $root, $temp,
- "__BACKUP_${pkg}.r" . $tlp->revision,
- $tlp->relocated);
+ my ($s, undef, $fullname) = $tlp->make_container("tar", $root,
+ destdir => $temp,
+ containername => "__BACKUP_${pkg}.r" . $tlp->revision,
+ relative => $tlp->relocated,
+ user => 1);
if ($s <= 0) {
tlwarn("\n$prg: creation of backup container failed for: $pkg\n");
tlwarn("$prg: continuing to update other packages, please retry...\n");
Modified: trunk/Master/tlpkg/TeXLive/TLPOBJ.pm
===================================================================
--- trunk/Master/tlpkg/TeXLive/TLPOBJ.pm 2019-11-26 22:24:38 UTC (rev 52930)
+++ trunk/Master/tlpkg/TeXLive/TLPOBJ.pm 2019-11-26 23:04:18 UTC (rev 52931)
@@ -559,7 +559,12 @@
sub make_container {
- my ($self,$type,$instroot,$destdir,$containername,$relative) = @_;
+ my ($self, $type, $instroot, %other) = @_;
+ my $destdir = ($other{'destdir'} || undef);
+ my $containername = ($other{'containername'} || undef);
+ my $relative = ($other{'relative'} || undef);
+ my $user = ($other{'user'} || undef);
+ my $copy_instead_of_link = ($other{'copy_instead_of_link'} || undef);
if (!($type eq 'tar' ||
TeXLive::TLUtils::member($type, @{$::progs{'working_compressors'}}))) {
tlwarn "$0: TLPOBJ supports @{$::progs{'working_compressors'}} and tar containers, not $type\n";
@@ -627,7 +632,11 @@
$selfcopy->writeout(\*TMP);
close(TMP);
push(@files, "$tlpobjdir/$self->{'name'}.tlpobj");
- $tarname = "$containername.tar";
+ # Switch to versioned containers
+ # $tarname = "$containername.tar";
+ $tarname = "$containername.r" . $self->revision . ".tar";
+ my $unversionedtar;
+ $unversionedtar = "$containername.tar" if (! $user);
# start the fun
my $tar = $::progs{'tar'};
@@ -649,7 +658,9 @@
# overflow standard tar format and result in special things being
# done. We don't want the GNU-specific special things.
#
- my $is_user_container = ( $containername =~ /\.r[0-9]/ );
+ # We use versioned containers throughout, user mode is determined by
+ # argument.
+ my $is_user_container = $user;
my @attrs
= $is_user_container
? ()
@@ -701,7 +712,7 @@
# A complication, as always. collapse_dirs returns absolute paths.
# We want to change them back to relative so that the backup tar
# has the same structure.
- # in relative mode we have to remove the texmf-dist prefix, too
+ # In relative mode we have to remove the texmf-dist prefix, too.
s,^$instroot/,, foreach @files_to_backup;
if ($relative) {
s,^$RelocTree/,, foreach @files_to_backup;
@@ -712,6 +723,7 @@
# Run tar. Unlink both here in case the container is also plain tar.
unlink("$destdir/$tarname");
+ unlink("$destdir/$unversionedtar") if (! $user);
unlink("$destdir/$containername");
xsystem(@cmdline);
@@ -743,6 +755,19 @@
# most strange cases.
unlink("$destdir/$tarname")
if ((-r "$destdir/$tarname") && (-r "$destdir/$containername"));
+ # in case of system containers also create the links to the
+ # versioned containers
+ if (! $user) {
+ my $linkname = "$destdir/$unversionedtar.$compressorextension";
+ unlink($linkname) if (-r $linkname);
+ if ($copy_instead_of_link) {
+ TeXLive::TLUtils::copy("-f", "$destdir/$containername", $linkname)
+ } else {
+ if (!symlink($containername, $linkname)) {
+ tlwarn("$0: Couldn't generate link $linkname -> $containername?\n");
+ }
+ }
+ }
} else {
tlwarn("$0: Couldn't find $destdir/$tarname to run $compressor\n");
return (0, 0, "");
@@ -1663,13 +1688,13 @@
is returned, otherwise an undefined value. That is also a check
whether a package is relocatable.
-=item C<make_container($type,$instroot[, $destdir[, $containername[, $relative]]])>
+=item C<make_container($type,$instroot, [ destdir => $destdir, containername => $containername, relative => 0|1, user => 0|1 ])>
creates a container file of the all files in the C<TLPOBJ>
in C<$destdir> (if not defined then C<< TLPOBJ->containerdir >> is used).
The C<$type> variable specifies the type of container to be used.
-Currently only C<zip> or C<xz> are allowed, and are generating
+Currently only C<zip> or C<xz> are allowed, and generate
zip files and tar.xz files, respectively.
The file name of the created container file is C<$containername.extension>,
@@ -1683,7 +1708,7 @@
The argument C<$instroot> specifies the root of the installation from
which the files should be taken.
-If the argument C<$relative> is present and true (perlish true) AND the
+If the argument C<$relative> is passed and true (perlish true) AND the
packages does not span multiple texmf trees (i.e., all the first path
components of all files are the same) then a relative packages is created,
i.e., the first path component is stripped. In this case the tlpobj file
@@ -1692,6 +1717,9 @@
This is used to distribute packages which can be installed in any arbitrary
texmf tree (of other distributions, too).
+If user is present and true, no extra arguments for container generation are
+passed to tar (to make sure that user tar doesn't break).
+
Return values are the size, the checksum, and the full name of the container.
=item C<recompute_sizes($tltree)>
Modified: trunk/Master/tlpkg/bin/tl-update-containers
===================================================================
--- trunk/Master/tlpkg/bin/tl-update-containers 2019-11-26 22:24:38 UTC (rev 52930)
+++ trunk/Master/tlpkg/bin/tl-update-containers 2019-11-26 23:04:18 UTC (rev 52931)
@@ -241,6 +241,10 @@
# get list of packages.
PACKS: for my $pkg (sort @todopacks) {
next if $pkg =~ /00texlive/;
+ # we make links from unversioned to versioned containers; this
+ # variable is an override for debugging:
+ my $copy_instead_of_link = 0;
+ #
foreach my $manualpkgre (@TeXLive::TLConfig::CriticalPackagesList) {
# we match the initial string of the package name, so that all the
# .arch packages are skipped, too
@@ -247,6 +251,8 @@
if ($pkg =~ m/^$manualpkgre/) {
if ($opt_all || $opt_recreate) {
tlwarn("$0: Updating critical $pkg due to -all\n");
+ # in general, we want links for everything, but for testing maybe:
+ #$copy_instead_of_link = 1;
last; # of the manualpkgre checks
} else {
@@ -299,7 +305,11 @@
$objcopy->clear_srcfiles;
if ($objsrc) {
my ($s,$m) = $objsrc->make_container($type, $Master,
- $opt_containerdir, "$pkg.source", $do_relative);
+ destdir => $opt_containerdir,
+ containername => "$pkg.source",
+ relative => $do_relative,
+ user => 0,
+ copy_instead_of_link => $copy_instead_of_link);
if ($s > 0) {
# something was created
# important, we have to add it to the original $obj
@@ -323,7 +333,11 @@
$objcopy->clear_docfiles;
if ($objdoc) {
my ($s,$m) = $objdoc->make_container($type, $Master,
- $opt_containerdir, "$pkg.doc", $do_relative);
+ destdir => $opt_containerdir,
+ containername => "$pkg.doc",
+ relative => $do_relative,
+ user => 0,
+ copy_instead_of_link => $copy_instead_of_link);
if ($s > 0) {
# something was created
$obj->doccontainersize($s);
@@ -341,8 +355,12 @@
push @removecontainers, "$pkg.doc";
}
if (!$opt_dry) {
- my ($s,$m) = $objcopy->make_container($type, $Master, $opt_containerdir,
- $pkg, $do_relative);
+ my ($s,$m) = $objcopy->make_container($type, $Master,
+ destdir => $opt_containerdir,
+ containername => $pkg,
+ relative => $do_relative,
+ user => 0,
+ copy_instead_of_link => $copy_instead_of_link);
if ($s > 0) {
$obj->containersize($s);
}
@@ -357,9 +375,9 @@
# add the updated (or new) TLPOBJ to NET TLPDB
# that way the other container sizes are not destroyed
$nettlpdb->add_tlpobj($obj) unless $opt_dry;
- }
+ } # end of big loop on all todopacks
- # remove old containers
+ # remove source/doc containers that have become obsolete.
for my $op (@removecontainers) {
if (-r "$opt_containerdir/$op.tar.xz") {
info("$0: $op container is old, removing it\n");
@@ -392,6 +410,9 @@
$count{"removed"}++;
}
+ # remove non-symlinked .rNNN files.
+ &rcheck_containerdir ($opt_containerdir);
+
if ($opt_recreate) {
info("$0: all packages recreated.\n");
} else {
@@ -400,7 +421,7 @@
info("$0: $count{new} new, $count{removed} removed, " .
"$count{updated} updated, $count{unchanged} unchanged.\n");
} else {
- info("$0: nothing to be done.\n");
+ info("$0: no containers to be updated.\n");
}
}
@@ -438,6 +459,56 @@
return 0;
}
+
+# With versioned containers, we have to remove older .rNNN tarballs when
+# a new NNN gets made and symlinked. Otherwise we'd be distributing
+# every version of every package.
+#
+# That is, if we previously had foo.tar.xz -> foo.r500.tar.xz,
+# but now we've just created foo.r501.tar.xz (and updated the symlink).
+# so we need to delete the .r500.
+#
+# We do this purely by looking at the contents of DIR: every file
+# whose name has a .rNNN pattern is checked for a corresponding symlink,
+# and if no symlink, it is deleted.
+#
+# There should be no stray files with other names, so we report them if
+# we find them, but do nothing with them.
+#
+sub rcheck_containerdir {
+ my ($dir) = @_;
+
+ opendir (my $dh, $dir) || die "opendir($dir) failed: $!";
+ for my $fname (sort readdir $dh) {
+ if ($fname =~ /(.*)\.r[0-9]+\./) {
+ my $linkname = "$dir/$1.tar.xz";
+ if (-l $linkname) {
+ my $linktarget = readlink ($linkname);
+ if ($linktarget ne $fname) {
+ if ($opt_dry) {
+ print "would unlink($fname) [because have $linktarget]\n";
+ } else {
+ debug("unlinking $dir/$fname [because have $linktarget]\n");
+ unlink("$dir/$fname")
+ || tlwarn("$0: non-symlinked unlink($dir/$fname) failed: $!");
+ }
+ } # (else it's the current version, do nothing.)
+ } else {
+ # this can happen if we accidentally created unlinked containers.
+ tlwarn("$0: have file $fname, but no link $linkname!\n");
+ }
+ } elsif ($fname =~ /^([^.]+(\.[^.]+)*\.tar\.xz|\.\.?)$/) {
+ ; # unversioned name, ok. Either . or .. or
+ # matching PKGNAME[.SUBPKG]*.tar.xz, where .SUBPKG might
+ # be .doc, .source, .x86_64-linux, etc., or
+ # .infra.x86_64-linux (for texlive.infra), win32.doc (for wintools).
+ } else {
+ tlwarn("$0: unexpected file: $fname (in $dir)\n");
+ }
+ }
+ closedir ($dh) || warn "closedir($dir) failed: $!";
+}
+
__END__
=head1 NAME
Modified: trunk/Master/tlpkg/doc/packages.txt
===================================================================
--- trunk/Master/tlpkg/doc/packages.txt 2019-11-26 22:24:38 UTC (rev 52930)
+++ trunk/Master/tlpkg/doc/packages.txt 2019-11-26 23:04:18 UTC (rev 52931)
@@ -44,10 +44,10 @@
rm -rf tlcritical
mkdir tlcritical
tl-update-tlcritical --recreate
-
-
-To branch before a release, in the unlikely event of separate
- development work before we want to switch trunk over:
+
+
+
To branch before a release,
+ in the unlikely event of separate development work before switching trunk:
svn copy -r 12205 -m'texlive-2009-dev tag based on r12205' \
svn://tug.org/texlive/trunk svn://tug.org/texlive/tags/texlive-2009-dev
svn copy -r 12205 -m'branch2009-dev branch based on r12205' \
@@ -55,3 +55,110 @@
(Use -dev instead of .something because - will sort first, keeping the
listings in a nice chronological order.)
+
+
+
To build tlnet in a separate location,
+ e.g., if making changes to its structure.
+ (e.g., in Oct 2019 we changed to versioned containers .rNNN.tar.xz.)
+
+# First we build the test repository and test installation from scratch.
+# Do it with no (significant) changes from the current source, for sanity.
+#
+# Use subdir of /home/ftp/texlive or another dir ignored for backups
+# (see /etc/rsnapshot.conf).
+destdir=/home/ftp/texlive/test-tlnet
+#
+# since we're building into a new hierarchy for testing, have to touch tlpdb.
+rm -rf $destdir
+mkdir -p $destdir/tlpkg
+touch !$/texlive.tlpdb
+
+# Update (populate, the first time) the new repo.
+# should take about 15 minutes; updating with the full scheme (which
+# should be done at least once) instead of infraonly takes about 35 min.
+tl-update-tlnet --critical --scheme infraonly --dest $destdir
+
+# if there are errors, the build hierarchy will not be moved to the
+# destination; can do it by hand if desired:
+cd /home/ftp
+rm -rf texlive/test-tlnet/ # just removing the empty file we created
+mv tlnet-trial-*/tlsrc.try texlive/test-tlnet/
+rmdir tlnet-trial-*
+
+# If ever desired, could recreate every package from scratch.
+# This will take hours.
+#tl-update-tlnet --critical --recreate --scheme infraonly --dest $destdir
+
+# Now we can test the new repo: do an installation into /tmp/ki (hardwired).
+# The default scheme here is again infraonly.
+tl-try-install --repo $destdir
+
+# We'll want to test network, local disk, and from-development repositories.
+# So move /tmp/ki, set the repository, and do it again.
+mv /tmp/ki /tmp/tldsk
+tl-try-install --repo $destdir
+/tmp/tldsk/bin/x86_64-linux/tlmgr option repo $destdir
+#
+# For ftp access, want to use ftpmirror to log in, since anonymous ftp
+# will often fail. See tug.org:~karl/.netrc or the ftpd setup on tug.
+mv /tmp/ki /tmp/tlftp
+/tmp/tlftp/bin/x86_64-linux/tlmgr option repo ftp://fm.tug.org/texlive/test-tlnet
+#
+# Create test installation from development tree.
+# This exercises yet another code path.
+Master=...
+$Master/install-tl --profile=$Master/tlpkg/dev/profiles/TLinfra.pro
+mv /tmp/ki /tmp/tldev
+
+# Now, from those repos, initial check with current tlmgr;
+# update list should be empty at first.
+/tmp/tldsk/bin/x86_64-linux/tlmgr update --list
+/tmp/tlftp/bin/x86_64-linux/tlmgr update --list
+/tmp/tldev/bin/x86_64-linux/tlmgr update --list
+
+# So much for the setup and sanity checks. Now it's time to test
+# whatever the real changes are. Make the changes (without
+# committing), and update our test repo just as above (no need to recreate):
+tl-update-tlnet --critical --scheme infraonly --dest $destdir
+
+# Then re-update the installations:
+/tmp/tldsk/bin/x86_64-linux/tlmgr update --self --all
+/tmp/tlftp/bin/x86_64-linux/tlmgr update --self --all
+/tmp/tldev/bin/x86_64-linux/tlmgr update --self --all
+
+# Also try an older tlmgr against the test repo (readonly, don't want to
+# update /usr/local/texlive/2019):
+env PATH=/usr/local/texlive/2019/bin/x86_64-linux:/usr/bin \
+ tlmgr -repo $destdir update --list | head
+#
+# Also must try creating and updating a 2019 install from the test-
+# repo; best to do a full install here for complete testing.
+rm -rf /tmp/ki
+pro=--profile=/home/texlive/trunk/Master/tlpkg/dev/profiles/TLfull.pro
+env PATH=/usr/bin \
+ /usr/local/texlive/2019/install-tl $pro
+test19=/tmp/tlt19
+mv /tmp/ki $test19 # so we don't accidentally lose it with rm /tmp/ki
+#
+env PATH=$test19/bin/x86_64-linux:/usr/bin \
+ tlmgr -repo $destdir update --self --all
+
+# To create the disaster recovery scripts in the test repo:
+tl-update-tlcritical --tlcrit $destdir --no-containers
+
+# Repeat until satisfied: all changes are in test-tlnet and
+# installations and working as intended. For the Nov 2019 update, that
+# means symlinks pointing to new files for updated packages, old
+# versioned containers removed, backups made, etc.
+
+# The tlpkg/bin/tltestnet script has minimal support for some of the
+# above in the November 2019 change - moving the dev files (to and
+# back), creating a 2019-original-release installation, etc. Take a look
+# for ideas and hacking; don't take it too seriously.
+
+# If you need to make changes to files without a nightly build
+# intervening, as will usually be the case, it is necessary to manually
+# edit the Master/tlpkg/texlive.tlpdb in the test- repo to
+# increment the "revision" number for texlive.infra, or whatever
+# packages need to be updated. Otherwise tl-update-containers will think
+# that nothing needs to be updated.
More information about the tex-live-commits
mailing list