texlive[54079] Master/tlpkg: TLPSRC.pm (from_file): try harder to
commits+karl at tug.org
commits+karl at tug.org
Wed Mar 4 23:37:58 CET 2020
Revision: 54079
http://tug.org/svn/texlive?view=revision&revision=54079
Author: karl
Date: 2020-03-04 23:37:58 +0100 (Wed, 04 Mar 2020)
Log Message:
-----------
TLPSRC.pm (from_file): try harder to return the same shortdesc and
longdesc strings from *.tlpsrc parsing as from tlpdb.
tl-update-tlpdb (main): clear $opt_catalogue_compare in addition to
$opt_fix_reverse_revisions when checking for the buggy tlpdb, since the
comparison tlpdb with old data (tlnet/tlpretest) can't get updated.
Also, insert commented-out kludge to dump and reread Catalogue data.
(compare_and_fix_tlpdbs): compare newtlpdb to the catalogue_compare
TLPDB, not oldtlpdb, although it's unlikely to make much difference.
(tlpobj_catalogue_equal): show shortdesc and longdesc strings.
Modified Paths:
--------------
trunk/Master/tlpkg/TeXLive/TLPSRC.pm
trunk/Master/tlpkg/bin/tl-update-tlpdb
Modified: trunk/Master/tlpkg/TeXLive/TLPSRC.pm
===================================================================
--- trunk/Master/tlpkg/TeXLive/TLPSRC.pm 2020-03-04 22:35:38 UTC (rev 54078)
+++ trunk/Master/tlpkg/TeXLive/TLPSRC.pm 2020-03-04 22:37:58 UTC (rev 54079)
@@ -184,7 +184,10 @@
$shortdesc = "";
} elsif ($line =~ /^longdesc$/) {
- $longdesc .= "\n";
+ # We need to use a space here instead of a newline so that strings
+ # read from *.tlpsrc and tlpdb come out the same; see $shortdesc
+ # and $longdesc assignments below.
+ $longdesc .= " ";
} elsif ($line =~ /^longdesc\s+(.*)$/) {
$longdesc .= "$1 ";
@@ -228,6 +231,16 @@
if ($name =~ m/^[[:space:]]*$/) {
die "Cannot deduce name from file argument and name tag not found";
}
+ #
+ # We should probably call TeXCatalogue::beautify(), but trailing
+ # whitespace seems to be the only thing that comes up in practice. We
+ # want the parsing from .tlpsrc to result in exactly the same string,
+ # including spaces, as parsing from texlive.tlpdb. Otherwise
+ # tl-update-tlpdb's tlpdb_catalogue_compare will think the strings
+ # are always different.
+ $shortdesc =~ s/\s+$//g;
+ $longdesc =~ s/\s+$//g;
+ #
$self->name($name);
$self->category($category);
$self->catalogue($catalogue) if $catalogue;
Modified: trunk/Master/tlpkg/bin/tl-update-tlpdb
===================================================================
--- trunk/Master/tlpkg/bin/tl-update-tlpdb 2020-03-04 22:35:38 UTC (rev 54078)
+++ trunk/Master/tlpkg/bin/tl-update-tlpdb 2020-03-04 22:37:58 UTC (rev 54079)
@@ -2,10 +2,11 @@
# $Id$
# Convert a TL tree and *.tlpsrc files into a (possibly updated) texlive.tlpdb.
#
-# Copyright 2007-2019 Norbert Preining
+# Copyright 2007-2020 Norbert Preining
# This file is licensed under the GNU General Public License version 2
# or any later version.
+#use warnings FATAL => 'all'; # can help in development
BEGIN {
$^W = 1;
chomp($mydir = `dirname $0`);
@@ -97,6 +98,20 @@
if (-r $opt_catalogue) {
info("$prg: reading TeX Catalogue $opt_catalogue ...\n");
$tlc = TeXLive::TeXCatalogue->new("location" => $opt_catalogue);
+ # way to dump the catalogue and then use it instead of reparsing.
+ # Maybe someday have the energy to make it a real option,
+ # and merge this with the same code in TLUtils as a common function.
+ #require "/home/karl/cat.pl";
+ #if (0) {
+ # require Data::Dumper;
+ # $Data::Dumper::Indent = 1;
+ # $Data::Dumper::Sortkeys = 1; # stable output
+ # $Data::Dumper::Purity = 1; # recursive structures must be safe
+ # print STDERR "DDdumping tlc(atalogue)\n";
+ # open (my $fh, ">/home/karl/cat.pl") || die "open(cat) failed: $!\n";
+ # print $fh Data::Dumper->Dump([$tlc], [qw(::tlc)]);
+ # close ($fh) || die "close(cat) failed: $!\n";
+ #}
} else {
tlwarn("$prg: reusing Catalogue data, since source is not readable: "
. "$opt_catalogue\n");
@@ -106,7 +121,7 @@
}
}
- # Create our working tlpdb with Catalogue data.
+ # Create our new working tlpdb with Catalogue data.
my ($newtlpdb,$pkgcount) = create_tlpdb($tlc);
my $oldtlpdb = undef;
@@ -115,7 +130,7 @@
$oldtlpdb = TeXLive::TLPDB->new("root" => $opt_master);
}
if (!defined($oldtlpdb)) {
- tlwarn("$prg: No revision check since no old tlpdb: $OLDTLPDB\n");
+ tlwarn("$prg: no revision check since no old tlpdb: $OLDTLPDB\n");
finish($newtlpdb, $pkgcount);
}
@@ -122,16 +137,16 @@
if ($opt_no_revision_check) {
if ($opt_keep_revisions) {
copy_revisions_over($oldtlpdb, $newtlpdb);
- info("$prg: Keeping old revisions due to --keep-revisions\n");
+ info("$prg: keeping old revisions due to --keep-revisions\n");
} else {
- info("$prg: Not checking revisions due to --no-reverse-revision-check\n");
+ info("$prg: not checking revisions due to --no-reverse-revision-check\n");
}
finish($newtlpdb, $pkgcount);
}
- # compare_and_fix_tlpdbs returns true if any reversed revisions were found.
- # If the option --fix-reverse-revisions (etc.) specify, it also tried
- # to fix them, by committing changes.
+ # compare_and_fix_tlpdbs returns true if any reversed revisions were
+ # found. If the option --fix-reverse-revisions (etc.) was specified,
+ # it also tried to fix them, by committing changes.
my $buggy_tlpdb = 0;
if (compare_and_fix_tlpdbs($oldtlpdb, $newtlpdb)) {
$buggy_tlpdb = 1;
@@ -141,21 +156,30 @@
#
# If we are running from svn (--from-files is not given),
# and we tried to fix revisions (--fix-reverse-revisions is given),
- # and we are committing the changes (--no_commit is not given),
+ # and we are committing the changes (--no-commit is not given),
# then we recreate the tlpdb, else we will report as buggy.
#
if (!$opt_fromfiles && $opt_fix_reverse_revisions && !$opt_no_commit) {
- info("$prg: Regenerating tlpdb after fixing, "
- . "without -fix-reverse-revisions\n");
+ info("$prg: regenerating tlpdb after fixing\n");
($newtlpdb,$pkgcount) = create_tlpdb($tlc);
#
# now recheck, but do NOT try to check in changes again, since it
# already failed: forcibly unset $opt_fix_reverse_revisions.
+ info("$prg: unsetting --fix-reverse-revisions for second check\n");
$opt_fix_reverse_revisions = 0;
+ #
+ # Also do not do catalogue checks, since the Catalogue comparison
+ # tree, which has old data, has not changed, so they would just
+ # fail again. (We should save and restore these values, or pass
+ # them as arguments, instead of just eradicating them, but in
+ # practice we do not use them again.)
+ info("$prg: unsetting --catalogue-compare for second check\n");
+ $opt_catalogue_compare = "";
+ #
if (compare_and_fix_tlpdbs($oldtlpdb, $newtlpdb)) {
- tlwarn("$prg: *** oops, should not happen!\n"
+ tlwarn("$prg: *** OOPS, should not happen!\n"
. " After one iteration of compare_tlpdbs\n"
- . " the new tlpdb is still going backwards in time;\n"
+ . " the new tlpdb still has changes;\n"
. " check output above and fix!\n");
# $buggy_tlpdb is already set to 1
} else {
@@ -175,7 +199,8 @@
tlwarn("$prg: saving it anyway due to --save-anyway\n");
$buggy_tlpdb = 0;
} else {
- tlwarn("$prg: not saving it, use --save-anyway if you want it\n");
+ tlwarn("$prg: not saving it, use --save-anyway or --output "
+ . "if you want it\n");
}
}
if (!$buggy_tlpdb) {
@@ -197,7 +222,7 @@
if (defined($oldtlp)) {
my $oldrev = $oldtlp->revision;
my $newrev = $newtlpdb->get_package($pkg)->revision;
- warn "xxxpkg $pkg: changing revision from $newrev to $oldrev\n";
+ debug("pkg $pkg: changing revision from $newrev to $oldrev\n");
$newtlpdb->get_package($pkg)->revision($oldtlp->revision);
}
}
@@ -206,9 +231,9 @@
# Compare OLDTLPDB and NEWTLPDB, returning whether any packages needed
# special fixing. Within the list of packages that differ, the
-# problematic ones are those where either files were added or removed,
-# or there were Catalogue changes (against --catalogue-compare) but the
-# revision did not increase.
+# problematic ones are those where either (a) files were added or
+# removed, or (b) NEWTLPDB has Catalogue data changes compared to the
+# --catalogue-compare tree, but the revision did not increase.
#
# If a package has been removed or added (or not changed at all) that is
# fine for here, as it will be dealt with in tl-update-containers.
@@ -246,7 +271,7 @@
# since we only register 'revision' key when there is a
# change, we have either rA < rB or rA > rB
if ($rA > $rB) {
- tlwarn("$prg: big warning for $p: file lists didn't change "
+ tlwarn("$prg: BIG WARNING for $p: file lists didn't change "
. "but revision is going backward! Very strange!\n");
$packages_needing_fixup{$p} = 1;
} # if strange revision change
@@ -257,7 +282,7 @@
# if requested, also may need to fix up packages with catalogue changs.
if ($opt_catalogue_compare) {
- my %cat_fixup = tlpdb_catalogue_compare($oldtlpdb, $opt_catalogue_compare);
+ my %cat_fixup = tlpdb_catalogue_compare($newtlpdb, $opt_catalogue_compare);
# hash slice to merge %cat_fixup into %packages_needing_fixup;
# all the values are 1, so that doesn't matter.
@packages_needing_fixup{ keys %cat_fixup } = values %cat_fixup;
@@ -270,7 +295,7 @@
tlwarn("$prg: package $p has different files but "
. "same/decreasing revision number",
$opt_fix_reverse_revisions ? "\n"
- : " [skipping fix since no fix_reverse_revisions]\n");
+ : " [skipping fix due to no fix_reverse_revisions]\n");
if ($opt_fix_reverse_revisions) {
my $fixfile = "$opt_master/tlpkg/tlpsrc/$p.tlpsrc";
# make sure that we actually fix a file of the right package
@@ -281,15 +306,16 @@
# in the packages
my $tlp = $newtlpdb->get_package($p);
if (!defined($tlp)) {
- tlwarn("Very strange, $p is mentioned as changed files,\n");
- tlwarn("but it is not found in new tlpdb.\n");
- tlwarn("Not trying to fix anything, don't know how!\n");
+ tlwarn("$prg: Very strange, $p is said to have changed files,\n");
+ tlwarn("but it is not found in new tlpdb;\n");
+ tlwarn("skipping $p.\n");
next;
}
my @allf = $tlp->all_files;
if ($#allf < 0) {
- tlwarn("No files found in $p, and no tlpsrc file present.\n");
- tlwarn("Don't know how to fix decreasing version in this case.\n");
+ tlwarn("$prg: no files found in $p, and no tlpsrc file present.\n");
+ tlwarn("don't know how to fix decreasing version in this case;\n");
+ tlwarn("skipping $p.\n");
next;
}
# still here, so use the first file in the list.
@@ -343,7 +369,7 @@
if (@ARGV) {
# if we have cmd line arguments act as if --tlpsrc-from-cmdline was given.
if (!$opt_tlpsrc_from_cmdline) {
- debug("$prg: generating tlpdb in /tmp/tlut for @ARGV.\n");
+ debug("$prg: generating tlpdb in /tmp/tlut from tlpsrc args: @ARGV\n");
$opt_tlpsrc_from_cmdline = 1;
}
}
@@ -478,20 +504,20 @@
# of packages that so differ.
#
# We do not worry about packages found in one tlpdb and not the other;
-# those will get found in other places. Our goal here is just to compare
+# those will get dealt with elsewhere. Our goal here is just to compare
# the Catalogue data in packages common to the two tlpdbs.
#
-# In practice, CMP_TLPDB is tlnet. This whole check is necessary because
-# we intentionally do not (normally) update packages in tlnet when only
-# Catalogue data has changed; that can easily lead to extra downloads
-# for a single real update of the package, since the metadata often
-# needs tweaking after an upload.
+# In practice, CMP_TLPDB is tlnet (or tlpretest). This whole check is
+# necessary because we intentionally do not (normally) update packages
+# in tlnet when only Catalogue data has changed; that can easily lead to
+# extra downloads for a single real update of the package, since the
+# metadata often needs tweaking after an upload.
#
# On the other hand, we do want to get current Catalogue data into TL
# sometimes, or it would grow stale forever; so that's what this
# comparison is for. We enable it once a month or so from cron.tl.
#
-# We do insert Catalogue updates into the texlive.tlpdb checked into the
+# We do update Catalogue data in the texlive.tlpdb checked into the
# repository (under Master/) nightly, but that is not what users
# (typically) use.
#
@@ -505,7 +531,7 @@
my %ret;
my $cmp_tlpdb = TeXLive::TLPDB->new("root" => $cmp_tlpdb_str);
- die "$prg: no tlpdb to compare under: $cmp_tlpdb_str\n"
+ die "$prg: could not get comparison tlpdb from: $cmp_tlpdb_str\n"
if ! defined $cmp_tlpdb;
for my $p ($master_tlpdb->list_packages()) {
@@ -539,13 +565,19 @@
my $pkg = $tlpA->name; # just for debugging output
debug("$pkg: comparing catalogue info:\n");
- # the shortdesc and longdesc usually come from the Catalogue,
- # so are not in the tlpsrc.
+ # The shortdesc and longdesc usually come from the Catalogue,
+ # but sometimes are in the tlpsrc.
if (! &equal_strings($tlpA->shortdesc, $tlpB->shortdesc)) {
- debug("$pkg: shortdesc changed\n");
+ # Maybe someday combine this silly code to avoid redundancy,
+ # and show strings (hashes) for all.
+ debug("$pkg: shortdesc changed: ", $tlpA->shortdesc, "\n",
+ " " x length($pkg),
+ " vs. ", $tlpB->shortdesc, "\n");
} elsif (! &equal_strings($tlpA->longdesc, $tlpB->longdesc)) {
- debug("$pkg: longdesc changed\n");
+ debug("$pkg: longdesc changed: ", $tlpA->longdesc, "\n",
+ " " x length($pkg),
+ " vs. ", $tlpB->longdesc, "\n");
} elsif (! &equal_strings($tlpA->catalogue, $tlpB->catalogue)) {
debug("$pkg: catalogue value changed\n");
@@ -688,11 +720,18 @@
By default, packages for which the only change is in the Catalogue data
are not marked as updated and written with new revisions. This option
-changes that (except that the catalogue-date field is ignored). We
-enable this from time to time to re-sync package information. This
-implies C<--reverse-revision-check>, since the same mechanism is used to
-force the new version numbers.
+checks the Catalogue data against that in I<tlpdb> (typically tlnet or
+tlpretest), and forces the package to be updated if there are
+differences. We enable this from time to time to re-sync the package
+information even if the package contents have not changed.
+As a consequence, after this has been invoked the first time, the same
+packages will continue to be found as different until I<tlpdb> is
+updated. Beware of confusion when testing/developing.
+
+This implies C<--reverse-revision-check>, since the same mechanism is
+used to force the new version numbers.
+
=item B<--fix-reverse-revisions>
If a package with revision number going backward is found, this option
More information about the tex-live-commits
mailing list.