texlive[47806] Master/tlpkg/TeXLive: more work, currently incomplete

commits+preining at tug.org commits+preining at tug.org
Wed May 23 04:33:13 CEST 2018


Revision: 47806
          http://tug.org/svn/texlive?view=revision&revision=47806
Author:   preining
Date:     2018-05-23 04:33:13 +0200 (Wed, 23 May 2018)
Log Message:
-----------
more work, currently incomplete

Modified Paths:
--------------
    trunk/Master/tlpkg/TeXLive/TLConfig.pm
    trunk/Master/tlpkg/TeXLive/TLUtils.pm

Modified: trunk/Master/tlpkg/TeXLive/TLConfig.pm
===================================================================
--- trunk/Master/tlpkg/TeXLive/TLConfig.pm	2018-05-23 02:33:01 UTC (rev 47805)
+++ trunk/Master/tlpkg/TeXLive/TLConfig.pm	2018-05-23 02:33:13 UTC (rev 47806)
@@ -22,6 +22,10 @@
     $MetaCategoriesRegexp
     $CategoriesRegexp
     $DefaultCategory
+    $DefaultFallbackDownloader
+    @AcceptedFallbackDownloaders
+    %FallbackDownloaderProgram
+    %FallbackDownloaderArgs
     $DefaultCompressorFormat
     $DefaultContainerExtension
     @AcceptedCompressors
@@ -110,6 +114,14 @@
   $CriticalPackagesRegexp = '^(texlive\.infra|tlperl\.win32$)';
 }
 
+#
+our $DefaultFallbackDownloader = "wget";
+our @AcceptedFallbackDownloaders = qw/wget curl/;
+our %FallbackDownloaderProgram = ( 'wget' => 'wget', 'curl' => 'curl');
+our %FallbackDownloaderArgs = (
+  'wget' => ['--user-agent=texlive/wget', '--tries=10', "--timeout=$NetworkTimeout", '-q', '-O'],
+  'curl' => ['--user-agent', 'texlive/curl', '--retry', '10', '--connect-timeout', "$NetworkTimeout", '--silent', '--output']
+);
 # the way we package things on the web
 our $DefaultCompressorFormat = "xz";
 our $DefaultContainerExtension = "tar.$DefaultCompressorFormat";

Modified: trunk/Master/tlpkg/TeXLive/TLUtils.pm
===================================================================
--- trunk/Master/tlpkg/TeXLive/TLUtils.pm	2018-05-23 02:33:01 UTC (rev 47805)
+++ trunk/Master/tlpkg/TeXLive/TLUtils.pm	2018-05-23 02:33:13 UTC (rev 47806)
@@ -54,7 +54,7 @@
   TeXLive::TLUtils::touch(@files);
   TeXLive::TLUtils::collapse_dirs(@files);
   TeXLive::TLUtils::removed_dirs(@files);
-  TeXLive::TLUtils::download_file($path, $destination [, $progs ]);
+  TeXLive::TLUtils::download_file($path, $destination);
   TeXLive::TLUtils::setup_programs($bindir, $platform);
   TeXLive::TLUtils::setup_compressor();
   TeXLive::TLUtils::tlcmp($file, $file);
@@ -2332,13 +2332,34 @@
   $::progs{'xz'} = "xz";
   $::progs{'tar'} = "tar";
   $::progs{'lz4'} = "lz4";
-
+  $::progs{'gzip'} = "gzip";
+  $::progs{'downloader'} = $::progs{$DefaultFallbackDownloader};
+  $::progs{'downloader-args'} = @{$FallbackDownloaderArgs{$DefaultFallbackDownloader}};
   if ($^O =~ /^MSWin/i) {
     $::progs{'wget'}  = conv_to_w32_path("$bindir/wget/wget.exe");
     $::progs{'tar'}   = conv_to_w32_path("$bindir/tar.exe");
     $::progs{'xz'}    = conv_to_w32_path("$bindir/xz/xz.exe");
     $::progs{'lz4'}   = conv_to_w32_path("$bindir/lz4/lz4.exe");
-    for my $prog ("xz", "wget") {
+    # we leave gzip as is, needs to be provided by the PATH
+    # for windows we don't expect to have curl available, use our own
+    # shipped wget in all cases
+    my $dltype = 'wget';
+    $::progs{'downloader'} = $FallbackDownloaderProgram{$dltype};
+    $::progs{'downloader-args'} = $FallbackDownloaderArgs{$dltype};
+    # allow for hit-her-to override env vars
+    if ($ENV{"TL_DOWNLOAD_PROGRAM"}) {
+      $::progs{'downloader'} = $ENV{"TL_DOWNLOAD_PROGRAM"};
+    }
+    if ($ENV{"TL_DOWNLOAD_ARGS"}) {
+      $::progs{'downloader-args'} = [ split (" ", $ENV{"TL_DOWNLOAD_ARGS"}) ];
+    }
+    my $comptype = setup_program_with_env('TEXLIVE_COMPRESSOR', $DefaultCompressorFormat, @AcceptedCompressors);
+    $::progs{'compressor'} = $CompressorProgram{$comptype}:
+    $::progs{'compressor-args'} = $CompressorArgs{$comptype}:
+    $::progs{'decompressor'} = $DecompressorProgram{$comptype}:
+    $::progs{'compressor-args'} = $DecompressorArgs{$comptype}:
+
+    for my $prog ("downloader", "compressor") {
       my $ret = system("$::progs{$prog} --version >nul 2>&1"); # on windows
       if ($ret != 0) {
         warn "TeXLive::TLUtils::setup_programs (w32) failed";  # no nl for perl
@@ -2358,7 +2379,37 @@
       $::installerdir = "$bindir/../..";
       $platform = platform();
     }
-    my $s = 0;
+    # setup of the fallback downloaders
+    for my $dltype (@AcceptedFallbackDownloaders) {
+      my $defprog = $FallbackDownloadProgram{$dltype};
+      # do not warn on errors
+      setup_unix_one($defprog, "$bindir/$dltype/$defprog.$platform", "--version", 1);
+    }
+    for my $comptype (@AcceptedCompressors) {
+      my $defprog = $CompressorProgram{$comptype};
+      # do not warn on errors
+      setup_unix_one($defprog, "$bindir/$comptype/$defprog.$platform", "--version", 1);
+    }
+    # in the case of downloaders keys and prog names agree
+    my $dltype = setup_program_with_env('TEXLIVE_DOWNLOADER', 
+                                        $DefaultFallbackDownloader,
+                                        @AcceptedFallbackDownloaders);
+    $::progs{'downloader'} = $FallbackDownloaderProgram{$dltype};
+    $::progs{'downloader-args'} = $FallbackDownloaderArgs{$dltype};
+    if ($ENV{"TL_DOWNLOAD_PROGRAM"}) {
+      $::progs{'downloader'} = $ENV{"TL_DOWNLOAD_PROGRAM"};
+    }
+    if ($ENV{"TL_DOWNLOAD_ARGS"}) {
+      $::progs{'downloader-args'} = [ split (" ", $ENV{"TL_DOWNLOAD_ARGS"}) ];
+    }
+
+    # TODO TODO TODO
+    die("NNED TO UPDATE MORE STUFF HERE IN setup_programs!");
+    my $comptype = setup_program_with_env('TEXLIVE_COMPRESSOR', $DefaultCompressorFormat, @AcceptedCompressors);
+
+
+
+
     $s += setup_unix_one('wget', "$bindir/wget/wget.$platform", "--version");
     $s += setup_unix_one('xz',   "$bindir/xz/xz.$platform", "notest");
     $ok = ($s == 2);  # failure return unless all are present.
@@ -2477,38 +2528,33 @@
 }
 
 

-=item C<setup_compressor()>
+=item C<setup_program_with_env($envvar, $default, @accepted)>
 
-Setup compression method for containers.
+Setup a program based on configuration in the environment of default.
 
 =cut
 
-# TODO TODO needs improvement, check also whether compressor prog is available!!!
-sub setup_compressor {
-  my $compressor = "xz";
-  if ($ENV{'TEXLIVE_COMPRESSOR'}) {
-    if (TeXLive::TLUtils::member($ENV{'TEXLIVE_COMPRESSOR'}, @TeXLive::TLConfig::AcceptedCompressors)) {
-      $compressor = $ENV{'TEXLIVE_COMPRESSOR'};
+sub setup_program_with_env {
+  my ($envvar, $default, @accepted) = @_;
+  my $sel = $default;
+  if ($ENV{$envvar}) {
+    if (TeXLive::TLUtils::member($ENV{$envvar}, @accepted)) {
+      $sel = $ENV{$envvar};
     } else {
-      tlwarn("$prog: unknown compressor in TEXLIVE_COMPRESSOR=$ENV{'TEXLIVE_COMPRESSOR'}, falling back to xz\n");
+      tlwarn("$prog: unknown prog in $envvar=$ENV{$envvar}, falling back to $default\n");
     }
   }
-  my $compressorextension = $TeXLive::TLConfig::CompressorExtension{$compressor};
-  return ($compressor, $compressorextension);
+  return $sel;
 }
 
 
-=item C<download_file( $relpath, $destination [, $progs ] )>
 
+=item C<download_file( $relpath, $destination )>
+
 Try to download the file given in C<$relpath> from C<$TeXLiveURL>
 into C<$destination>, which can be either
 a filename of simply C<|>. In the latter case a file handle is returned.
 
-The optional argument C<$progs> is a reference to a hash giving full
-paths to the respective programs, at least C<wget>.  If C<$progs> is not
-given the C<%::progs> hash is consulted, and if this also does not exist
-we try a literal C<wget>.
-
 Downloading honors two environment variables: C<TL_DOWNLOAD_PROGRAM> and
 C<TL_DOWNLOAD_ARGS>. The former overrides the above specification
 devolving to C<wget>, and the latter overrides the default wget
@@ -2521,11 +2567,9 @@
 =cut
 
 sub download_file {
-  my ($relpath, $dest, $progs) = @_;
+  my ($relpath, $dest) = @_;
   my $wget;
-  if (defined($progs) && defined($progs->{'wget'})) {
-    $wget = $progs->{'wget'};
-  } elsif (defined($::progs{'wget'})) {
+  if (defined($::progs{'wget'})) {
     $wget = $::progs{'wget'};
   } else {
     tlwarn ("download_file: Programs not set up, trying literal wget\n");



More information about the tex-live-commits mailing list