lfs-latest-git.php

   1 #! /usr/bin/php
   2 <?php
   3 $dirs = array();
   4 $vers = array();
   5
   6 date_default_timezone_set( "GMT" );
   7 $date = date( "Y-m-d H:i:s" );
   8
   9 // Special cases
  10 $exceptions = array();
  11 //$exceptions[ 'gmp' ] = "UPDIR=/.*(gmp-\d[\d\.-]*\d).*/:DOWNDIR=";
  12
  13 $regex = array();
  14 //$regex[ 'bzip2'    ] = "/^.*current version is ([\d\.]+).*$/";
  15 $regex[ 'intltool' ] = "/^.*Latest version is (\d[\d\.]+\d).*$/";
  16 $regex[ 'less'     ] = "/^.*current released version is less-(\d+).*$/";
  17 $regex[ 'mpfr'     ] = "/^mpfr-([\d\.]+)\.tar.*$/";
  18 $regex[ 'Python'   ] = "/^.*Latest Python 3.*Python (3[\d\.]+\d).*$/";
  19 //$regex[ 'sysvinit' ] = "/^.*sysvinit-([\d\.]+)dsf\.tar.*$/";
  20 $regex[ 'tzdata'   ] = "/^.*tzdata([\d]+[a-z]).*$/";
  21 $regex[ 'xz'       ] = "/^.*xz-([\d\.]*\d).*$/";
  22 $regex[ 'zlib'     ] = "/^.*zlib ([\d\.]*\d).*$/";
  23
  24 function find_max( $lines, $regex_match, $regex_replace )
  25 {
  26   $a = array();
  27   if ( ! is_array( $lines ) ) return -1;
  28
  29   foreach ( $lines as $line )
  30   {
  31      if ( ! preg_match( $regex_match, $line ) ) continue;
  32
  33      // Isolate the version and put in an array
  34      $slice = preg_replace( $regex_replace, "$1", $line );
  35      if ( strcmp( $slice, $line ) == 0 ) continue;
  36
  37      array_push( $a, $slice );
  38   }
  39
  40   // SORT_NATURAL requires php-5.4.0 or later
  41   rsort( $a, SORT_NATURAL );  // Max version is at the top
  42   return ( isset( $a[0] ) ) ? $a[0] : -2;
  43 }
  44
  45 function find_even_max( $lines, $regex_match, $regex_replace )
  46 {
  47   $a = array();
  48   foreach ( $lines as $line )
  49   {
  50      if ( ! preg_match( $regex_match, $line ) ) continue;
  51
  52      // Isolate the version and put in an array
  53      $slice = preg_replace( $regex_replace, "$1", $line );
  54
  55      if ( "x$slice" == "x$line" ) continue;
  56
  57      // Skip odd numbered minor versions and minors > 80
  58      list( $major, $minor, $rest ) = explode( ".", $slice . ".0" );
  59      if ( $minor % 2 == 1  ) continue;
  60      if ( $minor     >  80 ) continue;
  61      array_push( $a, $slice );
  62   }
  63
  64   rsort( $a, SORT_NATURAL );  // Max version is at the top
  65   return ( isset( $a[0] ) ) ? $a[0] : -2;
  66 }
  67
  68 function http_get_file( $url )
  69 {
  70   if ( preg_match( "/mpfr/", $url ) )
  71   {
  72     # There seems to be a problem with the mpfs certificate
  73     exec( "curl --location --silent --insecure --max-time 30 $url", $dir );
  74     $s   = implode( "\n", $dir );
  75     $dir = strip_tags( $s );
  76     return explode( "\n", $dir );
  77   }
  78
  79   if ( preg_match( "/sourceforge/", $url ) ||
  80        preg_match( "/psmisc/",      $url ) )
  81   {
  82     exec( "lynx -dump $url 2>/dev/null", $lines );
  83     return $lines;
  84   }
  85
  86   exec( "curl --location --silent --max-time 30 $url", $dir );
  87
  88   $dir = implode( "\n", $dir );
  89   if ( !preg_match( "/api.github.com/", $url) )
  90     $dir = strip_tags( $dir );
  91   return explode( "\n", $dir );
  92 }
  93
  94 function max_parent( $dirpath, $prefix )
  95 {
  96   // First, remove a directory
  97   $dirpath  = rtrim  ( $dirpath, "/" );    // Trim any trailing slash
  98   $position = strrpos( $dirpath, "/" );
  99   $dirpath  = substr ( $dirpath, 0, $position );
 100
 101   $lines = http_get_file( $dirpath );
 102
 103   $regex_match   = "#{$prefix}[\d\.]+/#";
 104   $regex_replace = "#^.*({$prefix}[\d\.]+)/.*$#";
 105   $max           = find_max( $lines, $regex_match, $regex_replace );
 106
 107   return "$dirpath/$max";
 108 }
 109
 110 function github( $path )
 111 {
 112   return "https://api.github.com/repos/$path/releases/latest";
 113 }
 114
 115 function get_packages( $package, $dirpath )
 116 {
 117   global $exceptions;
 118   global $regex;
 119
 120 //if ( $package != "psmisc" ) return 0; // debug
 121
 122 if ( $package == "bc"         ) $dirpath = github("gavinhoward/bc");
 123 if ( $package == "check"      ) $dirpath = github("libcheck/check");
 124 if ( $package == "e2fsprogs"  ) $dirpath = "https://sourceforge.net/projects/e2fsprogs/files/e2fsprogs";
 125 if ( $package == "expat"      ) $dirpath = "https://sourceforge.net/projects/expat/files";
 126 if ( $package == "elfutils"   ) $dirpath = "https://sourceware.org/ftp/elfutils";
 127 if ( $package == "expect"     ) $dirpath = "https://sourceforge.net/projects/expect/files";
 128 if ( $package == "file"       ) $dirpath = "https://github.com/file/file/tags";
 129 if ( $package == "flex"       ) $dirpath = github("westes/flex");
 130 if ( $package == "flit_core"  ) $dirpath = "https://pypi.org/project/flit-core/";
 131 if ( $package == "setuptools" ) $dirpath = "https://pypi.org/project/setuptools/";
 132 if ( $package == "gcc"        ) $dirpath = max_parent( $dirpath, "gcc-" );
 133 if ( $package == "iana-etc"   ) $dirpath = github("Mic92/iana-etc");
 134 if ( $package == "intltool"   ) $dirpath = "https://launchpad.net/intltool/trunk";
 135 if ( $package == "libffi"     ) $dirpath = github("libffi/libffi");
 136 if ( $package == "libxcrypt"  ) $dirpath = github("besser82/libxcrypt");
 137 if ( $package == "meson"      ) $dirpath = github("mesonbuild/meson");
 138 if ( $package == "mpc"        ) $dirpath = "https://ftp.gnu.org/gnu/mpc";
 139 if ( $package == "mpfr"       ) $dirpath = "https://mpfr.loria.fr/mpfr-current";
 140 if ( $package == "ncurses"    ) $dirpath = "https://invisible-mirror.net/archives/ncurses";
 141 if ( $package == "ninja"      ) $dirpath = github("ninja-build/ninja");
 142 if ( $package == "procps-ng"  ) $dirpath = "https://gitlab.com/procps-ng/procps/-/tags";
 143 if ( $package == "psmisc"     ) $dirpath = "https://gitlab.com/psmisc/psmisc/-/tags";
 144 if ( $package == "Python"     ) $dirpath = "https://www.python.org/downloads/source/";
 145 if ( $package == "shadow"     ) $dirpath = github("shadow-maint/shadow");
 146 if ( $package == "sysvinit"   ) $dirpath = github("slicer69/sysvinit");
 147 if ( $package == "MarkupSafe" ) $dirpath = "https://pypi.python.org/pypi/MarkupSafe/";
 148 if ( $package == "Jinja"      ) $dirpath = "https://pypi.python.org/pypi/Jinja2/";
 149 if ( $package == "systemd"    ) $dirpath = github("systemd/systemd");
 150 //if ( $package == "tcl"        ) $dirpath = "https://sourceforge.net/projects/tcl/files";
 151 if ( $package == "tcl"        ) $dirpath = "https://www.tcl.tk/software/tcltk/download.html";
 152 if ( $package == "util-linux" ) $dirpath = max_parent( $dirpath, "v." );
 153 if ( $package == "vim"        ) $dirpath = "https://github.com/vim/vim/tags";
 154 if ( $package == "wheel"      ) $dirpath = "https://pypi.org/project/wheel/#files";
 155 if ( $package == "zlib"       ) $dirpath = "https://www.zlib.net";
 156 if ( $package == "zstd"       ) $dirpath = github("facebook/zstd");
 157
 158   // Check for ftp
 159   if ( preg_match( "/^ftp/", $dirpath ) )
 160   {
 161     echo "ftp should not occur\n";
 162   /*
 163     $dirpath  = substr( $dirpath, 6 );           // Remove ftp://
 164     $dirpath  = rtrim ( $dirpath, "/" );         // Trim any trailing slash
 165     $position = strpos( $dirpath, "/" );         // Divide at first slash
 166     $server   = substr( $dirpath, 0, $position );
 167     $path     = substr( $dirpath, $position );
 168
 169     $conn = ftp_connect( $server );
 170     ftp_login( $conn, "anonymous", "" );
 171
 172     // See if we need special handling
 173     if ( isset( $exceptions[ $package ] ) )
 174     {
 175        $specials = explode( ":", $exceptions[ $package ] );
 176
 177        foreach ( $specials as $i )
 178        {
 179           list( $op, $regexp ) = explode( "=", $i );
 180
 181           switch ($op)
 182           {
 183             case "UPDIR":
 184               // Remove last dir from $path
 185               $position = strrpos( $path, "/" );
 186               $path = substr( $path, 0, $position );
 187
 188               // Get dir listing
 189               $lines = ftp_rawlist ($conn, $path);
 190               $max   = find_max( $lines, $regexp, $regexp );
 191               break;
 192
 193             case "DOWNDIR":
 194               // Append found directory
 195               $path .= "/$max";
 196               break;
 197
 198             default:
 199               echo "Error in specials array for $package\n";
 200               return -5;
 201               break;
 202           }
 203        }
 204     }
 205
 206     $lines = ftp_rawlist ($conn, $path);
 207     ftp_close( $conn );
 208 */
 209   }
 210   else // http(s)
 211   {
 212      // Customize http directories as needed
 213      if ( $package == "tzdata" )
 214      {
 215         // Remove two directories
 216         $dirpath  = rtrim  ( $dirpath, "/" );    // Trim any trailing slash
 217         $position = strrpos( $dirpath, "/" );
 218         $dirpath  = substr ( $dirpath, 0, $position );
 219         $position = strrpos( $dirpath, "/" );
 220         $dirpath  = substr ( $dirpath, 0, $position );
 221      }
 222
 223      $lines = http_get_file( $dirpath );
 224      if ( ! is_array( $lines ) ) return -6;
 225   } // End fetch
 226
 227   if ( isset( $regex[ $package ] ) )
 228   {
 229      // Custom search for latest package name
 230      foreach ( $lines as $l )
 231      {
 232         $ver = preg_replace( $regex[ $package ], "$1", $l );
 233         if ( $ver == $l ) continue;
 234         return $ver;  // Return first match of regex
 235      }
 236
 237      return -7;  // This is an error
 238   }
 239
 240   if ( $package == "perl" )  // Custom for perl
 241   {
 242      $tmp = array();
 243
 244      foreach ( $lines as $l )
 245      {
 246         if ( preg_match( "/sperl/", $l ) ) continue; // Don't want this
 247         $ver = preg_replace( "/^.*perl-([\d\.]+\d)\.tar.*$/", "$1", $l );
 248         if ( $ver == $l ) continue;
 249         list( $s1, $s2, $rest ) = explode( ".", $ver );
 250         if ( $s2 % 2 == 1 ) continue; // Remove odd minor versions
 251         array_push( $tmp, $l );
 252      }
 253
 254      $lines = $tmp;
 255   }
 256
 257   if ( $package == "attr" ||
 258        $package == "acl"  )
 259   {
 260      return find_max( $lines, "/$package/", "/^.*$package-([\d\.-]*\d).tar.*$/" );
 261   }
 262
 263   if ( $package == "e2fsprogs" )
 264      return find_max( $lines, "/v\d/", "/^.*v(\d[\d\.]+\d).*$/" );
 265
 266   if ( $package == "expect" )
 267      return find_max( $lines, "/expect/", "/^.*expect(\d[\d\.]+\d).tar.*$/" );
 268
 269   if ( $package == "elfutils" )
 270      return find_max( $lines, "/^\d/", "/^(\d[\d\.]+\d)\/.*$/" );
 271
 272   if ( $package == "XML-Parser" )
 273   {
 274      $max = find_max( $lines, "/$package/", "/^.*$package-([\d\._]*\d).tar.*$/" );
 275      # 2.44_01 is a developer release
 276      if ( $max == "2.44_01" ) { return "2.44"; }
 277      return $max;
 278   }
 279
 280   if ( $package == "tcl" )
 281      return find_max( $lines, "/tcl\d/", "/^.*tcl(\d\.[\d\.]*\d)-src.*$/" );
 282
 283   if ( $package == "gmp" )
 284      return find_max( $lines, "/$package/", "/^.*$package-([\d\._]*\d[a-z]?).tar.*$/" );
 285
 286   if ( $package == "dbus" )
 287      return find_even_max( $lines, "/$package/", "/^.*$package-([\d\.]+).tar.*$/" );
 288
 289   if ( $package == "file" )
 290   {
 291      $max = find_max( $lines, "/FILE5/", "/^.*FILE(5_\d+)*$/" );
 292      return str_replace( "_", ".", $max );
 293   }
 294
 295   if ( $package == "flit_core" )
 296      return find_max( $lines, "/flit-core /", "/^.*flit-core ([\d\.]+)$/" );
 297
 298   if ( $package == "setuptools" )
 299      return find_max( $lines, "/setuptools /", "/^.*setuptools ([\d\.]+)$/" );
 300
 301   if ( $package == "procps-ng" )
 302      return find_max( $lines, "/v\d/", "/^.*v([\d\.]+)$/" );
 303
 304   if ( $package == "psmisc" )
 305      return find_max( $lines, "/v\d/", "/^.*v([\d\.]+).tar.*$/" );
 306
 307   if ( $package == "grub" )
 308      return find_max( $lines, "/grub/", "/^.*grub-([\d\.]+).tar.xz.*$/" );
 309
 310   if ( $package == "Jinja" )
 311      return find_max( $lines, "/Jinja/", "/^.*Jinja2 ([\d\.]+).*$/" );
 312
 313   if ( $package == "openssl" )
 314      return find_max( $lines, "/openssl/", "/^.*openssl-([\d\.p]*\d.?).tar.*$/" );
 315
 316   if ( $package == "vim" )
 317      return find_max( $lines, "/v\d\./", "/^.*v([\d\.]+).*$/" );
 318
 319   if ( preg_match( "/api.github.com/", $dirpath) )
 320      return ltrim(json_decode(join("", $lines))->tag_name, "v");
 321
 322   // Most packages are in the form $package-n.n.n
 323   // Occasionally there are dashes (e.g. 201-1)
 324   return find_max( $lines, "/$package/", "/^.*$package-([\d\.-]*\d)\.tar.*$/" );
 325 }
 326
 327 function get_current()
 328 {
 329    global $dirs;
 330    global $vers;
 331
 332    // Fetech from git and get wget-list
 333    $current = array();
 334    #$lfssvn = "svn://svn.linuxfromscratch.org/LFS/trunk";
 335    $lfsgit = "https://git.linuxfromscratch.org/lfs.git";
 336
 337    $tmpdir = exec( "mktemp -d /tmp/lfscheck.XXXXXX" );
 338    $cdir   = getcwd();
 339    chdir( $tmpdir );
 340
 341    # git-version.sh needs the history since the rx.y tag.
 342    exec ( "git clone $lfsgit LFS --depth 1 --branch r11.2" );
 343    exec ( "git -C LFS pull origin trunk" );
 344
 345    # Make version.ent
 346    chdir( "$tmpdir/LFS" );
 347    exec ( "./git-version.sh systemd" );
 348
 349    chdir( $cdir );
 350
 351    $PAGE       = "$tmpdir/LFS/chapter03/chapter03.xml";
 352    $STYLESHEET = "$tmpdir/LFS/stylesheets/wget-list.xsl";
 353
 354    exec( "xsltproc --xinclude --nonet $STYLESHEET $PAGE", $current );
 355    exec( "rm -rf $tmpdir" );
 356
 357    foreach ( $current as $line )
 358    {
 359       $file = basename( $line ) . "\n";
 360       if ( preg_match( "/patch$/", $file ) ) { continue; } // Skip patches
 361
 362       $file = preg_replace( "/bz2/", '', $file ); // The 2 confusses the regex
 363
 364       $file        = rtrim( $file );
 365       $pkg_pattern = "/(\D*).*/";
 366       //$pattern     = "/\D*(\d.*\d)\D*/";
 367       $pattern     = "/\D*(\d.*\d)\D*/";
 368
 369       if ( preg_match( "/e2fsprogs/", $file ) )
 370       {
 371         $pattern = "/e2\D*(\d.*\d)\D*/";
 372         $pkg_pattern = "/(e2\D*).*/";
 373       }
 374
 375       else if ( preg_match( "/tzdata/", $file ) )
 376       {
 377         $pattern = "/\D*(\d.*[a-z])\.tar\D*/";
 378       }
 379
 380       else if ( preg_match( "/openssl/", $file ) )
 381       {
 382         $pattern = "/\D*(\d.*\d.*).tar.*$/";
 383       }
 384
 385       else if ( preg_match( "/gmp/", $file ) )
 386       {
 387         $pattern = "/\D*(\d.*[a-z]*)\.tar\D*/";
 388       }
 389
 390       else if ( preg_match( "/systemd-man-pages/", $file ) ) continue;
 391       else if ( preg_match( "/python/"         , $file ) ) continue;
 392
 393       $version = preg_replace( $pattern, "$1", $file );   // Isolate version
 394       $version = preg_replace( "/^\d-/", "", $version );  // Remove leading #-
 395
 396       // Touch up package names
 397       $pkg_name = preg_replace( $pkg_pattern, "$1", $file );
 398       $pkg_name = trim( $pkg_name, "-" );
 399
 400       if ( preg_match( "/bzip|iproute/", $pkg_name ) ) { $pkg_name .= "2"; }
 401       if ( preg_match( "/^m$/"         , $pkg_name ) ) { $pkg_name .= "4"; }
 402       if ( preg_match( "/shadow/"      , $pkg_name ) ) { $pkg_name  = "shadow"; }
 403
 404       $dirs[ $pkg_name ] = dirname( $line );
 405       $vers[ $pkg_name ] = $version;
 406    }
 407 }
 408
 409 function mail_to_lfs()
 410 {
 411    global $date;
 412    global $vers;
 413    global $dirs;
 414
 415    $to      = "lfs-book@lists.linuxfromscratch.org";
 416    $from    = "bdubbs@linuxfromscratch.org";
 417    $subject = "LFS Package Currency Check - $date GMT";
 418    $headers = "From: bdubbs@linuxfromscratch.org";
 419
 420    $message = "Package         LFS      Upstream  Flag\n\n";
 421
 422    foreach ( $dirs as $pkg => $dir )
 423    {
 424       //if ( $pkg != "gmp" ) continue;  //debug
 425       $v = get_packages( $pkg, $dir );
 426
 427       $flag = ( $vers[ $pkg ] != $v ) ? "*" : "";
 428
 429       // Pad for output
 430       $pad = "                ";
 431       $p   = substr( $pkg          . $pad, 0, 15 );
 432       $l   = substr( $vers[ $pkg ] . $pad, 0, 10 );
 433       $c   = substr( $v            . $pad, 0, 10 );
 434
 435       $message .= "$p $l $c $flag\n";
 436    }
 437
 438    exec ( "echo '$message' | mailx -r $from -s '$subject' $to" );
 439 }
 440
 441 function html()
 442 {
 443
 444    global $date;
 445    global $vers;
 446    global $dirs;
 447
 448    echo "<!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Strict//EN'
 449                       'http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd'>
 450 <html xmlns='http://www.w3.org/1999/xhtml' xml:lang='en' lang='en'>
 451 <head>
 452 <title>LFS Package Currency Check - $date</title>
 453 <style type='text/css'>
 454 h1, h2 {
 455    text-align      : center;
 456 }
 457
 458 table {
 459    border-width    : 1px;
 460    border-spacing  : 0px;
 461    border-style    : outset;
 462    border-color    : gray;
 463    border-collapse : separate;
 464    background-color: white;
 465    margin          : 0px auto;
 466 }
 467
 468 table th {
 469    border-width    : 1px;
 470    padding         : 2px;
 471    border-style    : inset;
 472    border-color    : gray;
 473    background-color: white;
 474 }
 475
 476 table td {
 477    border-width    : 1px;
 478    padding         : 2px;
 479    border-style    : inset;
 480    border-color    : gray;
 481    background-color: white;
 482 }
 483 </style>
 484
 485 </head>
 486 <body>
 487 <h1>LFS Package Currency Check</h1>
 488 <h2>As of $date GMT</h1>
 489
 490 <table>
 491 <tr><th>LFS Package</th> <th>LFS Version</th> <th>Latest</th> <th>Flag</th></tr>\n";
 492
 493    // Get the latest version of each package
 494    foreach ( $dirs as $pkg => $dir )
 495    {
 496       $v    = get_packages( $pkg, $dir );
 497       $flag = ( $vers[ $pkg ] != $v ) ? "*" : "";
 498       echo "<tr><td>$pkg</td> <td>{$vers[ $pkg ]}</td> <td>$v</td> <td>$flag</td></tr>\n";
 499    }
 500
 501    echo "</table>
 502 </body>
 503 </html>\n";
 504
 505 }
 506
 507 function write_to_stdout()
 508 {
 509
 510    global $date;
 511    global $vers;
 512    global $dirs;
 513
 514    echo "
 515 LFS Package Currency Check
 516 As of $date GMT
 517
 518 LFS Package     LFS Version Latest Flag\n";
 519
 520    // Get the latest version of each package
 521    foreach ( $dirs as $pkg => $dir )
 522    {
 523       $p_name    = sprintf( "%-15s", $pkg );       // package name formatted
 524
 525       $b_version = $vers[ $pkg ];                 // book version
 526       $b_string  = sprintf( "%-11s", $b_version ); // book version formatted
 527
 528       $latest    = get_packages( $pkg, $dir );    // latest version
 529       $l_string  = sprintf( "%-6s", $latest );     // latest version formatted
 530
 531       $flag = ( $b_version != $latest ) ? "*" : "";
 532       echo "$p_name $b_string $l_string  $flag\n";
 533    }
 534 }
 535
 536 get_current();  // Get what is in the book
 537 mail_to_lfs();
 538 //html();  // Write html output
 539 //write_to_stdout();  // For debugging
 540 ?>