testsuite/REQUIREMENTS

   1 #!/bin/bash
   2
   3 # SPDX-FileCopyrightText: 2002-2023 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
   4 #
   5 # SPDX-License-Identifier: GPL-3.0-or-later
   6
   7 install_packages() {
   8     # After first run, set this:
   9     #   export INSTALL=echo
  10     INSTALL=${INSTALL:-"sudo aptitude -y install"}
  11
  12     # The testsuite depends on this:
  13     test_pkgs="imagemagick expect autossh sshpass jq libpod-simple-perl"
  14     test_pkgs="$test_pkgs pod2pdf gawk lua5.3 clisp php-cli nodejs"
  15     test_pkgs="$test_pkgs mono-csharp-shell libevent-dev tcl libtext-csv-perl"
  16     test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386"
  17     test_pkgs="$test_pkgs libtest-nowarnings-perl xemacs21-bin"
  18
  19     # Debian package
  20     packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake-1.15"
  21     packaging_pkgs="$packaging_pkgs python3-m2crypto alien"
  22
  23     # Shebang Tools
  24     shebang_pkgs="gnuplot octave ruby r-base-core"
  25
  26     # SQL Tools
  27     sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl"
  28     sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap"
  29
  30     # Compression
  31     compression_pkgs="zstd clzip liblz4-tool lzop pigz pixz gzip plzip pbzip2"
  32     compression_pkgs="$compression_pkgs lzma xz-utils lzip bzip2 lbzip2 lrzip"
  33     compression_pkgs_missing="pxz"
  34
  35     # Shells
  36     # (csh = bsd-csh that is broken)
  37     shell_pkgs="ash dash fdclone fish fizsh ksh ksh93u+m ksh93 mksh posh rc rush sash"
  38     shell_pkgs="$shell_pkgs tcsh yash zsh busybox-static"
  39
  40     # Databases
  41     database_pkgs="postgresql mysql-server sqlite influxdb influxdb-client"
  42
  43     # Build Tools
  44     build_pkgs="bison libxxhash-dev libzstd-dev liblz4-dev libssl-dev"
  45     build_pkgs="$build_pkgs python3-cmarkgfm graphviz"
  46
  47     sudo dpkg --add-architecture i386; sudo apt update
  48     pip install sphinx-rtd-theme
  49     if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \
  50                 $compression_pkgs $shell_pkgs $database_pkgs $build_pkgs ; then
  51         # OK
  52         true
  53     else
  54         echo Packages missing
  55         return 1
  56     fi
  57     sudo apt update &
  58 }
  59
  60 install_oracle_client() {
  61     # https://salsa.debian.org/perl-team/modules/packages/libdbd-oracle-perl/blob/master/debian/README.Debian
  62     # sql oracle:// && return 0
  63     (cd /tmp
  64
  65      get_rpm_install_deb() {
  66          glob="$1"
  67          url="$oracleurl/$2"
  68          testglob="$3"
  69          if [ ! -e `echo $glob*rpm` ] ; then
  70              wget $url
  71          fi
  72          if [ ! -e `echo $glob*deb` ] ; then
  73              echo Convert `echo $glob*rpm` to deb
  74              fakeroot alien `echo $glob*rpm`
  75          fi
  76          if [ ! -e `echo $testglob` ] ; then
  77              echo Install `echo $glob*deb`
  78              sudo dpkg -i `echo $glob*deb`
  79          fi
  80      }
  81
  82      oracleurl=https://download.oracle.com/otn_software/linux/instantclient/19600
  83      client=oracle-instantclient19.6
  84      ver=19.6.0.0.0-1.x86_64.rpm
  85      get_rpm_install_deb 'oracle-instantclient*devel' $client-devel-$ver /usr/share/doc/oracle-instantclient*-devel/copyright
  86      get_rpm_install_deb 'oracle-instantclient*basic'   $client-basic-$ver /usr/share/oracle/*/client*/doc/BASIC_README
  87      get_rpm_install_deb 'oracle-instantclient*sqlplus' $client-sqlplus-$ver /usr/lib/oracle/*/client*/bin/sqlplus
  88     )
  89
  90     echo Add this to .bashrc:
  91     ORACLE_HOME=`echo /usr/lib/oracle/*/client*`
  92     echo ORACLE_HOME=$ORACLE_HOME
  93     if grep -q ORACLE_HOME=/ ~/.bashrc; then
  94         perl -i.old -pe "s:ORACLE_HOME=/.*:ORACLE_HOME=$ORACLE_HOME:" ~/.bashrc
  95     else
  96         echo ORACLE_HOME=$ORACLE_HOME >> ~/.bashrc
  97         echo 'PATH=$PATH:$ORACLE_HOME/bin' >> ~/.bashrc
  98         echo 'export ORACLE_HOME' >> ~/.bashrc
  99         echo 'export ORACLE_SID=XE' >> ~/.bashrc
 100     fi
 101     perl -MCPAN -e 'install DBD::Oracle'
 102     # TODO set up vagrant oracle server
 103     git clone https://github.com/oracle/vagrant-projects.git
 104     # TODO set up default passwd
 105     echo sudo su - oracle -c "'/home/oracle/setPassword.sh $oracle_password'" |
 106         vagrant ssh
 107     # test it works: sql oracle://
 108 }
 109
 110 setup_databases() {
 111     # DATABASES
 112     echo '# Create PostgreSQL'
 113     # Drop database and user if needed
 114     sudo su - postgres -c 'dropdb '`whoami`
 115     sudo su - postgres -c 'dropuser '`whoami`
 116     sudo su - postgres -c 'createdb '`whoami`
 117     sudo su - postgres -c 'createuser '`whoami`
 118     sudo su - postgres -c "sql pg:/// \"ALTER USER \\\"`whoami`\\\" WITH PASSWORD '`whoami`';\""
 119
 120     mysqlrootpass=${mysqlrootpass:-b+Ydjq4ejT4E}
 121     dburl=mysql://root:"$mysqlrootpass"@/mysql
 122     echo '# Create MySQL'
 123     sudo su mysql mysqladmin create `whoami`
 124     # Default Debian way of getting "root" access
 125     sudo mysql --defaults-file=/etc/mysql/debian.cnf mysql <<< "ALTER USER 'root'@'localhost' IDENTIFIED BY '$mysqlrootpass';"
 126
 127     # Drop database and user if needed
 128     sudo sql "$dburl" "DROP DATABASE `whoami`;DROP USER '`whoami`'@'localhost';"
 129     sudo sql "$dburl" "CREATE DATABASE `whoami`;CREATE USER '`whoami`'@'localhost' IDENTIFIED BY '`whoami`'; GRANT ALL ON `whoami`.* TO '`whoami`'@'localhost';"
 130 }
 131
 132 add_server_to_hosts() {
 133     # Insert 127.1.2.3 server in /etc/hosts
 134     if grep -q 127.1.2.3 /etc/hosts; then
 135         # skip
 136         true
 137     else
 138         printf "127.1.2.3\tserver\n" |
 139             sudo tee -a /etc/hosts
 140     fi
 141 }
 142
 143 shellsplus() {
 144     shells="bash sh csh ash dash tcsh zsh ksh ksh2020 ksh93 fish fizsh mksh"
 145     shells="$shells posh rc sash yash nopathbash nopathcsh"
 146     shellsplus="parallel $shells"
 147     parallel -k echo ::: $shellsplus
 148 }
 149
 150 create_shell_logins() {
 151     # SHELLS
 152     echo '# Create shell logins'
 153     touch ~/.zshrc
 154     del_add_user() {
 155         shell="$1"
 156         append-if-not-exists() {
 157             file="$1"
 158             shift
 159             string="$@"
 160             if grep -qF "$string" "$file"; then
 161                 # It is there already
 162                 echo "$string is already in $file"
 163             else
 164                 echo "$string" | sudo tee -a "$file"
 165             fi
 166         }
 167         SSHPASS=`goodpasswd`
 168         export SSHPASS
 169         append-if-not-exists /etc/shells $(which $shell ||
 170                                                which ${shell#"nopath"})
 171         sudo killall -u $shell
 172         sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM
 173         sudo groupdel $shell
 174         if echo $shell | grep -q parallel; then
 175             # User parallel should have /bin/bash
 176             loginshell=/bin/bash
 177         else
 178             # Other users should have `which $shell` with nopath removed
 179             loginshell=$(which $shell || which ${shell#"nopath"})
 180         fi
 181         sudo adduser --shell $loginshell --disabled-password --gecos "$shell for parallel,,," $shell &&
 182             echo "$shell:$SSHPASS" | sudo chpasswd &&
 183             sshpass -e ssh-copy-id $shell@lo &&
 184             echo "$shell created"
 185     }
 186     export -f del_add_user
 187
 188     echo '# (Re-)create user'
 189     # Racecondition: if multiple adds a group it will the same group ID
 190     (shellsplus; echo withpassword) |
 191         parallel --lb --halt soon,fail=1 --timeout 1000% --retries 5 --tag -j1 del_add_user ||
 192         (echo Creation failed: $?; false)
 193 }
 194
 195 copy_ssh_keys() {
 196     make_sshkey() {
 197         shell="$1"
 198         echo Add server keys for lo and server &&
 199             ssh $shell@lo 'rm -f .ssh/id_rsa;
 200                            ssh-keyscan lo >>.ssh/known_hosts;
 201                            ssh-keyscan server >> .ssh/known_hosts' &&
 202             echo Do ssh-keygen &&
 203             echo | ssh -t $shell@lo ssh-keygen -b 1024 -f .ssh/id_rsa &&
 204             echo Do ssh $shell@lo 'cat .ssh/id_rsa.pub >> .ssh/authorized_keys' &&
 205             ssh $shell@lo 'cat .ssh/id_rsa.pub | tee -a .ssh/authorized_keys' |
 206                 ssh parallel@lo 'cat >> .ssh/authorized_keys' &&
 207             ssh $shell@lo "echo ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxg+dh+BL1myqo6A+pHcQCKVV5v7cewdkN8xGtMDqm7xGgh+e5A44W7qKViIx641d6yoWb012XzDz2MKetG4Kosoma8Z/qkS27p6901RmI3ap2JFffzBESlpQtW1FyxQPlMyBfrd4ONy8xW6R/gEkjC3yOcXCQO2494/c46ouXs5gtE84Cusr3RsopR4bI7oloH1GQZ4vsHSFSakq8IwcujaSh1mmribMONLb2TjDpmE7tAY+yfOVWHPQ2J+EU1//Go60EZfSRKOu81oxW7SJ2uGgWfWcs2T1lRtT8Yh+TwVTz8UFV75kVtpZ10x5baN+ncsSpaBK+8sFLfoMvA9yQQ'==' tange@freebsd71.tange.dk >> .ssh/authorized_keys" &&
 208             echo Do env_parallel --install &&
 209             ssh $shell@lo 'mkdir -p .parallel;
 210                            touch .parallel/will-cite;
 211                            env_parallel --install' &&
 212             echo OK make_sshkey $shell &&
 213             echo >&2 &&
 214             echo OK make_sshkey $shell >&2 &&
 215             echo
 216     }
 217     export -f make_sshkey
 218
 219     echo '# Make and copy sshkey'
 220     shellsplus | parallel --timeout 20 --retries 5 --tag -j5 make_sshkey
 221
 222     ssh_a_to_b() {
 223         ssh $1@lo ssh $2@lo echo OK ||
 224             (echo failed && false)
 225     }
 226     export -f ssh_a_to_b
 227     echo "# copy id from X to X"
 228     shellsplus | parallel --timeout 10 --tag ssh_a_to_b {} {}
 229
 230     ssh_copy_id() {
 231         from="$1"
 232         to="$2"
 233         ssh $from cat .ssh/id_rsa.pub | ssh $to 'cat >> .ssh/authorized_keys'
 234     }
 235     export -f ssh_copy_id
 236
 237     echo '# copy id from any X to any Y'
 238     parallel -u --bar -j3 --timeout 3 --retries 10 --tag \
 239              ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus)
 240     echo '# Test the copying went well'
 241     parallel    --bar -j2 --timeout 9 --retries 10 --tag \
 242                 ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus)
 243
 244     echo '# change paths to no path'
 245     (
 246         ssh nopathbash@lo 'echo > .bashrc PATH=/bin:/usr/bin'
 247         ssh nopathbash@lo 'echo > .profile PATH=/bin:/usr/bin'
 248         # Remove env_parallel from .profile
 249         ssh nopathbash@lo 'perl -i.bak -pe s/.*env_parallel.*// .profile .bashrc'
 250         ssh nopathcsh@lo 'echo >> .cshrc setenv PATH /bin:/usr/bin'
 251         ssh nopathcsh@lo 'echo >> .login setenv PATH /bin:/usr/bin'
 252         ssh nopathbash@lo 'echo $PATH; echo 1 | parallel echo' ||
 253             echo Great - this should FAIL: parallel should not be found
 254         ssh nopathcsh@lo 'echo $PATH; echo 1 | parallel echo' ||
 255             echo Great - this should FAIL: parallel should not be found
 256     )
 257 }
 258
 259 lsh_setup() {
 260     ssh-keyscan localhost >>~/.ssh/known_hosts
 261     ssh-copy-id localhost
 262
 263     # SHELLS: lsh-client against openssh server
 264     $INSTALL lsh-client
 265     cd
 266     mkdir -p .lsh
 267     lsh-make-seed -o ".lsh/yarrow-seed-file"
 268     lsh -c aes256-ctr --sloppy-host-authentication \
 269         --capture-to ~/.lsh/host-acls lo echo Added host-auth
 270     lsh -c aes256-ctr --sloppy-host-authentication \
 271         --capture-to ~/.lsh/host-acls localhost echo Added host-auth
 272     lsh-keygen | lsh-writekey -c none
 273     export_key_to_local_users() {
 274         lsh-export-key --openssh < ~/.lsh/identity.pub |
 275             ssh -l $1 lo 'cat >>.ssh/authorized_keys'
 276     }
 277     export -f export_key_to_local_users
 278     shellsplus | parallel --bar --timeout 5 export_key_to_local_users
 279     shellsplus | parallel --bar --timeout 5 'lsh -l {} lo true || export_key_to_local_users {}'
 280     shellsplus | parallel --bar --timeout 5 'lsh -l {} lo true || echo Fail {}'
 281 }
 282
 283 add_freebsd() {
 284     echo "# Add public key to freebsd7.t"
 285     ssh freebsd7.t cat .ssh/id_rsa.pub |
 286         ssh parallel@localhost 'cat >>.ssh/authorized_keys'
 287
 288     echo Add:
 289     echo HostkeyAlgorithms +ssh-dss
 290     echo to .ssh/config if you get
 291     echo no matching host key type found. Their offer: ssh-dss
 292     echo
 293     echo Add to /etc/ssh/sshd_config
 294     echo Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc
 295     echo KexAlgorithms diffie-hellman-group-exchange-sha256,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,curve25519-sha256@libssh.org,diffie-hellman-group1-sha1,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1
 296     echo
 297     echo If you get:
 298     echo Unable to negotiate with server port 22: no matching cipher found.
 299     echo Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se
 300     echo add this to .ssh/config
 301     echo   Ciphers +aes256-cbc
 302 }
 303
 304 tmux_versions() {
 305     (cd /tmp
 306      git clone https://github.com/tmux/tmux.git
 307      listtags() {
 308          (cd tmux;
 309           git tag)
 310      }
 311      make_one() {
 312          rsync -a --delete tmux/ tmux$1/
 313          cd tmux$1
 314          git reset --hard
 315          git checkout $1
 316          ./autogen.sh &&
 317              ./configure &&
 318              make -j2 &&
 319              sudo cp tmux /usr/local/bin/tmux-$1
 320      }
 321      export -f make_one
 322      echo '# Building tmux'
 323      listtags |
 324          stdout parallel --lb --tag '/usr/local/bin/tmux-{} -V || make_one {}'
 325      listtags |
 326          parallel -k -v /usr/local/bin/tmux-{} -V
 327     )
 328 }
 329
 330 bash_versions() {
 331     echo You may have to kill conftest
 332     (cd /tmp
 333      git clone https://git.savannah.gnu.org/git/bash.git
 334      listtags() {
 335          (cd bash;
 336           git tag | grep -v -- '-.*-' | grep -v bash-4.1.11)
 337      }
 338      make_one() {
 339          rsync -a --delete bash/ $1/
 340          cd $1
 341          git reset --hard
 342          git checkout $1
 343          LDFLAGS=-static ./configure &&
 344              make -j2 &&
 345              sudo cp bash /usr/local/bin/$1
 346      }
 347      export -f make_one
 348      echo '# Building bash'
 349      listtags |
 350          stdout parallel --lb --tag \
 351                 '/usr/local/bin/{} --version || make_one {}'
 352      listtags |
 353          parallel -k -v --tag '/usr/local/bin/{} --version'
 354     )
 355 }
 356
 357 rsync_versions() {
 358     (cd /tmp
 359      git clone https://github.com/WayneD/rsync
 360      listtags() {
 361          (cd rsync;
 362           git tag | grep -v -- '-.*-' | grep -v pre)
 363      }
 364      make_one() {
 365          rsync -a --delete rsync/ rsync-$1/
 366          cd rsync-$1
 367          git reset --hard
 368          git checkout $1
 369          perl -i -pe 's/AC_DEFINE_UNQUOTED.HAVE_REMSH, .HAVE_REMSH./AC_DEFINE_UNQUOTED(HAVE_REMSH, \$HAVE_REMSH,[dummy])/;
 370          s/AC_DEFINE.HAVE_ERRNO_DECL.,/AC_DEFINE(HAVE_ERRNO_DECL,[1],[dummy]),/;
 371          s/AC_DEFINE.HAVE_FNMATCH.,/AC_DEFINE(HAVE_FNMATCH,[1],[dummy]),/;' configure.in
 372          autoreconf --install -W gnu
 373          # Make "lib/addrinfo.h" ?
 374          LDFLAGS=-static ./configure &&
 375              (make proto; make -j2) &&
 376              sudo cp rsync /usr/local/bin/rsync-$1
 377      }
 378      export -f make_one
 379      echo '# Building rsync'
 380      listtags |
 381          stdout parallel --lb --tag \
 382                 '/usr/local/bin/rsync-{} --version || make_one {}'
 383      listtags |
 384          parallel -k -v --tag '/usr/local/bin/rsync-{} --version'
 385     )
 386 }
 387
 388 install_tangetools() {
 389     git clone https://gitlab.com/ole.tange/tangetools.git
 390     cd tangetools &&
 391         make &&
 392         sudo make install
 393 }
 394
 395 misc() {
 396     parallel --record-env
 397
 398     # Build locale for LC_ALL=zh_HK.big5hkscs perl -e 1
 399     sudo locale-gen zh_HK
 400 }
 401
 402 run() {
 403     install_packages &&
 404         install_tangetools &&
 405         install_oracle_client &&
 406         setup_databases &&
 407         add_server_to_hosts &&
 408         create_shell_logins &&
 409         copy_ssh_keys &&
 410         lsh_setup &&
 411         add_freebsd &&
 412         tmux_versions &&
 413         bash_versions &&
 414         rsync_versions &&
 415         misc
 416 }