testsuite/REQUIREMENTS

   1 #!/bin/bash
   2
   3 # SPDX-FileCopyrightText: 2002-2024 Ole Tange, http://ole.tange.dk and Free Software and Foundation, Inc.
   4 #
   5 # SPDX-License-Identifier: GPL-3.0-or-later
   6
   7 install_packages() {
   8     # After first run, set this:
   9     #   export INSTALL=echo
  10     INSTALL=${INSTALL:-"sudo apt -y install"}
  11
  12     # The testsuite depends on this:
  13     test_pkgs="imagemagick expect pv zenity"
  14     test_pkgs="$test_pkgs autossh sshpass lsh-client lsh-utils jq"
  15     test_pkgs="$test_pkgs libpod-simple-perl libtext-csv-perl"
  16     test_pkgs="$test_pkgs libtest-nowarnings-perl pod2pdf"
  17     test_pkgs="$test_pkgs xemacs21"
  18     test_pkgs="$test_pkgs gawk lua5.3 clisp php-cli nodejs guile-3.0 fsharp"
  19     test_pkgs="$test_pkgs mono-csharp-shell octave openjdk-11-jdk-headless tcl"
  20     test_pkgs="$test_pkgs racket libevent-dev python-is-python3"
  21     test_pkgs="$test_pkgs xterm libc6-i386 libcrypt1:i386 virtualbox"
  22     test_pkgs="$test_pkgs torsocks net-tools"
  23     test_pkgs="$test_pkgs reiserfsprogs btrfs-progs exfatprogs f2fs-tools"
  24     test_pkgs="$test_pkgs xfsprogs"
  25
  26     test_snap_pkgs="groovy julia"
  27
  28     # Debian package
  29     packaging_pkgs="dpkg-dev build-essential debhelper osc cvs automake"
  30     packaging_pkgs="$packaging_pkgs python3-m2crypto alien python3-pip"
  31
  32     # Shebang Tools
  33     shebang_pkgs="gnuplot octave ruby r-base-core"
  34
  35     # SQL Tools
  36     sql_pkgs="libdbd-pg-perl libdbd-sqlite3-perl libdbd-csv-perl"
  37     sql_pkgs="$sql_pkgs libdbd-mysql-perl rlwrap"
  38
  39     # Compression
  40     compression_pkgs="zstd clzip lz4 lzop pigz pixz gzip plzip pbzip2"
  41     compression_pkgs="$compression_pkgs lzma xz-utils lzip bzip2 lbzip2 lrzip"
  42     compression_pkgs_missing="pxz"
  43
  44     # Shells
  45     # (csh = bsd-csh that is broken)
  46     # (ash ksh93 gone)
  47     shell_pkgs="dash fdclone fish fizsh ksh ksh93u+m mksh posh rc rush sash"
  48     shell_pkgs="$shell_pkgs tcsh yash zsh busybox-static"
  49
  50     # Databases
  51     database_pkgs="postgresql mysql-server influxdb influxdb-client"
  52
  53     # Build Tools
  54     build_pkgs="bison libxxhash-dev libzstd-dev liblz4-dev libssl-dev"
  55     build_pkgs="$build_pkgs python3-cmarkgfm libpod-pom-view-restructured-perl"
  56     build_pkgs="$build_pkgs graphviz yodl libreoffice"
  57
  58     # Alternatives
  59     alt_database="sqlite sqlite3"
  60     sudo dpkg --add-architecture i386; sudo apt update
  61     if $INSTALL $test_pkgs $packaging_pkgs $shebang_pkgs $sql_pkgs \
  62                 $compression_pkgs $shell_pkgs $database_pkgs $build_pkgs ; then
  63         # OK
  64         true
  65     else
  66         echo Packages missing
  67         return 1
  68     fi
  69     if sudo parallel -j1 snap install --classic ::: $test_snap_pkgs; then
  70         # OK
  71         true
  72     else
  73         echo Packages missing: $test_snap_pkgs
  74         return 1
  75     fi
  76     if parallel -j1 --tag --halt now,success=1 $INSTALL ::: $alt_database ; then
  77         # OK
  78         true
  79     else
  80         echo Cannot install: "$alt_database"
  81         return 1
  82     fi
  83     if \vagrant --help >/dev/null ; then
  84         # OK
  85         true
  86     else
  87         # Version 2.3.7 is the last under a free license
  88         wget -m https://releases.hashicorp.com/vagrant/2.3.7/vagrant_2.3.7-1_amd64.deb
  89         sudo dpkg -i releases.hashicorp.com/vagrant/2.3.7/vagrant_2.3.7-1_amd64.deb
  90     fi
  91     sudo apt install python3-sphinx-rtd-theme || pip install sphinx-rtd-theme
  92     echo zh_HK BIG5-HKSCS | sudo tee -a /etc/locale.gen
  93     sudo locale-gen zh_HK
  94     sudo apt update &
  95 }
  96
  97 install_sqlite2() {
  98     mkdir /tmp/sqlite2
  99     cd /tmp/sqlite2
 100     git clone https://github.com/sqlite/sqlite
 101     cd sqlite
 102     git checkout cc83b6e071ba69943f175a038d2625ae3d6abf47
 103     # New GCC needs <stdlib.h>, so add that
 104     perl -i -pe \
 105          's/echo (.#include <stdio.h>)(.)/printf $1\\n#include <stdlib.h>\\n$2/' \
 106          Makefile.in
 107     ./configure --prefix=/usr/local && make -j && sudo make install
 108 }
 109
 110 install_oracle_client() {
 111     # https://salsa.debian.org/perl-team/modules/packages/libdbd-oracle-perl/blob/master/debian/README.Debian
 112     # sql oracle:// && return 0
 113     (cd /tmp
 114
 115      get_rpm_install_deb() {
 116          glob="$1"
 117          url="$oracleurl/$2"
 118          testglob="$3"
 119          if [ ! -e `echo $glob*rpm` ] ; then
 120              wget $url
 121          fi
 122          if [ ! -e `echo $glob*deb` ] ; then
 123              echo Convert `echo $glob*rpm` to deb
 124              fakeroot alien `echo $glob*rpm`
 125          fi
 126          if [ ! -e `echo $testglob` ] ; then
 127              echo Install `echo $glob*deb`
 128              sudo dpkg -i `echo $glob*deb`
 129          fi
 130      }
 131
 132      oracleurl=https://download.oracle.com/otn_software/linux/instantclient/19600
 133      client=oracle-instantclient19.6
 134      ver=19.6.0.0.0-1.x86_64.rpm
 135      get_rpm_install_deb 'oracle-instantclient*devel' $client-devel-$ver /usr/share/doc/oracle-instantclient*-devel/copyright
 136      get_rpm_install_deb 'oracle-instantclient*basic'   $client-basic-$ver /usr/share/oracle/*/client*/doc/BASIC_README
 137      get_rpm_install_deb 'oracle-instantclient*sqlplus' $client-sqlplus-$ver /usr/lib/oracle/*/client*/bin/sqlplus
 138     )
 139
 140     echo Add this to .bashrc:
 141     ORACLE_HOME=`echo /usr/lib/oracle/*/client*`
 142     echo ORACLE_HOME=$ORACLE_HOME
 143     if grep -q ORACLE_HOME=/ ~/.bashrc; then
 144         perl -i.old -pe "s:ORACLE_HOME=/.*:ORACLE_HOME=$ORACLE_HOME:" ~/.bashrc
 145     else
 146         echo ORACLE_HOME=$ORACLE_HOME >> ~/.bashrc
 147         echo 'PATH=$PATH:$ORACLE_HOME/bin' >> ~/.bashrc
 148         echo 'export ORACLE_HOME' >> ~/.bashrc
 149         echo 'export ORACLE_SID=XE' >> ~/.bashrc
 150     fi
 151     # libaio
 152     sudo apt install libaio1 || sudo apt install libaio1t64
 153     sudo ln -s libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
 154     perl -MCPAN -e 'install DBD::Oracle'
 155     # TODO set up vagrant oracle server
 156     git clone https://github.com/oracle/vagrant-projects.git
 157     (
 158         cd vagrant-projects/OracleDatabase/23.4.0-Free
 159         echo export VM_ORACLE_PWD=`goodpasswd` >> ~/.passwords
 160         echo export 'ORACLE_PWD=$VM_ORACLE_PWD' >> ~/.passwords
 161         . ~/.passwords
 162         vagrant up
 163     )
 164     # test it works:
 165     sql --showtables oracle://SYSTEM:$VM_ORACLE_PWD@/FREE
 166 }
 167
 168 setup_databases() {
 169     # DATABASES
 170     setup_postgres() {
 171         echo '# Create PostgreSQL'
 172         (echo; echo "listen_addresses = '*'") |
 173             sudo tee -a /etc/postgresql/*/main/postgresql.conf
 174         sudo service postgresql restart
 175
 176         # Drop database and user if needed
 177         sudo su - postgres -c 'dropdb '$(whoami)
 178         sudo su - postgres -c 'dropuser '$(whoami)
 179         sudo su - postgres -c 'createdb '$(whoami)
 180         sudo su - postgres -c 'createuser '$(whoami)
 181         sudo su - postgres -c "sql pg:/// \"ALTER USER \\\"`whoami`\\\" WITH PASSWORD '$(whoami)';\""
 182         sudo su - postgres -c "psql -d $(whoami) -c 'GRANT CREATE ON SCHEMA public TO \"$(whoami)\";'"
 183         sql pg://// 'create table __test_foo_socket(i int)'
 184         sql pg://$me:$me@// 'create table __test_foo(i int)'
 185         sql pg://// 'drop table __test_foo_socket' &&
 186             sql pg://$me:$me@// 'drop table __test_foo' &&
 187             echo OK: PostgreSQL
 188     }
 189
 190     setup_mysql() {
 191         mysqlrootpass=${mysqlrootpass:-b+Ydjq4ejT4E}
 192         dburl=mysql://root:"$mysqlrootpass"@/mysql
 193         me=$(whoami)
 194         echo '# Create MySQL'
 195         sudo su mysql mysqladmin create $me
 196         # Default Debian way of getting "root" access
 197         sudo mysql --defaults-file=/etc/mysql/debian.cnf mysql <<< "ALTER USER 'root'@'localhost' IDENTIFIED BY '$mysqlrootpass';"
 198
 199         # Drop database and user if needed
 200         sudo sql "$dburl" "DROP DATABASE $me;DROP USER '$me'@'localhost';"
 201         sudo sql "$dburl" "CREATE DATABASE $me;"
 202         sudo sql "$dburl" "CREATE USER '$me'@'localhost' IDENTIFIED WITH auth_socket;"
 203         sudo sql "$dburl" "ALTER USER '$me'@'localhost' IDENTIFIED BY '$me';"
 204         sudo sql "$dburl" "GRANT ALL PRIVILEGES ON $me.* TO '$me'@'localhost';"
 205         sudo sql "$dburl" "FLUSH PRIVILEGES;"
 206
 207
 208         # Drop database and user if needed
 209         sql mysql://// 'create table __test_foo_socket(i int)'
 210         sql mysql://$me:$me@// 'create table __test_foo(i int)'
 211         sql mysql://// 'drop table __test_foo_socket' &&
 212             sql mysql://$me:$me@// 'drop table __test_foo' &&
 213             echo OK: MySQL
 214     }
 215     setup_postgres &&
 216         setup_mysql
 217 }
 218
 219 setup_virtual_box_network() {
 220     echo "Adding IP-range to virtualbox"
 221     sudo mkdir -p /etc/vbox
 222     sudo touch /etc/vbox/networks.conf
 223     (cat; echo '* 172.27.27.0/24') < /etc/vbox/networks.conf | uniq |
 224         sudo tee /etc/vbox/networks.conf
 225 }
 226
 227 add_server_to_hosts() {
 228     add_ssh_key_to_authorized() {
 229         (cat vagrant/authorized_keys; cat ~/.ssh/*.pub) |
 230             uniq > vagrant/authorized_keys.$$
 231         mv vagrant/authorized_keys.$$ vagrant/authorized_keys
 232         # Problem:
 233         # https://developer.hashicorp.com/vagrant/docs/v2.4.0/vagrantfile/ssh_settings
 234         # says:
 235         #
 236         #   (non-interactive SSH connections use the internal SSH
 237         #   communicator which is unaffected by this setting).
 238         #
 239         # We need to force Ruby's Net::SSH to use ~/.ssh/config
 240         # or -oKexAlgorithms=diffie-hellman-group1-sha1
 241         #
 242         # VAGRANT_PREFER_SYSTEM_BIN=1 ought to fix it, but does not.
 243
 244
 245         # Fix:
 246         # could not settle on kex algorithm
 247         # Server kex preferences: diffie-hellman-group-exchange-sha1,diffie-hellman-group1-sha1
 248         # Client kex preferences: ecdh-sha2-nistp521,ecdh-sha2-nistp384,ecdh-sha2-nistp256,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1>
 249 #debug2: KEX algorithms: curve25519-sha256,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,sntrup761x25519-sha512@openssh.com,diffie-hellman-group-exchange-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256,diffie-hellman-group1-sha1,ext-info-c
 250 #debug2: host key algorithms: ssh-dss,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,sk-ssh-ed25519-cert-v01@openssh.com,sk-ecdsa-sha2-nistp256-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-ed25519,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,rsa-sha2-512,rsa-sha2-256
 251 #debug2: ciphers ctos: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc
 252 #debug2: ciphers stoc: aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc
 253 #debug2: MACs ctos: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1
 254 #debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1
 255         #
 256
 257         # Fix: .ssh/config
 258         #Host centos3 127.0.0.1
 259         #  KexAlgorithms +diffie-hellman-group1-sha1
 260         #  Ciphers +3des-cbc,aes128-cbc,aes192-cbc,aes256-cbc
 261         #  HostKeyAlgorithms +ssh-rsa,ssh-dss
 262         # sshpass -p vagrant ssh -v -p 50136 vagrant@127.0.0.1
 263     }
 264
 265     insert_in_etc_hosts() {
 266         ip=$1
 267         host=$2
 268         if grep -q $1 /etc/hosts && grep -q $2 /etc/hosts ; then
 269             # skip
 270             true
 271         else
 272             echo "$@" |
 273                 sudo tee -a /etc/hosts
 274         fi
 275     }
 276     export -f insert_in_etc_hosts
 277     add_single_vagrant_to_etc_hosts() {
 278         vagrant_ip() {
 279             cd vagrant/*/$1
 280             # .vagrant can contain dirt
 281             rm -rf .vagrant
 282             grep 172 Vagrantfile | grep -v '#' |
 283                 perl -pe 's/.*(172.27[\.0-9]+).*/$1/';
 284         }
 285         insert_in_etc_hosts $(vagrant_ip $1) "$@"
 286     }
 287     export -f add_single_vagrant_to_etc_hosts
 288
 289     add_ssh_key_to_authorized
 290     insert_in_etc_hosts 127.1.2.3 server
 291
 292     insert_in_etc_hosts 127.0.0.2 lo
 293     parallel add_single_vagrant_to_etc_hosts {} parallel-server{#} ::: centos8 freebsd13 freebsd14 rhel8 centos3
 294 }
 295
 296 vagrant_up() {
 297     parallel -j0 --ll --tag 'cd {//};vagrant up' ::: vagrant/*/*/Vagrantfile
 298     # After this you should be able to 'vagrant ssh'
 299 }
 300
 301 vagrant_centos3() {
 302     echo 'Centos3 can only use an RSA key'
 303     if [ -e ~/.ssh/id_rsa ] ; then
 304         # OK
 305         true
 306     else
 307         ssh-keygen -P '' -t rsa -f ~/.ssh/id_rsa
 308     fi
 309     cat ~/.ssh/id_rsa.pub >> vagrant/authorized_keys
 310     (
 311         # Centos3 only works with some weak ciphers
 312         echo
 313         echo 'Host *'
 314         echo '  GSSAPIAuthentication no'
 315         echo '  KexAlgorithms +diffie-hellman-group1-sha1'
 316         echo '  Ciphers +3des-cbc,aes128-cbc,aes192-cbc,aes256-cbc'
 317         echo '  HostKeyAlgorithms +ssh-rsa,ssh-dss'
 318         echo '  PubkeyAcceptedKeyTypes +ssh-rsa,ssh-dss'
 319
 320     ) >> ~/.ssh/config
 321     (
 322         echo KexAlgorithms +diffie-hellman-group1-sha1,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1
 323         echo Ciphers +3des-cbc,aes128-cbc,aes192-cbc,aes256-cbc,aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm@openssh.com,aes256-gcm@openssh.com,chacha20-poly1305@openssh.com
 324         echo HostKeyAlgorithms +ssh-rsa,ssh-dss
 325         echo PubkeyAcceptedKeyTypes +ssh-rsa,ssh-dss
 326         echo MaxStartups 100:30:1000
 327     ) | sudo tee /etc/ssh/sshd_config.d/parallel.conf
 328     sudo /etc/init.d/ssh restart
 329     (
 330         cd vagrant/*/centos3 && vagrant up
 331         (cat ../../authorized_keys | vagrant ssh -c "cat >>.ssh/authorized_keys";
 332          vagrant ssh -c "sudo /sbin/ifconfig eth1 172.27.27.3")
 333         ssh-keygen -R centos3
 334         stdout wssh -oStrictHostKeyChecking=accept-new vagrant@centos3 echo OK: ssh centos3 started and works
 335         stdout ssh parallel@lo 'cat .ssh/id*pub' |
 336             ssh vagrant@centos3 "cat >>.ssh/authorized_keys"
 337         ssh vagrant@centos3 'yes n | ssh-keygen -t rsa -f .ssh/id_rsa -N "";echo '
 338         ssh vagrant@centos3 'cat .ssh/id*pub' |
 339             stdout ssh parallel@lo "cat >>.ssh/authorized_keys"
 340         ssh vagrant@centos3 ssh -oStrictHostKeyChecking=no parallel@172.27.27.1 echo OK: ssh vagrant@centos3 ssh parallel@172.27.27.1 works
 341     )
 342 }
 343
 344 add_key_to_auth_key() {
 345     sshaddvagrant() {
 346         ssh-keygen -f ~/.ssh/known_hosts -R $1
 347         cat ~/.ssh/*.pub | (cd vagrant/*/$1 && vagrant ssh -c 'cat >>.ssh/authorized_keys')
 348         sshpass -p vagrant ssh-copy-id -o StrictHostKeyChecking=accept-new vagrant@$1
 349         ssh vagrant@$1 echo vagrant@$1 OK
 350     }
 351     export -f sshaddvagrant
 352     sshaddlocal() {
 353         ssh-keygen -f ~/.ssh/known_hosts -R $1
 354         ssh-copy-id -o StrictHostKeyChecking=accept-new $1
 355         ssh $1 echo $1 OK
 356     }
 357     export -f sshaddlocal
 358     ipv6() {
 359         ifconfig | perl -nE '/inet6 ([0-9a-f:]+) .*(host|global)/ and say $1'
 360     }
 361     vagrant_names=$(ls vagrant/*/*/Vagrantfile| perl -pe 's:.*/(.*)/Vagrantfile:$1:')
 362     ipv6=$(ipv6)
 363     ips="172.27.27.1 127.0.0.1 ::1 $ipv6"
 364
 365     cat ~/.ssh/*pub >> ~/.ssh/authorized_keys
 366     parallel --tag --timeout 20 sshaddvagrant ::: $vagrant_names parallel-server{1..4}
 367     parallel --tag --timeout 20 sshaddlocal ::: $ips lo localhost
 368     parallel --tag --timeout 5 ssh vagrant@{} echo ssh vagrant@{} OK ::: $vagrant_names parallel-server{1..4}
 369     parallel ssh vagrant@{} 'echo "PATH=\$PATH:\$HOME/bin" >> .bashrc' ::: $vagrant_names
 370     parallel --tag --timeout 5 ssh {} echo ssh {} OK ::: $ips lo localhost
 371 }
 372
 373 install_rsync_on_vagrant() {
 374     parallel --tag ssh vagrant@{} "'
 375       sudo apt install rsync || sudo yum install -y rsync || sudo pkg install -y rsync;
 376       sudo apt install perl || sudo yum install -y perl || sudo pkg install -y perl;
 377     '" ::: parallel-server{1..4} centos3
 378 }
 379
 380 shellsplus() {
 381     shells="bash sh csh ash dash tcsh zsh ksh ksh93 fish fizsh mksh"
 382     shells="$shells posh rc sash yash nopathbash nopathcsh"
 383     shellsplus="parallel $shells"
 384     parallel -k echo ::: $shellsplus
 385 }
 386
 387 create_shell_logins() {
 388     # SHELLS
 389     echo '# Create shell logins'
 390     touch ~/.zshrc
 391     del_add_user() {
 392         shell="$1"
 393         append-if-not-exists() {
 394             file="$1"
 395             shift
 396             string="$@"
 397             if grep -qF "$string" "$file"; then
 398                 # It is there already
 399                 echo "$string is already in $file"
 400             else
 401                 echo "$string" | sudo tee -a "$file"
 402             fi
 403         }
 404         SSHPASS=`goodpasswd`
 405         export SSHPASS
 406         append-if-not-exists /etc/shells $(which $shell ||
 407                                                which ${shell#"nopath"})
 408         sudo killall -u $shell
 409         sudo deluser $shell && sudo mv /home/$shell /tmp/$shell.$RANDOM
 410         sudo groupdel $shell
 411         if echo $shell | grep -qE "parallel|withpassword"; then
 412             # User parallel+withpassword should have /bin/bash
 413             loginshell=/bin/bash
 414         else
 415             # Other users should have `which $shell` with nopath removed
 416             loginshell=$(which $shell || which ${shell#"nopath"})
 417         fi
 418         sudo adduser --shell $loginshell --disabled-password --gecos "$shell for parallel,,," $shell &&
 419             echo "$shell:$SSHPASS" | sudo chpasswd &&
 420             sshpass -e ssh-copy-id $shell@lo &&
 421             echo "$shell created"
 422     }
 423     export -f del_add_user
 424
 425     echo '# (Re-)create user'
 426     # Racecondition: if multiple adds a group it will the same group ID
 427     (shellsplus; echo withpassword) |
 428         parallel --lb --halt soon,fail=1 --timeout 1000% --retries 5 --tag -j1 del_add_user ||
 429         (echo Creation failed: $?; false)
 430     # Homedir must be readable by others
 431     umask 0002
 432     (shellsplus; echo withpassword) | sudo parallel chmod +rX ~{}
 433 }
 434
 435 copy_ssh_keys() {
 436     make_sshkey() {
 437         shell="$1"
 438         echo Add server keys for lo and server &&
 439             ssh $shell@lo 'rm -f .ssh/id_rsa;
 440                            ssh-keyscan lo >>.ssh/known_hosts;
 441                            ssh-keyscan server >> .ssh/known_hosts' &&
 442             echo Do ssh-keygen &&
 443             echo | ssh -t $shell@lo ssh-keygen -b 1024 -f .ssh/id_rsa &&
 444             echo Do ssh $shell@lo 'cat .ssh/id_rsa.pub >> .ssh/authorized_keys' &&
 445             ssh $shell@lo 'cat .ssh/id_rsa.pub | tee -a .ssh/authorized_keys' |
 446                 ssh parallel@lo 'cat >> .ssh/authorized_keys' &&
 447             ssh $shell@lo "echo ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAxg+dh+BL1myqo6A+pHcQCKVV5v7cewdkN8xGtMDqm7xGgh+e5A44W7qKViIx641d6yoWb012XzDz2MKetG4Kosoma8Z/qkS27p6901RmI3ap2JFffzBESlpQtW1FyxQPlMyBfrd4ONy8xW6R/gEkjC3yOcXCQO2494/c46ouXs5gtE84Cusr3RsopR4bI7oloH1GQZ4vsHSFSakq8IwcujaSh1mmribMONLb2TjDpmE7tAY+yfOVWHPQ2J+EU1//Go60EZfSRKOu81oxW7SJ2uGgWfWcs2T1lRtT8Yh+TwVTz8UFV75kVtpZ10x5baN+ncsSpaBK+8sFLfoMvA9yQQ'==' tange@freebsd71.tange.dk >> .ssh/authorized_keys" &&
 448             echo Do env_parallel --install &&
 449             ssh $shell@lo 'mkdir -p .parallel;
 450                            touch .parallel/will-cite;
 451                            env_parallel --install' &&
 452             echo OK make_sshkey $shell &&
 453             echo >&2 &&
 454             echo OK make_sshkey $shell >&2 &&
 455             echo
 456     }
 457     export -f make_sshkey
 458
 459     echo '# Make and copy sshkey'
 460     shellsplus | parallel --timeout 20 --retries 5 --tag -j5 make_sshkey
 461
 462     ssh_a_to_b() {
 463         ssh $1@lo ssh $2@lo echo OK ||
 464             (echo failed && false)
 465     }
 466     export -f ssh_a_to_b
 467     echo "# copy id from X to X"
 468     shellsplus | parallel --timeout 10 --tag ssh_a_to_b {} {}
 469
 470     ssh_copy_id() {
 471         from="$1"
 472         to="$2"
 473         ssh $from cat .ssh/id_rsa.pub | ssh $to 'cat >> .ssh/authorized_keys'
 474     }
 475     export -f ssh_copy_id
 476
 477     echo '# copy id from any X to any Y'
 478     parallel --bar -j3 --timeout 7 --retries 10 --tag \
 479              ssh_copy_id {1}@lo {2}@lo ::: $(shellsplus) ::: $(shellsplus)
 480     echo '# Test the copying went well'
 481     parallel --bar -j2 --timeout 9 --retries 10 --tag \
 482                 ssh_a_to_b ::: $(shellsplus) ::: $(shellsplus)
 483
 484     echo '# change paths to no path'
 485     (
 486         ssh nopathbash@lo 'echo > .bashrc PATH=/bin:/usr/bin'
 487         ssh nopathbash@lo 'echo > .profile PATH=/bin:/usr/bin'
 488         # Remove env_parallel from .profile
 489         ssh nopathbash@lo 'perl -i.bak -pe s/.*env_parallel.*// .profile .bashrc'
 490         ssh nopathcsh@lo 'echo >> .cshrc setenv PATH /bin:/usr/bin'
 491         ssh nopathcsh@lo 'echo >> .tcshrc setenv PATH /bin:/usr/bin'
 492         ssh nopathcsh@lo 'echo >> .login setenv PATH /bin:/usr/bin'
 493         ssh nopathbash@lo 'echo $PATH; echo NOT OK | parallel echo' ||
 494             echo Great - this should FAIL: parallel should not be found
 495         ssh nopathcsh@lo 'echo $PATH; echo NOT OK | parallel echo' ||
 496             echo Great - this should FAIL: parallel should not be found
 497     )
 498 }
 499
 500 parallel_versions() {
 501     install_version() {
 502         rm -r /tmp/parallel-$1
 503         (git clone . /tmp/parallel-$1 || git clone .. /tmp/parallel-$1) &&
 504             cd /tmp/parallel-$1 &&
 505             git checkout $1 &&
 506             sudo cp src/parallel /usr/local/bin/parallel-$1
 507     }
 508     export -f install_version
 509     git tag | grep -E '^20[0-9][0-9][0-9][0-9][0-9][0-9]$' | parallel install_version
 510 }
 511
 512 lsh_setup() {
 513     server=centos3
 514     user=vagrant
 515     ssh-keyscan $server >>~/.ssh/known_hosts
 516     ssh-copy-id $user@$server
 517
 518     # SHELLS: lsh-client against openssh server
 519     $INSTALL lsh-client
 520     (
 521         cd
 522         mkdir -p .lsh
 523         rm -f ".lsh/yarrow-seed-file"
 524         lsh-make-seed -o ".lsh/yarrow-seed-file"
 525         # TODO need to enter 'y' here
 526         lsh --sloppy-host-authentication \
 527             --capture-to ~/.lsh/host-acls -l $user $server echo Added host-auth
 528         rm -f .lsh/identity .lsh/identity.pub
 529         lsh-keygen | lsh-writekey -c none
 530         export_key_to_local_users() {
 531             lsh-export-key --openssh < ~/.lsh/identity.pub |
 532                 ssh -l $1 $2 'cat >>.ssh/authorized_keys'
 533         }
 534         export -f export_key_to_local_users
 535         # lsh: Protocol error: No common key exchange method.
 536         # It seems
 537         # $ lsh --list-algorithms
 538         # Supported hostkey algorithms: ssh-dss, spki, none
 539         #
 540         # $ nmap --script ssh2-enum-algos -sV -p 22 lo
 541         # |   server_host_key_algorithms: (4)
 542         # |       rsa-sha2-512
 543         # |       rsa-sha2-256
 544         # |       ecdsa-sha2-nistp256
 545         # |       ssh-ed25519
 546         # |
 547         #
 548         export_key_to_local_users $user $server
 549     )
 550     lsh -l $user $server echo lsh $user $server OK
 551 }
 552
 553 add_freebsd() {
 554     echo "# Add public keys to old servers"
 555     # freebsd7 is no longer supported as virtualbox
 556     # ssh freebsd7 cat .ssh/id_rsa.pub |
 557     #    ssh parallel@localhost 'cat >>.ssh/authorized_keys'
 558
 559     echo Add:
 560     echo HostkeyAlgorithms +ssh-dss
 561     echo to .ssh/config if you get
 562     echo no matching host key type found. Their offer: ssh-dss
 563     echo
 564     echo Add to /etc/ssh/sshd_config
 565     echo Ciphers aes128-ctr,aes192-ctr,aes256-ctr,aes256-cbc,aes192-cbc,aes128-cbc,3des-cbc
 566     echo KexAlgorithms diffie-hellman-group-exchange-sha256,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,curve25519-sha256@libssh.org,diffie-hellman-group1-sha1,diffie-hellman-group14-sha1,diffie-hellman-group-exchange-sha1
 567     echo
 568     echo If you get:
 569     echo Unable to negotiate with server port 22: no matching cipher found.
 570     echo Their offer: aes128-cbc,3des-cbc,blowfish-cbc,cast128-cbc,arcfour,aes192-cbc,aes256-cbc,rijndael-cbc@lysator.liu.se
 571     echo add this to .ssh/config
 572     echo   Ciphers +aes256-cbc
 573 }
 574
 575 tmux_versions() {
 576     (cd /tmp
 577      git clone https://github.com/tmux/tmux.git
 578      listtags() {
 579          (cd tmux;
 580           git tag)
 581      }
 582      make_one() {
 583          rsync -a --delete tmux/ tmux$1/
 584          cd tmux$1
 585          git reset --hard
 586          git checkout $1
 587          bash ./autogen.sh;
 588          ./configure &&
 589          make -j2 &&
 590          sudo cp tmux /usr/local/bin/tmux-$1
 591      }
 592      export -f make_one
 593      echo '# Building tmux'
 594      listtags |
 595          stdout parallel --lb --tag '/usr/local/bin/tmux-{} -V || make_one {}'
 596      listtags |
 597          parallel -k -v /usr/local/bin/tmux-{} -V
 598     )
 599 }
 600
 601 bash_versions() {
 602     echo You may have to kill conftest
 603     (cd /tmp
 604      git clone https://git.savannah.gnu.org/git/bash.git
 605      listtags() {
 606          (cd bash;
 607           git tag | grep -v -- '-.*-' | grep -v bash-4.1.11)
 608      }
 609      make_one() {
 610          rsync -a --delete bash/ $1/
 611          cd $1
 612          git reset --hard
 613          git checkout $1
 614          LDFLAGS=-static ./configure &&
 615              make -j2 &&
 616              sudo cp bash /usr/local/bin/$1
 617      }
 618      export -f make_one
 619      echo '# Building bash'
 620      listtags |
 621          stdout parallel --lb --tag \
 622                 '/usr/local/bin/{} --version || make_one {}'
 623      listtags |
 624          parallel -k -v --tag '/usr/local/bin/{} --version'
 625     )
 626 }
 627
 628 rsync_versions() {
 629     (cd /tmp
 630      git clone https://github.com/WayneD/rsync
 631      listtags() {
 632          (cd rsync;
 633           git tag | grep -v -- '-.*-' | grep -v pre)
 634      }
 635      make_one() {
 636          configure_make_install() {
 637              ./configure && make -j2 &&
 638                  sudo cp rsync /usr/local/bin/rsync-$1
 639          }
 640          configure_make_proto_install() {
 641              ./configure && make proto && make -j2 &&
 642                  sudo cp rsync /usr/local/bin/rsync-$1
 643          }
 644          edit_configure.in() {
 645              perl -i.bak -pe '
 646                  $printed++ || print "AC_DEFINE([HAVE_LONGLONG], [], [long long support])\n",
 647                               "AC_DEFINE([HAVE_UTIMBUF], [], [utimbuf is available])\n",
 648                               "AC_DEFINE([HAVE_FNMATCH], [], [dummy])\n";
 649                  s/AC_DEFINE_UNQUOTED.HAVE_REMSH, .HAVE_REMSH.*/AC_DEFINE_UNQUOTED(HAVE_REMSH, \$HAVE_REMSH,[dummy])/;
 650                  s/AC_DEFINE.HAVE_ERRNO_DECL.,/AC_DEFINE(HAVE_ERRNO_DECL,[1],[dummy]),/;
 651                  s/AC_CHECK_FUNC\((.*), , AC_LIBOBJ\((.*)\)\)/AC_CHECK_FUNC($1, , [AC_LIBOBJ($2)])/;
 652                  s/AC_CHECK_FUNCS\((.*), , AC_LIBOBJ\((.*)\)\)/AC_CHECK_FUNCS($1, , [AC_LIBOBJ($2)])/;
 653                  s/AC_DEFINE.HAVE_FNMATCH.,/AC_DEFINE(HAVE_FNMATCH,[1],[dummy]),/;' configure.in
 654          }
 655          build-older() {
 656              # rsync-mbp_bk_export0 rsync-v1.6.4 rsync-v1.6.5
 657              # rsync-v1.6.6 rsync-v1.6.7 rsync-v1.6.8 rsync-v1.6.9
 658              # rsync-v1.7.0 rsync-v1.7.1 rsync-v1.7.2 rsync-v1.7.3
 659              # rsync-v1.7.4 rsync-v2.0.0 rsync-v2.0.1 rsync-v2.0.2
 660              # rsync-v2.0.3 rsync-v2.0.4 rsync-v2.0.5 rsync-v2.0.6
 661              # rsync-v2.0.7 rsync-v2.0.8 rsync-v2.0.9 rsync-v2.0.10
 662              # rsync-v2.0.11 rsync-v2.0.12 rsync-v2.0.13 rsync-v2.0.14
 663              # rsync-v2.0.15 rsync-v2.0.16 rsync-v2.0.17 rsync-v2.0.18
 664              # rsync-v2.0.19 rsync-v2.1.0 rsync-v2.1.1 rsync-v2.2.0
 665              # rsync-v2.2.1 rsync-v2.3.0 rsync-v2.3.1 rsync-v2.3.2
 666              # rsync-v2.4.0 rsync-v2.4.1 rsync-v2.4.2 rsync-v2.4.3
 667              # rsync-v2.4.4 rsync-v2.4.5 rsync-v2.4.6 rsync-v2.5.1
 668              # rsync-v2.5.2 rsync-v2.5.3 rsync-v2.5.4 rsync-v2.5.5
 669              # rsync-v2.5.5.rc1 rsync-v2.5.6 rsync-v2.6.0 rsync-v2.6.1
 670              # rsync-v2.6.2 rsync-v2.6.3 rsync-v2.6.4 rsync-v2.6.5
 671              # rsync-v2.6.6 rsync-v2.6.7 rsync-v2.6.8 rsync-v2.6.9
 672              edit_configure.in
 673              mkdir -p lib
 674              touch lib/addrinfo.h lib/getopt.h
 675              autoreconf -fiv
 676              # strlcat strlcpy
 677              perl -i -pe 's/void (strlc..)/void _$1/' util.c
 678              # Force FNM_CASEFOLD to be set
 679              export CFLAGS='-Wno-incompatible-pointer-types -D_GNU_SOURCE=1'
 680              configure_make_proto_install $1
 681          }
 682          build-v3.0.0() {
 683              # rsync-v3.0.0 rsync-v3.0.1 rsync-v3.0.2 rsync-v3.0.3
 684              # rsync-v3.0.4 rsync-v3.0.5 rsync-v3.0.6 rsync-v3.0.7
 685              # rsync-v3.0.8 rsync-v3.0.9 rsync-v3.1.0 rsync-v3.1.1
 686              # rsync-v3.1.2 rsync-v3.1.3 rsync-v3.2.0 rsync-v3.2.1
 687              # rsync-v3.2.2 rsync-v3.2.3 rsync-v3.2.4 rsync-v3.2.5
 688              # rsync-v3.2.6 rsync-v3.2.7 rsync-v3.3.0
 689              configure_make_install $1
 690          }
 691          rsync -a --delete rsync/ rsync-$1/
 692          cd rsync-$1
 693          git reset --hard
 694          git checkout $1
 695
 696          build-v3.0.0 $1 || build-older $1
 697      }
 698      export -f make_one
 699      echo '# Building rsync'
 700      listtags |
 701          stdout parallel --lb --tag \
 702                 '/usr/local/bin/rsync-{} --version || make_one {}'
 703      listtags |
 704          parallel -k -v --tag '/usr/local/bin/rsync-{} --version'
 705     )
 706 }
 707
 708 install_tangetools() {
 709     (
 710         git clone https://git.data.coop/tange/tangetools
 711         cd tangetools &&
 712             make &&
 713             sudo make install
 714     )
 715 }
 716
 717 misc() {
 718     parallel --record-env
 719
 720     # Build locale for LC_ALL=zh_HK.big5hkscs perl -e 1
 721     sudo locale-gen zh_HK
 722
 723     p=$(goodpasswd)
 724     echo withpassword:$p | sudo chpasswd
 725     echo "export withpassword=$p" >> ~/.passwords
 726
 727
 728     echo TODO
 729     echo ssh til egen IP-adresse  192.168.100.94 ipv6 addr
 730     echo sshd_config
 731     echo HostKeyAlgorithms +ssh-rsa
 732     echo PubkeyAcceptedAlgorithms +ssh-rsa
 733     echo KexAlgorithms +diffie-hellman-group1-sha1,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,diffie-hellman-group-exchange-sha256,diffie-hellman-group14-sha1
 734     echo Ciphers +3des-cbc,aes128-cbc,aes192-cbc,aes256-cbc,aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm@openssh.com,aes256-gcm@openssh.com,chacha20-poly1305@openssh.com
 735     echo MaxStartups 100:30:1000
 736     echo
 737     echo strlncopy -> rsync
 738     echo sqlite2
 739     echo
 740     echo 'set $withpassword'
 741     echo 'set $mysqlrootpassword in .passwords'
 742     echo '/usr/local/bin/parallel-20120822'
 743     echo 'permission of ~shell/ must be +rx'
 744     echo '~/.sql/aliases'
 745     echo 'python install python-is-python3'
 746 }
 747
 748 run() {
 749     install_packages &&
 750         install_tangetools &&
 751         install_sqlite2 &&
 752         (install_oracle_client; # 20240622 broken
 753          true)  &&
 754         setup_databases &&
 755         setup_virtual_box_network &&
 756         add_server_to_hosts &&
 757         vagrant_up &&
 758         vagrant_centos3 &&
 759         add_key_to_auth_key &&
 760         create_shell_logins &&
 761         copy_ssh_keys &&
 762         lsh_setup &&
 763         add_freebsd &&
 764         parallel_versions &&
 765         tmux_versions &&
 766         bash_versions &&
 767         rsync_versions &&
 768         misc
 769 }