buildscripts/scripts/nightly-build.jenkins

   1 import java.text.SimpleDateFormat
   2
   3 def DISTRO_STR = ''
   4 if(JOB_BASE_NAME == 'nightly_build_containerized') {
   5     DISTRO_STR = DISTRO_LIST_MASTER
   6 } else {
   7     DISTRO_STR = DISTRO_LIST_TESTBUILD
   8 }
   9
  10 properties([
  11   parameters([
  12     string(name: 'DISTROS', defaultValue: DISTRO_STR, description: 'List of targeted distros' ),
  13     string(name: 'EDITION', defaultValue: 'enterprise', description: 'Edition: raw, enterprise or managed' )
  14   ])
  15 ])
  16
  17 def PACKAGE_BUILDS = [:]
  18 def AGENT_LIST = ["linux-64bit", "linux-32bit", "windows", "windows-legacy" ]
  19 def AGENT_BUILDS= [:]
  20
  21 DISTRO_LIST = DISTROS.split(' ');
  22
  23 println("Building for the following Distros:")
  24 println(DISTRO_LIST)
  25 currentBuild.description = '\nBuilding for the following Distros:\n' + DISTRO_LIST
  26
  27 def DOCKER_BUILDS = [:]
  28 def dateFormat = new SimpleDateFormat("yyyy.MM.dd")
  29 def date = new Date()
  30 def CMK_VERS = dateFormat.format(date)
  31 def DAILY_DATA = '/var/jenkins_home/daily-data/' + scm.branches[0].name.replaceAll("/","-") + '/' + CMK_VERS
  32
  33 def PACKAGE_DIR = '/bauwelt/download'
  34 def GPG_KEY_DIR = '/bauwelt/etc/.gnupg'
  35 def RELEASE_KEY_DIR = '/bauwelt/git/release/release.key'
  36
  37 node {
  38     docker.withRegistry(DOCKER_REGISTRY, 'nexus') {
  39         IMAGE = 'ubuntu-18.04-common'
  40         docker.image(IMAGE).pull()
  41         docker.image(IMAGE).inside('-u 0:0 -v /var/jenkins_home/daily-data/:/var/jenkins_home/daily-data/ --network ci_local-infra') {
  42             stage('checkout sources') {
  43                 sh "rm -rf ${DAILY_DATA}"
  44                 sh 'rm -rf *'
  45                 dir("${DAILY_DATA}/git") {
  46                     checkout(scm)
  47
  48                     switch(EDITION) {
  49                         case 'raw':
  50                             AGENT_LIST = ["windows", "windows-legacy"]
  51                             sh 'rm -rf enterprise managed'
  52                             // Workaround since scss does not support conditional includes
  53                             sh '''
  54                                 rm -rf web/htdocs/themes/classic/scss/{cme,cee}
  55                                 mkdir -p web/htdocs/themes/classic/scss/{cme,cee}
  56                                 echo '@mixin graphs {}' > web/htdocs/themes/classic/scss/cee/_graphs.scss
  57                                 echo '@mixin reporting {}' > web/htdocs/themes/classic/scss/cee/_reporting.scss
  58                                 echo '@mixin managed {}' > web/htdocs/themes/classic/scss/cme/_managed.scss
  59                             '''
  60                             break
  61                         case 'enterprise':
  62                             sh 'rm -rf  managed'
  63                             // Workaround since scss does not support conditional includes
  64                             sh '''
  65                                 rm -rf web/htdocs/themes/classic/scss/cme
  66                                 mkdir -p web/htdocs/themes/classic/scss/cme
  67                                 echo '@mixin managed {}' > web/htdocs/themes/classic/scss/cme/_managed.scss
  68                             '''
  69                             break
  70                     }
  71                     sh "make NEW_VERSION=${CMK_VERS} setversion"
  72
  73                     dir('agents/wnx') {
  74                         stash name: 'WinSRC', include: '**/*'
  75                     }
  76                     dir('agents/windows/dependencies/') {
  77                         stash name: 'WinDep', include: '**/*'
  78                     }
  79                     dir('agents/windows/plugins/') {
  80                         stash name: 'WinPlugins', include: '**/*'
  81                     }
  82                     dir('omd/packages/') {
  83                         stash name: 'WinDepOMD', include: 'googletest/*, simpleini/*'
  84                     }
  85                 }
  86             }
  87         }
  88     }
  89 }
  90
  91 AGENT_LIST.each { AGENT ->
  92     AGENT_BUILDS['build agent ' + AGENT] = {
  93         if (AGENT == 'windows') {
  94             node('windows') {
  95                 bat 'powershell Remove-Item .\\* -Recurse -Force'
  96                 dir('agents/wnx') {
  97                     unstash name: 'WinSRC'
  98                 }
  99                 dir('agents/windows/dependencies/') {
 100                     unstash name: 'WinDep'
 101                 }
 102                 dir('agents/windows/plugins/') {
 103                     unstash name: 'WinPlugins'
 104                 }
 105                 dir('omd/packages/') {
 106                     unstash name: 'WinDepOMD'
 107                 }
 108                 bat 'cd agents\\wnx && call build_release.cmd'
 109                 dir('artefacts') {
 110                     stash name: 'WinMSI', includes: 'check_mk_agent*'
 111                 }
 112             }
 113         }
 114         else {
 115             node {
 116                 docker.withRegistry(DOCKER_REGISTRY, 'nexus') {
 117                     docker.image('agent-builder-' + AGENT).pull()
 118                     docker.image('agent-builder-' + AGENT).inside("-u 0:0 -v ${DAILY_DATA}:${DAILY_DATA} --privileged --network ci_local-infra") {
 119                         stage('build agent ' + AGENT) {
 120                             if (AGENT == 'linux-64bit') {
 121                                 AGENT = ''
 122                             } else if (AGENT == 'linux-32bit') {
 123                                 AGENT = '-32'
 124                             } else {
 125                                 AGENT = '-windows'
 126                             }
 127
 128                             sh 'rm -rf *'
 129                             sh "mkdir agent${AGENT}-work agent${AGENT}-dest"
 130                             sh "mount -t overlay overlay -o lowerdir=${DAILY_DATA}/git,upperdir=${WORKSPACE}/agent${AGENT}-dest,workdir=${WORKSPACE}/agent${AGENT}-work ${WORKSPACE}/agent${AGENT}-dest"
 131                             if (AGENT == '-windows') {
 132                                 sh script: """
 133                                     mkdir -p ${DAILY_DATA}/agents
 134                                     cd agent${AGENT}-dest/agents/windows/
 135                                     CHROOT_BUILD_DIR=${WORKSPACE}/agent${AGENT}-dest \
 136                                         CHROOT_NAME="agent-builder-windows" \
 137                                         ./build-agent
 138                                     cp check_mk_agent-64.exe ${DAILY_DATA}/agents/check_mk_agent_legacy-64.exe
 139                                     cp check_mk_agent.exe ${DAILY_DATA}/agents/check_mk_agent_legacy.exe
 140                                     cp check_mk_agent.msi ${DAILY_DATA}/agents/check_mk_agent_legacy.msi
 141                                 """
 142                                 if(EDITION != 'raw') {
 143                                     sh script: """
 144                                         cd agent${AGENT}-dest/enterprise/agents/windows/plugins
 145                                         CHROOT_BUILD_DIR=${WORKSPACE}/agent${AGENT}-dest \
 146                                             make
 147                                         cp cmk-update-agent.exe ${DAILY_DATA}/agents/
 148                                     """
 149                                 }
 150                             } else { // Linux Agents (64 and 32 bit)
 151                                 sh script: """cd agent${AGENT}-dest/enterprise/agents/plugins \
 152                                     && CHROOT_BUILD_DIR=${WORKSPACE}/agent${AGENT}-dest \
 153                                         BITNESS_SUFFIX='${AGENT}' \
 154                                         make
 155                                 """
 156                                 sh """
 157                                     mkdir -p ${DAILY_DATA}/agents
 158                                     cp agent${AGENT}-dest/enterprise/agents/plugins/cmk-update-agent${AGENT} ${DAILY_DATA}/agents
 159                                 """
 160                             }
 161                             sh "until umount agent${AGENT}-dest; do sleep 1; done"
 162                         }
 163                     }
 164                 }
 165             }
 166         }
 167     }
 168 }
 169 parallel AGENT_BUILDS
 170
 171 node {
 172     docker.withRegistry(DOCKER_REGISTRY, 'nexus') {
 173         IMAGE = 'ubuntu-18.04-common'
 174         docker.image(IMAGE).pull()
 175         docker.image(IMAGE).inside("-u 0:0 -v ${DAILY_DATA}:${DAILY_DATA} --network ci_local-infra") {
 176             stage('create tarball') {
 177                sh 'rm -rf *'
 178                sh "pip install backports.functools_lru_cache"
 179                unstash name: 'WinMSI'
 180                sh "mv check_mk_agent* ${DAILY_DATA}/agents/"
 181                dir("${DAILY_DATA}/git") {
 182                     if(EDITION != 'raw') {
 183                         sh "cp ${DAILY_DATA}/agents/cmk-update-agent enterprise/agents/plugins/"
 184                         sh "cp ${DAILY_DATA}/agents/cmk-update-agent-32 enterprise/agents/plugins/"
 185                         sh "cp ${DAILY_DATA}/agents/cmk-update-agent.exe enterprise/agents/windows/plugins/"
 186                     }
 187                     sh "cp ${DAILY_DATA}/agents/check_mk_agent*.msi agents/windows"
 188                     sh "cp ${DAILY_DATA}/agents/check_mk_agent*.exe agents/windows"
 189                     sh 'make dist || cat /root/.npm/_logs/*-debug.log'
 190                 }
 191             }
 192         }
 193     }
 194 }
 195
 196 DISTRO_LIST.each { DISTRO ->
 197     PACKAGE_BUILDS[DISTRO] = {
 198         node {
 199             docker.withRegistry(DOCKER_REGISTRY, 'nexus') {
 200                 docker.image(DISTRO).pull()
 201                 docker.image(DISTRO).inside("-u 0:0 -v ${DAILY_DATA}:${DAILY_DATA} --privileged --network ci_local-infra --hostname ${DISTRO}") {
 202                     stage('build' + DISTRO + ' package') {
 203                         sh 'rm -rf *'
 204                         sh 'mkdir work dest'
 205                         sh "mount -t overlay overlay -o lowerdir=${DAILY_DATA}/git,upperdir=${WORKSPACE}/dest,workdir=${WORKSPACE}/work ${WORKSPACE}/dest"
 206                         dir('dest') {
 207                             sh """
 208                                 case $DISTRO in
 209                                     centos*|rh*|sles*|opensuse*)
 210                                         make -C omd rpm
 211                                         ;;
 212                                     cma*)
 213                                         make -C omd cma
 214                                         ;;
 215                                     *)
 216                                         DEBFULLNAME='Check_MK Team' DEBEMAIL='feedback@check-mk.org' make -C omd deb
 217                                         ;;
 218                                 esac
 219                             """
 220                         }
 221                         sh 'until umount dest; do sleep 1; done'
 222                         sh "rm -rf ${DAILY_DATA}/${DISTRO}-{dest,work}"
 223                         sh "mv dest ${DAILY_DATA}/${DISTRO}-dest"
 224                         sh "mv work ${DAILY_DATA}/${DISTRO}-work"
 225                         sh "ls -lah ${DAILY_DATA}"
 226                     }
 227                 }
 228             }
 229         }
 230     }
 231 }
 232 parallel PACKAGE_BUILDS
 233
 234 node {
 235     docker.withRegistry(DOCKER_REGISTRY, 'nexus') {
 236         IMAGE = 'ubuntu-18.04-common'
 237         docker.image(IMAGE).pull()
 238         docker.image(IMAGE).inside("-u 0:0 -v ${DAILY_DATA}:${DAILY_DATA} -v ${PACKAGE_DIR}:${PACKAGE_DIR} -v ${GPG_KEY_DIR}:${GPG_KEY_DIR}:ro -v ${RELEASE_KEY_DIR}:${RELEASE_KEY_DIR}:ro --network ci_local-infra") {
 239             stage('Archive artifacts') {
 240                 // Only master builds should end up on the Website
 241                 if (scm.branches[0].name == 'master' && JOB_BASE_NAME == 'nightly_build_containerized') {
 242                     sh """
 243                         mkdir -p /bauwelt/download/${CMK_VERS}
 244                         cd /bauwelt/download/${CMK_VERS}
 245                         cp ${DAILY_DATA}/*-dest/check-mk-${EDITION}-*_amd64.deb . || true
 246                         cp ${DAILY_DATA}/*-dest/check-mk-${EDITION}-*.x86_64.rpm . || true
 247                         cp ${DAILY_DATA}/*-dest/check-mk-${EDITION}-*x86_64.cma . || true
 248                         cp ${DAILY_DATA}/git/check-mk-${EDITION}-${CMK_VERS}.cee.tar.gz . || true
 249                     """
 250                     withCredentials([usernamePassword(credentialsId: '9d7aca31-0043-4cd0-abeb-26a249d68261', passwordVariable: 'GPG_PASSPHRASE', usernameVariable: 'GPG_USERNAME')]) {
 251                     sh """
 252                         cd /bauwelt/download/${CMK_VERS}
 253                         ${DAILY_DATA}/git/buildscripts/scripts/sign-packages.sh ${CMK_VERS}
 254                         rsync -av -e "ssh -o StrictHostKeyChecking=no -i ${RELEASE_KEY_DIR}" \
 255                                 ${PACKAGE_DIR}/${CMK_VERS} \
 256                                 bauwelt@mathias-kettner.de:${PACKAGE_DIR}
 257                     """
 258                     }
 259                 } else {
 260                     dir("${DAILY_DATA}/download") {
 261                         sh "cp ${DAILY_DATA}/*-dest/check-mk-${EDITION}-*_amd64.deb . || true"
 262                         sh "cp ${DAILY_DATA}/*-dest/check-mk-${EDITION}-*.x86_64.rpm . || true"
 263                         sh "cp ${DAILY_DATA}/*-dest/check-mk-${EDITION}-*x86_64.cma . || true"
 264                         sh "cp ${DAILY_DATA}/git/check-mk-${EDITION}-${CMK_VERS}.*.tar.gz . || true"
 265                         withCredentials([usernamePassword(credentialsId: '9d7aca31-0043-4cd0-abeb-26a249d68261', passwordVariable: 'GPG_PASSPHRASE', usernameVariable: 'GPG_USERNAME')]) {
 266                             sh "${DAILY_DATA}/git/buildscripts/scripts/sign-packages.sh ${CMK_VERS}"
 267                         }
 268                         archiveArtifacts "*"
 269                     }
 270                 }
 271             }
 272         }
 273     }
 274 }