eng/common/tools.sh

   1 #!/usr/bin/env bash
   2
   3 # Initialize variables if they aren't already defined.
   4
   5 # CI mode - set to true on CI server for PR validation build or official build.
   6 ci=${ci:-false}
   7
   8 # Set to true to use the pipelines logger which will enable Azure logging output.
   9 # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
  10 # This flag is meant as a temporary opt-opt for the feature while validate it across
  11 # our consumers. It will be deleted in the future.
  12 if [[ "$ci" == true ]]; then
  13   pipelines_log=${pipelines_log:-true}
  14 else
  15   pipelines_log=${pipelines_log:-false}
  16 fi
  17
  18 # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
  19 configuration=${configuration:-'Debug'}
  20
  21 # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
  22 # Binary log must be enabled on CI.
  23 binary_log=${binary_log:-$ci}
  24
  25 # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
  26 prepare_machine=${prepare_machine:-false}
  27
  28 # True to restore toolsets and dependencies.
  29 restore=${restore:-true}
  30
  31 # Adjusts msbuild verbosity level.
  32 verbosity=${verbosity:-'minimal'}
  33
  34 # Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
  35 if [[ "$ci" == true ]]; then
  36   node_reuse=${node_reuse:-false}
  37 else
  38   node_reuse=${node_reuse:-true}
  39 fi
  40
  41 # Configures warning treatment in msbuild.
  42 warn_as_error=${warn_as_error:-true}
  43
  44 # True to attempt using .NET Core already that meets requirements specified in global.json
  45 # installed on the machine instead of downloading one.
  46 use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
  47
  48 # Enable repos to use a particular version of the on-line dotnet-install scripts.
  49 #    default URL: https://dot.net/v1/dotnet-install.sh
  50 dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
  51
  52 # True to use global NuGet cache instead of restoring packages to repository-local directory.
  53 if [[ "$ci" == true ]]; then
  54   use_global_nuget_cache=${use_global_nuget_cache:-false}
  55 else
  56   use_global_nuget_cache=${use_global_nuget_cache:-true}
  57 fi
  58
  59 # Resolve any symlinks in the given path.
  60 function ResolvePath {
  61   local path=$1
  62
  63   while [[ -h $path ]]; do
  64     local dir="$( cd -P "$( dirname "$path" )" && pwd )"
  65     path="$(readlink "$path")"
  66
  67     # if $path was a relative symlink, we need to resolve it relative to the path where the
  68     # symlink file was located
  69     [[ $path != /* ]] && path="$dir/$path"
  70   done
  71
  72   # return value
  73   _ResolvePath="$path"
  74 }
  75
  76 # ReadVersionFromJson [json key]
  77 function ReadGlobalVersion {
  78   local key=$1
  79
  80   local line=`grep -m 1 "$key" "$global_json_file"`
  81   local pattern="\"$key\" *: *\"(.*)\""
  82
  83   if [[ ! $line =~ $pattern ]]; then
  84     Write-PipelineTelemetryError -category 'Build' "Error: Cannot find \"$key\" in $global_json_file"
  85     ExitWithExitCode 1
  86   fi
  87
  88   # return value
  89   _ReadGlobalVersion=${BASH_REMATCH[1]}
  90 }
  91
  92 function InitializeDotNetCli {
  93   if [[ -n "${_InitializeDotNetCli:-}" ]]; then
  94     return
  95   fi
  96
  97   local install=$1
  98
  99   # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
 100   export DOTNET_MULTILEVEL_LOOKUP=0
 101
 102   # Disable first run since we want to control all package sources
 103   export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 104
 105   # Disable telemetry on CI
 106   if [[ $ci == true ]]; then
 107     export DOTNET_CLI_TELEMETRY_OPTOUT=1
 108   fi
 109
 110   # LTTNG is the logging infrastructure used by Core CLR. Need this variable set
 111   # so it doesn't output warnings to the console.
 112   export LTTNG_HOME="$HOME"
 113
 114   # Source Build uses DotNetCoreSdkDir variable
 115   if [[ -n "${DotNetCoreSdkDir:-}" ]]; then
 116     export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir"
 117   fi
 118
 119   # Find the first path on $PATH that contains the dotnet.exe
 120   if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
 121     local dotnet_path=`command -v dotnet`
 122     if [[ -n "$dotnet_path" ]]; then
 123       ResolvePath "$dotnet_path"
 124       export DOTNET_INSTALL_DIR=`dirname "$_ResolvePath"`
 125     fi
 126   fi
 127
 128   ReadGlobalVersion "dotnet"
 129   local dotnet_sdk_version=$_ReadGlobalVersion
 130   local dotnet_root=""
 131
 132   # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
 133   # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
 134   if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
 135     dotnet_root="$DOTNET_INSTALL_DIR"
 136   else
 137     dotnet_root="$repo_root/.dotnet"
 138
 139     export DOTNET_INSTALL_DIR="$dotnet_root"
 140
 141     if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
 142       if [[ "$install" == true ]]; then
 143         InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version"
 144       else
 145         Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'"
 146         ExitWithExitCode 1
 147       fi
 148     fi
 149   fi
 150
 151   # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
 152   # build steps from using anything other than what we've downloaded.
 153   Write-PipelinePrependPath -path "$dotnet_root"
 154
 155   Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
 156   Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1"
 157
 158   # return value
 159   _InitializeDotNetCli="$dotnet_root"
 160 }
 161
 162 function InstallDotNetSdk {
 163   local root=$1
 164   local version=$2
 165   local architecture=""
 166   if [[ $# == 3 ]]; then
 167     architecture=$3
 168   fi
 169   InstallDotNet "$root" "$version" $architecture
 170 }
 171
 172 function InstallDotNet {
 173   local root=$1
 174   local version=$2
 175
 176   GetDotNetInstallScript "$root"
 177   local install_script=$_GetDotNetInstallScript
 178
 179   local archArg=''
 180   if [[ -n "${3:-}" ]]; then
 181     archArg="--architecture $3"
 182   fi
 183   local runtimeArg=''
 184   if [[ -n "${4:-}" ]]; then
 185     runtimeArg="--runtime $4"
 186   fi
 187
 188   local skipNonVersionedFilesArg=""
 189   if [[ "$#" -ge "5" ]]; then
 190     skipNonVersionedFilesArg="--skip-non-versioned-files"
 191   fi
 192   bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
 193     local exit_code=$?
 194     Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
 195
 196     if [[ -n "$runtimeArg" ]]; then
 197       local runtimeSourceFeed=''
 198       if [[ -n "${6:-}" ]]; then
 199         runtimeSourceFeed="--azure-feed $6"
 200       fi
 201
 202       local runtimeSourceFeedKey=''
 203       if [[ -n "${7:-}" ]]; then
 204         decodedFeedKey=`echo $7 | base64 --decode`
 205         runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
 206       fi
 207
 208       if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
 209         bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
 210           local exit_code=$?
 211           Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
 212           ExitWithExitCode $exit_code
 213         }
 214       else
 215         ExitWithExitCode $exit_code
 216       fi
 217     fi
 218   }
 219 }
 220
 221 function GetDotNetInstallScript {
 222   local root=$1
 223   local install_script="$root/dotnet-install.sh"
 224   local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh"
 225
 226   if [[ ! -a "$install_script" ]]; then
 227     mkdir -p "$root"
 228
 229     echo "Downloading '$install_script_url'"
 230
 231     # Use curl if available, otherwise use wget
 232     if command -v curl > /dev/null; then
 233       curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
 234         local exit_code=$?
 235         Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
 236         ExitWithExitCode $exit_code
 237       }
 238     else
 239       wget -q -O "$install_script" "$install_script_url" || {
 240         local exit_code=$?
 241         Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
 242         ExitWithExitCode $exit_code
 243       }
 244     fi
 245   fi
 246   # return value
 247   _GetDotNetInstallScript="$install_script"
 248 }
 249
 250 function InitializeBuildTool {
 251   if [[ -n "${_InitializeBuildTool:-}" ]]; then
 252     return
 253   fi
 254
 255   InitializeDotNetCli $restore
 256
 257   # return values
 258   _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
 259   _InitializeBuildToolCommand="msbuild"
 260   _InitializeBuildToolFramework="netcoreapp2.1"
 261 }
 262
 263 function GetNuGetPackageCachePath {
 264   if [[ -z ${NUGET_PACKAGES:-} ]]; then
 265     if [[ "$use_global_nuget_cache" == true ]]; then
 266       export NUGET_PACKAGES="$HOME/.nuget/packages"
 267     else
 268       export NUGET_PACKAGES="$repo_root/.packages"
 269     fi
 270   fi
 271
 272   # return value
 273   _GetNuGetPackageCachePath=$NUGET_PACKAGES
 274 }
 275
 276 function InitializeNativeTools() {
 277   if [[ -n "${DisableNativeToolsetInstalls:-}" ]]; then
 278     return
 279   fi
 280   if grep -Fq "native-tools" $global_json_file
 281   then
 282     local nativeArgs=""
 283     if [[ "$ci" == true ]]; then
 284       nativeArgs="--installDirectory $tools_dir"
 285     fi
 286     "$_script_dir/init-tools-native.sh" $nativeArgs
 287   fi
 288 }
 289
 290 function InitializeToolset {
 291   if [[ -n "${_InitializeToolset:-}" ]]; then
 292     return
 293   fi
 294
 295   GetNuGetPackageCachePath
 296
 297   ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
 298
 299   local toolset_version=$_ReadGlobalVersion
 300   local toolset_location_file="$toolset_dir/$toolset_version.txt"
 301
 302   if [[ -a "$toolset_location_file" ]]; then
 303     local path=`cat "$toolset_location_file"`
 304     if [[ -a "$path" ]]; then
 305       # return value
 306       _InitializeToolset="$path"
 307       return
 308     fi
 309   fi
 310
 311   if [[ "$restore" != true ]]; then
 312     Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored."
 313     ExitWithExitCode 2
 314   fi
 315
 316   local proj="$toolset_dir/restore.proj"
 317
 318   local bl=""
 319   if [[ "$binary_log" == true ]]; then
 320     bl="/bl:$log_dir/ToolsetRestore.binlog"
 321   fi
 322
 323   echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj"
 324   MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file"
 325
 326   local toolset_build_proj=`cat "$toolset_location_file"`
 327
 328   if [[ ! -a "$toolset_build_proj" ]]; then
 329     Write-PipelineTelemetryError -category 'Build' "Invalid toolset path: $toolset_build_proj"
 330     ExitWithExitCode 3
 331   fi
 332
 333   # return value
 334   _InitializeToolset="$toolset_build_proj"
 335 }
 336
 337 function ExitWithExitCode {
 338   if [[ "$ci" == true && "$prepare_machine" == true ]]; then
 339     StopProcesses
 340   fi
 341   exit $1
 342 }
 343
 344 function StopProcesses {
 345   echo "Killing running build processes..."
 346   pkill -9 "dotnet" || true
 347   pkill -9 "vbcscompiler" || true
 348   return 0
 349 }
 350
 351 function MSBuild {
 352   local args=$@
 353   if [[ "$pipelines_log" == true ]]; then
 354     InitializeBuildTool
 355     InitializeToolset
 356
 357     # Work around issues with Azure Artifacts credential provider
 358     # https://github.com/dotnet/arcade/issues/3932
 359     if [[ "$ci" == true ]]; then
 360       "$_InitializeBuildTool" nuget locals http-cache -c
 361
 362       export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
 363       export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
 364       Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
 365       Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
 366     fi
 367
 368     local toolset_dir="${_InitializeToolset%/*}"
 369     local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll"
 370     args=( "${args[@]}" "-logger:$logger_path" )
 371   fi
 372
 373   MSBuild-Core ${args[@]}
 374 }
 375
 376 function MSBuild-Core {
 377   if [[ "$ci" == true ]]; then
 378     if [[ "$binary_log" != true ]]; then
 379       Write-PipelineTelemetryError -category 'Build'  "Binary log must be enabled in CI build."
 380       ExitWithExitCode 1
 381     fi
 382
 383     if [[ "$node_reuse" == true ]]; then
 384       Write-PipelineTelemetryError -category 'Build'  "Node reuse must be disabled in CI build."
 385       ExitWithExitCode 1
 386     fi
 387   fi
 388
 389   InitializeBuildTool
 390
 391   local warnaserror_switch=""
 392   if [[ $warn_as_error == true ]]; then
 393     warnaserror_switch="/warnaserror"
 394   fi
 395
 396   "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || {
 397     local exit_code=$?
 398     Write-PipelineTelemetryError -category 'Build'  "Build failed (exit code '$exit_code')."
 399     ExitWithExitCode $exit_code
 400   }
 401 }
 402
 403 ResolvePath "${BASH_SOURCE[0]}"
 404 _script_dir=`dirname "$_ResolvePath"`
 405
 406 . "$_script_dir/pipeline-logging-functions.sh"
 407
 408 eng_root=`cd -P "$_script_dir/.." && pwd`
 409 repo_root=`cd -P "$_script_dir/../.." && pwd`
 410 artifacts_dir="$repo_root/artifacts"
 411 toolset_dir="$artifacts_dir/toolset"
 412 tools_dir="$repo_root/.tools"
 413 log_dir="$artifacts_dir/log/$configuration"
 414 temp_dir="$artifacts_dir/tmp/$configuration"
 415
 416 global_json_file="$repo_root/global.json"
 417 # determine if global.json contains a "runtimes" entry
 418 global_json_has_runtimes=false
 419 dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true
 420 if [[ -n "$dotnetlocal_key" ]]; then
 421   global_json_has_runtimes=true
 422 fi
 423
 424 # HOME may not be defined in some scenarios, but it is required by NuGet
 425 if [[ -z $HOME ]]; then
 426   export HOME="$repo_root/artifacts/.home/"
 427   mkdir -p "$HOME"
 428 fi
 429
 430 mkdir -p "$toolset_dir"
 431 mkdir -p "$temp_dir"
 432 mkdir -p "$log_dir"
 433
 434 Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir"
 435 Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir"
 436 Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir"
 437 Write-PipelineSetVariable -name "Temp" -value "$temp_dir"
 438 Write-PipelineSetVariable -name "TMP" -value "$temp_dir"
 439
 440 # Import custom tools configuration, if present in the repo.
 441 if [ -z "${disable_configure_toolset_import:-}" ]; then
 442   configure_toolset_script="$eng_root/configure-toolset.sh"
 443   if [[ -a "$configure_toolset_script" ]]; then
 444     . "$configure_toolset_script"
 445   fi
 446 fi
 447
 448 # TODO: https://github.com/dotnet/arcade/issues/1468
 449 # Temporary workaround to avoid breaking change.
 450 # Remove once repos are updated.
 451 if [[ -n "${useInstalledDotNetCli:-}" ]]; then
 452   use_installed_dotnet_cli="$useInstalledDotNetCli"
 453 fi