eng/common/tools.sh

   1 #!/usr/bin/env bash
   2
   3 # Initialize variables if they aren't already defined.
   4
   5 # CI mode - set to true on CI server for PR validation build or official build.
   6 ci=${ci:-false}
   7 disable_configure_toolset_import=${disable_configure_toolset_import:-null}
   8
   9 # Set to true to use the pipelines logger which will enable Azure logging output.
  10 # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
  11 # This flag is meant as a temporary opt-opt for the feature while validate it across
  12 # our consumers. It will be deleted in the future.
  13 if [[ "$ci" == true ]]; then
  14   pipelines_log=${pipelines_log:-true}
  15 else
  16   pipelines_log=${pipelines_log:-false}
  17 fi
  18
  19 # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
  20 configuration=${configuration:-'Debug'}
  21
  22 # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
  23 # Binary log must be enabled on CI.
  24 binary_log=${binary_log:-$ci}
  25
  26 # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
  27 prepare_machine=${prepare_machine:-false}
  28
  29 # True to restore toolsets and dependencies.
  30 restore=${restore:-true}
  31
  32 # Adjusts msbuild verbosity level.
  33 verbosity=${verbosity:-'minimal'}
  34
  35 # Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
  36 if [[ "$ci" == true ]]; then
  37   node_reuse=${node_reuse:-false}
  38 else
  39   node_reuse=${node_reuse:-true}
  40 fi
  41
  42 # Configures warning treatment in msbuild.
  43 warn_as_error=${warn_as_error:-true}
  44
  45 # True to attempt using .NET Core already that meets requirements specified in global.json
  46 # installed on the machine instead of downloading one.
  47 use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
  48
  49 # Enable repos to use a particular version of the on-line dotnet-install scripts.
  50 #    default URL: https://dot.net/v1/dotnet-install.sh
  51 dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'}
  52
  53 # True to use global NuGet cache instead of restoring packages to repository-local directory.
  54 if [[ "$ci" == true ]]; then
  55   use_global_nuget_cache=${use_global_nuget_cache:-false}
  56 else
  57   use_global_nuget_cache=${use_global_nuget_cache:-true}
  58 fi
  59
  60 # Resolve any symlinks in the given path.
  61 function ResolvePath {
  62   local path=$1
  63
  64   while [[ -h $path ]]; do
  65     local dir="$( cd -P "$( dirname "$path" )" && pwd )"
  66     path="$(readlink "$path")"
  67
  68     # if $path was a relative symlink, we need to resolve it relative to the path where the
  69     # symlink file was located
  70     [[ $path != /* ]] && path="$dir/$path"
  71   done
  72
  73   # return value
  74   _ResolvePath="$path"
  75 }
  76
  77 # ReadVersionFromJson [json key]
  78 function ReadGlobalVersion {
  79   local key=$1
  80
  81   local line=`grep -m 1 "$key" "$global_json_file"`
  82   local pattern="\"$key\" *: *\"(.*)\""
  83
  84   if [[ ! $line =~ $pattern ]]; then
  85     Write-PipelineTelemetryError -category 'InitializeToolset' "Error: Cannot find \"$key\" in $global_json_file"
  86     ExitWithExitCode 1
  87   fi
  88
  89   # return value
  90   _ReadGlobalVersion=${BASH_REMATCH[1]}
  91 }
  92
  93 function InitializeDotNetCli {
  94   if [[ -n "${_InitializeDotNetCli:-}" ]]; then
  95     return
  96   fi
  97
  98   local install=$1
  99
 100   # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
 101   export DOTNET_MULTILEVEL_LOOKUP=0
 102
 103   # Disable first run since we want to control all package sources
 104   export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 105
 106   # Disable telemetry on CI
 107   if [[ $ci == true ]]; then
 108     export DOTNET_CLI_TELEMETRY_OPTOUT=1
 109   fi
 110
 111   # LTTNG is the logging infrastructure used by Core CLR. Need this variable set
 112   # so it doesn't output warnings to the console.
 113   export LTTNG_HOME="$HOME"
 114
 115   # Source Build uses DotNetCoreSdkDir variable
 116   if [[ -n "${DotNetCoreSdkDir:-}" ]]; then
 117     export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir"
 118   fi
 119
 120   # Find the first path on $PATH that contains the dotnet.exe
 121   if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
 122     local dotnet_path=`command -v dotnet`
 123     if [[ -n "$dotnet_path" ]]; then
 124       ResolvePath "$dotnet_path"
 125       export DOTNET_INSTALL_DIR=`dirname "$_ResolvePath"`
 126     fi
 127   fi
 128
 129   ReadGlobalVersion "dotnet"
 130   local dotnet_sdk_version=$_ReadGlobalVersion
 131   local dotnet_root=""
 132
 133   # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
 134   # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
 135   if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
 136     dotnet_root="$DOTNET_INSTALL_DIR"
 137   else
 138     dotnet_root="$repo_root/.dotnet"
 139
 140     export DOTNET_INSTALL_DIR="$dotnet_root"
 141
 142     if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
 143       if [[ "$install" == true ]]; then
 144         InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version"
 145       else
 146         Write-PipelineTelemetryError -category 'InitializeToolset' "Unable to find dotnet with SDK version '$dotnet_sdk_version'"
 147         ExitWithExitCode 1
 148       fi
 149     fi
 150   fi
 151
 152   # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
 153   # build steps from using anything other than what we've downloaded.
 154   Write-PipelinePrependPath -path "$dotnet_root"
 155
 156   Write-PipelineSetVariable -name "DOTNET_MULTILEVEL_LOOKUP" -value "0"
 157   Write-PipelineSetVariable -name "DOTNET_SKIP_FIRST_TIME_EXPERIENCE" -value "1"
 158
 159   # return value
 160   _InitializeDotNetCli="$dotnet_root"
 161 }
 162
 163 function InstallDotNetSdk {
 164   local root=$1
 165   local version=$2
 166   local architecture=""
 167   if [[ $# == 3 ]]; then
 168     architecture=$3
 169   fi
 170   InstallDotNet "$root" "$version" $architecture
 171 }
 172
 173 function InstallDotNet {
 174   local root=$1
 175   local version=$2
 176
 177   GetDotNetInstallScript "$root"
 178   local install_script=$_GetDotNetInstallScript
 179
 180   local archArg=''
 181   if [[ -n "${3:-}" ]]; then
 182     archArg="--architecture $3"
 183   fi
 184   local runtimeArg=''
 185   if [[ -n "${4:-}" ]]; then
 186     runtimeArg="--runtime $4"
 187   fi
 188
 189   local skipNonVersionedFilesArg=""
 190   if [[ "$#" -ge "5" ]]; then
 191     skipNonVersionedFilesArg="--skip-non-versioned-files"
 192   fi
 193   bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
 194     local exit_code=$?
 195     Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
 196
 197     if [[ -n "$runtimeArg" ]]; then
 198       local runtimeSourceFeed=''
 199       if [[ -n "${6:-}" ]]; then
 200         runtimeSourceFeed="--azure-feed $6"
 201       fi
 202
 203       local runtimeSourceFeedKey=''
 204       if [[ -n "${7:-}" ]]; then
 205         decodedFeedKey=`echo $7 | base64 --decode`
 206         runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
 207       fi
 208
 209       if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
 210         bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
 211           local exit_code=$?
 212           Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
 213           ExitWithExitCode $exit_code
 214         }
 215       fi
 216     fi
 217   }
 218 }
 219
 220 function GetDotNetInstallScript {
 221   local root=$1
 222   local install_script="$root/dotnet-install.sh"
 223   local install_script_url="https://dot.net/$dotnetInstallScriptVersion/dotnet-install.sh"
 224
 225   if [[ ! -a "$install_script" ]]; then
 226     mkdir -p "$root"
 227
 228     echo "Downloading '$install_script_url'"
 229
 230     # Use curl if available, otherwise use wget
 231     if command -v curl > /dev/null; then
 232       curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
 233         local exit_code=$?
 234         Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
 235         ExitWithExitCode $exit_code
 236       }
 237     else
 238       wget -q -O "$install_script" "$install_script_url" || {
 239         local exit_code=$?
 240         Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
 241         ExitWithExitCode $exit_code
 242       }
 243     fi
 244   fi
 245   # return value
 246   _GetDotNetInstallScript="$install_script"
 247 }
 248
 249 function InitializeBuildTool {
 250   if [[ -n "${_InitializeBuildTool:-}" ]]; then
 251     return
 252   fi
 253
 254   InitializeDotNetCli $restore
 255
 256   # return values
 257   _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
 258   _InitializeBuildToolCommand="msbuild"
 259   _InitializeBuildToolFramework="netcoreapp2.1"
 260 }
 261
 262 function GetNuGetPackageCachePath {
 263   if [[ -z ${NUGET_PACKAGES:-} ]]; then
 264     if [[ "$use_global_nuget_cache" == true ]]; then
 265       export NUGET_PACKAGES="$HOME/.nuget/packages"
 266     else
 267       export NUGET_PACKAGES="$repo_root/.packages"
 268     fi
 269   fi
 270
 271   # return value
 272   _GetNuGetPackageCachePath=$NUGET_PACKAGES
 273 }
 274
 275 function InitializeNativeTools() {
 276   if [[ -z "${DisableNativeToolsetInstalls:-}" ]]; then
 277     return
 278   fi
 279   if grep -Fq "native-tools" $global_json_file
 280   then
 281     local nativeArgs=""
 282     if [[ "$ci" == true ]]; then
 283       nativeArgs="--installDirectory $tools_dir"
 284     fi
 285     "$_script_dir/init-tools-native.sh" $nativeArgs
 286   fi
 287 }
 288
 289 function InitializeToolset {
 290   if [[ -n "${_InitializeToolset:-}" ]]; then
 291     return
 292   fi
 293
 294   GetNuGetPackageCachePath
 295
 296   ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
 297
 298   local toolset_version=$_ReadGlobalVersion
 299   local toolset_location_file="$toolset_dir/$toolset_version.txt"
 300
 301   if [[ -a "$toolset_location_file" ]]; then
 302     local path=`cat "$toolset_location_file"`
 303     if [[ -a "$path" ]]; then
 304       # return value
 305       _InitializeToolset="$path"
 306       return
 307     fi
 308   fi
 309
 310   if [[ "$restore" != true ]]; then
 311     Write-PipelineTelemetryError -category 'InitializeToolset' "Toolset version $toolset_version has not been restored."
 312     ExitWithExitCode 2
 313   fi
 314
 315   local proj="$toolset_dir/restore.proj"
 316
 317   local bl=""
 318   if [[ "$binary_log" == true ]]; then
 319     bl="/bl:$log_dir/ToolsetRestore.binlog"
 320   fi
 321
 322   echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj"
 323   MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file"
 324
 325   local toolset_build_proj=`cat "$toolset_location_file"`
 326
 327   if [[ ! -a "$toolset_build_proj" ]]; then
 328     Write-PipelineTelemetryError -category 'InitializeToolset' "Invalid toolset path: $toolset_build_proj"
 329     ExitWithExitCode 3
 330   fi
 331
 332   # return value
 333   _InitializeToolset="$toolset_build_proj"
 334 }
 335
 336 function ExitWithExitCode {
 337   if [[ "$ci" == true && "$prepare_machine" == true ]]; then
 338     StopProcesses
 339   fi
 340   exit $1
 341 }
 342
 343 function StopProcesses {
 344   echo "Killing running build processes..."
 345   pkill -9 "dotnet" || true
 346   pkill -9 "vbcscompiler" || true
 347   return 0
 348 }
 349
 350 function MSBuild {
 351   local args=$@
 352   if [[ "$pipelines_log" == true ]]; then
 353     InitializeBuildTool
 354     InitializeToolset
 355
 356     # Work around issues with Azure Artifacts credential provider
 357     # https://github.com/dotnet/arcade/issues/3932
 358     if [[ "$ci" == true ]]; then
 359       "$_InitializeBuildTool" nuget locals http-cache -c
 360
 361       export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
 362       export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
 363       Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
 364       Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
 365     fi
 366
 367     local toolset_dir="${_InitializeToolset%/*}"
 368     local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll"
 369     args=( "${args[@]}" "-logger:$logger_path" )
 370   fi
 371
 372   MSBuild-Core ${args[@]}
 373 }
 374
 375 function MSBuild-Core {
 376   if [[ "$ci" == true ]]; then
 377     if [[ "$binary_log" != true ]]; then
 378       Write-PipelineTaskError "Binary log must be enabled in CI build."
 379       ExitWithExitCode 1
 380     fi
 381
 382     if [[ "$node_reuse" == true ]]; then
 383       Write-PipelineTaskError "Node reuse must be disabled in CI build."
 384       ExitWithExitCode 1
 385     fi
 386   fi
 387
 388   InitializeBuildTool
 389
 390   local warnaserror_switch=""
 391   if [[ $warn_as_error == true ]]; then
 392     warnaserror_switch="/warnaserror"
 393   fi
 394
 395   "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || {
 396     local exit_code=$?
 397     Write-PipelineTaskError "Build failed (exit code '$exit_code')."
 398     ExitWithExitCode $exit_code
 399   }
 400 }
 401
 402 ResolvePath "${BASH_SOURCE[0]}"
 403 _script_dir=`dirname "$_ResolvePath"`
 404
 405 . "$_script_dir/pipeline-logging-functions.sh"
 406
 407 eng_root=`cd -P "$_script_dir/.." && pwd`
 408 repo_root=`cd -P "$_script_dir/../.." && pwd`
 409 artifacts_dir="$repo_root/artifacts"
 410 toolset_dir="$artifacts_dir/toolset"
 411 tools_dir="$repo_root/.tools"
 412 log_dir="$artifacts_dir/log/$configuration"
 413 temp_dir="$artifacts_dir/tmp/$configuration"
 414
 415 global_json_file="$repo_root/global.json"
 416 # determine if global.json contains a "runtimes" entry
 417 global_json_has_runtimes=false
 418 dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true
 419 if [[ -n "$dotnetlocal_key" ]]; then
 420   global_json_has_runtimes=true
 421 fi
 422
 423 # HOME may not be defined in some scenarios, but it is required by NuGet
 424 if [[ -z $HOME ]]; then
 425   export HOME="$repo_root/artifacts/.home/"
 426   mkdir -p "$HOME"
 427 fi
 428
 429 mkdir -p "$toolset_dir"
 430 mkdir -p "$temp_dir"
 431 mkdir -p "$log_dir"
 432
 433 Write-PipelineSetVariable -name "Artifacts" -value "$artifacts_dir"
 434 Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir"
 435 Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir"
 436 Write-PipelineSetVariable -name "Temp" -value "$temp_dir"
 437 Write-PipelineSetVariable -name "TMP" -value "$temp_dir"
 438
 439 # Import custom tools configuration, if present in the repo.
 440 if [[ "$disable_configure_toolset_import" != null ]]; then
 441   configure_toolset_script="$eng_root/configure-toolset.sh"
 442   if [[ -a "$configure_toolset_script" ]]; then
 443     . "$configure_toolset_script"
 444   fi
 445 fi
 446
 447 # TODO: https://github.com/dotnet/arcade/issues/1468
 448 # Temporary workaround to avoid breaking change.
 449 # Remove once repos are updated.
 450 if [[ -n "${useInstalledDotNetCli:-}" ]]; then
 451   use_installed_dotnet_cli="$useInstalledDotNetCli"
 452 fi