eng/common/tools.sh

   1 #!/usr/bin/env bash
   2
   3 # Initialize variables if they aren't already defined.
   4
   5 # CI mode - set to true on CI server for PR validation build or official build.
   6 ci=${ci:-false}
   7
   8 # Set to true to use the pipelines logger which will enable Azure logging output.
   9 # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
  10 # This flag is meant as a temporary opt-opt for the feature while validate it across
  11 # our consumers. It will be deleted in the future.
  12 if [[ "$ci" == true ]]; then
  13   pipelines_log=${pipelines_log:-true}
  14 else
  15   pipelines_log=${pipelines_log:-false}
  16 fi
  17
  18 # Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
  19 configuration=${configuration:-'Debug'}
  20
  21 # Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
  22 # Binary log must be enabled on CI.
  23 binary_log=${binary_log:-$ci}
  24
  25 # Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
  26 prepare_machine=${prepare_machine:-false}
  27
  28 # True to restore toolsets and dependencies.
  29 restore=${restore:-true}
  30
  31 # Adjusts msbuild verbosity level.
  32 verbosity=${verbosity:-'minimal'}
  33
  34 # Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
  35 if [[ "$ci" == true ]]; then
  36   node_reuse=${node_reuse:-false}
  37 else
  38   node_reuse=${node_reuse:-true}
  39 fi
  40
  41 # Configures warning treatment in msbuild.
  42 warn_as_error=${warn_as_error:-true}
  43
  44 # True to attempt using .NET Core already that meets requirements specified in global.json
  45 # installed on the machine instead of downloading one.
  46 use_installed_dotnet_cli=${use_installed_dotnet_cli:-true}
  47
  48 # True to use global NuGet cache instead of restoring packages to repository-local directory.
  49 if [[ "$ci" == true ]]; then
  50   use_global_nuget_cache=${use_global_nuget_cache:-false}
  51 else
  52   use_global_nuget_cache=${use_global_nuget_cache:-true}
  53 fi
  54
  55 function EmitError {
  56   if [[ "$ci" != true ]]; then
  57     echo "$@" >&2
  58     return
  59   fi
  60
  61   message_type="error"
  62   sourcepath=''
  63   linenumber=''
  64   columnnumber=''
  65   error_code=''
  66
  67   while [[ $# -gt 0 ]]; do
  68     opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
  69     case "$opt" in
  70       -type|-t)
  71         message_type=$2
  72         shift
  73         ;;
  74       -sourcepath|-s)
  75         sourcepath=$2
  76         shift
  77         ;;
  78       -linenumber|-l)
  79         linenumber=$2
  80         shift
  81         ;;
  82       -columnnumber|-col)
  83         columnnumber=$2
  84         shift
  85         ;;
  86       -code|-c)
  87         error_code=$2
  88         shift
  89         ;;
  90       *)
  91         break
  92         ;;
  93     esac
  94
  95     shift
  96   done
  97
  98   message='##vso[task.logissue'
  99
 100   message="$message type=$message_type"
 101
 102   if [ -n "$sourcepath" ]; then
 103     message="$message;sourcepath=$sourcepath"
 104   else
 105     message="$message;sourcepath=${BASH_SOURCE[1]}"
 106   fi
 107
 108   if [ -n "$linenumber" ]; then
 109     message="$message;linenumber=$linenumber"
 110   else
 111     message="$message;linenumber=${BASH_LINENO[0]}"
 112   fi
 113
 114   if [ -n "$columnnumber" ]; then
 115     message="$message;columnnumber=$columnnumber"
 116   fi
 117
 118   if [ -n "$error_code" ]; then
 119     message="$message;code=$error_code"
 120   fi
 121
 122   message="$message]$*"
 123
 124   echo "$message"
 125 }
 126
 127 # Resolve any symlinks in the given path.
 128 function ResolvePath {
 129   local path=$1
 130
 131   while [[ -h $path ]]; do
 132     local dir="$( cd -P "$( dirname "$path" )" && pwd )"
 133     path="$(readlink "$path")"
 134
 135     # if $path was a relative symlink, we need to resolve it relative to the path where the
 136     # symlink file was located
 137     [[ $path != /* ]] && path="$dir/$path"
 138   done
 139
 140   # return value
 141   _ResolvePath="$path"
 142 }
 143
 144 # ReadVersionFromJson [json key]
 145 function ReadGlobalVersion {
 146   local key=$1
 147
 148   local line=`grep -m 1 "$key" "$global_json_file"`
 149   local pattern="\"$key\" *: *\"(.*)\""
 150
 151   if [[ ! $line =~ $pattern ]]; then
 152     EmitError "Error: Cannot find \"$key\" in $global_json_file"
 153     ExitWithExitCode 1
 154   fi
 155
 156   # return value
 157   _ReadGlobalVersion=${BASH_REMATCH[1]}
 158 }
 159
 160 function InitializeDotNetCli {
 161   if [[ -n "${_InitializeDotNetCli:-}" ]]; then
 162     return
 163   fi
 164
 165   local install=$1
 166
 167   # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
 168   export DOTNET_MULTILEVEL_LOOKUP=0
 169
 170   # Disable first run since we want to control all package sources
 171   export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 172
 173   # Disable telemetry on CI
 174   if [[ $ci == true ]]; then
 175     export DOTNET_CLI_TELEMETRY_OPTOUT=1
 176   fi
 177
 178   # LTTNG is the logging infrastructure used by Core CLR. Need this variable set
 179   # so it doesn't output warnings to the console.
 180   export LTTNG_HOME="$HOME"
 181
 182   # Source Build uses DotNetCoreSdkDir variable
 183   if [[ -n "${DotNetCoreSdkDir:-}" ]]; then
 184     export DOTNET_INSTALL_DIR="$DotNetCoreSdkDir"
 185   fi
 186
 187   # Find the first path on $PATH that contains the dotnet.exe
 188   if [[ "$use_installed_dotnet_cli" == true && $global_json_has_runtimes == false && -z "${DOTNET_INSTALL_DIR:-}" ]]; then
 189     local dotnet_path=`command -v dotnet`
 190     if [[ -n "$dotnet_path" ]]; then
 191       ResolvePath "$dotnet_path"
 192       export DOTNET_INSTALL_DIR=`dirname "$_ResolvePath"`
 193     fi
 194   fi
 195
 196   ReadGlobalVersion "dotnet"
 197   local dotnet_sdk_version=$_ReadGlobalVersion
 198   local dotnet_root=""
 199
 200   # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
 201   # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
 202   if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
 203     dotnet_root="$DOTNET_INSTALL_DIR"
 204   else
 205     dotnet_root="$repo_root/.dotnet"
 206
 207     export DOTNET_INSTALL_DIR="$dotnet_root"
 208
 209     if [[ ! -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then
 210       if [[ "$install" == true ]]; then
 211         InstallDotNetSdk "$dotnet_root" "$dotnet_sdk_version"
 212       else
 213         EmitError "Unable to find dotnet with SDK version '$dotnet_sdk_version'"
 214         ExitWithExitCode 1
 215       fi
 216     fi
 217   fi
 218
 219   # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
 220   # build steps from using anything other than what we've downloaded.
 221   export PATH="$dotnet_root:$PATH"
 222
 223   if [[ $ci == true ]]; then
 224     # Make Sure that our bootstrapped dotnet cli is avaliable in future steps of the Azure Pipelines build
 225     echo "##vso[task.prependpath]$dotnet_root"
 226     echo "##vso[task.setvariable variable=DOTNET_MULTILEVEL_LOOKUP]0"
 227     echo "##vso[task.setvariable variable=DOTNET_SKIP_FIRST_TIME_EXPERIENCE]1"
 228   fi
 229
 230   # return value
 231   _InitializeDotNetCli="$dotnet_root"
 232 }
 233
 234 function InstallDotNetSdk {
 235   local root=$1
 236   local version=$2
 237   local architecture=""
 238   if [[ $# == 3 ]]; then
 239     architecture=$3
 240   fi
 241   InstallDotNet "$root" "$version" $architecture
 242 }
 243
 244 function InstallDotNet {
 245   local root=$1
 246   local version=$2
 247
 248   GetDotNetInstallScript "$root"
 249   local install_script=$_GetDotNetInstallScript
 250
 251   local archArg=''
 252   if [[ -n "${3:-}" ]]; then
 253     archArg="--architecture $3"
 254   fi
 255   local runtimeArg=''
 256   if [[ -n "${4:-}" ]]; then
 257     runtimeArg="--runtime $4"
 258   fi
 259
 260   local skipNonVersionedFilesArg=""
 261   if [[ "$#" -ge "5" ]]; then
 262     skipNonVersionedFilesArg="--skip-non-versioned-files"
 263   fi
 264   bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
 265     local exit_code=$?
 266     EmitError "Failed to install dotnet SDK (exit code '$exit_code')."
 267     ExitWithExitCode $exit_code
 268   }
 269 }
 270
 271 function GetDotNetInstallScript {
 272   local root=$1
 273   local install_script="$root/dotnet-install.sh"
 274   local install_script_url="https://dot.net/v1/dotnet-install.sh"
 275
 276   if [[ ! -a "$install_script" ]]; then
 277     mkdir -p "$root"
 278
 279     echo "Downloading '$install_script_url'"
 280
 281     # Use curl if available, otherwise use wget
 282     if command -v curl > /dev/null; then
 283       curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script"
 284     else
 285       wget -q -O "$install_script" "$install_script_url"
 286     fi
 287   fi
 288
 289   # return value
 290   _GetDotNetInstallScript="$install_script"
 291 }
 292
 293 function InitializeBuildTool {
 294   if [[ -n "${_InitializeBuildTool:-}" ]]; then
 295     return
 296   fi
 297
 298   InitializeDotNetCli $restore
 299
 300   # return values
 301   _InitializeBuildTool="$_InitializeDotNetCli/dotnet"
 302   _InitializeBuildToolCommand="msbuild"
 303   _InitializeBuildToolFramework="netcoreapp2.1"
 304 }
 305
 306 function GetNuGetPackageCachePath {
 307   if [[ -z ${NUGET_PACKAGES:-} ]]; then
 308     if [[ "$use_global_nuget_cache" == true ]]; then
 309       export NUGET_PACKAGES="$HOME/.nuget/packages"
 310     else
 311       export NUGET_PACKAGES="$repo_root/.packages"
 312     fi
 313   fi
 314
 315   # return value
 316   _GetNuGetPackageCachePath=$NUGET_PACKAGES
 317 }
 318
 319 function InitializeNativeTools() {
 320   if grep -Fq "native-tools" $global_json_file
 321   then
 322     local nativeArgs=""
 323     if [[ "$ci" == true ]]; then
 324       nativeArgs="-InstallDirectory $tools_dir"
 325     fi
 326     "$_script_dir/init-tools-native.sh" $nativeArgs
 327   fi
 328 }
 329
 330 function InitializeToolset {
 331   if [[ -n "${_InitializeToolset:-}" ]]; then
 332     return
 333   fi
 334
 335   GetNuGetPackageCachePath
 336
 337   ReadGlobalVersion "Microsoft.DotNet.Arcade.Sdk"
 338
 339   local toolset_version=$_ReadGlobalVersion
 340   local toolset_location_file="$toolset_dir/$toolset_version.txt"
 341
 342   if [[ -a "$toolset_location_file" ]]; then
 343     local path=`cat "$toolset_location_file"`
 344     if [[ -a "$path" ]]; then
 345       # return value
 346       _InitializeToolset="$path"
 347       return
 348     fi
 349   fi
 350
 351   if [[ "$restore" != true ]]; then
 352     EmitError "Toolset version $toolsetVersion has not been restored."
 353     ExitWithExitCode 2
 354   fi
 355
 356   local proj="$toolset_dir/restore.proj"
 357
 358   local bl=""
 359   if [[ "$binary_log" == true ]]; then
 360     bl="/bl:$log_dir/ToolsetRestore.binlog"
 361   fi
 362
 363   echo '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' > "$proj"
 364   MSBuild-Core "$proj" $bl /t:__WriteToolsetLocation /clp:ErrorsOnly\;NoSummary /p:__ToolsetLocationOutputFile="$toolset_location_file"
 365
 366   local toolset_build_proj=`cat "$toolset_location_file"`
 367
 368   if [[ ! -a "$toolset_build_proj" ]]; then
 369     EmitError "Invalid toolset path: $toolset_build_proj"
 370     ExitWithExitCode 3
 371   fi
 372
 373   # return value
 374   _InitializeToolset="$toolset_build_proj"
 375 }
 376
 377 function ExitWithExitCode {
 378   if [[ "$ci" == true && "$prepare_machine" == true ]]; then
 379     StopProcesses
 380   fi
 381   exit $1
 382 }
 383
 384 function StopProcesses {
 385   echo "Killing running build processes..."
 386   pkill -9 "dotnet" || true
 387   pkill -9 "vbcscompiler" || true
 388   return 0
 389 }
 390
 391 function MSBuild {
 392   local args=$@
 393   if [[ "$pipelines_log" == true ]]; then
 394     InitializeBuildTool
 395     InitializeToolset
 396     local toolset_dir="${_InitializeToolset%/*}"
 397     local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll"
 398     args=( "${args[@]}" "-logger:$logger_path" )
 399   fi
 400
 401   MSBuild-Core ${args[@]}
 402 }
 403
 404 function MSBuild-Core {
 405   if [[ "$ci" == true ]]; then
 406     if [[ "$binary_log" != true ]]; then
 407       EmitError "Binary log must be enabled in CI build."
 408       ExitWithExitCode 1
 409     fi
 410
 411     if [[ "$node_reuse" == true ]]; then
 412       EmitError "Node reuse must be disabled in CI build."
 413       ExitWithExitCode 1
 414     fi
 415   fi
 416
 417   InitializeBuildTool
 418
 419   local warnaserror_switch=""
 420   if [[ $warn_as_error == true ]]; then
 421     warnaserror_switch="/warnaserror"
 422   fi
 423
 424   "$_InitializeBuildTool" "$_InitializeBuildToolCommand" /m /nologo /clp:Summary /v:$verbosity /nr:$node_reuse $warnaserror_switch /p:TreatWarningsAsErrors=$warn_as_error /p:ContinuousIntegrationBuild=$ci "$@" || {
 425     local exit_code=$?
 426     EmitError "Build failed (exit code '$exit_code')."
 427     ExitWithExitCode $exit_code
 428   }
 429 }
 430
 431 ResolvePath "${BASH_SOURCE[0]}"
 432 _script_dir=`dirname "$_ResolvePath"`
 433
 434 eng_root=`cd -P "$_script_dir/.." && pwd`
 435 repo_root=`cd -P "$_script_dir/../.." && pwd`
 436 artifacts_dir="$repo_root/artifacts"
 437 toolset_dir="$artifacts_dir/toolset"
 438 tools_dir="$repo_root/.tools"
 439 log_dir="$artifacts_dir/log/$configuration"
 440 temp_dir="$artifacts_dir/tmp/$configuration"
 441
 442 global_json_file="$repo_root/global.json"
 443 # determine if global.json contains a "runtimes" entry
 444 global_json_has_runtimes=false
 445 dotnetlocal_key=`grep -m 1 "runtimes" "$global_json_file"` || true
 446 if [[ -n "$dotnetlocal_key" ]]; then
 447   global_json_has_runtimes=true
 448 fi
 449
 450 # HOME may not be defined in some scenarios, but it is required by NuGet
 451 if [[ -z $HOME ]]; then
 452   export HOME="$repo_root/artifacts/.home/"
 453   mkdir -p "$HOME"
 454 fi
 455
 456 mkdir -p "$toolset_dir"
 457 mkdir -p "$temp_dir"
 458 mkdir -p "$log_dir"
 459
 460 if [[ $ci == true ]]; then
 461   export TEMP="$temp_dir"
 462   export TMP="$temp_dir"
 463 fi