Tests/RunCMake/RunCMake.cmake

   1 # Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
   2 # file Copyright.txt or https://cmake.org/licensing for details.
   3
   4 foreach(
   5   arg
   6   IN ITEMS
   7     RunCMake_GENERATOR
   8     RunCMake_SOURCE_DIR
   9     RunCMake_BINARY_DIR
  10   )
  11   if(NOT DEFINED ${arg})
  12     message(FATAL_ERROR "${arg} not given!")
  13   endif()
  14 endforeach()
  15
  16 function(run_cmake test)
  17   if(DEFINED ENV{RunCMake_TEST_FILTER})
  18     set(test_and_variant "${test}${RunCMake_TEST_VARIANT_DESCRIPTION}")
  19     if(NOT test_and_variant MATCHES "$ENV{RunCMake_TEST_FILTER}")
  20       return()
  21     endif()
  22     unset(test_and_variant)
  23   endif()
  24
  25   set(top_src "${RunCMake_SOURCE_DIR}")
  26   set(top_bin "${RunCMake_BINARY_DIR}")
  27   if(EXISTS ${top_src}/${test}-result.txt)
  28     file(READ ${top_src}/${test}-result.txt expect_result)
  29     string(REGEX REPLACE "\n+$" "" expect_result "${expect_result}")
  30   elseif(DEFINED RunCMake_TEST_EXPECT_RESULT)
  31     set(expect_result "${RunCMake_TEST_EXPECT_RESULT}")
  32   else()
  33     set(expect_result 0)
  34   endif()
  35
  36   string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} platform_name)
  37   #remove all additional bits from cygwin/msys name
  38   if(platform_name MATCHES cygwin)
  39     set(platform_name cygwin)
  40   endif()
  41   if(platform_name MATCHES msys)
  42     set(platform_name msys)
  43   endif()
  44
  45   foreach(o IN ITEMS stdout stderr config)
  46     if(RunCMake-${o}-file AND EXISTS ${top_src}/${RunCMake-${o}-file})
  47       file(READ ${top_src}/${RunCMake-${o}-file} expect_${o})
  48       string(REGEX REPLACE "\n+$" "" expect_${o} "${expect_${o}}")
  49     elseif(EXISTS ${top_src}/${test}-${o}-${platform_name}.txt)
  50       file(READ ${top_src}/${test}-${o}-${platform_name}.txt expect_${o})
  51       string(REGEX REPLACE "\n+$" "" expect_${o} "${expect_${o}}")
  52     elseif(EXISTS ${top_src}/${test}-${o}.txt)
  53       file(READ ${top_src}/${test}-${o}.txt expect_${o})
  54       string(REGEX REPLACE "\n+$" "" expect_${o} "${expect_${o}}")
  55     elseif(DEFINED RunCMake_TEST_EXPECT_${o})
  56       string(REGEX REPLACE "\n+$" "" expect_${o} "${RunCMake_TEST_EXPECT_${o}}")
  57     else()
  58       unset(expect_${o})
  59     endif()
  60   endforeach()
  61   foreach(o IN ITEMS stdout stderr config)
  62     if(DEFINED RunCMake_TEST_NOT_EXPECT_${o})
  63       string(REGEX REPLACE "\n+$" "" not_expect_${o} "${RunCMake_TEST_NOT_EXPECT_${o}}")
  64     endif()
  65   endforeach()
  66   if (NOT expect_stderr)
  67     if (NOT RunCMake_DEFAULT_stderr)
  68       set(RunCMake_DEFAULT_stderr "^$")
  69     endif()
  70     set(expect_stderr ${RunCMake_DEFAULT_stderr})
  71   endif()
  72
  73   if (NOT RunCMake_TEST_SOURCE_DIR)
  74     set(RunCMake_TEST_SOURCE_DIR "${top_src}")
  75   endif()
  76   if(NOT RunCMake_TEST_BINARY_DIR)
  77     set(RunCMake_TEST_BINARY_DIR "${top_bin}/${test}-build")
  78   endif()
  79   if(NOT RunCMake_TEST_NO_CLEAN)
  80     file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}")
  81   endif()
  82   file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
  83   if(RunCMake-prep-file AND EXISTS ${top_src}/${RunCMake-prep-file})
  84     include(${top_src}/${RunCMake-prep-file})
  85   else()
  86     include(${top_src}/${test}-prep.cmake OPTIONAL)
  87   endif()
  88   if(RunCMake_TEST_OUTPUT_MERGE)
  89     set(actual_stderr_var actual_stdout)
  90     set(actual_stderr "")
  91   else()
  92     set(actual_stderr_var actual_stderr)
  93   endif()
  94   if(DEFINED RunCMake_TEST_TIMEOUT)
  95     set(maybe_timeout TIMEOUT ${RunCMake_TEST_TIMEOUT})
  96   else()
  97     set(maybe_timeout "")
  98   endif()
  99   if(RunCMake-stdin-file AND EXISTS ${top_src}/${RunCMake-stdin-file})
 100     set(maybe_input_file INPUT_FILE ${top_src}/${RunCMake-stdin-file})
 101   elseif(EXISTS ${top_src}/${test}-stdin.txt)
 102     set(maybe_input_file INPUT_FILE ${top_src}/${test}-stdin.txt)
 103   else()
 104     set(maybe_input_file "")
 105   endif()
 106   if(NOT RunCMake_TEST_COMMAND)
 107     if(NOT DEFINED RunCMake_TEST_OPTIONS)
 108       set(RunCMake_TEST_OPTIONS "")
 109     endif()
 110     if(APPLE)
 111       list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_POLICY_DEFAULT_CMP0025=NEW)
 112     endif()
 113     if(RunCMake_TEST_LCC AND NOT RunCMake_TEST_NO_CMP0129)
 114       list(APPEND RunCMake_TEST_OPTIONS -DCMAKE_POLICY_DEFAULT_CMP0129=NEW)
 115     endif()
 116     if(RunCMake_MAKE_PROGRAM)
 117       list(APPEND RunCMake_TEST_OPTIONS "-DCMAKE_MAKE_PROGRAM=${RunCMake_MAKE_PROGRAM}")
 118     endif()
 119     set(RunCMake_TEST_COMMAND ${CMAKE_COMMAND})
 120     if(NOT RunCMake_TEST_NO_SOURCE_DIR)
 121       list(APPEND RunCMake_TEST_COMMAND "${RunCMake_TEST_SOURCE_DIR}")
 122     endif()
 123     list(APPEND RunCMake_TEST_COMMAND -G "${RunCMake_GENERATOR}")
 124     if(RunCMake_GENERATOR_PLATFORM)
 125       list(APPEND RunCMake_TEST_COMMAND -A "${RunCMake_GENERATOR_PLATFORM}")
 126     endif()
 127     if(RunCMake_GENERATOR_TOOLSET)
 128       list(APPEND RunCMake_TEST_COMMAND -T "${RunCMake_GENERATOR_TOOLSET}")
 129     endif()
 130     if(RunCMake_GENERATOR_INSTANCE)
 131       list(APPEND RunCMake_TEST_COMMAND "-DCMAKE_GENERATOR_INSTANCE=${RunCMake_GENERATOR_INSTANCE}")
 132     endif()
 133     list(APPEND RunCMake_TEST_COMMAND
 134       -DRunCMake_TEST=${test}
 135       --no-warn-unused-cli
 136       )
 137   else()
 138     set(RunCMake_TEST_OPTIONS "")
 139   endif()
 140   if(NOT DEFINED RunCMake_TEST_RAW_ARGS)
 141     set(RunCMake_TEST_RAW_ARGS "")
 142   endif()
 143   if(NOT RunCMake_TEST_COMMAND_WORKING_DIRECTORY)
 144     set(RunCMake_TEST_COMMAND_WORKING_DIRECTORY "${RunCMake_TEST_BINARY_DIR}")
 145   endif()
 146   string(CONCAT _code [[execute_process(
 147     COMMAND ${RunCMake_TEST_COMMAND}
 148             ${RunCMake_TEST_OPTIONS}
 149             ]] "${RunCMake_TEST_RAW_ARGS}\n" [[
 150     WORKING_DIRECTORY "${RunCMake_TEST_COMMAND_WORKING_DIRECTORY}"
 151     OUTPUT_VARIABLE actual_stdout
 152     ERROR_VARIABLE ${actual_stderr_var}
 153     RESULT_VARIABLE actual_result
 154     ENCODING UTF8
 155     ${maybe_timeout}
 156     ${maybe_input_file}
 157     )]])
 158   if(DEFINED ENV{PWD})
 159     set(old_pwd "$ENV{PWD}")
 160   else()
 161     set(old_pwd)
 162   endif()
 163   # Emulate a shell using this directory.
 164   set(ENV{PWD} "${RunCMake_TEST_COMMAND_WORKING_DIRECTORY}")
 165   cmake_language(EVAL CODE "${_code}")
 166   if(DEFINED old_pwd)
 167     set(ENV{PWD} "${old_pwd}")
 168   else()
 169     set(ENV{PWD})
 170   endif()
 171   set(msg "")
 172   if(NOT "${actual_result}" MATCHES "${expect_result}")
 173     string(APPEND msg "Result is [${actual_result}], not [${expect_result}].\n")
 174   endif()
 175   set(config_file "${RunCMake_TEST_COMMAND_WORKING_DIRECTORY}/CMakeFiles/CMakeConfigureLog.yaml")
 176   if(EXISTS "${config_file}")
 177     file(READ "${config_file}" actual_config)
 178   else()
 179     set(actual_config "")
 180   endif()
 181
 182   # Special case: remove ninja no-op line from stderr, but not stdout.
 183   # Test cases that look for it should use RunCMake_TEST_OUTPUT_MERGE.
 184   string(REGEX REPLACE "(^|\r?\n)ninja: no work to do\\.\r?\n" "\\1" actual_stderr "${actual_stderr}")
 185
 186   # Remove incidental content from both stdout and stderr.
 187   string(CONCAT ignore_line_regex
 188     "(^|\n)((==[0-9]+=="
 189     "|BullseyeCoverage"
 190     "|[a-z]+\\([0-9]+\\) malloc:"
 191     "|clang[^:]*: warning: the object size sanitizer has no effect at -O0, but is explicitly enabled:"
 192     "|flang-new: warning: argument unused during compilation: .-flang-experimental-exec."
 193     "|icp?x: remark: Note that use of .-g. without any optimization-level option will turn off most compiler optimizations"
 194     "|ifx: remark #10440: Note that use of a debug option without any optimization-level option will turnoff most compiler optimizations"
 195     "|lld-link: warning: procedure symbol record for .* refers to PDB item index [0-9A-Fa-fx]+ which is not a valid function ID record"
 196     "|Error kstat returned"
 197     "|Hit xcodebuild bug"
 198     "|Recompacting log\\.\\.\\."
 199
 200     "|LICENSE WARNING:"
 201     "|Your license to use PGI[^\n]*expired"
 202     "|Please obtain a new version at"
 203     "|contact PGI Sales at"
 204     "|ic(p?c|l): remark #10441: The Intel\\(R\\) C\\+\\+ Compiler Classic \\(ICC\\) is deprecated"
 205
 206     "|[^\n]*install_name_tool: warning: changes being made to the file will invalidate the code signature in:"
 207     "|[^\n]*(createItemModels|_NSMainThread|Please file a bug at)"
 208     "|[^\n]*xcodebuild[^\n]*DVTAssertions: Warning"
 209     "|[^\n]*xcodebuild[^\n]*DVTCoreDeviceEnabledState: DVTCoreDeviceEnabledState_Disabled set via user default"
 210     "|[^\n]*xcodebuild[^\n]*DVTPlugInManager"
 211     "|[^\n]*xcodebuild[^\n]*DVTSDK: Warning: SDK path collision for path"
 212     "|[^\n]*xcodebuild[^\n]*Requested but did not find extension point with identifier"
 213     "|[^\n]*xcodebuild[^\n]*nil host used in call to allows.*HTTPSCertificateForHost"
 214     "|[^\n]*xcodebuild[^\n]*warning: file type[^\n]*is based on missing file type"
 215     "|[^\n]*objc[^\n]*: Class [^\n]* One of the two will be used. Which one is undefined."
 216     "|[^\n]*is a member of multiple groups"
 217     "|[^\n]*offset in archive not a multiple of 8"
 218     "|[^\n]*from Time Machine by path"
 219     "|[^\n]*Bullseye Testing Technology"
 220     ${RunCMake_TEST_EXTRA_IGNORE_LINE_REGEX}
 221     ")[^\n]*\n)+"
 222     )
 223   if(RunCMake_IGNORE_POLICY_VERSION_DEPRECATION)
 224     string(REGEX REPLACE [[
 225 ^CMake Deprecation Warning at [^
 226 ]*CMakeLists.txt:1 \(cmake_minimum_required\):
 227   Compatibility with CMake < 3\.10 will be removed from a future version of
 228   CMake.
 229
 230   Update the VERSION argument <min> value or use a \.\.\.<max> suffix to tell
 231   CMake that the project does not need compatibility with older versions\.
 232 +
 233 ]] "" actual_stderr "${actual_stderr}")
 234   endif()
 235   foreach(o IN ITEMS stdout stderr config)
 236     string(REGEX REPLACE "\r\n" "\n" actual_${o} "${actual_${o}}")
 237     string(REGEX REPLACE "${ignore_line_regex}" "\\1" actual_${o} "${actual_${o}}")
 238     string(REGEX REPLACE "\n+$" "" actual_${o} "${actual_${o}}")
 239     if(DEFINED expect_${o})
 240       if(NOT "${actual_${o}}" MATCHES "${expect_${o}}")
 241         string(APPEND msg "${o} does not match that expected.\n")
 242       endif()
 243     endif()
 244     if(DEFINED not_expect_${o})
 245       if("${actual_${o}}" MATCHES "${not_expect_${o}}")
 246         string(APPEND msg "${o} matches that not expected.\n")
 247       endif()
 248     endif()
 249   endforeach()
 250   unset(RunCMake_TEST_FAILED)
 251   unset(RunCMake_TEST_FAILURE_MESSAGE)
 252   if(RunCMake-check-file AND EXISTS ${top_src}/${RunCMake-check-file})
 253     include(${top_src}/${RunCMake-check-file})
 254   else()
 255     include(${top_src}/${test}-check.cmake OPTIONAL)
 256   endif()
 257   if(RunCMake_TEST_FAILED)
 258     set(msg "${RunCMake_TEST_FAILED}\n${msg}")
 259   endif()
 260   if(msg)
 261     string(REPLACE ";" "\" \"" command "\"${RunCMake_TEST_COMMAND}\"")
 262     if(RunCMake_TEST_OPTIONS)
 263       string(REPLACE ";" "\" \"" options "\"${RunCMake_TEST_OPTIONS}\"")
 264       string(APPEND command " ${options}")
 265     endif()
 266     if(RunCMake_TEST_RAW_ARGS)
 267       string(APPEND command " ${RunCMake_TEST_RAW_ARGS}")
 268     endif()
 269     string(APPEND msg "Command was:\n command> ${command}\n")
 270   endif()
 271   if(msg)
 272     foreach(o IN ITEMS stdout stderr config)
 273       if(DEFINED expect_${o})
 274         string(REGEX REPLACE "\n" "\n expect-${o}> " expect_${o} " expect-${o}> ${expect_${o}}")
 275         string(APPEND msg "Expected ${o} to match:\n${expect_${o}}\n")
 276       endif()
 277       if(NOT o STREQUAL "config" OR DEFINED expect_${o})
 278         string(REGEX REPLACE "\n" "\n actual-${o}> " actual_${o} " actual-${o}> ${actual_${o}}")
 279         string(APPEND msg "Actual ${o}:\n${actual_${o}}\n")
 280       endif()
 281     endforeach()
 282     if(RunCMake_TEST_FAILURE_MESSAGE)
 283       string(APPEND msg "${RunCMake_TEST_FAILURE_MESSAGE}")
 284     endif()
 285     message(SEND_ERROR "${test}${RunCMake_TEST_VARIANT_DESCRIPTION} - FAILED:\n${msg}")
 286   else()
 287     message(STATUS "${test}${RunCMake_TEST_VARIANT_DESCRIPTION} - PASSED")
 288   endif()
 289 endfunction()
 290
 291 function(run_cmake_command test)
 292   set(RunCMake_TEST_COMMAND "${ARGN}")
 293   run_cmake(${test})
 294 endfunction()
 295
 296 function(run_cmake_script test)
 297   set(RunCMake_TEST_COMMAND ${CMAKE_COMMAND} ${ARGN} -P ${RunCMake_SOURCE_DIR}/${test}.cmake)
 298   run_cmake(${test})
 299 endfunction()
 300
 301 function(run_cmake_with_options test)
 302   set(RunCMake_TEST_OPTIONS "${ARGN}")
 303   run_cmake(${test})
 304 endfunction()
 305
 306 function(run_cmake_with_raw_args test args)
 307   set(RunCMake_TEST_RAW_ARGS "${args}")
 308   run_cmake(${test})
 309 endfunction()
 310
 311 function(ensure_files_match expected_file actual_file)
 312   if(NOT EXISTS "${expected_file}")
 313     message(FATAL_ERROR "Expected file does not exist:\n  ${expected_file}")
 314   endif()
 315   if(NOT EXISTS "${actual_file}")
 316     message(FATAL_ERROR "Actual file does not exist:\n  ${actual_file}")
 317   endif()
 318   file(READ "${expected_file}" expected_file_content)
 319   file(READ "${actual_file}" actual_file_content)
 320   if(NOT "${expected_file_content}" STREQUAL "${actual_file_content}")
 321     message(FATAL_ERROR "Actual file content does not match expected:\n
 322     \n
 323       expected file: ${expected_file}\n
 324       expected content:\n
 325       ${expected_file_content}\n
 326     \n
 327       actual file: ${actual_file}\n
 328       actual content:\n
 329       ${actual_file_content}\n
 330     ")
 331   endif()
 332 endfunction()
 333
 334 # Get the user id on unix if possible.
 335 function(get_unix_uid var)
 336   set("${var}" "" PARENT_SCOPE)
 337   if(UNIX)
 338     set(ID "id")
 339     if(CMAKE_SYSTEM_NAME STREQUAL "SunOS" AND EXISTS "/usr/xpg4/bin/id")
 340       set (ID "/usr/xpg4/bin/id")
 341     endif()
 342     execute_process(COMMAND ${ID} -u $ENV{USER} OUTPUT_VARIABLE uid ERROR_QUIET
 343                     RESULT_VARIABLE status OUTPUT_STRIP_TRAILING_WHITESPACE)
 344     if(status EQUAL 0)
 345       set("${var}" "${uid}" PARENT_SCOPE)
 346     endif()
 347   endif()
 348 endfunction()
 349
 350 # Protect RunCMake tests from calling environment.
 351 unset(ENV{MAKEFLAGS})