usr.bin/dfregress/runlist.c

   1 /*
   2  * Copyright (c) 2011 Alex Hornung <alex@alexhornung.com>.
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in
  13  *    the documentation and/or other materials provided with the
  14  *    distribution.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  17  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  18  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  19  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  20  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  22  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  23  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  24  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  25  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  26  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  27  * SUCH DAMAGE.
  28  */
  29
  30 #include <sys/resource.h>
  31 #include <sys/time.h>
  32 #include <sys/types.h>
  33 #include <sys/wait.h>
  34
  35 #include <errno.h>
  36 #include <limits.h>
  37 #include <fcntl.h>
  38 #include <signal.h>
  39 #include <stdio.h>
  40 #include <stdlib.h>
  41 #include <stdint.h>
  42 #include <string.h>
  43 #include <unistd.h>
  44 #include <pwd.h>
  45
  46 #include <err.h>
  47
  48 #include <libprop/proplib.h>
  49
  50 #include "parser.h"
  51 #include "testcase.h"
  52 #include "runlist.h"
  53 #include "userland.h"
  54 #include "kernel.h"
  55 #include <dfregress.h>
  56
  57 char output_file[PATH_MAX+1];
  58 char testcase_dir[PATH_MAX+1];
  59 char prepost_dir[PATH_MAX+1];
  60
  61 prop_array_t
  62 runlist_load_from_text(const char *runlist_file)
  63 {
  64         prop_array_t runlist;
  65
  66         runlist = prop_array_create_with_capacity(2048);
  67
  68         process_file(runlist_file, testcase_entry_parser, runlist, NULL);
  69
  70         return runlist;
  71 }
  72
  73 prop_array_t
  74 runlist_load(const char *runlist_file)
  75 {
  76         return prop_array_internalize_from_file(runlist_file);
  77 }
  78
  79 int
  80 runlist_save(const char *runlist_file, prop_array_t runlist)
  81 {
  82         return !prop_array_externalize_to_file(runlist, runlist_file);
  83 }
  84
  85 int
  86 runlist_iterate(prop_array_t runlist, runlist_iterator_t iterator, void *arg)
  87 {
  88         prop_object_iterator_t it;
  89         prop_dictionary_t testcase;
  90         int r = 0;
  91
  92         it = prop_array_iterator(runlist);
  93         if (it == NULL)
  94                 err(1, "could not get runlist iterator");
  95
  96         while ((testcase = prop_object_iterator_next(it)) != NULL) {
  97                 r = iterator(arg, testcase);
  98                 if (r != 0)
  99                         break;
 100         }
 101
 102         prop_object_iterator_release(it);
 103         return r;
 104 }
 105
 106 int
 107 runlist_run_test(void *arg, prop_dictionary_t testcase)
 108 {
 109         prop_array_t runlist = (prop_array_t)arg;
 110         struct testcase_result tr;
 111         char testcase_path[FILENAME_MAX+2];
 112         char testcase_dir_only[FILENAME_MAX];
 113         char prepost_path[FILENAME_MAX+2];
 114         char errbuf[FILENAME_MAX*2];
 115         int r, nopre, nopost;
 116         char *str;
 117
 118         sprintf(testcase_path, "%s/%s", testcase_dir,
 119             testcase_get_name(testcase));
 120         strcpy(testcase_dir_only, testcase_path);
 121         str = strrchr(testcase_dir_only, '/');
 122         if (str != NULL)
 123                 *str = '\0';
 124
 125         printf("Running testcase %s... ", testcase_get_name(testcase));
 126         fflush(stdout);
 127
 128         /* Switch to testcase directory */
 129         r = chdir(testcase_dir_only);
 130         if (r < 0) {
 131                 sprintf(errbuf, "could not switch working directory to %s: %s\n",
 132                     testcase_dir_only, strerror(errno));
 133                 testcase_set_result(testcase, RESULT_PREFAIL);
 134                 testcase_set_sys_buf(testcase, errbuf);
 135                 goto out;
 136         }
 137
 138         /* build unless nobuild flag is set */
 139         if ((testcase_get_flags(testcase) & TESTCASE_NOBUILD) == 0) {
 140                 r = run_simple_cmd(testcase_get_make_cmd(testcase), NULL,
 141                     errbuf, sizeof(errbuf), &tr);
 142                 if (r != 0) {
 143                         testcase_set_sys_buf(testcase, errbuf);
 144                         testcase_set_result(testcase, RESULT_PREFAIL);
 145                         goto out;
 146                 }
 147
 148                 if (tr.stdout_buf != NULL) {
 149                         testcase_set_build_buf(testcase, tr.stdout_buf);
 150                         free(tr.stdout_buf);
 151                 }
 152
 153                 if (tr.result != RESULT_PASS) {
 154                         if (testcase_get_type(testcase)
 155                             == TESTCASE_TYPE_BUILDONLY)
 156                                 testcase_set_result(testcase, tr.result);
 157                         else
 158                                 testcase_set_result(testcase, RESULT_BUILDFAIL);
 159
 160                         testcase_set_exit_value(testcase, tr.exit_value);
 161                         testcase_set_signal(testcase, tr.signal);
 162
 163                         goto out;
 164                 }
 165         }
 166
 167
 168         /* Pre-execution run */
 169         switch (testcase_get_precmd_type(testcase)) {
 170         case TESTCASE_INT_PRE:
 171                 /* Test case has internal but explicit PRE - code */
 172                 r = run_simple_cmd(testcase_path, "pre", errbuf,
 173                     sizeof(errbuf), &tr);
 174                 nopre = 0;
 175                 break;
 176
 177         case TESTCASE_CUSTOM_PRE:
 178                 /* Test case uses external and explicit PRE command */
 179                 sprintf(prepost_path, "%s/%s", prepost_dir,
 180                     testcase_get_custom_precmd(testcase));
 181
 182                 r = run_simple_cmd(prepost_path, NULL, errbuf, sizeof(errbuf),
 183                     &tr);
 184                 nopre = 0;
 185                 break;
 186
 187         default:
 188                 nopre = 1;
 189                 r = 0;
 190                 break;
 191         }
 192
 193         if (!nopre) {
 194                 if (r != 0) {
 195                         testcase_set_sys_buf(testcase, errbuf);
 196                         testcase_set_result(testcase, RESULT_PREFAIL);
 197                         goto out;
 198                 }
 199
 200                 if (tr.stdout_buf != NULL) {
 201                         testcase_set_precmd_buf(testcase, tr.stdout_buf);
 202                         free(tr.stdout_buf);
 203                 }
 204
 205                 if (tr.result != RESULT_PASS) {
 206                         testcase_set_result(testcase, RESULT_PREFAIL);
 207                         goto out;
 208                 }
 209         }
 210
 211         switch (testcase_get_type(testcase)) {
 212         case TESTCASE_TYPE_BUILDONLY:
 213                 testcase_set_result(testcase, RESULT_PASS);
 214                 testcase_set_exit_value(testcase, 0);
 215                 break;
 216
 217         case TESTCASE_TYPE_USERLAND:
 218                 /* Main testcase execution */
 219                 r = run_userland(testcase_path,
 220                     testcase_get_argc(testcase),
 221                     testcase_get_args(testcase),
 222                     testcase_get_interpreter_noexit(testcase),
 223                     testcase_needs_setuid(testcase),
 224                     testcase_get_runas_uid(testcase),
 225                     testcase_get_timeout(testcase),
 226                     testcase_get_rc(testcase),
 227                     0, errbuf, sizeof(errbuf), &tr);
 228
 229                 if (r == 0) {
 230                         testcase_set_result(testcase, tr.result);
 231                         testcase_set_exit_value(testcase, tr.exit_value);
 232                         testcase_set_signal(testcase, tr.signal);
 233
 234                         if (tr.stdout_buf != NULL) {
 235                                 testcase_set_stdout_buf(testcase, tr.stdout_buf);
 236                                 free(tr.stdout_buf);
 237                         }
 238
 239                         if (tr.stderr_buf != NULL) {
 240                                 testcase_set_stderr_buf(testcase, tr.stderr_buf);
 241                                 free(tr.stderr_buf);
 242                         }
 243                 } else {
 244                         /* driver/monitor error */
 245                         testcase_set_sys_buf(testcase, errbuf);
 246                 }
 247
 248                 break;
 249
 250         case TESTCASE_TYPE_KERNEL:
 251                 run_kernel(testcase_path, testcase);
 252                 break;
 253         }
 254
 255
 256         /* Post-execution run */
 257         switch (testcase_get_postcmd_type(testcase)) {
 258         case TESTCASE_INT_POST:
 259                 /* Test case has internal but explicit POST - code */
 260                 r = run_simple_cmd(testcase_path, "post", errbuf,
 261                     sizeof(errbuf), &tr);
 262                 nopost = 0;
 263                 break;
 264
 265         case TESTCASE_CUSTOM_POST:
 266                 /* Test case uses external and explicit POST command */
 267                 sprintf(prepost_path, "%s/%s", prepost_dir,
 268                     testcase_get_custom_postcmd(testcase));
 269
 270                 r = run_simple_cmd(prepost_path, NULL, errbuf, sizeof(errbuf),
 271                     &tr);
 272                 nopost = 0;
 273                 break;
 274
 275         default:
 276                 r = 0;
 277                 nopost = 1;
 278                 break;
 279         }
 280
 281         if (!nopost) {
 282                 if (r != 0) {
 283                         testcase_set_sys_buf(testcase, errbuf);
 284                         testcase_set_result(testcase, RESULT_POSTFAIL);
 285                         goto out;
 286                 }
 287
 288                 if (tr.stdout_buf != NULL) {
 289                         testcase_set_postcmd_buf(testcase, tr.stdout_buf);
 290                         free(tr.stdout_buf);
 291                 }
 292
 293                 if (tr.result != RESULT_PASS) {
 294                         testcase_set_result(testcase, RESULT_POSTFAIL);
 295                         goto out;
 296                 }
 297         }
 298
 299
 300
 301 out:
 302         /* clean build unless nobuild flag is set */
 303         if ((testcase_get_flags(testcase) & TESTCASE_NOBUILD) == 0) {
 304                 r = run_simple_cmd(testcase_get_make_cmd(testcase), "clean",
 305                     errbuf, sizeof(errbuf), &tr);
 306
 307                 if (tr.stdout_buf != NULL) {
 308                         testcase_set_cleanup_buf(testcase, tr.stdout_buf);
 309                         free(tr.stdout_buf);
 310                 }
 311
 312                 if (r != 0)
 313                         testcase_set_cleanup_buf(testcase, errbuf);
 314         }
 315
 316
 317         /* ... and save results */
 318         runlist_save(output_file, runlist);
 319
 320         printf("done.\n");
 321         return 0;
 322 }
 323