uspace/app/testrunner/main.c

   1 /*
   2  * Copyright (c) 2018 Jiří Zárevúcky
   3  * All rights reserved.
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions
   7  * are met:
   8  *
   9  * - Redistributions of source code must retain the above copyright
  10  *   notice, this list of conditions and the following disclaimer.
  11  * - Redistributions in binary form must reproduce the above copyright
  12  *   notice, this list of conditions and the following disclaimer in the
  13  *   documentation and/or other materials provided with the distribution.
  14  * - The name of the author may not be used to endorse or promote products
  15  *   derived from this software without specific prior written permission.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  */
  28
  29 #include <errno.h>
  30 #include <stdio.h>
  31 #include <stdlib.h>
  32 #include <str_error.h>
  33 #include <task.h>
  34 #include <vfs/vfs.h>
  35 #include <dirent.h>
  36 #include <str.h>
  37
  38 static errno_t run_test(const char *logfile, const char *logmode,
  39     const char *path, const char *const args[], task_exit_t *ex, int *retval)
  40 {
  41         FILE *f = fopen(logfile, logmode);
  42         if (!f) {
  43                 fprintf(stderr, "Can't open file %s: %s\n",
  44                     logfile, str_error(errno));
  45                 return errno;
  46         }
  47
  48         int h;
  49         errno_t rc = vfs_fhandle(f, &h);
  50         if (rc != EOK) {
  51                 fprintf(stderr, "Error getting file handle: %s\n",
  52                     str_error_name(rc));
  53                 fclose(f);
  54                 return rc;
  55         }
  56
  57         task_id_t id;
  58         task_wait_t wait;
  59
  60         rc = task_spawnvf(&id, &wait, path, args, -1, h, h);
  61         if (rc != EOK) {
  62                 fprintf(stderr, "Task spawning failed: %s\n",
  63                     str_error_name(rc));
  64                 fclose(f);
  65                 return rc;
  66         }
  67
  68         rc = task_wait(&wait, ex, retval);
  69         if (rc != EOK) {
  70                 fprintf(stderr, "Task wait failed: %s\n",
  71                     str_error_name(rc));
  72                 fclose(f);
  73                 return rc;
  74         }
  75
  76         // TODO: check that we are managing resources correctly
  77         fclose(f);
  78         return EOK;
  79 }
  80
  81 static void run_tester(const char *logfile)
  82 {
  83         const char *const tests[] = {
  84                 "thread1",
  85                 "setjmp1",
  86                 "print1",
  87                 "print2",
  88                 "print3",
  89                 "print4",
  90                 "print5",
  91                 "print6",
  92                 "stdio1",
  93                 "stdio2",
  94                 "logger1",
  95                 "fault1",
  96                 "fault2",
  97                 "fault3",
  98                 "float1",
  99                 "float2",
 100                 "vfs1",
 101                 "ping_pong",
 102                 "malloc1",
 103                 // FIXME: malloc2 doesn't work as expected
 104                 // "malloc2",
 105                 "malloc3",
 106                 "mapping1",
 107                 "pager1",
 108                 // "serial1",
 109                 // "chardev1",
 110         };
 111
 112         int tests_count = sizeof(tests) / sizeof(const char *);
 113
 114         task_exit_t ex;
 115         int retval;
 116         int failed = 0;
 117
 118         const char *app = "/app/tester";
 119
 120         for (int i = 0; i < tests_count; i++) {
 121                 const char *const args[] = { app, tests[i], NULL };
 122                 errno_t rc = run_test(logfile, "a", app, args, &ex, &retval);
 123                 if (rc != EOK) {
 124                         /* Reason already printed in run_test(). */
 125                         continue;
 126                 }
 127
 128                 if (ex != TASK_EXIT_NORMAL) {
 129                         fprintf(stderr, "tester %s CRASHED\n",
 130                             tests[i]);
 131                         failed++;
 132                         continue;
 133                 }
 134
 135                 if (retval == 0) {
 136                         printf("tester %s ok\n", tests[i]);
 137                 } else {
 138                         printf("tester %s FAILED\n", tests[i]);
 139                         failed++;
 140                 }
 141         }
 142
 143         printf("tester: %d failed tests\n", failed);
 144 }
 145
 146 static void run_tester_fault(const char *logfile)
 147 {
 148         task_exit_t ex;
 149         int retval;
 150
 151         const char *app = "/app/tester";
 152         const char *const args[] = { app, "fault1", NULL };
 153         errno_t rc = run_test(logfile, "w", app, args, &ex, &retval);
 154         if (rc != EOK) {
 155                 /* Reason already printed in run_test(). */
 156                 return;
 157         }
 158
 159         if (ex != TASK_EXIT_UNEXPECTED) {
 160                 fprintf(stderr, "`tester fault1` unexpectedly"
 161                     " didn't terminate unexpectedly\n");
 162                 return;
 163         }
 164
 165         printf("`tester fault1`: terminated as expected\n");
 166 }
 167
 168 static void run_pcut_tests(void)
 169 {
 170         printf("Running all pcut tests...\n");
 171
 172         DIR *d = opendir("/test");
 173         if (!d)
 174                 return;
 175
 176         struct dirent *e;
 177
 178         while ((e = readdir(d))) {
 179                 if (str_lcmp(e->d_name, "test-", 5) != 0)
 180                         continue;
 181
 182                 task_exit_t ex;
 183                 int retval;
 184
 185                 char *bin;
 186                 if (asprintf(&bin, "/test/%s", e->d_name) < 0) {
 187                         fprintf(stderr, "out of memory\n");
 188                         exit(EXIT_FAILURE);
 189                 }
 190
 191                 char *logfile;
 192                 if (asprintf(&logfile, "/data/web/result-%s.txt", e->d_name) < 0) {
 193                         fprintf(stderr, "out of memory\n");
 194                         exit(EXIT_FAILURE);
 195                 }
 196
 197                 const char *const args[] = { bin, NULL };
 198                 errno_t rc = run_test(logfile, "w", bin, args, &ex, &retval);
 199                 free(bin);
 200                 free(logfile);
 201
 202                 if (rc != EOK) {
 203                         /* Reason already printed in run_test(). */
 204                         return;
 205                 }
 206
 207                 if (ex != TASK_EXIT_NORMAL) {
 208                         fprintf(stderr, "%s CRASHED\n", e->d_name);
 209                         return;
 210                 }
 211
 212                 if (retval == 0) {
 213                         printf("%s ok\n", e->d_name);
 214                 } else {
 215                         printf("%s FAILED\n", e->d_name);
 216                 }
 217         }
 218
 219         closedir(d);
 220 }
 221
 222 static void gen_index(const char *fname)
 223 {
 224         FILE *f = fopen(fname, "w");
 225         if (!f) {
 226                 fprintf(stderr, "Can't open %s for writing: %s\n", fname,
 227                     str_error_name(errno));
 228                 return;
 229         }
 230
 231         fprintf(f, "<html><head><title>HelenOS test results"
 232             "</title></head><body>\n");
 233         fprintf(f, "<h1>HelenOS test results</h1><ul>\n");
 234
 235         fprintf(f, "<li><a href=\"result-tester.txt\">tester</a></li>\n");
 236
 237         DIR *d = opendir("/test");
 238         if (d) {
 239                 struct dirent *e;
 240
 241                 while ((e = readdir(d))) {
 242                         if (str_lcmp(e->d_name, "test-", 5) != 0)
 243                                 continue;
 244
 245                         fprintf(f, "<li><a href=\"result-%s.txt\">%s"
 246                             "</a></li>\n", e->d_name, e->d_name);
 247                 }
 248         }
 249
 250         fprintf(f, "</ul></body></html>\n");
 251         fclose(f);
 252 }
 253
 254 int main(int argc, char **argv)
 255 {
 256         run_tester("/data/web/result-tester.txt");
 257         run_tester_fault("/tmp/tester_fault.log");
 258         run_pcut_tests();
 259
 260         const char *fname = "/data/web/test.html";
 261         printf("Generating HTML report in %s\n", fname);
 262         gen_index(fname);
 263         return EXIT_SUCCESS;
 264 }