block/curl.c

   1 /*
   2  * QEMU Block driver for CURL images
   3  *
   4  * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24 #include "qemu-common.h"
  25 #include "block_int.h"
  26 #include <curl/curl.h>
  27
  28 // #define DEBUG
  29 // #define DEBUG_VERBOSE
  30
  31 #ifdef DEBUG_CURL
  32 #define DPRINTF(fmt, ...) do { printf(fmt, ## __VA_ARGS__); } while (0)
  33 #else
  34 #define DPRINTF(fmt, ...) do { } while (0)
  35 #endif
  36
  37 #define CURL_NUM_STATES 8
  38 #define CURL_NUM_ACB    8
  39 #define SECTOR_SIZE     512
  40 #define READ_AHEAD_SIZE (256 * 1024)
  41
  42 #define FIND_RET_NONE   0
  43 #define FIND_RET_OK     1
  44 #define FIND_RET_WAIT   2
  45
  46 struct BDRVCURLState;
  47
  48 typedef struct CURLAIOCB {
  49     BlockDriverAIOCB common;
  50     QEMUBH *bh;
  51     QEMUIOVector *qiov;
  52
  53     int64_t sector_num;
  54     int nb_sectors;
  55
  56     size_t start;
  57     size_t end;
  58 } CURLAIOCB;
  59
  60 typedef struct CURLState
  61 {
  62     struct BDRVCURLState *s;
  63     CURLAIOCB *acb[CURL_NUM_ACB];
  64     CURL *curl;
  65     char *orig_buf;
  66     size_t buf_start;
  67     size_t buf_off;
  68     size_t buf_len;
  69     char range[128];
  70     char errmsg[CURL_ERROR_SIZE];
  71     char in_use;
  72 } CURLState;
  73
  74 typedef struct BDRVCURLState {
  75     CURLM *multi;
  76     size_t len;
  77     CURLState states[CURL_NUM_STATES];
  78     char *url;
  79     size_t readahead_size;
  80 } BDRVCURLState;
  81
  82 static void curl_clean_state(CURLState *s);
  83 static void curl_multi_do(void *arg);
  84 static int curl_aio_flush(void *opaque);
  85
  86 static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,
  87                         void *s, void *sp)
  88 {
  89     DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, fd);
  90     switch (action) {
  91         case CURL_POLL_IN:
  92             qemu_aio_set_fd_handler(fd, curl_multi_do, NULL, curl_aio_flush,
  93                                     NULL, s);
  94             break;
  95         case CURL_POLL_OUT:
  96             qemu_aio_set_fd_handler(fd, NULL, curl_multi_do, curl_aio_flush,
  97                                     NULL, s);
  98             break;
  99         case CURL_POLL_INOUT:
 100             qemu_aio_set_fd_handler(fd, curl_multi_do, curl_multi_do,
 101                                     curl_aio_flush, NULL, s);
 102             break;
 103         case CURL_POLL_REMOVE:
 104             qemu_aio_set_fd_handler(fd, NULL, NULL, NULL, NULL, NULL);
 105             break;
 106     }
 107
 108     return 0;
 109 }
 110
 111 static size_t curl_size_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
 112 {
 113     CURLState *s = ((CURLState*)opaque);
 114     size_t realsize = size * nmemb;
 115     size_t fsize;
 116
 117     if(sscanf(ptr, "Content-Length: %zd", &fsize) == 1) {
 118         s->s->len = fsize;
 119     }
 120
 121     return realsize;
 122 }
 123
 124 static size_t curl_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
 125 {
 126     CURLState *s = ((CURLState*)opaque);
 127     size_t realsize = size * nmemb;
 128     int i;
 129
 130     DPRINTF("CURL: Just reading %zd bytes\n", realsize);
 131
 132     if (!s || !s->orig_buf)
 133         goto read_end;
 134
 135     memcpy(s->orig_buf + s->buf_off, ptr, realsize);
 136     s->buf_off += realsize;
 137
 138     for(i=0; i<CURL_NUM_ACB; i++) {
 139         CURLAIOCB *acb = s->acb[i];
 140
 141         if (!acb)
 142             continue;
 143
 144         if ((s->buf_off >= acb->end)) {
 145             qemu_iovec_from_buffer(acb->qiov, s->orig_buf + acb->start,
 146                                    acb->end - acb->start);
 147             acb->common.cb(acb->common.opaque, 0);
 148             qemu_aio_release(acb);
 149             s->acb[i] = NULL;
 150         }
 151     }
 152
 153 read_end:
 154     return realsize;
 155 }
 156
 157 static int curl_find_buf(BDRVCURLState *s, size_t start, size_t len,
 158                          CURLAIOCB *acb)
 159 {
 160     int i;
 161     size_t end = start + len;
 162
 163     for (i=0; i<CURL_NUM_STATES; i++) {
 164         CURLState *state = &s->states[i];
 165         size_t buf_end = (state->buf_start + state->buf_off);
 166         size_t buf_fend = (state->buf_start + state->buf_len);
 167
 168         if (!state->orig_buf)
 169             continue;
 170         if (!state->buf_off)
 171             continue;
 172
 173         // Does the existing buffer cover our section?
 174         if ((start >= state->buf_start) &&
 175             (start <= buf_end) &&
 176             (end >= state->buf_start) &&
 177             (end <= buf_end))
 178         {
 179             char *buf = state->orig_buf + (start - state->buf_start);
 180
 181             qemu_iovec_from_buffer(acb->qiov, buf, len);
 182             acb->common.cb(acb->common.opaque, 0);
 183
 184             return FIND_RET_OK;
 185         }
 186
 187         // Wait for unfinished chunks
 188         if ((start >= state->buf_start) &&
 189             (start <= buf_fend) &&
 190             (end >= state->buf_start) &&
 191             (end <= buf_fend))
 192         {
 193             int j;
 194
 195             acb->start = start - state->buf_start;
 196             acb->end = acb->start + len;
 197
 198             for (j=0; j<CURL_NUM_ACB; j++) {
 199                 if (!state->acb[j]) {
 200                     state->acb[j] = acb;
 201                     return FIND_RET_WAIT;
 202                 }
 203             }
 204         }
 205     }
 206
 207     return FIND_RET_NONE;
 208 }
 209
 210 static void curl_multi_do(void *arg)
 211 {
 212     BDRVCURLState *s = (BDRVCURLState *)arg;
 213     int running;
 214     int r;
 215     int msgs_in_queue;
 216
 217     if (!s->multi)
 218         return;
 219
 220     do {
 221         r = curl_multi_socket_all(s->multi, &running);
 222     } while(r == CURLM_CALL_MULTI_PERFORM);
 223
 224     /* Try to find done transfers, so we can free the easy
 225      * handle again. */
 226     do {
 227         CURLMsg *msg;
 228         msg = curl_multi_info_read(s->multi, &msgs_in_queue);
 229
 230         if (!msg)
 231             break;
 232         if (msg->msg == CURLMSG_NONE)
 233             break;
 234
 235         switch (msg->msg) {
 236             case CURLMSG_DONE:
 237             {
 238                 CURLState *state = NULL;
 239                 curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char**)&state);
 240
 241                 /* ACBs for successful messages get completed in curl_read_cb */
 242                 if (msg->data.result != CURLE_OK) {
 243                     int i;
 244                     for (i = 0; i < CURL_NUM_ACB; i++) {
 245                         CURLAIOCB *acb = state->acb[i];
 246
 247                         if (acb == NULL) {
 248                             continue;
 249                         }
 250
 251                         acb->common.cb(acb->common.opaque, -EIO);
 252                         qemu_aio_release(acb);
 253                         state->acb[i] = NULL;
 254                     }
 255                 }
 256
 257                 curl_clean_state(state);
 258                 break;
 259             }
 260             default:
 261                 msgs_in_queue = 0;
 262                 break;
 263         }
 264     } while(msgs_in_queue);
 265 }
 266
 267 static CURLState *curl_init_state(BDRVCURLState *s)
 268 {
 269     CURLState *state = NULL;
 270     int i, j;
 271
 272     do {
 273         for (i=0; i<CURL_NUM_STATES; i++) {
 274             for (j=0; j<CURL_NUM_ACB; j++)
 275                 if (s->states[i].acb[j])
 276                     continue;
 277             if (s->states[i].in_use)
 278                 continue;
 279
 280             state = &s->states[i];
 281             state->in_use = 1;
 282             break;
 283         }
 284         if (!state) {
 285             usleep(100);
 286             curl_multi_do(s);
 287         }
 288     } while(!state);
 289
 290     if (state->curl)
 291         goto has_curl;
 292
 293     state->curl = curl_easy_init();
 294     if (!state->curl)
 295         return NULL;
 296     curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
 297     curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
 298     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
 299     curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void *)state);
 300     curl_easy_setopt(state->curl, CURLOPT_PRIVATE, (void *)state);
 301     curl_easy_setopt(state->curl, CURLOPT_AUTOREFERER, 1);
 302     curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1);
 303     curl_easy_setopt(state->curl, CURLOPT_NOSIGNAL, 1);
 304     curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
 305     curl_easy_setopt(state->curl, CURLOPT_FAILONERROR, 1);
 306
 307 #ifdef DEBUG_VERBOSE
 308     curl_easy_setopt(state->curl, CURLOPT_VERBOSE, 1);
 309 #endif
 310
 311 has_curl:
 312
 313     state->s = s;
 314
 315     return state;
 316 }
 317
 318 static void curl_clean_state(CURLState *s)
 319 {
 320     if (s->s->multi)
 321         curl_multi_remove_handle(s->s->multi, s->curl);
 322     s->in_use = 0;
 323 }
 324
 325 static int curl_open(BlockDriverState *bs, const char *filename, int flags)
 326 {
 327     BDRVCURLState *s = bs->opaque;
 328     CURLState *state = NULL;
 329     double d;
 330
 331     #define RA_OPTSTR ":readahead="
 332     char *file;
 333     char *ra;
 334     const char *ra_val;
 335     int parse_state = 0;
 336
 337     static int inited = 0;
 338
 339     file = g_strdup(filename);
 340     s->readahead_size = READ_AHEAD_SIZE;
 341
 342     /* Parse a trailing ":readahead=#:" param, if present. */
 343     ra = file + strlen(file) - 1;
 344     while (ra >= file) {
 345         if (parse_state == 0) {
 346             if (*ra == ':')
 347                 parse_state++;
 348             else
 349                 break;
 350         } else if (parse_state == 1) {
 351             if (*ra > '9' || *ra < '0') {
 352                 char *opt_start = ra - strlen(RA_OPTSTR) + 1;
 353                 if (opt_start > file &&
 354                     strncmp(opt_start, RA_OPTSTR, strlen(RA_OPTSTR)) == 0) {
 355                     ra_val = ra + 1;
 356                     ra -= strlen(RA_OPTSTR) - 1;
 357                     *ra = '\0';
 358                     s->readahead_size = atoi(ra_val);
 359                     break;
 360                 } else {
 361                     break;
 362                 }
 363             }
 364         }
 365         ra--;
 366     }
 367
 368     if ((s->readahead_size & 0x1ff) != 0) {
 369         fprintf(stderr, "HTTP_READAHEAD_SIZE %zd is not a multiple of 512\n",
 370                 s->readahead_size);
 371         goto out_noclean;
 372     }
 373
 374     if (!inited) {
 375         curl_global_init(CURL_GLOBAL_ALL);
 376         inited = 1;
 377     }
 378
 379     DPRINTF("CURL: Opening %s\n", file);
 380     s->url = file;
 381     state = curl_init_state(s);
 382     if (!state)
 383         goto out_noclean;
 384
 385     // Get file size
 386
 387     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1);
 388     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_size_cb);
 389     if (curl_easy_perform(state->curl))
 390         goto out;
 391     curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
 392     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
 393     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0);
 394     if (d)
 395         s->len = (size_t)d;
 396     else if(!s->len)
 397         goto out;
 398     DPRINTF("CURL: Size = %zd\n", s->len);
 399
 400     curl_clean_state(state);
 401     curl_easy_cleanup(state->curl);
 402     state->curl = NULL;
 403
 404     // Now we know the file exists and its size, so let's
 405     // initialize the multi interface!
 406
 407     s->multi = curl_multi_init();
 408     curl_multi_setopt( s->multi, CURLMOPT_SOCKETDATA, s);
 409     curl_multi_setopt( s->multi, CURLMOPT_SOCKETFUNCTION, curl_sock_cb );
 410     curl_multi_do(s);
 411
 412     return 0;
 413
 414 out:
 415     fprintf(stderr, "CURL: Error opening file: %s\n", state->errmsg);
 416     curl_easy_cleanup(state->curl);
 417     state->curl = NULL;
 418 out_noclean:
 419     g_free(file);
 420     return -EINVAL;
 421 }
 422
 423 static int curl_aio_flush(void *opaque)
 424 {
 425     BDRVCURLState *s = opaque;
 426     int i, j;
 427
 428     for (i=0; i < CURL_NUM_STATES; i++) {
 429         for(j=0; j < CURL_NUM_ACB; j++) {
 430             if (s->states[i].acb[j]) {
 431                 return 1;
 432             }
 433         }
 434     }
 435     return 0;
 436 }
 437
 438 static void curl_aio_cancel(BlockDriverAIOCB *blockacb)
 439 {
 440     // Do we have to implement canceling? Seems to work without...
 441 }
 442
 443 static AIOPool curl_aio_pool = {
 444     .aiocb_size         = sizeof(CURLAIOCB),
 445     .cancel             = curl_aio_cancel,
 446 };
 447
 448
 449 static void curl_readv_bh_cb(void *p)
 450 {
 451     CURLState *state;
 452
 453     CURLAIOCB *acb = p;
 454     BDRVCURLState *s = acb->common.bs->opaque;
 455
 456     qemu_bh_delete(acb->bh);
 457     acb->bh = NULL;
 458
 459     size_t start = acb->sector_num * SECTOR_SIZE;
 460     size_t end;
 461
 462     // In case we have the requested data already (e.g. read-ahead),
 463     // we can just call the callback and be done.
 464     switch (curl_find_buf(s, start, acb->nb_sectors * SECTOR_SIZE, acb)) {
 465         case FIND_RET_OK:
 466             qemu_aio_release(acb);
 467             // fall through
 468         case FIND_RET_WAIT:
 469             return;
 470         default:
 471             break;
 472     }
 473
 474     // No cache found, so let's start a new request
 475     state = curl_init_state(s);
 476     if (!state) {
 477         acb->common.cb(acb->common.opaque, -EIO);
 478         qemu_aio_release(acb);
 479         return;
 480     }
 481
 482     acb->start = 0;
 483     acb->end = (acb->nb_sectors * SECTOR_SIZE);
 484
 485     state->buf_off = 0;
 486     if (state->orig_buf)
 487         g_free(state->orig_buf);
 488     state->buf_start = start;
 489     state->buf_len = acb->end + s->readahead_size;
 490     end = MIN(start + state->buf_len, s->len) - 1;
 491     state->orig_buf = g_malloc(state->buf_len);
 492     state->acb[0] = acb;
 493
 494     snprintf(state->range, 127, "%zd-%zd", start, end);
 495     DPRINTF("CURL (AIO): Reading %d at %zd (%s)\n",
 496             (acb->nb_sectors * SECTOR_SIZE), start, state->range);
 497     curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);
 498
 499     curl_multi_add_handle(s->multi, state->curl);
 500     curl_multi_do(s);
 501
 502 }
 503
 504 static BlockDriverAIOCB *curl_aio_readv(BlockDriverState *bs,
 505         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
 506         BlockDriverCompletionFunc *cb, void *opaque)
 507 {
 508     CURLAIOCB *acb;
 509
 510     acb = qemu_aio_get(&curl_aio_pool, bs, cb, opaque);
 511
 512     acb->qiov = qiov;
 513     acb->sector_num = sector_num;
 514     acb->nb_sectors = nb_sectors;
 515
 516     acb->bh = qemu_bh_new(curl_readv_bh_cb, acb);
 517
 518     if (!acb->bh) {
 519         DPRINTF("CURL: qemu_bh_new failed\n");
 520         return NULL;
 521     }
 522
 523     qemu_bh_schedule(acb->bh);
 524     return &acb->common;
 525 }
 526
 527 static void curl_close(BlockDriverState *bs)
 528 {
 529     BDRVCURLState *s = bs->opaque;
 530     int i;
 531
 532     DPRINTF("CURL: Close\n");
 533     for (i=0; i<CURL_NUM_STATES; i++) {
 534         if (s->states[i].in_use)
 535             curl_clean_state(&s->states[i]);
 536         if (s->states[i].curl) {
 537             curl_easy_cleanup(s->states[i].curl);
 538             s->states[i].curl = NULL;
 539         }
 540         if (s->states[i].orig_buf) {
 541             g_free(s->states[i].orig_buf);
 542             s->states[i].orig_buf = NULL;
 543         }
 544     }
 545     if (s->multi)
 546         curl_multi_cleanup(s->multi);
 547     if (s->url)
 548         free(s->url);
 549 }
 550
 551 static int64_t curl_getlength(BlockDriverState *bs)
 552 {
 553     BDRVCURLState *s = bs->opaque;
 554     return s->len;
 555 }
 556
 557 static BlockDriver bdrv_http = {
 558     .format_name     = "http",
 559     .protocol_name   = "http",
 560
 561     .instance_size   = sizeof(BDRVCURLState),
 562     .bdrv_file_open  = curl_open,
 563     .bdrv_close      = curl_close,
 564     .bdrv_getlength  = curl_getlength,
 565
 566     .bdrv_aio_readv  = curl_aio_readv,
 567 };
 568
 569 static BlockDriver bdrv_https = {
 570     .format_name     = "https",
 571     .protocol_name   = "https",
 572
 573     .instance_size   = sizeof(BDRVCURLState),
 574     .bdrv_file_open  = curl_open,
 575     .bdrv_close      = curl_close,
 576     .bdrv_getlength  = curl_getlength,
 577
 578     .bdrv_aio_readv  = curl_aio_readv,
 579 };
 580
 581 static BlockDriver bdrv_ftp = {
 582     .format_name     = "ftp",
 583     .protocol_name   = "ftp",
 584
 585     .instance_size   = sizeof(BDRVCURLState),
 586     .bdrv_file_open  = curl_open,
 587     .bdrv_close      = curl_close,
 588     .bdrv_getlength  = curl_getlength,
 589
 590     .bdrv_aio_readv  = curl_aio_readv,
 591 };
 592
 593 static BlockDriver bdrv_ftps = {
 594     .format_name     = "ftps",
 595     .protocol_name   = "ftps",
 596
 597     .instance_size   = sizeof(BDRVCURLState),
 598     .bdrv_file_open  = curl_open,
 599     .bdrv_close      = curl_close,
 600     .bdrv_getlength  = curl_getlength,
 601
 602     .bdrv_aio_readv  = curl_aio_readv,
 603 };
 604
 605 static BlockDriver bdrv_tftp = {
 606     .format_name     = "tftp",
 607     .protocol_name   = "tftp",
 608
 609     .instance_size   = sizeof(BDRVCURLState),
 610     .bdrv_file_open  = curl_open,
 611     .bdrv_close      = curl_close,
 612     .bdrv_getlength  = curl_getlength,
 613
 614     .bdrv_aio_readv  = curl_aio_readv,
 615 };
 616
 617 static void curl_block_init(void)
 618 {
 619     bdrv_register(&bdrv_http);
 620     bdrv_register(&bdrv_https);
 621     bdrv_register(&bdrv_ftp);
 622     bdrv_register(&bdrv_ftps);
 623     bdrv_register(&bdrv_tftp);
 624 }
 625
 626 block_init(curl_block_init);