block/curl.c

   1 /*
   2  * QEMU Block driver for CURL images
   3  *
   4  * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24 #include "qemu-common.h"
  25 #include "block_int.h"
  26 #include <curl/curl.h>
  27
  28 // #define DEBUG
  29 // #define DEBUG_VERBOSE
  30
  31 #ifdef DEBUG_CURL
  32 #define DPRINTF(fmt, ...) do { printf(fmt, ## __VA_ARGS__); } while (0)
  33 #else
  34 #define DPRINTF(fmt, ...) do { } while (0)
  35 #endif
  36
  37 #define CURL_NUM_STATES 8
  38 #define CURL_NUM_ACB    8
  39 #define SECTOR_SIZE     512
  40 #define READ_AHEAD_SIZE (256 * 1024)
  41
  42 #define FIND_RET_NONE   0
  43 #define FIND_RET_OK     1
  44 #define FIND_RET_WAIT   2
  45
  46 struct BDRVCURLState;
  47
  48 typedef struct CURLAIOCB {
  49     BlockDriverAIOCB common;
  50     QEMUBH *bh;
  51     QEMUIOVector *qiov;
  52
  53     int64_t sector_num;
  54     int nb_sectors;
  55
  56     size_t start;
  57     size_t end;
  58 } CURLAIOCB;
  59
  60 typedef struct CURLState
  61 {
  62     struct BDRVCURLState *s;
  63     CURLAIOCB *acb[CURL_NUM_ACB];
  64     CURL *curl;
  65     char *orig_buf;
  66     size_t buf_start;
  67     size_t buf_off;
  68     size_t buf_len;
  69     char range[128];
  70     char errmsg[CURL_ERROR_SIZE];
  71     char in_use;
  72 } CURLState;
  73
  74 typedef struct BDRVCURLState {
  75     CURLM *multi;
  76     size_t len;
  77     CURLState states[CURL_NUM_STATES];
  78     char *url;
  79     size_t readahead_size;
  80 } BDRVCURLState;
  81
  82 static void curl_clean_state(CURLState *s);
  83 static void curl_multi_do(void *arg);
  84 static int curl_aio_flush(void *opaque);
  85
  86 static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,
  87                         void *s, void *sp)
  88 {
  89     DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, fd);
  90     switch (action) {
  91         case CURL_POLL_IN:
  92             qemu_aio_set_fd_handler(fd, curl_multi_do, NULL, curl_aio_flush, s);
  93             break;
  94         case CURL_POLL_OUT:
  95             qemu_aio_set_fd_handler(fd, NULL, curl_multi_do, curl_aio_flush, s);
  96             break;
  97         case CURL_POLL_INOUT:
  98             qemu_aio_set_fd_handler(fd, curl_multi_do, curl_multi_do,
  99                                     curl_aio_flush, s);
 100             break;
 101         case CURL_POLL_REMOVE:
 102             qemu_aio_set_fd_handler(fd, NULL, NULL, NULL, NULL);
 103             break;
 104     }
 105
 106     return 0;
 107 }
 108
 109 static size_t curl_size_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
 110 {
 111     CURLState *s = ((CURLState*)opaque);
 112     size_t realsize = size * nmemb;
 113     size_t fsize;
 114
 115     if(sscanf(ptr, "Content-Length: %zd", &fsize) == 1) {
 116         s->s->len = fsize;
 117     }
 118
 119     return realsize;
 120 }
 121
 122 static size_t curl_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
 123 {
 124     CURLState *s = ((CURLState*)opaque);
 125     size_t realsize = size * nmemb;
 126     int i;
 127
 128     DPRINTF("CURL: Just reading %zd bytes\n", realsize);
 129
 130     if (!s || !s->orig_buf)
 131         goto read_end;
 132
 133     memcpy(s->orig_buf + s->buf_off, ptr, realsize);
 134     s->buf_off += realsize;
 135
 136     for(i=0; i<CURL_NUM_ACB; i++) {
 137         CURLAIOCB *acb = s->acb[i];
 138
 139         if (!acb)
 140             continue;
 141
 142         if ((s->buf_off >= acb->end)) {
 143             qemu_iovec_from_buf(acb->qiov, 0, s->orig_buf + acb->start,
 144                                 acb->end - acb->start);
 145             acb->common.cb(acb->common.opaque, 0);
 146             qemu_aio_release(acb);
 147             s->acb[i] = NULL;
 148         }
 149     }
 150
 151 read_end:
 152     return realsize;
 153 }
 154
 155 static int curl_find_buf(BDRVCURLState *s, size_t start, size_t len,
 156                          CURLAIOCB *acb)
 157 {
 158     int i;
 159     size_t end = start + len;
 160
 161     for (i=0; i<CURL_NUM_STATES; i++) {
 162         CURLState *state = &s->states[i];
 163         size_t buf_end = (state->buf_start + state->buf_off);
 164         size_t buf_fend = (state->buf_start + state->buf_len);
 165
 166         if (!state->orig_buf)
 167             continue;
 168         if (!state->buf_off)
 169             continue;
 170
 171         // Does the existing buffer cover our section?
 172         if ((start >= state->buf_start) &&
 173             (start <= buf_end) &&
 174             (end >= state->buf_start) &&
 175             (end <= buf_end))
 176         {
 177             char *buf = state->orig_buf + (start - state->buf_start);
 178
 179             qemu_iovec_from_buf(acb->qiov, 0, buf, len);
 180             acb->common.cb(acb->common.opaque, 0);
 181
 182             return FIND_RET_OK;
 183         }
 184
 185         // Wait for unfinished chunks
 186         if ((start >= state->buf_start) &&
 187             (start <= buf_fend) &&
 188             (end >= state->buf_start) &&
 189             (end <= buf_fend))
 190         {
 191             int j;
 192
 193             acb->start = start - state->buf_start;
 194             acb->end = acb->start + len;
 195
 196             for (j=0; j<CURL_NUM_ACB; j++) {
 197                 if (!state->acb[j]) {
 198                     state->acb[j] = acb;
 199                     return FIND_RET_WAIT;
 200                 }
 201             }
 202         }
 203     }
 204
 205     return FIND_RET_NONE;
 206 }
 207
 208 static void curl_multi_do(void *arg)
 209 {
 210     BDRVCURLState *s = (BDRVCURLState *)arg;
 211     int running;
 212     int r;
 213     int msgs_in_queue;
 214
 215     if (!s->multi)
 216         return;
 217
 218     do {
 219         r = curl_multi_socket_all(s->multi, &running);
 220     } while(r == CURLM_CALL_MULTI_PERFORM);
 221
 222     /* Try to find done transfers, so we can free the easy
 223      * handle again. */
 224     do {
 225         CURLMsg *msg;
 226         msg = curl_multi_info_read(s->multi, &msgs_in_queue);
 227
 228         if (!msg)
 229             break;
 230         if (msg->msg == CURLMSG_NONE)
 231             break;
 232
 233         switch (msg->msg) {
 234             case CURLMSG_DONE:
 235             {
 236                 CURLState *state = NULL;
 237                 curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char**)&state);
 238
 239                 /* ACBs for successful messages get completed in curl_read_cb */
 240                 if (msg->data.result != CURLE_OK) {
 241                     int i;
 242                     for (i = 0; i < CURL_NUM_ACB; i++) {
 243                         CURLAIOCB *acb = state->acb[i];
 244
 245                         if (acb == NULL) {
 246                             continue;
 247                         }
 248
 249                         acb->common.cb(acb->common.opaque, -EIO);
 250                         qemu_aio_release(acb);
 251                         state->acb[i] = NULL;
 252                     }
 253                 }
 254
 255                 curl_clean_state(state);
 256                 break;
 257             }
 258             default:
 259                 msgs_in_queue = 0;
 260                 break;
 261         }
 262     } while(msgs_in_queue);
 263 }
 264
 265 static CURLState *curl_init_state(BDRVCURLState *s)
 266 {
 267     CURLState *state = NULL;
 268     int i, j;
 269
 270     do {
 271         for (i=0; i<CURL_NUM_STATES; i++) {
 272             for (j=0; j<CURL_NUM_ACB; j++)
 273                 if (s->states[i].acb[j])
 274                     continue;
 275             if (s->states[i].in_use)
 276                 continue;
 277
 278             state = &s->states[i];
 279             state->in_use = 1;
 280             break;
 281         }
 282         if (!state) {
 283             g_usleep(100);
 284             curl_multi_do(s);
 285         }
 286     } while(!state);
 287
 288     if (state->curl)
 289         goto has_curl;
 290
 291     state->curl = curl_easy_init();
 292     if (!state->curl)
 293         return NULL;
 294     curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
 295     curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
 296     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
 297     curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void *)state);
 298     curl_easy_setopt(state->curl, CURLOPT_PRIVATE, (void *)state);
 299     curl_easy_setopt(state->curl, CURLOPT_AUTOREFERER, 1);
 300     curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1);
 301     curl_easy_setopt(state->curl, CURLOPT_NOSIGNAL, 1);
 302     curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
 303     curl_easy_setopt(state->curl, CURLOPT_FAILONERROR, 1);
 304
 305 #ifdef DEBUG_VERBOSE
 306     curl_easy_setopt(state->curl, CURLOPT_VERBOSE, 1);
 307 #endif
 308
 309 has_curl:
 310
 311     state->s = s;
 312
 313     return state;
 314 }
 315
 316 static void curl_clean_state(CURLState *s)
 317 {
 318     if (s->s->multi)
 319         curl_multi_remove_handle(s->s->multi, s->curl);
 320     s->in_use = 0;
 321 }
 322
 323 static int curl_open(BlockDriverState *bs, const char *filename, int flags)
 324 {
 325     BDRVCURLState *s = bs->opaque;
 326     CURLState *state = NULL;
 327     double d;
 328
 329     #define RA_OPTSTR ":readahead="
 330     char *file;
 331     char *ra;
 332     const char *ra_val;
 333     int parse_state = 0;
 334
 335     static int inited = 0;
 336
 337     file = g_strdup(filename);
 338     s->readahead_size = READ_AHEAD_SIZE;
 339
 340     /* Parse a trailing ":readahead=#:" param, if present. */
 341     ra = file + strlen(file) - 1;
 342     while (ra >= file) {
 343         if (parse_state == 0) {
 344             if (*ra == ':')
 345                 parse_state++;
 346             else
 347                 break;
 348         } else if (parse_state == 1) {
 349             if (*ra > '9' || *ra < '0') {
 350                 char *opt_start = ra - strlen(RA_OPTSTR) + 1;
 351                 if (opt_start > file &&
 352                     strncmp(opt_start, RA_OPTSTR, strlen(RA_OPTSTR)) == 0) {
 353                     ra_val = ra + 1;
 354                     ra -= strlen(RA_OPTSTR) - 1;
 355                     *ra = '\0';
 356                     s->readahead_size = atoi(ra_val);
 357                     break;
 358                 } else {
 359                     break;
 360                 }
 361             }
 362         }
 363         ra--;
 364     }
 365
 366     if ((s->readahead_size & 0x1ff) != 0) {
 367         fprintf(stderr, "HTTP_READAHEAD_SIZE %zd is not a multiple of 512\n",
 368                 s->readahead_size);
 369         goto out_noclean;
 370     }
 371
 372     if (!inited) {
 373         curl_global_init(CURL_GLOBAL_ALL);
 374         inited = 1;
 375     }
 376
 377     DPRINTF("CURL: Opening %s\n", file);
 378     s->url = file;
 379     state = curl_init_state(s);
 380     if (!state)
 381         goto out_noclean;
 382
 383     // Get file size
 384
 385     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1);
 386     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_size_cb);
 387     if (curl_easy_perform(state->curl))
 388         goto out;
 389     curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
 390     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
 391     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0);
 392     if (d)
 393         s->len = (size_t)d;
 394     else if(!s->len)
 395         goto out;
 396     DPRINTF("CURL: Size = %zd\n", s->len);
 397
 398     curl_clean_state(state);
 399     curl_easy_cleanup(state->curl);
 400     state->curl = NULL;
 401
 402     // Now we know the file exists and its size, so let's
 403     // initialize the multi interface!
 404
 405     s->multi = curl_multi_init();
 406     curl_multi_setopt( s->multi, CURLMOPT_SOCKETDATA, s);
 407     curl_multi_setopt( s->multi, CURLMOPT_SOCKETFUNCTION, curl_sock_cb );
 408     curl_multi_do(s);
 409
 410     return 0;
 411
 412 out:
 413     fprintf(stderr, "CURL: Error opening file: %s\n", state->errmsg);
 414     curl_easy_cleanup(state->curl);
 415     state->curl = NULL;
 416 out_noclean:
 417     g_free(file);
 418     return -EINVAL;
 419 }
 420
 421 static int curl_aio_flush(void *opaque)
 422 {
 423     BDRVCURLState *s = opaque;
 424     int i, j;
 425
 426     for (i=0; i < CURL_NUM_STATES; i++) {
 427         for(j=0; j < CURL_NUM_ACB; j++) {
 428             if (s->states[i].acb[j]) {
 429                 return 1;
 430             }
 431         }
 432     }
 433     return 0;
 434 }
 435
 436 static void curl_aio_cancel(BlockDriverAIOCB *blockacb)
 437 {
 438     // Do we have to implement canceling? Seems to work without...
 439 }
 440
 441 static AIOPool curl_aio_pool = {
 442     .aiocb_size         = sizeof(CURLAIOCB),
 443     .cancel             = curl_aio_cancel,
 444 };
 445
 446
 447 static void curl_readv_bh_cb(void *p)
 448 {
 449     CURLState *state;
 450
 451     CURLAIOCB *acb = p;
 452     BDRVCURLState *s = acb->common.bs->opaque;
 453
 454     qemu_bh_delete(acb->bh);
 455     acb->bh = NULL;
 456
 457     size_t start = acb->sector_num * SECTOR_SIZE;
 458     size_t end;
 459
 460     // In case we have the requested data already (e.g. read-ahead),
 461     // we can just call the callback and be done.
 462     switch (curl_find_buf(s, start, acb->nb_sectors * SECTOR_SIZE, acb)) {
 463         case FIND_RET_OK:
 464             qemu_aio_release(acb);
 465             // fall through
 466         case FIND_RET_WAIT:
 467             return;
 468         default:
 469             break;
 470     }
 471
 472     // No cache found, so let's start a new request
 473     state = curl_init_state(s);
 474     if (!state) {
 475         acb->common.cb(acb->common.opaque, -EIO);
 476         qemu_aio_release(acb);
 477         return;
 478     }
 479
 480     acb->start = 0;
 481     acb->end = (acb->nb_sectors * SECTOR_SIZE);
 482
 483     state->buf_off = 0;
 484     if (state->orig_buf)
 485         g_free(state->orig_buf);
 486     state->buf_start = start;
 487     state->buf_len = acb->end + s->readahead_size;
 488     end = MIN(start + state->buf_len, s->len) - 1;
 489     state->orig_buf = g_malloc(state->buf_len);
 490     state->acb[0] = acb;
 491
 492     snprintf(state->range, 127, "%zd-%zd", start, end);
 493     DPRINTF("CURL (AIO): Reading %d at %zd (%s)\n",
 494             (acb->nb_sectors * SECTOR_SIZE), start, state->range);
 495     curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);
 496
 497     curl_multi_add_handle(s->multi, state->curl);
 498     curl_multi_do(s);
 499
 500 }
 501
 502 static BlockDriverAIOCB *curl_aio_readv(BlockDriverState *bs,
 503         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
 504         BlockDriverCompletionFunc *cb, void *opaque)
 505 {
 506     CURLAIOCB *acb;
 507
 508     acb = qemu_aio_get(&curl_aio_pool, bs, cb, opaque);
 509
 510     acb->qiov = qiov;
 511     acb->sector_num = sector_num;
 512     acb->nb_sectors = nb_sectors;
 513
 514     acb->bh = qemu_bh_new(curl_readv_bh_cb, acb);
 515
 516     if (!acb->bh) {
 517         DPRINTF("CURL: qemu_bh_new failed\n");
 518         return NULL;
 519     }
 520
 521     qemu_bh_schedule(acb->bh);
 522     return &acb->common;
 523 }
 524
 525 static void curl_close(BlockDriverState *bs)
 526 {
 527     BDRVCURLState *s = bs->opaque;
 528     int i;
 529
 530     DPRINTF("CURL: Close\n");
 531     for (i=0; i<CURL_NUM_STATES; i++) {
 532         if (s->states[i].in_use)
 533             curl_clean_state(&s->states[i]);
 534         if (s->states[i].curl) {
 535             curl_easy_cleanup(s->states[i].curl);
 536             s->states[i].curl = NULL;
 537         }
 538         if (s->states[i].orig_buf) {
 539             g_free(s->states[i].orig_buf);
 540             s->states[i].orig_buf = NULL;
 541         }
 542     }
 543     if (s->multi)
 544         curl_multi_cleanup(s->multi);
 545     g_free(s->url);
 546 }
 547
 548 static int64_t curl_getlength(BlockDriverState *bs)
 549 {
 550     BDRVCURLState *s = bs->opaque;
 551     return s->len;
 552 }
 553
 554 static BlockDriver bdrv_http = {
 555     .format_name     = "http",
 556     .protocol_name   = "http",
 557
 558     .instance_size   = sizeof(BDRVCURLState),
 559     .bdrv_file_open  = curl_open,
 560     .bdrv_close      = curl_close,
 561     .bdrv_getlength  = curl_getlength,
 562
 563     .bdrv_aio_readv  = curl_aio_readv,
 564 };
 565
 566 static BlockDriver bdrv_https = {
 567     .format_name     = "https",
 568     .protocol_name   = "https",
 569
 570     .instance_size   = sizeof(BDRVCURLState),
 571     .bdrv_file_open  = curl_open,
 572     .bdrv_close      = curl_close,
 573     .bdrv_getlength  = curl_getlength,
 574
 575     .bdrv_aio_readv  = curl_aio_readv,
 576 };
 577
 578 static BlockDriver bdrv_ftp = {
 579     .format_name     = "ftp",
 580     .protocol_name   = "ftp",
 581
 582     .instance_size   = sizeof(BDRVCURLState),
 583     .bdrv_file_open  = curl_open,
 584     .bdrv_close      = curl_close,
 585     .bdrv_getlength  = curl_getlength,
 586
 587     .bdrv_aio_readv  = curl_aio_readv,
 588 };
 589
 590 static BlockDriver bdrv_ftps = {
 591     .format_name     = "ftps",
 592     .protocol_name   = "ftps",
 593
 594     .instance_size   = sizeof(BDRVCURLState),
 595     .bdrv_file_open  = curl_open,
 596     .bdrv_close      = curl_close,
 597     .bdrv_getlength  = curl_getlength,
 598
 599     .bdrv_aio_readv  = curl_aio_readv,
 600 };
 601
 602 static BlockDriver bdrv_tftp = {
 603     .format_name     = "tftp",
 604     .protocol_name   = "tftp",
 605
 606     .instance_size   = sizeof(BDRVCURLState),
 607     .bdrv_file_open  = curl_open,
 608     .bdrv_close      = curl_close,
 609     .bdrv_getlength  = curl_getlength,
 610
 611     .bdrv_aio_readv  = curl_aio_readv,
 612 };
 613
 614 static void curl_block_init(void)
 615 {
 616     bdrv_register(&bdrv_http);
 617     bdrv_register(&bdrv_https);
 618     bdrv_register(&bdrv_ftp);
 619     bdrv_register(&bdrv_ftps);
 620     bdrv_register(&bdrv_tftp);
 621 }
 622
 623 block_init(curl_block_init);