block/curl.c

   1 /*
   2  * QEMU Block driver for CURL images
   3  *
   4  * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24 #include "qemu-common.h"
  25 #include "block/block_int.h"
  26 #include <curl/curl.h>
  27
  28 // #define DEBUG
  29 // #define DEBUG_VERBOSE
  30
  31 #ifdef DEBUG_CURL
  32 #define DPRINTF(fmt, ...) do { printf(fmt, ## __VA_ARGS__); } while (0)
  33 #else
  34 #define DPRINTF(fmt, ...) do { } while (0)
  35 #endif
  36
  37 #define PROTOCOLS (CURLPROTO_HTTP | CURLPROTO_HTTPS | \
  38                    CURLPROTO_FTP | CURLPROTO_FTPS | \
  39                    CURLPROTO_TFTP)
  40
  41 #define CURL_NUM_STATES 8
  42 #define CURL_NUM_ACB    8
  43 #define SECTOR_SIZE     512
  44 #define READ_AHEAD_SIZE (256 * 1024)
  45
  46 #define FIND_RET_NONE   0
  47 #define FIND_RET_OK     1
  48 #define FIND_RET_WAIT   2
  49
  50 struct BDRVCURLState;
  51
  52 typedef struct CURLAIOCB {
  53     BlockDriverAIOCB common;
  54     QEMUBH *bh;
  55     QEMUIOVector *qiov;
  56
  57     int64_t sector_num;
  58     int nb_sectors;
  59
  60     size_t start;
  61     size_t end;
  62 } CURLAIOCB;
  63
  64 typedef struct CURLState
  65 {
  66     struct BDRVCURLState *s;
  67     CURLAIOCB *acb[CURL_NUM_ACB];
  68     CURL *curl;
  69     char *orig_buf;
  70     size_t buf_start;
  71     size_t buf_off;
  72     size_t buf_len;
  73     char range[128];
  74     char errmsg[CURL_ERROR_SIZE];
  75     char in_use;
  76 } CURLState;
  77
  78 typedef struct BDRVCURLState {
  79     CURLM *multi;
  80     size_t len;
  81     CURLState states[CURL_NUM_STATES];
  82     char *url;
  83     size_t readahead_size;
  84 } BDRVCURLState;
  85
  86 static void curl_clean_state(CURLState *s);
  87 static void curl_multi_do(void *arg);
  88 static int curl_aio_flush(void *opaque);
  89
  90 static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,
  91                         void *s, void *sp)
  92 {
  93     DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, fd);
  94     switch (action) {
  95         case CURL_POLL_IN:
  96             qemu_aio_set_fd_handler(fd, curl_multi_do, NULL, curl_aio_flush, s);
  97             break;
  98         case CURL_POLL_OUT:
  99             qemu_aio_set_fd_handler(fd, NULL, curl_multi_do, curl_aio_flush, s);
 100             break;
 101         case CURL_POLL_INOUT:
 102             qemu_aio_set_fd_handler(fd, curl_multi_do, curl_multi_do,
 103                                     curl_aio_flush, s);
 104             break;
 105         case CURL_POLL_REMOVE:
 106             qemu_aio_set_fd_handler(fd, NULL, NULL, NULL, NULL);
 107             break;
 108     }
 109
 110     return 0;
 111 }
 112
 113 static size_t curl_size_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
 114 {
 115     CURLState *s = ((CURLState*)opaque);
 116     size_t realsize = size * nmemb;
 117     size_t fsize;
 118
 119     if(sscanf(ptr, "Content-Length: %zd", &fsize) == 1) {
 120         s->s->len = fsize;
 121     }
 122
 123     return realsize;
 124 }
 125
 126 static size_t curl_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
 127 {
 128     CURLState *s = ((CURLState*)opaque);
 129     size_t realsize = size * nmemb;
 130     int i;
 131
 132     DPRINTF("CURL: Just reading %zd bytes\n", realsize);
 133
 134     if (!s || !s->orig_buf)
 135         goto read_end;
 136
 137     memcpy(s->orig_buf + s->buf_off, ptr, realsize);
 138     s->buf_off += realsize;
 139
 140     for(i=0; i<CURL_NUM_ACB; i++) {
 141         CURLAIOCB *acb = s->acb[i];
 142
 143         if (!acb)
 144             continue;
 145
 146         if ((s->buf_off >= acb->end)) {
 147             qemu_iovec_from_buf(acb->qiov, 0, s->orig_buf + acb->start,
 148                                 acb->end - acb->start);
 149             acb->common.cb(acb->common.opaque, 0);
 150             qemu_aio_release(acb);
 151             s->acb[i] = NULL;
 152         }
 153     }
 154
 155 read_end:
 156     return realsize;
 157 }
 158
 159 static int curl_find_buf(BDRVCURLState *s, size_t start, size_t len,
 160                          CURLAIOCB *acb)
 161 {
 162     int i;
 163     size_t end = start + len;
 164
 165     for (i=0; i<CURL_NUM_STATES; i++) {
 166         CURLState *state = &s->states[i];
 167         size_t buf_end = (state->buf_start + state->buf_off);
 168         size_t buf_fend = (state->buf_start + state->buf_len);
 169
 170         if (!state->orig_buf)
 171             continue;
 172         if (!state->buf_off)
 173             continue;
 174
 175         // Does the existing buffer cover our section?
 176         if ((start >= state->buf_start) &&
 177             (start <= buf_end) &&
 178             (end >= state->buf_start) &&
 179             (end <= buf_end))
 180         {
 181             char *buf = state->orig_buf + (start - state->buf_start);
 182
 183             qemu_iovec_from_buf(acb->qiov, 0, buf, len);
 184             acb->common.cb(acb->common.opaque, 0);
 185
 186             return FIND_RET_OK;
 187         }
 188
 189         // Wait for unfinished chunks
 190         if ((start >= state->buf_start) &&
 191             (start <= buf_fend) &&
 192             (end >= state->buf_start) &&
 193             (end <= buf_fend))
 194         {
 195             int j;
 196
 197             acb->start = start - state->buf_start;
 198             acb->end = acb->start + len;
 199
 200             for (j=0; j<CURL_NUM_ACB; j++) {
 201                 if (!state->acb[j]) {
 202                     state->acb[j] = acb;
 203                     return FIND_RET_WAIT;
 204                 }
 205             }
 206         }
 207     }
 208
 209     return FIND_RET_NONE;
 210 }
 211
 212 static void curl_multi_do(void *arg)
 213 {
 214     BDRVCURLState *s = (BDRVCURLState *)arg;
 215     int running;
 216     int r;
 217     int msgs_in_queue;
 218
 219     if (!s->multi)
 220         return;
 221
 222     do {
 223         r = curl_multi_socket_all(s->multi, &running);
 224     } while(r == CURLM_CALL_MULTI_PERFORM);
 225
 226     /* Try to find done transfers, so we can free the easy
 227      * handle again. */
 228     do {
 229         CURLMsg *msg;
 230         msg = curl_multi_info_read(s->multi, &msgs_in_queue);
 231
 232         if (!msg)
 233             break;
 234         if (msg->msg == CURLMSG_NONE)
 235             break;
 236
 237         switch (msg->msg) {
 238             case CURLMSG_DONE:
 239             {
 240                 CURLState *state = NULL;
 241                 curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char**)&state);
 242
 243                 /* ACBs for successful messages get completed in curl_read_cb */
 244                 if (msg->data.result != CURLE_OK) {
 245                     int i;
 246                     for (i = 0; i < CURL_NUM_ACB; i++) {
 247                         CURLAIOCB *acb = state->acb[i];
 248
 249                         if (acb == NULL) {
 250                             continue;
 251                         }
 252
 253                         acb->common.cb(acb->common.opaque, -EIO);
 254                         qemu_aio_release(acb);
 255                         state->acb[i] = NULL;
 256                     }
 257                 }
 258
 259                 curl_clean_state(state);
 260                 break;
 261             }
 262             default:
 263                 msgs_in_queue = 0;
 264                 break;
 265         }
 266     } while(msgs_in_queue);
 267 }
 268
 269 static CURLState *curl_init_state(BDRVCURLState *s)
 270 {
 271     CURLState *state = NULL;
 272     int i, j;
 273
 274     do {
 275         for (i=0; i<CURL_NUM_STATES; i++) {
 276             for (j=0; j<CURL_NUM_ACB; j++)
 277                 if (s->states[i].acb[j])
 278                     continue;
 279             if (s->states[i].in_use)
 280                 continue;
 281
 282             state = &s->states[i];
 283             state->in_use = 1;
 284             break;
 285         }
 286         if (!state) {
 287             g_usleep(100);
 288             curl_multi_do(s);
 289         }
 290     } while(!state);
 291
 292     if (state->curl)
 293         goto has_curl;
 294
 295     state->curl = curl_easy_init();
 296     if (!state->curl)
 297         return NULL;
 298     curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
 299     curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
 300     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
 301     curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void *)state);
 302     curl_easy_setopt(state->curl, CURLOPT_PRIVATE, (void *)state);
 303     curl_easy_setopt(state->curl, CURLOPT_AUTOREFERER, 1);
 304     curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1);
 305     curl_easy_setopt(state->curl, CURLOPT_NOSIGNAL, 1);
 306     curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
 307     curl_easy_setopt(state->curl, CURLOPT_FAILONERROR, 1);
 308
 309     /* Restrict supported protocols to avoid security issues in the more
 310      * obscure protocols.  For example, do not allow POP3/SMTP/IMAP see
 311      * CVE-2013-0249.
 312      *
 313      * Restricting protocols is only supported from 7.19.4 upwards.
 314      */
 315 #if LIBCURL_VERSION_NUM >= 0x071304
 316     curl_easy_setopt(state->curl, CURLOPT_PROTOCOLS, PROTOCOLS);
 317     curl_easy_setopt(state->curl, CURLOPT_REDIR_PROTOCOLS, PROTOCOLS);
 318 #endif
 319
 320 #ifdef DEBUG_VERBOSE
 321     curl_easy_setopt(state->curl, CURLOPT_VERBOSE, 1);
 322 #endif
 323
 324 has_curl:
 325
 326     state->s = s;
 327
 328     return state;
 329 }
 330
 331 static void curl_clean_state(CURLState *s)
 332 {
 333     if (s->s->multi)
 334         curl_multi_remove_handle(s->s->multi, s->curl);
 335     s->in_use = 0;
 336 }
 337
 338 static int curl_open(BlockDriverState *bs, const char *filename,
 339                      QDict *options, int flags)
 340 {
 341     BDRVCURLState *s = bs->opaque;
 342     CURLState *state = NULL;
 343     double d;
 344
 345     #define RA_OPTSTR ":readahead="
 346     char *file;
 347     char *ra;
 348     const char *ra_val;
 349     int parse_state = 0;
 350
 351     static int inited = 0;
 352
 353     file = g_strdup(filename);
 354     s->readahead_size = READ_AHEAD_SIZE;
 355
 356     /* Parse a trailing ":readahead=#:" param, if present. */
 357     ra = file + strlen(file) - 1;
 358     while (ra >= file) {
 359         if (parse_state == 0) {
 360             if (*ra == ':')
 361                 parse_state++;
 362             else
 363                 break;
 364         } else if (parse_state == 1) {
 365             if (*ra > '9' || *ra < '0') {
 366                 char *opt_start = ra - strlen(RA_OPTSTR) + 1;
 367                 if (opt_start > file &&
 368                     strncmp(opt_start, RA_OPTSTR, strlen(RA_OPTSTR)) == 0) {
 369                     ra_val = ra + 1;
 370                     ra -= strlen(RA_OPTSTR) - 1;
 371                     *ra = '\0';
 372                     s->readahead_size = atoi(ra_val);
 373                     break;
 374                 } else {
 375                     break;
 376                 }
 377             }
 378         }
 379         ra--;
 380     }
 381
 382     if ((s->readahead_size & 0x1ff) != 0) {
 383         fprintf(stderr, "HTTP_READAHEAD_SIZE %zd is not a multiple of 512\n",
 384                 s->readahead_size);
 385         goto out_noclean;
 386     }
 387
 388     if (!inited) {
 389         curl_global_init(CURL_GLOBAL_ALL);
 390         inited = 1;
 391     }
 392
 393     DPRINTF("CURL: Opening %s\n", file);
 394     s->url = file;
 395     state = curl_init_state(s);
 396     if (!state)
 397         goto out_noclean;
 398
 399     // Get file size
 400
 401     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1);
 402     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_size_cb);
 403     if (curl_easy_perform(state->curl))
 404         goto out;
 405     curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
 406     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
 407     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0);
 408     if (d)
 409         s->len = (size_t)d;
 410     else if(!s->len)
 411         goto out;
 412     DPRINTF("CURL: Size = %zd\n", s->len);
 413
 414     curl_clean_state(state);
 415     curl_easy_cleanup(state->curl);
 416     state->curl = NULL;
 417
 418     // Now we know the file exists and its size, so let's
 419     // initialize the multi interface!
 420
 421     s->multi = curl_multi_init();
 422     curl_multi_setopt( s->multi, CURLMOPT_SOCKETDATA, s);
 423     curl_multi_setopt( s->multi, CURLMOPT_SOCKETFUNCTION, curl_sock_cb );
 424     curl_multi_do(s);
 425
 426     return 0;
 427
 428 out:
 429     fprintf(stderr, "CURL: Error opening file: %s\n", state->errmsg);
 430     curl_easy_cleanup(state->curl);
 431     state->curl = NULL;
 432 out_noclean:
 433     g_free(file);
 434     return -EINVAL;
 435 }
 436
 437 static int curl_aio_flush(void *opaque)
 438 {
 439     BDRVCURLState *s = opaque;
 440     int i, j;
 441
 442     for (i=0; i < CURL_NUM_STATES; i++) {
 443         for(j=0; j < CURL_NUM_ACB; j++) {
 444             if (s->states[i].acb[j]) {
 445                 return 1;
 446             }
 447         }
 448     }
 449     return 0;
 450 }
 451
 452 static void curl_aio_cancel(BlockDriverAIOCB *blockacb)
 453 {
 454     // Do we have to implement canceling? Seems to work without...
 455 }
 456
 457 static const AIOCBInfo curl_aiocb_info = {
 458     .aiocb_size         = sizeof(CURLAIOCB),
 459     .cancel             = curl_aio_cancel,
 460 };
 461
 462
 463 static void curl_readv_bh_cb(void *p)
 464 {
 465     CURLState *state;
 466
 467     CURLAIOCB *acb = p;
 468     BDRVCURLState *s = acb->common.bs->opaque;
 469
 470     qemu_bh_delete(acb->bh);
 471     acb->bh = NULL;
 472
 473     size_t start = acb->sector_num * SECTOR_SIZE;
 474     size_t end;
 475
 476     // In case we have the requested data already (e.g. read-ahead),
 477     // we can just call the callback and be done.
 478     switch (curl_find_buf(s, start, acb->nb_sectors * SECTOR_SIZE, acb)) {
 479         case FIND_RET_OK:
 480             qemu_aio_release(acb);
 481             // fall through
 482         case FIND_RET_WAIT:
 483             return;
 484         default:
 485             break;
 486     }
 487
 488     // No cache found, so let's start a new request
 489     state = curl_init_state(s);
 490     if (!state) {
 491         acb->common.cb(acb->common.opaque, -EIO);
 492         qemu_aio_release(acb);
 493         return;
 494     }
 495
 496     acb->start = 0;
 497     acb->end = (acb->nb_sectors * SECTOR_SIZE);
 498
 499     state->buf_off = 0;
 500     if (state->orig_buf)
 501         g_free(state->orig_buf);
 502     state->buf_start = start;
 503     state->buf_len = acb->end + s->readahead_size;
 504     end = MIN(start + state->buf_len, s->len) - 1;
 505     state->orig_buf = g_malloc(state->buf_len);
 506     state->acb[0] = acb;
 507
 508     snprintf(state->range, 127, "%zd-%zd", start, end);
 509     DPRINTF("CURL (AIO): Reading %d at %zd (%s)\n",
 510             (acb->nb_sectors * SECTOR_SIZE), start, state->range);
 511     curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);
 512
 513     curl_multi_add_handle(s->multi, state->curl);
 514     curl_multi_do(s);
 515
 516 }
 517
 518 static BlockDriverAIOCB *curl_aio_readv(BlockDriverState *bs,
 519         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
 520         BlockDriverCompletionFunc *cb, void *opaque)
 521 {
 522     CURLAIOCB *acb;
 523
 524     acb = qemu_aio_get(&curl_aiocb_info, bs, cb, opaque);
 525
 526     acb->qiov = qiov;
 527     acb->sector_num = sector_num;
 528     acb->nb_sectors = nb_sectors;
 529
 530     acb->bh = qemu_bh_new(curl_readv_bh_cb, acb);
 531
 532     if (!acb->bh) {
 533         DPRINTF("CURL: qemu_bh_new failed\n");
 534         return NULL;
 535     }
 536
 537     qemu_bh_schedule(acb->bh);
 538     return &acb->common;
 539 }
 540
 541 static void curl_close(BlockDriverState *bs)
 542 {
 543     BDRVCURLState *s = bs->opaque;
 544     int i;
 545
 546     DPRINTF("CURL: Close\n");
 547     for (i=0; i<CURL_NUM_STATES; i++) {
 548         if (s->states[i].in_use)
 549             curl_clean_state(&s->states[i]);
 550         if (s->states[i].curl) {
 551             curl_easy_cleanup(s->states[i].curl);
 552             s->states[i].curl = NULL;
 553         }
 554         if (s->states[i].orig_buf) {
 555             g_free(s->states[i].orig_buf);
 556             s->states[i].orig_buf = NULL;
 557         }
 558     }
 559     if (s->multi)
 560         curl_multi_cleanup(s->multi);
 561     g_free(s->url);
 562 }
 563
 564 static int64_t curl_getlength(BlockDriverState *bs)
 565 {
 566     BDRVCURLState *s = bs->opaque;
 567     return s->len;
 568 }
 569
 570 static BlockDriver bdrv_http = {
 571     .format_name     = "http",
 572     .protocol_name   = "http",
 573
 574     .instance_size   = sizeof(BDRVCURLState),
 575     .bdrv_file_open  = curl_open,
 576     .bdrv_close      = curl_close,
 577     .bdrv_getlength  = curl_getlength,
 578
 579     .bdrv_aio_readv  = curl_aio_readv,
 580 };
 581
 582 static BlockDriver bdrv_https = {
 583     .format_name     = "https",
 584     .protocol_name   = "https",
 585
 586     .instance_size   = sizeof(BDRVCURLState),
 587     .bdrv_file_open  = curl_open,
 588     .bdrv_close      = curl_close,
 589     .bdrv_getlength  = curl_getlength,
 590
 591     .bdrv_aio_readv  = curl_aio_readv,
 592 };
 593
 594 static BlockDriver bdrv_ftp = {
 595     .format_name     = "ftp",
 596     .protocol_name   = "ftp",
 597
 598     .instance_size   = sizeof(BDRVCURLState),
 599     .bdrv_file_open  = curl_open,
 600     .bdrv_close      = curl_close,
 601     .bdrv_getlength  = curl_getlength,
 602
 603     .bdrv_aio_readv  = curl_aio_readv,
 604 };
 605
 606 static BlockDriver bdrv_ftps = {
 607     .format_name     = "ftps",
 608     .protocol_name   = "ftps",
 609
 610     .instance_size   = sizeof(BDRVCURLState),
 611     .bdrv_file_open  = curl_open,
 612     .bdrv_close      = curl_close,
 613     .bdrv_getlength  = curl_getlength,
 614
 615     .bdrv_aio_readv  = curl_aio_readv,
 616 };
 617
 618 static BlockDriver bdrv_tftp = {
 619     .format_name     = "tftp",
 620     .protocol_name   = "tftp",
 621
 622     .instance_size   = sizeof(BDRVCURLState),
 623     .bdrv_file_open  = curl_open,
 624     .bdrv_close      = curl_close,
 625     .bdrv_getlength  = curl_getlength,
 626
 627     .bdrv_aio_readv  = curl_aio_readv,
 628 };
 629
 630 static void curl_block_init(void)
 631 {
 632     bdrv_register(&bdrv_http);
 633     bdrv_register(&bdrv_https);
 634     bdrv_register(&bdrv_ftp);
 635     bdrv_register(&bdrv_ftps);
 636     bdrv_register(&bdrv_tftp);
 637 }
 638
 639 block_init(curl_block_init);