stream/stream.c

   1 /*
   2  * This file is part of MPlayer.
   3  *
   4  * MPlayer is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * MPlayer is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along
  15  * with MPlayer; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  17  */
  18
  19 #include <stdio.h>
  20 #include <stdlib.h>
  21 #include <unistd.h>
  22
  23 #include <sys/types.h>
  24 #include <sys/stat.h>
  25 #ifndef __MINGW32__
  26 #include <sys/ioctl.h>
  27 #include <sys/wait.h>
  28 #endif
  29 #include <fcntl.h>
  30 #include <strings.h>
  31 #include <assert.h>
  32
  33 #include "talloc.h"
  34
  35 #include "config.h"
  36
  37 #if HAVE_WINSOCK2_H
  38 #include <winsock2.h>
  39 #endif
  40
  41 #include <libavutil/common.h>
  42
  43 #include "mp_msg.h"
  44 #include "osdep/shmem.h"
  45 #include "osdep/timer.h"
  46 #include "network.h"
  47 #include "stream.h"
  48 #include "libmpdemux/demuxer.h"
  49 #include "ffmpeg_files/intreadwrite.h"
  50
  51 #include "m_option.h"
  52 #include "m_struct.h"
  53
  54 #include "cache2.h"
  55
  56 struct input_ctx;
  57 static int (*stream_check_interrupt_cb)(struct input_ctx *ctx, int time);
  58 static struct input_ctx *stream_check_interrupt_ctx;
  59
  60 extern const stream_info_t stream_info_vcd;
  61 extern const stream_info_t stream_info_cdda;
  62 extern const stream_info_t stream_info_netstream;
  63 extern const stream_info_t stream_info_pnm;
  64 extern const stream_info_t stream_info_asf;
  65 extern const stream_info_t stream_info_rtsp;
  66 extern const stream_info_t stream_info_rtp;
  67 extern const stream_info_t stream_info_udp;
  68 extern const stream_info_t stream_info_http1;
  69 extern const stream_info_t stream_info_http2;
  70 extern const stream_info_t stream_info_dvb;
  71 extern const stream_info_t stream_info_tv;
  72 extern const stream_info_t stream_info_radio;
  73 extern const stream_info_t stream_info_pvr;
  74 extern const stream_info_t stream_info_ftp;
  75 extern const stream_info_t stream_info_vstream;
  76 extern const stream_info_t stream_info_dvdnav;
  77 extern const stream_info_t stream_info_smb;
  78 extern const stream_info_t stream_info_sdp;
  79 extern const stream_info_t stream_info_rtsp_sip;
  80
  81 extern const stream_info_t stream_info_cue;
  82 extern const stream_info_t stream_info_null;
  83 extern const stream_info_t stream_info_mf;
  84 extern const stream_info_t stream_info_ffmpeg;
  85 extern const stream_info_t stream_info_file;
  86 extern const stream_info_t stream_info_ifo;
  87 extern const stream_info_t stream_info_dvd;
  88 extern const stream_info_t stream_info_bluray;
  89
  90 static const stream_info_t* const auto_open_streams[] = {
  91 #ifdef CONFIG_VCD
  92   &stream_info_vcd,
  93 #endif
  94 #ifdef CONFIG_CDDA
  95   &stream_info_cdda,
  96 #endif
  97 #ifdef CONFIG_NETWORKING
  98   &stream_info_netstream,
  99   &stream_info_http1,
 100   &stream_info_asf,
 101   &stream_info_pnm,
 102   &stream_info_rtsp,
 103 #ifdef CONFIG_LIVE555
 104   &stream_info_sdp,
 105   &stream_info_rtsp_sip,
 106 #endif
 107   &stream_info_rtp,
 108   &stream_info_udp,
 109   &stream_info_http2,
 110 #endif
 111 #ifdef CONFIG_DVBIN
 112   &stream_info_dvb,
 113 #endif
 114 #ifdef CONFIG_TV
 115   &stream_info_tv,
 116 #endif
 117 #ifdef CONFIG_RADIO
 118   &stream_info_radio,
 119 #endif
 120 #ifdef CONFIG_PVR
 121   &stream_info_pvr,
 122 #endif
 123 #ifdef CONFIG_FTP
 124   &stream_info_ftp,
 125 #endif
 126 #ifdef CONFIG_VSTREAM
 127   &stream_info_vstream,
 128 #endif
 129 #ifdef CONFIG_LIBSMBCLIENT
 130   &stream_info_smb,
 131 #endif
 132   &stream_info_cue,
 133 #ifdef CONFIG_DVDREAD
 134   &stream_info_ifo,
 135   &stream_info_dvd,
 136 #endif
 137 #ifdef CONFIG_DVDNAV
 138   &stream_info_dvdnav,
 139 #endif
 140 #ifdef CONFIG_LIBBLURAY
 141   &stream_info_bluray,
 142 #endif
 143   &stream_info_ffmpeg,
 144
 145   &stream_info_null,
 146   &stream_info_mf,
 147   &stream_info_file,
 148   NULL
 149 };
 150
 151 static stream_t *open_stream_plugin(const stream_info_t *sinfo,
 152                                     const char *filename,
 153                                     int mode, struct MPOpts *options,
 154                                     int *file_format, int *ret,
 155                                     char **redirected_url)
 156 {
 157   void* arg = NULL;
 158   stream_t* s;
 159   m_struct_t* desc = (m_struct_t*)sinfo->opts;
 160
 161   // Parse options
 162   if(desc) {
 163     arg = m_struct_alloc(desc);
 164     if(sinfo->opts_url) {
 165       m_option_t url_opt =
 166         { "stream url", arg , CONF_TYPE_CUSTOM_URL, 0, 0 ,0, (void *)sinfo->opts };
 167       if (m_option_parse(&url_opt, bstr("stream url"), bstr(filename), false, arg) < 0) {
 168         mp_tmsg(MSGT_OPEN,MSGL_ERR, "URL parsing failed on url %s\n",filename);
 169         m_struct_free(desc,arg);
 170         return NULL;
 171       }
 172     }
 173   }
 174   s = new_stream(-2,-2);
 175   s->opts = options;
 176   s->url=strdup(filename);
 177   s->flags |= mode;
 178   *ret = sinfo->open(s,mode,arg,file_format);
 179   if((*ret) != STREAM_OK) {
 180 #ifdef CONFIG_NETWORKING
 181     if (*ret == STREAM_REDIRECTED && redirected_url) {
 182         if (s->streaming_ctrl && s->streaming_ctrl->url
 183             && s->streaming_ctrl->url->url)
 184           *redirected_url = strdup(s->streaming_ctrl->url->url);
 185         else
 186           *redirected_url = NULL;
 187     }
 188     streaming_ctrl_free(s->streaming_ctrl);
 189 #endif
 190     free(s->url);
 191     free(s);
 192     return NULL;
 193   }
 194   if(s->type <= -2)
 195     mp_msg(MSGT_OPEN,MSGL_WARN, "Warning streams need a type !!!!\n");
 196   if(s->flags & MP_STREAM_SEEK && !s->seek)
 197     s->flags &= ~MP_STREAM_SEEK;
 198   if(s->seek && !(s->flags & MP_STREAM_SEEK))
 199     s->flags |= MP_STREAM_SEEK;
 200
 201   s->mode = mode;
 202
 203   mp_msg(MSGT_OPEN,MSGL_V, "STREAM: [%s] %s\n",sinfo->name,filename);
 204   mp_msg(MSGT_OPEN,MSGL_V, "STREAM: Description: %s\n",sinfo->info);
 205   mp_msg(MSGT_OPEN,MSGL_V, "STREAM: Author: %s\n", sinfo->author);
 206   mp_msg(MSGT_OPEN,MSGL_V, "STREAM: Comment: %s\n", sinfo->comment);
 207
 208   return s;
 209 }
 210
 211
 212 stream_t *open_stream_full(const char *filename, int mode,
 213                            struct MPOpts *options, int *file_format)
 214 {
 215   int i,j,l,r;
 216   const stream_info_t* sinfo;
 217   stream_t* s;
 218   char *redirected_url = NULL;
 219
 220   for(i = 0 ; auto_open_streams[i] ; i++) {
 221     sinfo = auto_open_streams[i];
 222     if(!sinfo->protocols) {
 223       mp_msg(MSGT_OPEN,MSGL_WARN, "Stream type %s has protocols == NULL, it's a bug\n", sinfo->name);
 224       continue;
 225     }
 226     for(j = 0 ; sinfo->protocols[j] ; j++) {
 227       l = strlen(sinfo->protocols[j]);
 228       // l == 0 => Don't do protocol matching (ie network and filenames)
 229       if((l == 0 && !strstr(filename, "://")) ||
 230          ((strncasecmp(sinfo->protocols[j],filename,l) == 0) &&
 231                       (strncmp("://",filename+l,3) == 0))) {
 232         *file_format = DEMUXER_TYPE_UNKNOWN;
 233         s = open_stream_plugin(sinfo,filename,mode,options,file_format,&r,
 234                                 &redirected_url);
 235         if(s) return s;
 236         if(r == STREAM_REDIRECTED && redirected_url) {
 237           mp_msg(MSGT_OPEN,MSGL_V, "[%s] open %s redirected to %s\n",
 238                  sinfo->info, filename, redirected_url);
 239           s = open_stream_full(redirected_url, mode, options, file_format);
 240           free(redirected_url);
 241           return s;
 242         }
 243         else if(r != STREAM_UNSUPPORTED) {
 244           mp_tmsg(MSGT_OPEN,MSGL_ERR, "Failed to open %s.\n",filename);
 245           return NULL;
 246         }
 247         break;
 248       }
 249     }
 250   }
 251
 252   mp_tmsg(MSGT_OPEN,MSGL_ERR, "No stream found to handle url %s\n", filename);
 253   return NULL;
 254 }
 255
 256 stream_t *open_output_stream(const char *filename, struct MPOpts *options)
 257 {
 258   int file_format; //unused
 259   if(!filename) {
 260     mp_msg(MSGT_OPEN,MSGL_ERR,"open_output_stream(), NULL filename, report this bug\n");
 261     return NULL;
 262   }
 263
 264   return open_stream_full(filename,STREAM_WRITE,options,&file_format);
 265 }
 266
 267 //=================== STREAMER =========================
 268
 269 void stream_capture_do(stream_t *s)
 270 {
 271   if (fwrite(s->buffer, s->buf_len, 1, s->capture_file) < 1) {
 272     mp_tmsg(MSGT_GLOBAL, MSGL_ERR, "Error writing capture file: %s\n",
 273             strerror(errno));
 274     fclose(s->capture_file);
 275     s->capture_file = NULL;
 276   }
 277 }
 278
 279 int stream_read_internal(stream_t *s, void *buf, int len)
 280 {
 281   int orig_len = len;
 282   // we will retry even if we already reached EOF previously.
 283   switch(s->type){
 284   case STREAMTYPE_STREAM:
 285 #ifdef CONFIG_NETWORKING
 286     if( s->streaming_ctrl!=NULL && s->streaming_ctrl->streaming_read ) {
 287       len=s->streaming_ctrl->streaming_read(s->fd, buf, len, s->streaming_ctrl);
 288       if (s->streaming_ctrl->status == streaming_stopped_e)
 289         s->eof = 1;
 290     } else
 291 #endif
 292     if (s->fill_buffer)
 293       len = s->fill_buffer(s, buf, len);
 294     else
 295       len = read(s->fd, buf, len);
 296     break;
 297   case STREAMTYPE_DS:
 298     len = demux_read_data((demux_stream_t*)s->priv, buf, len);
 299     break;
 300
 301
 302   default:
 303     len= s->fill_buffer ? s->fill_buffer(s, buf, len) : 0;
 304   }
 305   if(len<=0){
 306     off_t pos = s->pos;
 307     // do not retry if this looks like proper eof
 308     if (s->eof || (s->end_pos && pos == s->end_pos))
 309       goto eof_out;
 310     // dvdnav has some horrible hacks to "suspend" reads,
 311     // we need to skip this code or seeks will hang.
 312     if (s->type == STREAMTYPE_DVDNAV)
 313       goto eof_out;
 314
 315     // just in case this is an error e.g. due to network
 316     // timeout reset and retry
 317     // Seeking is used as a hack to make network streams
 318     // reopen the connection, ideally they would implement
 319     // e.g. a STREAM_CTRL_RECONNECT to do this
 320     s->eof=1;
 321     stream_reset(s);
 322     if (stream_seek_internal(s, pos) >= 0 || s->pos != pos) // seek failed
 323       goto eof_out;
 324     // make sure EOF is set to ensure no endless loops
 325     s->eof=1;
 326     return stream_read_internal(s, buf, orig_len);
 327
 328 eof_out:
 329     s->eof=1;
 330     return 0;
 331   }
 332   // When reading succeeded we are obviously not at eof.
 333   // This e.g. avoids issues with eof getting stuck when lavf seeks in MPEG-TS
 334   s->eof=0;
 335   s->pos+=len;
 336   return len;
 337 }
 338
 339 int stream_fill_buffer(stream_t *s){
 340   int len = stream_read_internal(s, s->buffer, STREAM_BUFFER_SIZE);
 341   if (len <= 0)
 342     return 0;
 343   s->buf_pos=0;
 344   s->buf_len=len;
 345 //  printf("[%d]",len);fflush(stdout);
 346   if (s->capture_file)
 347     stream_capture_do(s);
 348   return len;
 349 }
 350
 351 int stream_write_buffer(stream_t *s, unsigned char *buf, int len) {
 352   int rd;
 353   if(!s->write_buffer)
 354     return -1;
 355   rd = s->write_buffer(s, buf, len);
 356   if(rd < 0)
 357     return -1;
 358   s->pos += rd;
 359   assert(rd == len && "stream_write_buffer(): unexpected short write");
 360   return rd;
 361 }
 362
 363 int stream_seek_internal(stream_t *s, off_t newpos)
 364 {
 365 if(newpos==0 || newpos!=s->pos){
 366   switch(s->type){
 367   case STREAMTYPE_STREAM:
 368     //s->pos=newpos; // real seek
 369     // Some streaming protocol allow to seek backward and forward
 370     // A function call that return -1 can tell that the protocol
 371     // doesn't support seeking.
 372 #ifdef CONFIG_NETWORKING
 373     if(s->seek) { // new stream seek is much cleaner than streaming_ctrl one
 374       if(!s->seek(s,newpos)) {
 375         mp_tmsg(MSGT_STREAM,MSGL_ERR, "Seek failed\n");
 376         return 0;
 377       }
 378       break;
 379     }
 380
 381     if( s->streaming_ctrl!=NULL && s->streaming_ctrl->streaming_seek ) {
 382       if( s->streaming_ctrl->streaming_seek( s->fd, newpos, s->streaming_ctrl )<0 ) {
 383         mp_tmsg(MSGT_STREAM,MSGL_INFO,"Stream not seekable!\n");
 384         return 1;
 385       }
 386       break;
 387     }
 388 #endif
 389     if(newpos<s->pos){
 390       mp_tmsg(MSGT_STREAM, MSGL_INFO,
 391               "Cannot seek backward in linear streams!\n");
 392       return 1;
 393     }
 394     break;
 395   default:
 396     // This should at the beginning as soon as all streams are converted
 397     if(!s->seek)
 398       return 0;
 399     // Now seek
 400     if(!s->seek(s,newpos)) {
 401       mp_tmsg(MSGT_STREAM,MSGL_ERR, "Seek failed\n");
 402       return 0;
 403     }
 404   }
 405 //   putchar('.');fflush(stdout);
 406 //} else {
 407 //   putchar('%');fflush(stdout);
 408 }
 409   return -1;
 410 }
 411
 412 int stream_seek_long(stream_t *s,off_t pos){
 413   int res;
 414 off_t newpos=0;
 415
 416 //  if( mp_msg_test(MSGT_STREAM,MSGL_DBG3) ) printf("seek_long to 0x%X\n",(unsigned int)pos);
 417
 418   s->buf_pos=s->buf_len=0;
 419
 420   if(s->mode == STREAM_WRITE) {
 421     if(!s->seek || !s->seek(s,pos))
 422       return 0;
 423     return 1;
 424   }
 425
 426   if(s->sector_size)
 427       newpos = (pos/s->sector_size)*s->sector_size;
 428   else
 429       newpos = pos&(~((off_t)STREAM_BUFFER_SIZE-1));
 430
 431 if( mp_msg_test(MSGT_STREAM,MSGL_DBG3) ){
 432   mp_msg(MSGT_STREAM,MSGL_DBG3, "s->pos=%"PRIX64"  newpos=%"PRIX64"  new_bufpos=%"PRIX64"  buflen=%X  \n",
 433     (int64_t)s->pos,(int64_t)newpos,(int64_t)pos,s->buf_len);
 434 }
 435   pos-=newpos;
 436
 437   res = stream_seek_internal(s, newpos);
 438   if (res >= 0)
 439     return res;
 440
 441   while(s->pos<newpos){
 442     if(stream_fill_buffer(s)<=0) break; // EOF
 443   }
 444
 445 s->eof = 0; // EOF reset when seek succeeds.
 446 while (stream_fill_buffer(s) > 0) {
 447   if(pos<=s->buf_len){
 448     s->buf_pos=pos; // byte position in sector
 449     return 1;
 450   }
 451   pos -= s->buf_len;
 452 }
 453 // Fill failed, but seek still is a success.
 454 s->pos += pos;
 455 s->buf_pos = 0;
 456 s->buf_len = 0;
 457
 458 mp_msg(MSGT_STREAM,MSGL_V,
 459        "stream_seek: Seek to/past EOF: no buffer preloaded.\n");
 460 return 1;
 461 }
 462
 463
 464 void stream_reset(stream_t *s){
 465   if(s->eof){
 466     s->pos=0;
 467     s->buf_pos=s->buf_len=0;
 468     s->eof=0;
 469   }
 470   if(s->control) s->control(s,STREAM_CTRL_RESET,NULL);
 471   //stream_seek(s,0);
 472 }
 473
 474 int stream_control(stream_t *s, int cmd, void *arg){
 475   if(!s->control) return STREAM_UNSUPPORTED;
 476 #ifdef CONFIG_STREAM_CACHE
 477   if (s->cache_pid)
 478     return cache_do_control(s, cmd, arg);
 479 #endif
 480   return s->control(s, cmd, arg);
 481 }
 482
 483 stream_t* new_memory_stream(unsigned char* data,int len){
 484   stream_t *s;
 485
 486   if(len < 0)
 487     return NULL;
 488   s=calloc(1, sizeof(stream_t)+len);
 489   s->fd=-1;
 490   s->type=STREAMTYPE_MEMORY;
 491   s->buf_pos=0; s->buf_len=len;
 492   s->start_pos=0; s->end_pos=len;
 493   stream_reset(s);
 494   s->pos=len;
 495   memcpy(s->buffer,data,len);
 496   return s;
 497 }
 498
 499 stream_t* new_stream(int fd,int type){
 500   stream_t *s=calloc(1, sizeof(stream_t));
 501   if(s==NULL) return NULL;
 502
 503 #if HAVE_WINSOCK2_H
 504   {
 505     WSADATA wsdata;
 506     int temp = WSAStartup(0x0202, &wsdata); // there might be a better place for this (-> later)
 507     mp_msg(MSGT_STREAM,MSGL_V,"WINSOCK2 init: %i\n", temp);
 508   }
 509 #endif
 510
 511   s->fd=fd;
 512   s->type=type;
 513   s->buf_pos=s->buf_len=0;
 514   s->start_pos=s->end_pos=0;
 515   s->priv=NULL;
 516   s->url=NULL;
 517   s->cache_pid=0;
 518   stream_reset(s);
 519   return s;
 520 }
 521
 522 void free_stream(stream_t *s){
 523 //  printf("\n*** free_stream() called ***\n");
 524 #ifdef CONFIG_STREAM_CACHE
 525     cache_uninit(s);
 526 #endif
 527   if (s->capture_file) {
 528     fclose(s->capture_file);
 529     s->capture_file = NULL;
 530   }
 531
 532   if(s->close) s->close(s);
 533   if(s->fd>0){
 534     /* on unix we define closesocket to close
 535        on windows however we have to distinguish between
 536        network socket and file */
 537     if(s->url && strstr(s->url,"://"))
 538       closesocket(s->fd);
 539     else close(s->fd);
 540   }
 541 #if HAVE_WINSOCK2_H
 542   mp_msg(MSGT_STREAM,MSGL_V,"WINSOCK2 uninit\n");
 543   WSACleanup(); // there might be a better place for this (-> later)
 544 #endif
 545   // Disabled atm, i don't like that. s->priv can be anything after all
 546   // streams should destroy their priv on close
 547   //free(s->priv);
 548   free(s->url);
 549   free(s);
 550 }
 551
 552 stream_t* new_ds_stream(demux_stream_t *ds) {
 553   stream_t* s = new_stream(-1,STREAMTYPE_DS);
 554   s->priv = ds;
 555   return s;
 556 }
 557
 558 void stream_set_interrupt_callback(int (*cb)(struct input_ctx *, int),
 559                                    struct input_ctx *ctx)
 560 {
 561     stream_check_interrupt_cb = cb;
 562     stream_check_interrupt_ctx = ctx;
 563 }
 564
 565 int stream_check_interrupt(int time) {
 566     if(!stream_check_interrupt_cb) {
 567         usec_sleep(time * 1000);
 568         return 0;
 569     }
 570     return stream_check_interrupt_cb(stream_check_interrupt_ctx, time);
 571 }
 572
 573 /**
 574  * Helper function to read 16 bits little-endian and advance pointer
 575  */
 576 static uint16_t get_le16_inc(const uint8_t **buf)
 577 {
 578   uint16_t v = AV_RL16(*buf);
 579   *buf += 2;
 580   return v;
 581 }
 582
 583 /**
 584  * Helper function to read 16 bits big-endian and advance pointer
 585  */
 586 static uint16_t get_be16_inc(const uint8_t **buf)
 587 {
 588   uint16_t v = AV_RB16(*buf);
 589   *buf += 2;
 590   return v;
 591 }
 592
 593 /**
 594  * Find a newline character in buffer
 595  * \param buf buffer to search
 596  * \param len amount of bytes to search in buffer, may not overread
 597  * \param utf16 chose between UTF-8/ASCII/other and LE and BE UTF-16
 598  *              0 = UTF-8/ASCII/other, 1 = UTF-16-LE, 2 = UTF-16-BE
 599  */
 600 static const uint8_t *find_newline(const uint8_t *buf, int len, int utf16)
 601 {
 602   uint32_t c;
 603   const uint8_t *end = buf + len;
 604   switch (utf16) {
 605   case 0:
 606     return (uint8_t *)memchr(buf, '\n', len);
 607   case 1:
 608     while (buf < end - 1) {
 609       GET_UTF16(c, buf < end - 1 ? get_le16_inc(&buf) : 0, return NULL;)
 610       if (buf <= end && c == '\n')
 611         return buf - 1;
 612     }
 613     break;
 614   case 2:
 615     while (buf < end - 1) {
 616       GET_UTF16(c, buf < end - 1 ? get_be16_inc(&buf) : 0, return NULL;)
 617       if (buf <= end && c == '\n')
 618         return buf - 1;
 619     }
 620     break;
 621   }
 622   return NULL;
 623 }
 624
 625 /**
 626  * Copy a number of bytes, converting to UTF-8 if input is UTF-16
 627  * \param dst buffer to copy to
 628  * \param dstsize size of dst buffer
 629  * \param src buffer to copy from
 630  * \param len amount of bytes to copy from src
 631  * \param utf16 chose between UTF-8/ASCII/other and LE and BE UTF-16
 632  *              0 = UTF-8/ASCII/other, 1 = UTF-16-LE, 2 = UTF-16-BE
 633  */
 634 static int copy_characters(uint8_t *dst, int dstsize,
 635                            const uint8_t *src, int *len, int utf16)
 636 {
 637   uint32_t c;
 638   uint8_t *dst_end = dst + dstsize;
 639   const uint8_t *end = src + *len;
 640   switch (utf16) {
 641   case 0:
 642     if (*len > dstsize)
 643       *len = dstsize;
 644     memcpy(dst, src, *len);
 645     return *len;
 646   case 1:
 647     while (src < end - 1 && dst_end - dst > 8) {
 648       uint8_t tmp;
 649       GET_UTF16(c, src < end - 1 ? get_le16_inc(&src) : 0, ;)
 650       PUT_UTF8(c, tmp, *dst++ = tmp;)
 651     }
 652     *len -= end - src;
 653     return dstsize - (dst_end - dst);
 654   case 2:
 655     while (src < end - 1 && dst_end - dst > 8) {
 656       uint8_t tmp;
 657       GET_UTF16(c, src < end - 1 ? get_be16_inc(&src) : 0, ;)
 658       PUT_UTF8(c, tmp, *dst++ = tmp;)
 659     }
 660     *len -= end - src;
 661     return dstsize - (dst_end - dst);
 662   }
 663   return 0;
 664 }
 665
 666 unsigned char* stream_read_line(stream_t *s,unsigned char* mem, int max, int utf16) {
 667   int len;
 668   const unsigned char *end;
 669   unsigned char *ptr = mem;
 670   if (max < 1) return NULL;
 671   max--; // reserve one for 0-termination
 672   do {
 673     len = s->buf_len-s->buf_pos;
 674     // try to fill the buffer
 675     if(len <= 0 &&
 676        (!cache_stream_fill_buffer(s) ||
 677         (len = s->buf_len-s->buf_pos) <= 0)) break;
 678     end = find_newline(s->buffer+s->buf_pos, len, utf16);
 679     if(end) len = end - (s->buffer+s->buf_pos) + 1;
 680     if(len > 0 && max > 0) {
 681       int l = copy_characters(ptr, max, s->buffer+s->buf_pos, &len, utf16);
 682       max -= l;
 683       ptr += l;
 684       if (!len)
 685         break;
 686     }
 687     s->buf_pos += len;
 688   } while(!end);
 689   ptr[0] = 0;
 690   if(s->eof && ptr == mem) return NULL;
 691   return mem;
 692 }
 693
 694 struct bstr stream_read_complete(struct stream *s, void *talloc_ctx,
 695                                  int max_size, int padding_bytes)
 696 {
 697     if (max_size > 1000000000)
 698         abort();
 699
 700     int bufsize;
 701     int total_read = 0;
 702     int padding = FFMAX(padding_bytes, 1);
 703     char *buf = NULL;
 704     if (s->end_pos > max_size)
 705         return (struct bstr){NULL, 0};
 706     if (s->end_pos > 0)
 707         bufsize = s->end_pos + padding;
 708     else
 709         bufsize = 1000;
 710     while (1) {
 711         buf = talloc_realloc_size(talloc_ctx, buf, bufsize);
 712         int readsize = stream_read(s, buf + total_read, bufsize - total_read);
 713         total_read += readsize;
 714         if (total_read < bufsize)
 715             break;
 716         if (bufsize > max_size) {
 717             talloc_free(buf);
 718             return (struct bstr){NULL, 0};
 719         }
 720         bufsize = FFMIN(bufsize + (bufsize >> 1), max_size + padding);
 721     }
 722     buf = talloc_realloc_size(talloc_ctx, buf, total_read + padding);
 723     return (struct bstr){buf, total_read};
 724 }