stream/stream.c

   1 /*
   2  * This file is part of MPlayer.
   3  *
   4  * MPlayer is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * MPlayer is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along
  15  * with MPlayer; if not, write to the Free Software Foundation, Inc.,
  16  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  17  */
  18
  19 #include <stdio.h>
  20 #include <stdlib.h>
  21 #include <unistd.h>
  22
  23 #include <sys/types.h>
  24 #include <sys/stat.h>
  25 #ifndef __MINGW32__
  26 #include <sys/ioctl.h>
  27 #include <sys/wait.h>
  28 #endif
  29 #include <fcntl.h>
  30 #include <strings.h>
  31 #include <assert.h>
  32
  33 #include <libavutil/intreadwrite.h>
  34 #include <libavutil/common.h>
  35
  36 #include "talloc.h"
  37
  38 #include "config.h"
  39
  40 #if HAVE_WINSOCK2_H
  41 #include <winsock2.h>
  42 #endif
  43
  44 #include "mp_msg.h"
  45 #include "osdep/shmem.h"
  46 #include "osdep/timer.h"
  47 #include "network.h"
  48 #include "stream.h"
  49 #include "libmpdemux/demuxer.h"
  50 #include "options.h"
  51
  52 #include "m_option.h"
  53 #include "m_struct.h"
  54
  55 #include "cache2.h"
  56
  57 char* cdrom_device=NULL;
  58 char* dvd_device=NULL;
  59 int dvd_title=0;
  60
  61 struct input_ctx;
  62 static int (*stream_check_interrupt_cb)(struct input_ctx *ctx, int time);
  63 static struct input_ctx *stream_check_interrupt_ctx;
  64
  65 extern const stream_info_t stream_info_vcd;
  66 extern const stream_info_t stream_info_cdda;
  67 extern const stream_info_t stream_info_pnm;
  68 extern const stream_info_t stream_info_asf;
  69 extern const stream_info_t stream_info_rtp;
  70 extern const stream_info_t stream_info_udp;
  71 extern const stream_info_t stream_info_http1;
  72 extern const stream_info_t stream_info_http2;
  73 extern const stream_info_t stream_info_dvb;
  74 extern const stream_info_t stream_info_tv;
  75 extern const stream_info_t stream_info_radio;
  76 extern const stream_info_t stream_info_pvr;
  77 extern const stream_info_t stream_info_ftp;
  78 extern const stream_info_t stream_info_vstream;
  79 extern const stream_info_t stream_info_dvdnav;
  80 extern const stream_info_t stream_info_smb;
  81
  82 extern const stream_info_t stream_info_cue;
  83 extern const stream_info_t stream_info_null;
  84 extern const stream_info_t stream_info_mf;
  85 extern const stream_info_t stream_info_ffmpeg;
  86 extern const stream_info_t stream_info_file;
  87 extern const stream_info_t stream_info_ifo;
  88 extern const stream_info_t stream_info_dvd;
  89 extern const stream_info_t stream_info_bluray;
  90
  91 static const stream_info_t* const auto_open_streams[] = {
  92 #ifdef CONFIG_VCD
  93   &stream_info_vcd,
  94 #endif
  95 #ifdef CONFIG_CDDA
  96   &stream_info_cdda,
  97 #endif
  98   &stream_info_ffmpeg,  // use for rstp:// before http fallback
  99 #ifdef CONFIG_NETWORKING
 100   &stream_info_http1,
 101   &stream_info_asf,
 102   &stream_info_pnm,
 103   &stream_info_rtp,
 104   &stream_info_udp,
 105   &stream_info_http2,
 106 #endif
 107 #ifdef CONFIG_DVBIN
 108   &stream_info_dvb,
 109 #endif
 110 #ifdef CONFIG_TV
 111   &stream_info_tv,
 112 #endif
 113 #ifdef CONFIG_RADIO
 114   &stream_info_radio,
 115 #endif
 116 #ifdef CONFIG_PVR
 117   &stream_info_pvr,
 118 #endif
 119 #ifdef CONFIG_FTP
 120   &stream_info_ftp,
 121 #endif
 122 #ifdef CONFIG_VSTREAM
 123   &stream_info_vstream,
 124 #endif
 125 #ifdef CONFIG_LIBSMBCLIENT
 126   &stream_info_smb,
 127 #endif
 128   &stream_info_cue,
 129 #ifdef CONFIG_DVDREAD
 130   &stream_info_ifo,
 131   &stream_info_dvd,
 132 #endif
 133 #ifdef CONFIG_DVDNAV
 134   &stream_info_dvdnav,
 135 #endif
 136 #ifdef CONFIG_LIBBLURAY
 137   &stream_info_bluray,
 138 #endif
 139
 140   &stream_info_null,
 141   &stream_info_mf,
 142   &stream_info_file,
 143   NULL
 144 };
 145
 146 static stream_t *open_stream_plugin(const stream_info_t *sinfo,
 147                                     const char *filename,
 148                                     int mode, struct MPOpts *options,
 149                                     int *file_format, int *ret,
 150                                     char **redirected_url)
 151 {
 152   void* arg = NULL;
 153   stream_t* s;
 154   m_struct_t* desc = (m_struct_t*)sinfo->opts;
 155
 156   // Parse options
 157   if(desc) {
 158     arg = m_struct_alloc(desc);
 159     if(sinfo->opts_url) {
 160       m_option_t url_opt =
 161         { "stream url", arg , CONF_TYPE_CUSTOM_URL, 0, 0 ,0, (void *)sinfo->opts };
 162       if (m_option_parse(&url_opt, bstr("stream url"), bstr(filename), false, arg) < 0) {
 163         mp_tmsg(MSGT_OPEN,MSGL_ERR, "URL parsing failed on url %s\n",filename);
 164         m_struct_free(desc,arg);
 165         return NULL;
 166       }
 167     }
 168   }
 169   s = new_stream(-2,-2);
 170   s->opts = options;
 171   s->url=strdup(filename);
 172   s->flags |= mode;
 173   *ret = sinfo->open(s,mode,arg,file_format);
 174   if((*ret) != STREAM_OK) {
 175 #ifdef CONFIG_NETWORKING
 176     if (*ret == STREAM_REDIRECTED && redirected_url) {
 177         if (s->streaming_ctrl && s->streaming_ctrl->url
 178             && s->streaming_ctrl->url->url)
 179           *redirected_url = strdup(s->streaming_ctrl->url->url);
 180         else
 181           *redirected_url = NULL;
 182     }
 183     streaming_ctrl_free(s->streaming_ctrl);
 184 #endif
 185     free(s->url);
 186     free(s);
 187     return NULL;
 188   }
 189
 190     s->cache_size = 0;
 191     if (s->streaming_ctrl && s->streaming_ctrl->buffering) {
 192         // Set default cache size to use if user does not specify it.
 193         // buffer in KBytes, *5 assuming the prefill is 20% of the buffer.
 194         s->cache_size = s->streaming_ctrl->prebuffer_size / 1024 * 5;
 195         if (s->cache_size < 64)
 196             s->cache_size = 64;
 197     }
 198
 199   if(s->type <= -2)
 200     mp_msg(MSGT_OPEN,MSGL_WARN, "Warning streams need a type !!!!\n");
 201   if(s->flags & MP_STREAM_SEEK && !s->seek)
 202     s->flags &= ~MP_STREAM_SEEK;
 203   if(s->seek && !(s->flags & MP_STREAM_SEEK))
 204     s->flags |= MP_STREAM_SEEK;
 205
 206   s->mode = mode;
 207
 208   mp_msg(MSGT_OPEN,MSGL_V, "STREAM: [%s] %s\n",sinfo->name,filename);
 209   mp_msg(MSGT_OPEN,MSGL_V, "STREAM: Description: %s\n",sinfo->info);
 210   mp_msg(MSGT_OPEN,MSGL_V, "STREAM: Author: %s\n", sinfo->author);
 211   mp_msg(MSGT_OPEN,MSGL_V, "STREAM: Comment: %s\n", sinfo->comment);
 212
 213   return s;
 214 }
 215
 216
 217 static stream_t *open_stream_full(const char *filename, int mode,
 218                                   struct MPOpts *options, int *file_format)
 219 {
 220   int i,j,l,r;
 221   const stream_info_t* sinfo;
 222   stream_t* s;
 223   char *redirected_url = NULL;
 224
 225   for(i = 0 ; auto_open_streams[i] ; i++) {
 226     sinfo = auto_open_streams[i];
 227     if(!sinfo->protocols) {
 228       mp_msg(MSGT_OPEN,MSGL_WARN, "Stream type %s has protocols == NULL, it's a bug\n", sinfo->name);
 229       continue;
 230     }
 231     for(j = 0 ; sinfo->protocols[j] ; j++) {
 232       l = strlen(sinfo->protocols[j]);
 233       // l == 0 => Don't do protocol matching (ie network and filenames)
 234       if((l == 0 && !strstr(filename, "://")) ||
 235          ((strncasecmp(sinfo->protocols[j],filename,l) == 0) &&
 236                       (strncmp("://",filename+l,3) == 0))) {
 237         *file_format = DEMUXER_TYPE_UNKNOWN;
 238         s = open_stream_plugin(sinfo,filename,mode,options,file_format,&r,
 239                                 &redirected_url);
 240         if(s) return s;
 241         if(r == STREAM_REDIRECTED && redirected_url) {
 242           mp_msg(MSGT_OPEN,MSGL_V, "[%s] open %s redirected to %s\n",
 243                  sinfo->info, filename, redirected_url);
 244           s = open_stream_full(redirected_url, mode, options, file_format);
 245           free(redirected_url);
 246           return s;
 247         }
 248         else if(r != STREAM_UNSUPPORTED) {
 249           mp_tmsg(MSGT_OPEN,MSGL_ERR, "Failed to open %s.\n",filename);
 250           return NULL;
 251         }
 252         break;
 253       }
 254     }
 255   }
 256
 257   mp_tmsg(MSGT_OPEN,MSGL_ERR, "No stream found to handle url %s\n", filename);
 258   return NULL;
 259 }
 260
 261 stream_t *open_stream(const char *filename, struct MPOpts *options,
 262                       int *file_format)
 263 {
 264     if (!file_format)
 265         file_format = &(int){DEMUXER_TYPE_UNKNOWN};
 266     if (*file_format != DEMUXER_TYPE_PLAYLIST)
 267         *file_format = DEMUXER_TYPE_UNKNOWN;
 268
 269     return open_stream_full(filename, STREAM_READ, options, file_format);
 270 }
 271
 272 stream_t *open_output_stream(const char *filename, struct MPOpts *options)
 273 {
 274   int file_format; //unused
 275   if(!filename) {
 276     mp_msg(MSGT_OPEN,MSGL_ERR,"open_output_stream(), NULL filename, report this bug\n");
 277     return NULL;
 278   }
 279
 280   return open_stream_full(filename,STREAM_WRITE,options,&file_format);
 281 }
 282
 283 //=================== STREAMER =========================
 284
 285 void stream_capture_do(stream_t *s)
 286 {
 287   if (fwrite(s->buffer, s->buf_len, 1, s->capture_file) < 1) {
 288     mp_tmsg(MSGT_GLOBAL, MSGL_ERR, "Error writing capture file: %s\n",
 289             strerror(errno));
 290     fclose(s->capture_file);
 291     s->capture_file = NULL;
 292   }
 293 }
 294
 295 int stream_read_internal(stream_t *s, void *buf, int len)
 296 {
 297   int orig_len = len;
 298   // we will retry even if we already reached EOF previously.
 299   switch(s->type){
 300   case STREAMTYPE_STREAM:
 301 #ifdef CONFIG_NETWORKING
 302     if( s->streaming_ctrl!=NULL && s->streaming_ctrl->streaming_read ) {
 303       len=s->streaming_ctrl->streaming_read(s->fd, buf, len, s->streaming_ctrl);
 304       if (s->streaming_ctrl->status == streaming_stopped_e)
 305         s->eof = 1;
 306     } else
 307 #endif
 308     if (s->fill_buffer)
 309       len = s->fill_buffer(s, buf, len);
 310     else
 311       len = read(s->fd, buf, len);
 312     break;
 313   case STREAMTYPE_DS:
 314     len = demux_read_data((demux_stream_t*)s->priv, buf, len);
 315     break;
 316
 317
 318   default:
 319     len= s->fill_buffer ? s->fill_buffer(s, buf, len) : 0;
 320   }
 321   if(len<=0){
 322     off_t pos = s->pos;
 323     // do not retry if this looks like proper eof
 324     if (s->eof || (s->end_pos && pos == s->end_pos))
 325       goto eof_out;
 326     // dvdnav has some horrible hacks to "suspend" reads,
 327     // we need to skip this code or seeks will hang.
 328     if (s->type == STREAMTYPE_DVDNAV)
 329       goto eof_out;
 330
 331     // just in case this is an error e.g. due to network
 332     // timeout reset and retry
 333     // Seeking is used as a hack to make network streams
 334     // reopen the connection, ideally they would implement
 335     // e.g. a STREAM_CTRL_RECONNECT to do this
 336     s->eof=1;
 337     stream_reset(s);
 338     if (stream_seek_internal(s, pos) >= 0 || s->pos != pos) // seek failed
 339       goto eof_out;
 340     // make sure EOF is set to ensure no endless loops
 341     s->eof=1;
 342     return stream_read_internal(s, buf, orig_len);
 343
 344 eof_out:
 345     s->eof=1;
 346     return 0;
 347   }
 348   // When reading succeeded we are obviously not at eof.
 349   // This e.g. avoids issues with eof getting stuck when lavf seeks in MPEG-TS
 350   s->eof=0;
 351   s->pos+=len;
 352   return len;
 353 }
 354
 355 int stream_fill_buffer(stream_t *s){
 356   int len = stream_read_internal(s, s->buffer, STREAM_BUFFER_SIZE);
 357   if (len <= 0)
 358     return 0;
 359   s->buf_pos=0;
 360   s->buf_len=len;
 361 //  printf("[%d]",len);fflush(stdout);
 362   if (s->capture_file)
 363     stream_capture_do(s);
 364   return len;
 365 }
 366
 367 int stream_write_buffer(stream_t *s, unsigned char *buf, int len) {
 368   int rd;
 369   if(!s->write_buffer)
 370     return -1;
 371   rd = s->write_buffer(s, buf, len);
 372   if(rd < 0)
 373     return -1;
 374   s->pos += rd;
 375   assert(rd == len && "stream_write_buffer(): unexpected short write");
 376   return rd;
 377 }
 378
 379 int stream_seek_internal(stream_t *s, off_t newpos)
 380 {
 381 if(newpos==0 || newpos!=s->pos){
 382   switch(s->type){
 383   case STREAMTYPE_STREAM:
 384     //s->pos=newpos; // real seek
 385     // Some streaming protocol allow to seek backward and forward
 386     // A function call that return -1 can tell that the protocol
 387     // doesn't support seeking.
 388 #ifdef CONFIG_NETWORKING
 389     if(s->seek) { // new stream seek is much cleaner than streaming_ctrl one
 390       if(!s->seek(s,newpos)) {
 391         mp_tmsg(MSGT_STREAM,MSGL_ERR, "Seek failed\n");
 392         return 0;
 393       }
 394       break;
 395     }
 396
 397     if( s->streaming_ctrl!=NULL && s->streaming_ctrl->streaming_seek ) {
 398       if( s->streaming_ctrl->streaming_seek( s->fd, newpos, s->streaming_ctrl )<0 ) {
 399         mp_tmsg(MSGT_STREAM,MSGL_INFO,"Stream not seekable!\n");
 400         return 1;
 401       }
 402       break;
 403     }
 404 #endif
 405     if(newpos<s->pos){
 406       mp_tmsg(MSGT_STREAM, MSGL_INFO,
 407               "Cannot seek backward in linear streams!\n");
 408       return 1;
 409     }
 410     break;
 411   default:
 412     // This should at the beginning as soon as all streams are converted
 413     if(!s->seek)
 414       return 0;
 415     // Now seek
 416     if(!s->seek(s,newpos)) {
 417       mp_tmsg(MSGT_STREAM,MSGL_ERR, "Seek failed\n");
 418       return 0;
 419     }
 420   }
 421 //   putchar('.');fflush(stdout);
 422 //} else {
 423 //   putchar('%');fflush(stdout);
 424 }
 425   return -1;
 426 }
 427
 428 int stream_seek_long(stream_t *s,off_t pos){
 429   int res;
 430 off_t newpos=0;
 431
 432 //  if( mp_msg_test(MSGT_STREAM,MSGL_DBG3) ) printf("seek_long to 0x%X\n",(unsigned int)pos);
 433
 434   s->buf_pos=s->buf_len=0;
 435
 436   if(s->mode == STREAM_WRITE) {
 437     if(!s->seek || !s->seek(s,pos))
 438       return 0;
 439     return 1;
 440   }
 441
 442   if(s->sector_size)
 443       newpos = (pos/s->sector_size)*s->sector_size;
 444   else
 445       newpos = pos&(~((off_t)STREAM_BUFFER_SIZE-1));
 446
 447 if( mp_msg_test(MSGT_STREAM,MSGL_DBG3) ){
 448   mp_msg(MSGT_STREAM,MSGL_DBG3, "s->pos=%"PRIX64"  newpos=%"PRIX64"  new_bufpos=%"PRIX64"  buflen=%X  \n",
 449     (int64_t)s->pos,(int64_t)newpos,(int64_t)pos,s->buf_len);
 450 }
 451   pos-=newpos;
 452
 453   res = stream_seek_internal(s, newpos);
 454   if (res >= 0)
 455     return res;
 456
 457   while(s->pos<newpos){
 458     if(stream_fill_buffer(s)<=0) break; // EOF
 459   }
 460
 461 s->eof = 0; // EOF reset when seek succeeds.
 462 while (stream_fill_buffer(s) > 0) {
 463   if(pos<=s->buf_len){
 464     s->buf_pos=pos; // byte position in sector
 465     return 1;
 466   }
 467   pos -= s->buf_len;
 468 }
 469 // Fill failed, but seek still is a success.
 470 s->pos += pos;
 471 s->buf_pos = 0;
 472 s->buf_len = 0;
 473
 474 mp_msg(MSGT_STREAM,MSGL_V,
 475        "stream_seek: Seek to/past EOF: no buffer preloaded.\n");
 476 return 1;
 477 }
 478
 479
 480 void stream_reset(stream_t *s){
 481   if(s->eof){
 482     s->pos=0;
 483     s->buf_pos=s->buf_len=0;
 484     s->eof=0;
 485   }
 486   if(s->control) s->control(s,STREAM_CTRL_RESET,NULL);
 487   //stream_seek(s,0);
 488 }
 489
 490 int stream_control(stream_t *s, int cmd, void *arg){
 491   if(!s->control) return STREAM_UNSUPPORTED;
 492 #ifdef CONFIG_STREAM_CACHE
 493   if (s->cache_pid)
 494     return cache_do_control(s, cmd, arg);
 495 #endif
 496   return s->control(s, cmd, arg);
 497 }
 498
 499 stream_t* new_memory_stream(unsigned char* data,int len){
 500   stream_t *s;
 501
 502   if(len < 0)
 503     return NULL;
 504   s=calloc(1, sizeof(stream_t)+len);
 505   s->fd=-1;
 506   s->type=STREAMTYPE_MEMORY;
 507   s->buf_pos=0; s->buf_len=len;
 508   s->start_pos=0; s->end_pos=len;
 509   stream_reset(s);
 510   s->pos=len;
 511   memcpy(s->buffer,data,len);
 512   return s;
 513 }
 514
 515 stream_t* new_stream(int fd,int type){
 516   stream_t *s=calloc(1, sizeof(stream_t));
 517   if(s==NULL) return NULL;
 518
 519 #if HAVE_WINSOCK2_H
 520   {
 521     WSADATA wsdata;
 522     int temp = WSAStartup(0x0202, &wsdata); // there might be a better place for this (-> later)
 523     mp_msg(MSGT_STREAM,MSGL_V,"WINSOCK2 init: %i\n", temp);
 524   }
 525 #endif
 526
 527   s->fd=fd;
 528   s->type=type;
 529   s->buf_pos=s->buf_len=0;
 530   s->start_pos=s->end_pos=0;
 531   s->priv=NULL;
 532   s->url=NULL;
 533   s->cache_pid=0;
 534   stream_reset(s);
 535   return s;
 536 }
 537
 538 void free_stream(stream_t *s){
 539 //  printf("\n*** free_stream() called ***\n");
 540 #ifdef CONFIG_STREAM_CACHE
 541     cache_uninit(s);
 542 #endif
 543   if (s->capture_file) {
 544     fclose(s->capture_file);
 545     s->capture_file = NULL;
 546   }
 547
 548   if(s->close) s->close(s);
 549   if(s->fd>0){
 550     /* on unix we define closesocket to close
 551        on windows however we have to distinguish between
 552        network socket and file */
 553     if(s->url && strstr(s->url,"://"))
 554       closesocket(s->fd);
 555     else close(s->fd);
 556   }
 557 #if HAVE_WINSOCK2_H
 558   mp_msg(MSGT_STREAM,MSGL_V,"WINSOCK2 uninit\n");
 559   WSACleanup(); // there might be a better place for this (-> later)
 560 #endif
 561   // Disabled atm, i don't like that. s->priv can be anything after all
 562   // streams should destroy their priv on close
 563   //free(s->priv);
 564   free(s->url);
 565   free(s);
 566 }
 567
 568 stream_t* new_ds_stream(demux_stream_t *ds) {
 569   stream_t* s = new_stream(-1,STREAMTYPE_DS);
 570   s->priv = ds;
 571   return s;
 572 }
 573
 574 void stream_set_interrupt_callback(int (*cb)(struct input_ctx *, int),
 575                                    struct input_ctx *ctx)
 576 {
 577     stream_check_interrupt_cb = cb;
 578     stream_check_interrupt_ctx = ctx;
 579 }
 580
 581 int stream_check_interrupt(int time) {
 582     if(!stream_check_interrupt_cb) {
 583         usec_sleep(time * 1000);
 584         return 0;
 585     }
 586     return stream_check_interrupt_cb(stream_check_interrupt_ctx, time);
 587 }
 588
 589 /**
 590  * Helper function to read 16 bits little-endian and advance pointer
 591  */
 592 static uint16_t get_le16_inc(const uint8_t **buf)
 593 {
 594   uint16_t v = AV_RL16(*buf);
 595   *buf += 2;
 596   return v;
 597 }
 598
 599 /**
 600  * Helper function to read 16 bits big-endian and advance pointer
 601  */
 602 static uint16_t get_be16_inc(const uint8_t **buf)
 603 {
 604   uint16_t v = AV_RB16(*buf);
 605   *buf += 2;
 606   return v;
 607 }
 608
 609 /**
 610  * Find a newline character in buffer
 611  * \param buf buffer to search
 612  * \param len amount of bytes to search in buffer, may not overread
 613  * \param utf16 chose between UTF-8/ASCII/other and LE and BE UTF-16
 614  *              0 = UTF-8/ASCII/other, 1 = UTF-16-LE, 2 = UTF-16-BE
 615  */
 616 static const uint8_t *find_newline(const uint8_t *buf, int len, int utf16)
 617 {
 618   uint32_t c;
 619   const uint8_t *end = buf + len;
 620   switch (utf16) {
 621   case 0:
 622     return (uint8_t *)memchr(buf, '\n', len);
 623   case 1:
 624     while (buf < end - 1) {
 625       GET_UTF16(c, buf < end - 1 ? get_le16_inc(&buf) : 0, return NULL;)
 626       if (buf <= end && c == '\n')
 627         return buf - 1;
 628     }
 629     break;
 630   case 2:
 631     while (buf < end - 1) {
 632       GET_UTF16(c, buf < end - 1 ? get_be16_inc(&buf) : 0, return NULL;)
 633       if (buf <= end && c == '\n')
 634         return buf - 1;
 635     }
 636     break;
 637   }
 638   return NULL;
 639 }
 640
 641 /**
 642  * Copy a number of bytes, converting to UTF-8 if input is UTF-16
 643  * \param dst buffer to copy to
 644  * \param dstsize size of dst buffer
 645  * \param src buffer to copy from
 646  * \param len amount of bytes to copy from src
 647  * \param utf16 chose between UTF-8/ASCII/other and LE and BE UTF-16
 648  *              0 = UTF-8/ASCII/other, 1 = UTF-16-LE, 2 = UTF-16-BE
 649  */
 650 static int copy_characters(uint8_t *dst, int dstsize,
 651                            const uint8_t *src, int *len, int utf16)
 652 {
 653   uint32_t c;
 654   uint8_t *dst_end = dst + dstsize;
 655   const uint8_t *end = src + *len;
 656   switch (utf16) {
 657   case 0:
 658     if (*len > dstsize)
 659       *len = dstsize;
 660     memcpy(dst, src, *len);
 661     return *len;
 662   case 1:
 663     while (src < end - 1 && dst_end - dst > 8) {
 664       uint8_t tmp;
 665       GET_UTF16(c, src < end - 1 ? get_le16_inc(&src) : 0, ;)
 666       PUT_UTF8(c, tmp, *dst++ = tmp;)
 667     }
 668     *len -= end - src;
 669     return dstsize - (dst_end - dst);
 670   case 2:
 671     while (src < end - 1 && dst_end - dst > 8) {
 672       uint8_t tmp;
 673       GET_UTF16(c, src < end - 1 ? get_be16_inc(&src) : 0, ;)
 674       PUT_UTF8(c, tmp, *dst++ = tmp;)
 675     }
 676     *len -= end - src;
 677     return dstsize - (dst_end - dst);
 678   }
 679   return 0;
 680 }
 681
 682 unsigned char* stream_read_line(stream_t *s,unsigned char* mem, int max, int utf16) {
 683   int len;
 684   const unsigned char *end;
 685   unsigned char *ptr = mem;
 686   if (max < 1) return NULL;
 687   max--; // reserve one for 0-termination
 688   do {
 689     len = s->buf_len-s->buf_pos;
 690     // try to fill the buffer
 691     if(len <= 0 &&
 692        (!cache_stream_fill_buffer(s) ||
 693         (len = s->buf_len-s->buf_pos) <= 0)) break;
 694     end = find_newline(s->buffer+s->buf_pos, len, utf16);
 695     if(end) len = end - (s->buffer+s->buf_pos) + 1;
 696     if(len > 0 && max > 0) {
 697       int l = copy_characters(ptr, max, s->buffer+s->buf_pos, &len, utf16);
 698       max -= l;
 699       ptr += l;
 700       if (!len)
 701         break;
 702     }
 703     s->buf_pos += len;
 704   } while(!end);
 705   ptr[0] = 0;
 706   if(s->eof && ptr == mem) return NULL;
 707   return mem;
 708 }
 709
 710 struct bstr stream_read_complete(struct stream *s, void *talloc_ctx,
 711                                  int max_size, int padding_bytes)
 712 {
 713     if (max_size > 1000000000)
 714         abort();
 715
 716     int bufsize;
 717     int total_read = 0;
 718     int padding = FFMAX(padding_bytes, 1);
 719     char *buf = NULL;
 720     if (s->end_pos > max_size)
 721         return (struct bstr){NULL, 0};
 722     if (s->end_pos > 0)
 723         bufsize = s->end_pos + padding;
 724     else
 725         bufsize = 1000;
 726     while (1) {
 727         buf = talloc_realloc_size(talloc_ctx, buf, bufsize);
 728         int readsize = stream_read(s, buf + total_read, bufsize - total_read);
 729         total_read += readsize;
 730         if (total_read < bufsize)
 731             break;
 732         if (bufsize > max_size) {
 733             talloc_free(buf);
 734             return (struct bstr){NULL, 0};
 735         }
 736         bufsize = FFMIN(bufsize + (bufsize >> 1), max_size + padding);
 737     }
 738     buf = talloc_realloc_size(talloc_ctx, buf, total_read + padding);
 739     return (struct bstr){buf, total_read};
 740 }