packetizer: hevc: don't double store poc prev msb/lsb
[vlc.git] / modules / stream_extractor / archive.c
blob4fdb3b13786192f4a4bcc87c5bd0ee9ea869f429
1 /*****************************************************************************
2 * archive.c: libarchive based stream filter
3 *****************************************************************************
4 * Copyright (C) 2016 VLC authors and VideoLAN
5 * $Id$
7 * Authors: Filip Roséen <filip@atch.se>
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU Lesser General Public License as published by
11 * the Free Software Foundation; either version 2.1 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public License
20 * along with this program; if not, write to the Free Software Foundation,
21 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22 *****************************************************************************/
24 #ifdef HAVE_CONFIG_H
25 # include "config.h"
26 #endif
28 #include <vlc_common.h>
29 #include <vlc_plugin.h>
30 #include <vlc_stream.h>
31 #include <vlc_stream_extractor.h>
32 #include <vlc_dialog.h>
33 #include <vlc_input_item.h>
35 #include <assert.h>
36 #include <archive.h>
37 #include <archive_entry.h>
39 #if ARCHIVE_VERSION_NUMBER < 3002000
40 typedef __LA_INT64_T la_int64_t;
41 typedef __LA_SSIZE_T la_ssize_t;
42 #endif
44 static int ExtractorOpen( vlc_object_t* );
45 static void ExtractorClose( vlc_object_t* );
47 static int DirectoryOpen( vlc_object_t* );
48 static void DirectoryClose( vlc_object_t* );
50 vlc_module_begin()
51 set_category( CAT_INPUT )
52 set_subcategory( SUBCAT_INPUT_STREAM_FILTER )
53 set_capability( "stream_directory", 99 )
54 set_description( N_( "libarchive based stream directory" ) )
55 set_callbacks( DirectoryOpen, DirectoryClose );
57 add_submodule()
58 set_description( N_( "libarchive based stream extractor" ) )
59 set_capability( "stream_extractor", 99 )
60 set_callbacks( ExtractorOpen, ExtractorClose );
62 vlc_module_end()
64 typedef struct libarchive_callback_t libarchive_callback_t;
65 typedef struct private_sys_t private_sys_t;
66 typedef struct archive libarchive_t;
68 struct private_sys_t
70 libarchive_t* p_archive;
71 vlc_object_t* p_obj;
72 stream_t* source;
74 struct archive_entry* p_entry;
75 bool b_dead;
77 uint64_t i_offset;
79 uint8_t buffer[ 8192 ];
80 bool b_seekable_source;
81 bool b_seekable_archive;
83 libarchive_callback_t** pp_callback_data;
84 size_t i_callback_data;
87 struct libarchive_callback_t {
88 private_sys_t* p_sys;
89 stream_t* p_source;
90 char* psz_url;
93 /* ------------------------------------------------------------------------- */
95 static int libarchive_exit_cb( libarchive_t* p_arc, void* p_obj )
97 VLC_UNUSED( p_arc );
99 libarchive_callback_t* p_cb = (libarchive_callback_t*)p_obj;
101 if( p_cb->p_sys->source == p_cb->p_source )
102 { /* DO NOT CLOSE OUR MOTHER STREAM */
103 if( !p_cb->p_sys->b_dead && vlc_stream_Seek( p_cb->p_source, 0 ) )
104 return ARCHIVE_FATAL;
106 else if( p_cb->p_source )
108 vlc_stream_Delete( p_cb->p_source );
109 p_cb->p_source = NULL;
112 return ARCHIVE_OK;
115 static int libarchive_jump_cb( libarchive_t* p_arc, void* p_obj_current,
116 void* p_obj_next )
118 libarchive_callback_t* p_current = (libarchive_callback_t*)p_obj_current;
119 libarchive_callback_t* p_next = (libarchive_callback_t*)p_obj_next;
121 if( libarchive_exit_cb( p_arc, p_current ) )
122 return ARCHIVE_FATAL;
124 if( p_next->p_source == NULL )
125 p_next->p_source = vlc_stream_NewURL( p_next->p_sys->p_obj,
126 p_next->psz_url );
128 return p_next->p_source ? ARCHIVE_OK : ARCHIVE_FATAL;
132 static la_int64_t libarchive_skip_cb( libarchive_t* p_arc, void* p_obj,
133 off_t i_request )
135 VLC_UNUSED( p_arc );
137 libarchive_callback_t* p_cb = (libarchive_callback_t*)p_obj;
139 stream_t* p_source = p_cb->p_source;
140 private_sys_t* p_sys = p_cb->p_sys;
142 /* TODO: fix b_seekable_source on libarchive_callback_t */
144 if( p_sys->b_seekable_source )
146 if( vlc_stream_Seek( p_source, vlc_stream_Tell( p_source ) + i_request ) )
147 return ARCHIVE_FATAL;
149 return i_request;
152 ssize_t i_read = vlc_stream_Read( p_source, NULL, i_request );
153 return i_read >= 0 ? i_read : ARCHIVE_FATAL;
156 static la_int64_t libarchive_seek_cb( libarchive_t* p_arc, void* p_obj,
157 la_int64_t offset, int whence )
159 VLC_UNUSED( p_arc );
161 libarchive_callback_t* p_cb = (libarchive_callback_t*)p_obj;
162 stream_t* p_source = p_cb->p_source;
164 ssize_t whence_pos;
166 switch( whence )
168 case SEEK_SET: whence_pos = 0; break;
169 case SEEK_CUR: whence_pos = vlc_stream_Tell( p_source ); break;
170 case SEEK_END: whence_pos = stream_Size( p_source ); break;
171 default: vlc_assert_unreachable();
175 if( whence_pos < 0 || vlc_stream_Seek( p_source, whence_pos + offset ) )
176 return ARCHIVE_FATAL;
178 return vlc_stream_Tell( p_source );
181 static la_ssize_t libarchive_read_cb( libarchive_t* p_arc, void* p_obj,
182 const void** pp_dst )
184 VLC_UNUSED( p_arc );
186 libarchive_callback_t* p_cb = (libarchive_callback_t*)p_obj;
188 stream_t* p_source = p_cb->p_source;
189 private_sys_t* p_sys = p_cb->p_sys;
191 ssize_t i_ret = vlc_stream_Read( p_source, &p_sys->buffer,
192 sizeof( p_sys->buffer ) );
194 if( i_ret < 0 )
196 archive_set_error( p_sys->p_archive, ARCHIVE_FATAL,
197 "libarchive_read_cb failed = %" PRId64, i_ret );
199 return ARCHIVE_FATAL;
202 *pp_dst = &p_sys->buffer;
203 return i_ret;
206 /* ------------------------------------------------------------------------- */
208 static int archive_push_resource( private_sys_t* p_sys,
209 stream_t* p_source, char const* psz_url )
211 libarchive_callback_t** pp_callback_data;
212 libarchive_callback_t* p_callback_data;
214 /* INCREASE BUFFER SIZE */
216 pp_callback_data = realloc( p_sys->pp_callback_data,
217 sizeof( *p_sys->pp_callback_data ) * ( p_sys->i_callback_data + 1 ) );
219 if( unlikely( !pp_callback_data ) )
220 goto error;
222 /* CREATE NEW NODE */
224 p_callback_data = malloc( sizeof( *p_callback_data ) );
226 if( unlikely( !p_callback_data ) )
227 goto error;
229 /* INITIALIZE AND APPEND */
231 p_callback_data->psz_url = psz_url ? strdup( psz_url ) : NULL;
232 p_callback_data->p_source = p_source;
233 p_callback_data->p_sys = p_sys;
235 if( unlikely( !p_callback_data->psz_url && psz_url ) )
237 free( p_callback_data );
238 goto error;
241 pp_callback_data[ p_sys->i_callback_data++ ] = p_callback_data;
242 p_sys->pp_callback_data = pp_callback_data;
244 return VLC_SUCCESS;
246 error:
247 free( pp_callback_data );
248 return VLC_ENOMEM;
251 static int archive_init( private_sys_t* p_sys, stream_t* source )
253 /* CREATE ARCHIVE HANDLE */
255 p_sys->p_archive = archive_read_new();
257 if( unlikely( !p_sys->p_archive ) )
259 msg_Dbg( p_sys->p_obj, "unable to create libarchive handle" );
260 return VLC_EGENERIC;
263 /* SETUP SEEKING */
265 p_sys->b_seekable_archive = false;
267 if( vlc_stream_Control( source, STREAM_CAN_SEEK,
268 &p_sys->b_seekable_source ) )
270 msg_Warn( p_sys->p_obj, "unable to query whether source stream can seek" );
271 p_sys->b_seekable_source = false;
274 if( p_sys->b_seekable_source )
276 if( archive_read_set_seek_callback( p_sys->p_archive,
277 libarchive_seek_cb ) )
279 msg_Err( p_sys->p_obj, "archive_read_set_callback failed, aborting." );
280 return VLC_EGENERIC;
284 /* ENABLE ALL FORMATS/FILTERS */
286 archive_read_support_filter_all( p_sys->p_archive );
287 archive_read_support_format_all( p_sys->p_archive );
289 /* REGISTER CALLBACK DATA */
291 if( archive_read_set_switch_callback( p_sys->p_archive,
292 libarchive_jump_cb ) )
294 msg_Err( p_sys->p_obj, "archive_read_set_switch_callback failed, aborting." );
295 return VLC_EGENERIC;
298 for( size_t i = 0; i < p_sys->i_callback_data; ++i )
300 if( archive_read_append_callback_data( p_sys->p_archive,
301 p_sys->pp_callback_data[i] ) )
303 return VLC_EGENERIC;
307 /* OPEN THE ARCHIVE */
309 if( archive_read_open2( p_sys->p_archive, p_sys->pp_callback_data[0], NULL,
310 libarchive_read_cb, libarchive_skip_cb, libarchive_exit_cb ) )
312 msg_Dbg( p_sys->p_obj, "libarchive: %s",
313 archive_error_string( p_sys->p_archive ) );
315 return VLC_EGENERIC;
318 return VLC_SUCCESS;
321 static int archive_clean( private_sys_t* p_sys )
323 libarchive_t* p_arc = p_sys->p_archive;
325 if( p_sys->p_entry )
326 archive_entry_free( p_sys->p_entry );
328 if( p_arc )
329 archive_read_free( p_arc );
331 p_sys->p_entry = NULL;
332 p_sys->p_archive = NULL;
334 return VLC_SUCCESS;
337 static int archive_seek_subentry( private_sys_t* p_sys, char const* psz_subentry )
339 libarchive_t* p_arc = p_sys->p_archive;
341 struct archive_entry* entry;
342 int archive_status;
344 while( !( archive_status = archive_read_next_header( p_arc, &entry ) ) )
346 char const* entry_path = archive_entry_pathname( entry );
348 if( strcmp( entry_path, psz_subentry ) == 0 )
350 p_sys->p_entry = archive_entry_clone( entry );
352 if( unlikely( !p_sys->p_entry ) )
353 return VLC_ENOMEM;
355 break;
358 archive_read_data_skip( p_arc );
361 switch( archive_status )
363 case ARCHIVE_WARN:
364 msg_Warn( p_sys->p_obj,
365 "libarchive: %s", archive_error_string( p_arc ) );
367 case ARCHIVE_EOF:
368 case ARCHIVE_FATAL:
369 case ARCHIVE_RETRY:
370 archive_set_error( p_arc, ARCHIVE_FATAL,
371 "archive does not contain >>> %s <<<", psz_subentry );
373 return VLC_EGENERIC;
376 /* check if seeking is supported */
378 if( p_sys->b_seekable_source )
380 if( archive_seek_data( p_sys->p_archive, 0, SEEK_CUR ) >= 0 )
381 p_sys->b_seekable_archive = true;
384 return VLC_SUCCESS;
387 /* ------------------------------------------------------------------------- */
389 static private_sys_t* setup( vlc_object_t* obj, stream_t* source )
391 private_sys_t* p_sys = calloc( 1, sizeof( *p_sys ) );
392 char* psz_files = var_InheritString( obj, "concat-list" );
394 if( unlikely( !p_sys ) )
395 goto error;
397 if( archive_push_resource( p_sys, source, NULL ) )
398 goto error;
400 if( psz_files )
402 for( char* state,
403 * path = strtok_r( psz_files, ",", &state );
404 path; path = strtok_r( NULL, ",", &state ) )
406 if( path == psz_files )
407 continue;
409 if( archive_push_resource( p_sys, NULL, path ) )
410 goto error;
413 free( psz_files );
416 p_sys->source = source;
417 p_sys->p_obj = obj;
419 return p_sys;
421 error:
422 free( psz_files );
423 free( p_sys );
424 return NULL;
427 static int probe( stream_t* source )
429 struct
431 uint16_t i_offset;
432 uint8_t i_length;
433 char const * p_bytes;
434 } const magicbytes[] = {
435 /* keep heaviest at top */
436 { 257, 5, "ustar" }, //TAR
437 { 0, 7, "Rar!\x1A\x07" }, //RAR
438 { 0, 6, "7z\xBC\xAF\x27\x1C" }, //7z
439 { 0, 4, "xar!" }, //XAR
440 { 0, 4, "PK\x03\x04" }, //ZIP
441 { 0, 4, "PK\x05\x06" }, //ZIP
442 { 0, 4, "PK\x07\x08" }, //ZIP
443 { 2, 3, "-lh" }, //LHA/LHZ
444 { 0, 3, "\x1f\x8b\x08" }, // Gzip
445 { 0, 3, "PAX" }, //PAX
446 { 0, 6, "070707" }, //CPIO
447 { 0, 6, "070701" }, //CPIO
448 { 0, 6, "070702" }, //CPIO
449 { 0, 4, "MSCH" }, //CAB
452 const uint8_t *p_peek;
454 int i_peek = vlc_stream_Peek( source, &p_peek,
455 magicbytes[0].i_offset + magicbytes[0].i_length);
457 for(unsigned i=0; i < ARRAY_SIZE( magicbytes ); i++)
459 if (i_peek < magicbytes[i].i_offset + magicbytes[i].i_length)
460 continue;
462 if ( !memcmp(p_peek + magicbytes[i].i_offset,
463 magicbytes[i].p_bytes, magicbytes[i].i_length) )
464 return VLC_SUCCESS;
467 return VLC_EGENERIC;
470 /* ------------------------------------------------------------------------- */
472 static int Control( stream_extractor_t* p_extractor, int i_query, va_list args )
474 private_sys_t* p_sys = p_extractor->p_sys;
476 if( p_sys->b_dead )
477 return VLC_EGENERIC;
479 switch( i_query )
481 case STREAM_CAN_FASTSEEK:
482 *va_arg( args, bool* ) = false;
483 break;
485 case STREAM_CAN_SEEK:
486 *va_arg( args, bool* ) = p_sys->b_seekable_source;
487 break;
489 case STREAM_GET_SIZE:
490 if( p_sys->p_entry == NULL )
491 return VLC_EGENERIC;
493 *va_arg( args, uint64_t* ) = archive_entry_size( p_sys->p_entry );
494 break;
496 default:
497 return vlc_stream_vaControl( p_extractor->source, i_query, args );
500 return VLC_SUCCESS;
503 static int ReadDir( stream_directory_t* p_directory, input_item_node_t* p_node )
505 private_sys_t* p_sys = p_directory->p_sys;
506 libarchive_t* p_arc = p_sys->p_archive;
508 struct vlc_readdir_helper rdh;
509 vlc_readdir_helper_init( &rdh, p_directory, p_node);
510 struct archive_entry* entry;
511 int archive_status;
513 while( !( archive_status = archive_read_next_header( p_arc, &entry ) ) )
515 if( archive_entry_filetype( entry ) == AE_IFDIR )
516 continue;
518 char const* path = archive_entry_pathname( entry );
519 char* mrl = vlc_stream_extractor_CreateMRL( p_directory, path );
521 if( unlikely( !mrl ) )
522 break;
524 if( vlc_readdir_helper_additem( &rdh, mrl, path, NULL, ITEM_TYPE_FILE,
525 ITEM_LOCAL ) )
527 free( mrl );
528 break;
530 free( mrl );
532 if( archive_read_data_skip( p_arc ) )
533 break;
536 vlc_readdir_helper_finish( &rdh, archive_status == ARCHIVE_EOF );
537 return archive_status == ARCHIVE_EOF ? VLC_SUCCESS : VLC_EGENERIC;
540 static ssize_t Read( stream_extractor_t *p_extractor, void* p_data, size_t i_size )
542 char dummy_buffer[ 8192 ];
544 private_sys_t* p_sys = p_extractor->p_sys;
545 libarchive_t* p_arc = p_sys->p_archive;
546 ssize_t i_ret;
548 if( p_sys->b_dead || p_sys->p_entry == NULL )
549 return 0;
551 i_ret = archive_read_data( p_arc,
552 p_data ? p_data : dummy_buffer,
553 p_data ? i_size : __MIN( i_size, sizeof( dummy_buffer ) ) );
555 switch( i_ret )
557 case ARCHIVE_RETRY:
558 case ARCHIVE_FAILED:
559 msg_Dbg( p_extractor, "libarchive: %s", archive_error_string( p_arc ) );
560 return -1;
562 case ARCHIVE_WARN:
563 msg_Warn( p_extractor, "libarchive: %s", archive_error_string( p_arc ) );
564 return -1;
566 case ARCHIVE_FATAL:
567 p_sys->b_dead = true;
568 msg_Err( p_extractor, "libarchive: %s", archive_error_string( p_arc ) );
569 return 0;
572 p_sys->i_offset += i_ret;
573 return i_ret;
576 static int Seek( stream_extractor_t* p_extractor, uint64_t i_req )
578 private_sys_t* p_sys = p_extractor->p_sys;
580 if( p_sys->b_dead )
581 return VLC_EGENERIC;
583 if( !p_sys->p_entry )
584 return VLC_EGENERIC;
586 if( !p_sys->b_seekable_source )
587 return VLC_EGENERIC;
589 if( !p_sys->b_seekable_archive
590 || archive_seek_data( p_sys->p_archive, i_req, SEEK_SET ) < 0 )
592 msg_Dbg( p_extractor, "libarchive intrinsic seek failed:"
593 " '%s' (falling back to dumb seek)",
594 archive_error_string( p_sys->p_archive ) );
596 uint64_t i_offset = p_sys->i_offset;
597 uint64_t i_skip = i_req - i_offset;
599 /* RECREATE LIBARCHIE HANDLE IF WE ARE SEEKING BACKWARDS */
601 if( i_req < i_offset )
603 if( archive_clean( p_sys ) )
604 return VLC_EGENERIC;
606 if( archive_init( p_sys, p_extractor->source ) ||
607 archive_seek_subentry( p_sys, p_extractor->identifier ) )
609 msg_Err( p_extractor, "unable to recreate libarchive handle" );
610 return VLC_EGENERIC;
613 i_skip = i_req;
614 i_offset = 0;
617 /* SKIP _DECOMPRESSED_ DATA */
619 while( i_skip )
621 ssize_t i_read = Read( p_extractor, NULL, i_skip );
623 if( i_read < 1 )
624 return VLC_EGENERIC;
626 i_offset += i_read;
627 i_skip -= i_read;
631 p_sys->i_offset = i_req;
632 return VLC_SUCCESS;
636 static void CommonClose( private_sys_t* p_sys )
638 p_sys->b_dead = true;
639 archive_clean( p_sys );
641 for( size_t i = 0; i < p_sys->i_callback_data; ++i )
643 free( p_sys->pp_callback_data[i]->psz_url );
644 free( p_sys->pp_callback_data[i] );
647 free( p_sys->pp_callback_data );
648 free( p_sys );
651 static void DirectoryClose( vlc_object_t* p_obj )
653 stream_directory_t* p_directory = (void*)p_obj;
654 return CommonClose( p_directory->p_sys );
657 static void ExtractorClose( vlc_object_t* p_obj )
659 stream_extractor_t* p_extractor = (void*)p_obj;
660 return CommonClose( p_extractor->p_sys );
663 static private_sys_t* CommonOpen( vlc_object_t* p_obj, stream_t* source )
665 if( probe( source ) )
666 return NULL;
668 private_sys_t* p_sys = setup( p_obj, source );
670 if( p_sys == NULL )
671 return NULL;
673 if( archive_init( p_sys, source ) )
675 CommonClose( p_sys );
676 return NULL;
679 return p_sys;
682 static int DirectoryOpen( vlc_object_t* p_obj )
684 stream_directory_t* p_directory = (void*)p_obj;
685 private_sys_t* p_sys = CommonOpen( p_obj, p_directory->source );
687 if( p_sys == NULL )
688 return VLC_EGENERIC;
690 p_directory->p_sys = p_sys;
691 p_directory->pf_readdir = ReadDir;
693 return VLC_SUCCESS;
696 static int ExtractorOpen( vlc_object_t* p_obj )
698 stream_extractor_t* p_extractor = (void*)p_obj;
699 private_sys_t* p_sys = CommonOpen( p_obj, p_extractor->source );
701 if( p_sys == NULL )
702 return VLC_EGENERIC;
704 if( archive_seek_subentry( p_sys, p_extractor->identifier ) )
706 CommonClose( p_sys );
707 return VLC_EGENERIC;
710 p_extractor->p_sys = p_sys;
711 p_extractor->pf_read = Read;
712 p_extractor->pf_control = Control;
713 p_extractor->pf_seek = Seek;
715 return VLC_SUCCESS;