2 # -*- coding: utf-8 -*-
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/31/2024 Ver. 0.12.0 RC 1 - Author: cooldude2k $
20 from __future__
import absolute_import
, division
, print_function
, unicode_literals
;
21 import io
, os
, re
, sys
, time
, stat
, zlib
, base64
, shutil
, socket
, hashlib
, datetime
, logging
, binascii
, zipfile
, platform
;
23 from backports
import tempfile
;
29 from ftplib
import FTP
, FTP_TLS
;
32 from ftplib
import FTP
;
35 import simplejson
as json
;
41 from urllib
.parse
import urlparse
, urlunparse
;
43 from urlparse
import urlparse
, urlunparse
;
45 # Windows-specific setup
47 if sys
.version_info
[0] == 2:
49 sys
.stdout
= codecs
.getwriter('utf-8')(sys
.stdout
);
50 sys
.stderr
= codecs
.getwriter('utf-8')(sys
.stderr
);
52 sys
.stdout
= io
.TextIOWrapper(sys
.stdout
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
53 sys
.stderr
= io
.TextIOWrapper(sys
.stderr
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
55 hashlib_guaranteed
= False;
57 os
.environ
["PYTHONIOENCODING"] = "UTF-8";
58 os
.environ
["LC_CTYPE"] = "UTF-8";
60 # Reload sys to set default encoding to UTF-8 (Python 2 only)
61 if sys
.version_info
[0] == 2:
64 sys
.setdefaultencoding('UTF-8');
65 except (NameError, AttributeError):
70 from zlib
import crc32
;
72 from binascii
import crc32
;
74 # Define FileNotFoundError for Python 2
78 FileNotFoundError
= IOError;
81 rarfile_support
= False;
84 rarfile_support
= True;
89 py7zr_support
= False;
98 from xtarfile
import is_tarfile
;
101 from safetar
import is_tarfile
;
103 from tarfile
import is_tarfile
;
107 import xtarfile
as tarfile
;
110 import safetar
as tarfile
;
115 haveparamiko
= False;
131 haverequests
= False;
136 logging
.getLogger("urllib3").setLevel(logging
.WARNING
);
140 # HTTP and URL parsing
142 from urllib
.request
import Request
, build_opener
, HTTPBasicAuthHandler
;
143 from urllib
.parse
import urlparse
;
145 from urllib2
import Request
, build_opener
, HTTPBasicAuthHandler
;
146 from urlparse
import urlparse
;
148 # StringIO and BytesIO
150 from io
import StringIO
, BytesIO
;
153 from cStringIO
import StringIO
;
154 from cStringIO
import StringIO
as BytesIO
;
156 from StringIO
import StringIO
;
157 from StringIO
import StringIO
as BytesIO
;
159 __use_pysftp__
= False;
161 __use_pysftp__
= False;
162 __file_format_name__
= "CatFile";
163 __program_name__
= "Py"+__file_format_name__
;
164 __file_format_lower__
= __file_format_name__
.lower();
165 __file_format_magic__
= __file_format_name__
;
166 __file_format_len__
= len(__file_format_magic__
);
167 __file_format_hex__
= binascii
.hexlify(__file_format_magic__
.encode("UTF-8")).decode("UTF-8");
168 __file_format_delimiter__
= "\x00";
169 __file_format_ver__
= "001";
170 __use_new_style__
= True;
171 __use_advanced_list__
= True;
172 __use_alt_inode__
= False;
173 __file_format_extension__
= ".cat";
175 __file_format_name__ = "FastArchive";
176 __program_name__ = "Py" + __file_format_name__;
177 __file_format_lower__ = __file_format_name__.lower();
178 __file_format_magic__ = "FstArch";
179 __file_format_len__ = len(__file_format_magic__);
180 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
181 __file_format_delimiter__ = "\x1F"; # Using a non-printable ASCII character as delimiter
182 __file_format_ver__ = "001";
183 __use_new_style__ = True;
184 __use_advanced_list__ = False;
185 __use_alt_inode__ = False;
186 __file_format_extension__ = ".fast";
188 __file_format_list__
= [__file_format_name__
, __file_format_magic__
, __file_format_lower__
, __file_format_len__
, __file_format_hex__
, __file_format_delimiter__
, __file_format_ver__
, __use_new_style__
, __use_advanced_list__
, __use_alt_inode__
];
189 __file_format_dict__
= {'format_name': __file_format_name__
, 'format_magic': __file_format_magic__
, 'format_lower': __file_format_lower__
, 'format_len': __file_format_len__
, 'format_hex': __file_format_hex__
, 'format_delimiter': __file_format_delimiter__
, 'format_ver': __file_format_ver__
, 'new_style': __use_new_style__
, 'use_advanced_list': __use_advanced_list__
, 'use_alt_inode': __use_alt_inode__
};
190 __project__
= __program_name__
;
191 __project_url__
= "https://github.com/GameMaker2k/PyCatFile";
192 __version_info__
= (0, 12, 0, "RC 1", 1);
193 __version_date_info__
= (2024, 5, 31, "RC 1", 1);
194 __version_date__
= str(__version_date_info__
[0]) + "." + str(__version_date_info__
[1]).zfill(2) + "." + str(__version_date_info__
[2]).zfill(2);
195 __revision__
= __version_info__
[3];
196 __revision_id__
= "$Id$";
197 if(__version_info__
[4] is not None):
198 __version_date_plusrc__
= __version_date__
+ "-" + str(__version_date_info__
[4]);
199 if(__version_info__
[4] is None):
200 __version_date_plusrc__
= __version_date__
;
201 if(__version_info__
[3] is not None):
202 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]) + " " + str(__version_info__
[3]);
203 if(__version_info__
[3] is None):
204 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]);
206 PyBitness
= platform
.architecture();
207 if(PyBitness
=="32bit" or PyBitness
=="32"):
209 elif(PyBitness
=="64bit" or PyBitness
=="64"):
214 geturls_ua_pycatfile_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
215 if(platform
.python_implementation()!=""):
216 py_implementation
= platform
.python_implementation();
217 if(platform
.python_implementation()==""):
218 py_implementation
= "Python";
219 geturls_ua_pycatfile_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
220 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
221 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
222 geturls_headers_pycatfile_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
223 geturls_headers_pycatfile_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
224 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
225 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
227 def CompressionSupport():
228 compression_list
= [];
231 compression_list
.append("gz");
232 compression_list
.append("gzip");
237 compression_list
.append("bz2");
238 compression_list
.append("bzip2");
243 compression_list
.append("lz4");
248 compression_list
.append("lzo");
249 compression_list
.append("lzop");
254 compression_list
.append("zstd");
255 compression_list
.append("zstandard");
260 compression_list
.append("lzma");
261 compression_list
.append("xz");
265 from backports
import lzma
;
266 compression_list
.append("lzma");
267 compression_list
.append("xz");
270 return compression_list
;
272 compressionsupport
= CompressionSupport();
273 compressionlist
= ['auto'];
274 compressionlistalt
= [];
277 if('gzip' in compressionsupport
):
278 compressionlist
.append('gzip');
279 compressionlistalt
.append('gzip');
280 outextlist
.append('gz');
281 outextlistwd
.append('.gz');
282 if('bzip2' in compressionsupport
):
283 compressionlist
.append('bzip2');
284 compressionlistalt
.append('bzip2');
285 outextlist
.append('bz2');
286 outextlistwd
.append('.bz2');
287 if('zstd' in compressionsupport
):
288 compressionlist
.append('zstd');
289 compressionlistalt
.append('zstd');
290 outextlist
.append('zst');
291 outextlistwd
.append('.zst');
292 if('lz4' in compressionsupport
):
293 compressionlist
.append('lz4');
294 compressionlistalt
.append('lz4');
295 outextlist
.append('lz4');
296 outextlistwd
.append('.lz4');
297 if('lzo' in compressionsupport
):
298 compressionlist
.append('lzo');
299 compressionlistalt
.append('lzo');
300 outextlist
.append('lzo');
301 outextlistwd
.append('.lzo');
302 if('lzop' in compressionsupport
):
303 compressionlist
.append('lzop');
304 compressionlistalt
.append('lzop');
305 outextlist
.append('lzop');
306 outextlistwd
.append('.lzop');
307 if('lzma' in compressionsupport
):
308 compressionlist
.append('lzma');
309 compressionlistalt
.append('lzma');
310 outextlist
.append('lzma');
311 outextlistwd
.append('.lzma');
312 if('xz' in compressionsupport
):
313 compressionlist
.append('xz');
314 compressionlistalt
.append('xz');
315 outextlist
.append('xz');
316 outextlistwd
.append('.xz');
318 tarfile_mimetype
= "application/tar";
319 tarfile_tar_mimetype
= tarfile_mimetype
;
320 zipfile_mimetype
= "application/zip";
321 zipfile_zip_mimetype
= zipfile_mimetype
;
322 rarfile_mimetype
= "application/rar";
323 rarfile_rar_mimetype
= rarfile_mimetype
;
324 archivefile_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"";
325 archivefile_cat_mimetype
= archivefile_mimetype
;
326 archivefile_gzip_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+gzip";
327 archivefile_gz_mimetype
= archivefile_gzip_mimetype
;
328 archivefile_bzip2_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+bzip2";
329 archivefile_bz2_mimetype
= archivefile_bzip2_mimetype
;
330 archivefile_lz4_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lz4";
331 archivefile_lzop_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzop";
332 archivefile_lzo_mimetype
= archivefile_lzop_mimetype
;
333 archivefile_zstandard_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+zstandard";
334 archivefile_zstd_mimetype
= archivefile_zstandard_mimetype
;
335 archivefile_lzma_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzma";
336 archivefile_xz_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+xz";
337 archivefile_extensions
= [__file_format_extension__
, __file_format_extension__
+".gz", __file_format_extension__
+".bz2", __file_format_extension__
+".zst", __file_format_extension__
+".lz4", __file_format_extension__
+".lzo", __file_format_extension__
+".lzop", __file_format_extension__
+".lzma", __file_format_extension__
+".xz"];
339 if __name__
== "__main__":
341 curscrpath
= os
.path
.dirname(sys
.argv
[0]);
345 curscrpath
= curscrpath
.replace(os
.sep
, "/");
346 curscrpath
= curscrpath
+ "/";
347 scrfile
= curscrpath
+ "catfile.py";
348 if(os
.path
.exists(scrfile
) and os
.path
.isfile(scrfile
)):
349 scrcmd
= subprocess
.Popen([sys
.executable
, scrfile
] + sys
.argv
[1:]);
352 def VerbosePrintOut(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
358 "warning": logging
.warning
,
359 "error": logging
.error
,
360 "critical": logging
.critical
,
361 "exception": logging
.exception
,
362 "logalt": lambda x
: logging
.log(dgblevel
, x
),
363 "debug": logging
.debug
365 log_function
= log_functions
.get(outtype
);
367 log_function(dbgtxt
);
371 def VerbosePrintOutReturn(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
372 VerbosePrintOut(dbgtxt
, outtype
, dbgenable
, dgblevel
);
375 def RemoveWindowsPath(dpath
):
379 dpath
= dpath
.replace(os
.path
.sep
, "/");
380 dpath
= dpath
.rstrip("/");
381 if(dpath
=="." or dpath
==".."):
385 def NormalizeRelativePath(inpath
):
386 inpath
= RemoveWindowsPath(inpath
);
387 if(os
.path
.isabs(inpath
)):
390 if(inpath
.startswith("./") or inpath
.startswith("../")):
393 outpath
= "./" + inpath
;
396 def PrependPath(base_dir
, child_path
):
397 # Check if base_dir is None or empty, if so, return child_path as is
400 # Ensure base_dir ends with exactly one slash
401 if not base_dir
.endswith('/'):
403 # Check if child_path starts with ./ or ../ (indicating a relative path)
404 if child_path
.startswith('./') or child_path
.startswith('../'):
405 # For relative paths, we don't alter the child_path
406 return base_dir
+ child_path
;
408 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
409 return base_dir
+ child_path
.lstrip('/');
411 def ListDir(dirpath
, followlink
=False, duplicates
=False):
412 if isinstance(dirpath
, (list, tuple, )):
413 dirpath
= list(filter(None, dirpath
));
414 elif isinstance(dirpath
, str):
415 dirpath
= list(filter(None, [dirpath
]));
417 fs_encoding
= sys
.getfilesystemencoding();
418 for mydirfile
in dirpath
:
419 if not os
.path
.exists(mydirfile
):
421 mydirfile
= NormalizeRelativePath(mydirfile
);
422 if os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
):
423 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
));
424 if os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
):
425 for root
, dirs
, filenames
in os
.walk(mydirfile
):
427 dpath
= RemoveWindowsPath(dpath
);
428 if fs_encoding
!= 'utf-8':
429 dpath
= dpath
.encode(fs_encoding
).decode('utf-8');
430 if dpath
not in retlist
and not duplicates
:
431 retlist
.append(dpath
);
433 retlist
.append(dpath
);
434 for file in filenames
:
435 fpath
= os
.path
.join(root
, file);
436 fpath
= RemoveWindowsPath(fpath
);
437 if fs_encoding
!= 'utf-8':
438 fpath
= fpath
.encode(fs_encoding
).decode('utf-8');
439 if fpath
not in retlist
and not duplicates
:
440 retlist
.append(fpath
);
442 retlist
.append(fpath
);
444 path
= RemoveWindowsPath(mydirfile
);
445 if fs_encoding
!= 'utf-8':
446 path
= path
.encode(fs_encoding
).decode('utf-8');
447 retlist
.append(path
);
450 def ListDirAdvanced(dirpath
, followlink
=False, duplicates
=False):
451 if isinstance(dirpath
, (list, tuple, )):
452 dirpath
= list(filter(None, dirpath
));
453 elif isinstance(dirpath
, str):
454 dirpath
= list(filter(None, [dirpath
]));
456 fs_encoding
= sys
.getfilesystemencoding();
457 for mydirfile
in dirpath
:
458 if not os
.path
.exists(mydirfile
):
460 mydirfile
= NormalizeRelativePath(mydirfile
);
461 if os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
) and followlink
:
462 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
));
463 if os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
):
464 for root
, dirs
, filenames
in os
.walk(mydirfile
):
465 # Sort dirs and filenames alphabetically in place
466 dirs
.sort(key
=lambda x
: x
.lower());
467 filenames
.sort(key
=lambda x
: x
.lower());
468 dpath
= RemoveWindowsPath(root
);
469 if fs_encoding
!= 'utf-8':
470 dpath
= dpath
.encode(fs_encoding
).decode('utf-8');
471 if not duplicates
and dpath
not in retlist
:
472 retlist
.append(dpath
);
474 retlist
.append(dpath
);
475 for file in filenames
:
476 fpath
= os
.path
.join(root
, file);
477 fpath
= RemoveWindowsPath(fpath
);
478 if fs_encoding
!= 'utf-8':
479 fpath
= fpath
.encode(fs_encoding
).decode('utf-8');
480 if not duplicates
and fpath
not in retlist
:
481 retlist
.append(fpath
);
483 retlist
.append(fpath
);
485 path
= RemoveWindowsPath(mydirfile
);
486 if fs_encoding
!= 'utf-8':
487 path
= path
.encode(fs_encoding
).decode('utf-8');
488 retlist
.append(path
);
491 def create_alias_function(prefix
, base_name
, suffix
, target_function
):
492 # Define a new function that wraps the target function
493 def alias_function(*args
, **kwargs
):
494 return target_function(*args
, **kwargs
);
495 # Create the function name by combining the prefix, base name, and the suffix
496 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
);
497 # Add the new function to the global namespace
498 globals()[function_name
] = alias_function
;
500 def create_alias_function_alt(prefix
, base_name
, suffix
, target_function
):
501 # Create the function name by combining the prefix, base name, and the suffix
502 # Use the format method for string formatting, compatible with Python 2 and 3
503 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
)
504 # Add the new function (alias of the target_function) to the global namespace
505 # This line is compatible as-is with both Python 2 and 3
506 globals()[function_name
] = target_function
508 def FormatSpecsListToDict(formatspecs
=__file_format_list__
):
509 if(isinstance(formatspecs
, (list, tuple, ))):
510 return {'format_name': formatspecs
[0], 'format_magic': formatspecs
[1], 'format_lower': formatspecs
[2], 'format_len': formatspecs
[3], 'format_hex': formatspecs
[4], 'format_delimiter': formatspecs
[5], 'format_ver': formatspecs
[6], 'new_style': formatspecs
[7], 'use_advanced_list': formatspecs
[8], 'use_alt_inode': formatspecs
[9]};
511 elif(isinstance(formatspecs
, (dict, ))):
514 return __file_format_dict__
;
515 return __file_format_dict__
;
517 def TarFileCheck(infile
):
519 if is_tarfile(infile
):
525 # Check if the input is a file object
526 if hasattr(infile
, 'read'):
527 # Save the current file position
528 current_position
= infile
.tell();
529 # Attempt to open the file object as a tar file
530 tar
= tarfile
.open(fileobj
=infile
);
532 # Restore the file position
533 infile
.seek(current_position
);
536 # Assume it's a filename
537 tar
= tarfile
.open(name
=infile
);
540 except tarfile
.TarError
:
543 # initial_value can be 0xFFFF or 0x0000
544 def crc16_ansi(msg
, initial_value
=0xFFFF):
545 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
546 poly
= 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
547 crc
= initial_value
; # Initial value
549 crc ^
= b
<< 8; # XOR byte into CRC top byte
550 for _
in range(8): # Process each bit
551 if crc
& 0x8000: # If the top bit is set
552 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
554 crc
= crc
<< 1; # Just shift left
555 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
558 # initial_value can be 0xFFFF or 0x0000
559 def crc16_ibm(msg
, initial_value
=0xFFFF):
560 return crc16_ansi(msg
, initial_value
);
562 # initial_value is 0xFFFF
564 return crc16_ansi(msg
, 0xFFFF);
566 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
567 def crc16_ccitt(msg
, initial_value
=0xFFFF):
568 # CRC-16-CCITT polynomial
569 poly
= 0x1021; # Polynomial for CRC-16-CCITT
570 # Use the specified initial value
573 crc ^
= b
<< 8; # XOR byte into CRC top byte
574 for _
in range(8): # Process each bit
575 if crc
& 0x8000: # If the top bit is set
576 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
578 crc
= crc
<< 1; # Just shift left
579 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
582 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
583 def crc64_ecma(msg
, initial_value
=0x0000000000000000):
584 # CRC-64-ECMA polynomial and initial value
585 poly
= 0x42F0E1EBA9EA3693;
586 crc
= initial_value
; # Initial value for CRC-64-ECMA
588 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
589 for _
in range(8): # Process each bit
590 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
591 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
593 crc
<<= 1; # Just shift left if the MSB is 0
594 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
597 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
598 def crc64_iso(msg
, initial_value
=0xFFFFFFFFFFFFFFFF):
599 # CRC-64-ISO polynomial and initial value
600 poly
= 0x000000000000001B;
601 crc
= initial_value
; # Common initial value for CRC-64-ISO
603 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
604 for _
in range(8): # Process each bit
605 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
606 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
608 crc
<<= 1; # Just shift left if the MSB is 0
609 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
612 def GetDataFromArray(data
, path
, default
=None):
616 element
= element
[key
];
618 except (KeyError, TypeError, IndexError):
621 def GetDataFromArrayAlt(structure
, path
, default
=None):
624 if isinstance(element
, dict) and key
in element
:
625 element
= element
[key
];
626 elif isinstance(element
, list) and isinstance(key
, int) and -len(element
) <= key
< len(element
):
627 element
= element
[key
];
632 def GetHeaderChecksum(inlist
=[], checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
633 formatspecs
= FormatSpecsListToDict(formatspecs
);
634 fileheader
= AppendNullBytes(inlist
, formatspecs
['format_delimiter']) if isinstance(inlist
, list) else AppendNullByte(inlist
, formatspecs
['format_delimiter']);
636 fileheader
= fileheader
.encode('UTF-8');
638 "crc16": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
639 "crc16_ansi": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
640 "crc16_ibm": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
641 "crc16_ccitt": lambda data
: format(crc16_ccitt(data
) & 0xffff, '04x').lower(),
642 "adler32": lambda data
: format(zlib
.adler32(data
) & 0xffffffff, '08x').lower(),
643 "crc32": lambda data
: format(crc32(data
) & 0xffffffff, '08x').lower(),
644 "crc64_ecma": lambda data
: format(crc64_ecma(data
) & 0xffffffffffffffff, '016x').lower(),
645 "crc64": lambda data
: format(crc64_iso(data
) & 0xffffffffffffffff, '016x').lower(),
646 "crc64_iso": lambda data
: format(crc64_iso(data
) & 0xffffffffffffffff, '016x').lower(),
648 if checksumtype
in checksum_methods
:
649 return checksum_methods
[checksumtype
](fileheader
);
650 elif CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
):
651 checksumoutstr
= hashlib
.new(checksumtype
);
652 checksumoutstr
.update(fileheader
);
653 return checksumoutstr
.hexdigest().lower();
654 return format(0, 'x').lower();
656 def GetFileChecksum(instr
, checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
657 formatspecs
= FormatSpecsListToDict(formatspecs
);
659 instr
= instr
.encode('UTF-8');
661 "crc16": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
662 "crc16_ansi": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
663 "crc16_ibm": lambda data
: format(crc16(data
) & 0xffff, '04x').lower(),
664 "crc16_ccitt": lambda data
: format(crc16_ccitt(data
) & 0xffff, '04x').lower(),
665 "adler32": lambda data
: format(zlib
.adler32(data
) & 0xffffffff, '08x').lower(),
666 "crc32": lambda data
: format(crc32(data
) & 0xffffffff, '08x').lower(),
667 "crc64_ecma": lambda data
: format(crc64_ecma(data
) & 0xffffffffffffffff, '016x').lower(),
668 "crc64": lambda data
: format(crc64_iso(data
) & 0xffffffffffffffff, '016x').lower(),
669 "crc64_iso": lambda data
: format(crc64_iso(data
) & 0xffffffffffffffff, '016x').lower(),
671 if checksumtype
in checksum_methods
:
672 return checksum_methods
[checksumtype
](instr
);
673 elif CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
):
674 checksumoutstr
= hashlib
.new(checksumtype
);
675 checksumoutstr
.update(instr
);
676 return checksumoutstr
.hexdigest().lower();
677 return format(0, 'x').lower();
679 def ValidateHeaderChecksum(inlist
=[], checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
680 formatspecs
= FormatSpecsListToDict(formatspecs
);
681 catfileheadercshex
= GetHeaderChecksum(inlist
, checksumtype
, True, formatspecs
).lower();
682 return inchecksum
.lower() == catfileheadercshex
;
684 def ValidateFileChecksum(infile
, checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
685 formatspecs
= FormatSpecsListToDict(formatspecs
);
686 catinfilecshex
= GetFileChecksum(infile
, checksumtype
, True, formatspecs
).lower();
687 return inchecksum
.lower() == catinfilecshex
;
689 def ReadTillNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
692 nullbyte
= delimiter
.encode("UTF-8");
694 curbyte
= fp
.read(1);
695 if(curbyte
==nullbyte
or not curbyte
):
697 curfullbyte
= curfullbyte
+ curbyte
;
698 return curfullbyte
.decode('UTF-8');
700 def ReadUntilNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
701 return ReadTillNullByteOld(fp
, delimiter
);
703 def ReadTillNullByteAlt(fp
, delimiter
=__file_format_dict__
['format_delimiter'], chunk_size
=1024, max_read
=1024000):
704 delimiter
= delimiter
.encode('UTF-8') # Ensure the delimiter is in bytes
705 buffer = bytearray();
707 delimiter_length
= len(delimiter
);
709 chunk
= fp
.read(chunk_size
)
711 # End of file reached without finding the delimiter
713 buffer.extend(chunk
);
714 total_read
+= len(chunk
);
715 if delimiter
in buffer:
716 # Delimiter found, calculate where to reset the file pointer
717 index
= buffer.find(delimiter
);
718 # Calculate how many extra bytes were read after the delimiter
719 extra_bytes_read
= len(buffer) - (index
+ delimiter_length
);
720 # Move the file pointer back to just after the delimiter
721 fp
.seek(-extra_bytes_read
, 1);
722 buffer = buffer[:index
];
724 if total_read
>= max_read
:
725 # Stop reading if max limit is reached to prevent excessive memory usage
726 raise MemoryError("Maximum read limit reached without finding the delimiter.");
727 # Check for incomplete UTF-8 sequences at the end of the buffer
728 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
729 # This suggests that the last byte might be the start of a multi-byte character
730 # Try to read one more byte to complete the character
731 extra_byte
= fp
.read(1);
733 buffer.extend(extra_byte
);
735 # No more data available
738 return buffer.decode('UTF-8', errors
='replace');
739 except UnicodeDecodeError:
740 return buffer.decode('UTF-8', errors
='replace');
742 def ReadUntilNullByteAlt(fp
, delimiter
=__file_format_dict__
['format_delimiter'], chunk_size
=1024, max_read
=1024000):
743 return ReadTillNullByteAlt(fp
, delimiter
, chunk_size
, max_read
);
745 def ReadTillNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
746 curfullbyte
= bytearray();
747 nullbyte
= delimiter
.encode("UTF-8");
748 total_read
= 0; # Track the total number of bytes read
750 curbyte
= fp
.read(1);
751 if curbyte
== nullbyte
or not curbyte
:
753 curfullbyte
.extend(curbyte
);
755 if total_read
>= max_read
:
756 raise MemoryError("Maximum read limit reached without finding the delimiter.");
757 # Decode the full byte array to string once out of the loop
759 return curfullbyte
.decode('UTF-8');
760 except UnicodeDecodeError:
761 # Handle potential partial UTF-8 characters
762 for i
in range(1, 4):
764 return curfullbyte
[:-i
].decode('UTF-8');
765 except UnicodeDecodeError:
767 raise; # Re-raise if decoding fails even after trimming
769 def ReadUntilNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
770 return ReadTillNullByte(fp
, delimiter
, max_read
);
772 def ReadTillNullByteByNum(fp
, delimiter
=__file_format_dict__
['format_delimiter'], num_delimiters
=1, chunk_size
=1024, max_read
=1024000):
773 delimiter
= delimiter
.encode('UTF-8'); # Ensure the delimiter is in bytes
774 buffer = bytearray();
776 delimiter_length
= len(delimiter
);
778 while len(results
) < num_delimiters
:
779 chunk
= fp
.read(chunk_size
);
781 # End of file reached; decode whatever is collected if it's the last needed part
783 results
.append(buffer.decode('UTF-8', errors
='replace'));
786 total_read
+= len(chunk
);
787 # Check if we have found the delimiter
788 while delimiter
in buffer:
789 index
= buffer.find(delimiter
);
790 # Decode the section before the delimiter
791 results
.append(buffer[:index
].decode('UTF-8', errors
='replace'));
792 # Remove the processed part from the buffer
793 buffer = buffer[index
+ delimiter_length
:];
794 if len(results
) == num_delimiters
:
795 # If reached the required number of delimiters, adjust the file pointer and stop
796 fp
.seek(-len(buffer), 1);
798 if total_read
>= max_read
:
799 # Stop reading if max limit is reached to prevent excessive memory usage
800 raise MemoryError("Maximum read limit reached without finding the delimiter.");
801 # Check for incomplete UTF-8 sequences at the end of the buffer
802 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
803 # This suggests that the last byte might be the start of a multi-byte character
804 # Try to read one more byte to complete the character
805 extra_byte
= fp
.read(1);
807 buffer.extend(extra_byte
);
809 # No more data available
811 # Process remaining buffer if less than the required number of delimiters were found
812 if len(buffer) > 0 and len(results
) < num_delimiters
:
813 results
.append(buffer.decode('UTF-8', errors
='replace'));
816 def ReadUntilNullByteByNum(fp
, delimiter
=__file_format_dict__
['format_delimiter'], num_delimiters
=1, chunk_size
=1024, max_read
=1024000):
817 return ReadTillNullByteByNum(fp
, delimiter
, num_delimiters
, chunk_size
, max_read
);
819 def SeekToEndOfFile(fp
):
823 if(lasttell
==fp
.tell()):
825 lasttell
= fp
.tell();
828 def ReadFileHeaderData(fp
, rounds
=0, delimiter
=__file_format_dict__
['format_delimiter']):
832 while(rocount
<roend
):
833 HeaderOut
.append(ReadTillNullByte(fp
, delimiter
));
834 rocount
= rocount
+ 1;
837 def ReadFileHeaderDataBySize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
838 headerpresize
= ReadTillNullByte(fp
, delimiter
);
839 headersize
= int(headerpresize
, 16);
842 headercontent
= str(fp
.read(headersize
).decode('UTF-8')).split(delimiter
);
845 roend
= int(len(headercontent
));
846 HeaderOut
= [headerpresize
];
847 while(rocount
<roend
):
848 HeaderOut
.append(headercontent
[rocount
]);
849 rocount
= rocount
+ 1;
852 def ReadFileHeaderDataWoSize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
853 preheaderdata
= ReadFileHeaderData(fp
, 2, delimiter
);
854 headersize
= int(preheaderdata
[0], 16);
855 headernumfields
= int(preheaderdata
[1], 16);
856 if(headersize
<=0 or headernumfields
<=0):
858 headerdata
= ReadTillNullByteByNum(fp
, delimiter
, headernumfields
);
859 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
860 HeaderOut
= preheaderdata
+ headerdata
;
863 def ReadFileHeaderDataBySizeWithContent(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
864 formatspecs
= FormatSpecsListToDict(formatspecs
);
865 delimiter
= formatspecs
['format_delimiter'];
866 fheaderstart
= fp
.tell();
867 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
868 if(len(HeaderOut
)==0):
870 if(re
.findall(r
"^[.|/]", HeaderOut
[3])):
871 fname
= HeaderOut
[3];
873 fname
= "./"+HeaderOut
[3];
874 fcs
= HeaderOut
[-2].lower();
875 fccs
= HeaderOut
[-1].lower();
876 fsize
= int(HeaderOut
[5], 16);
877 fcompression
= HeaderOut
[12];
878 fcsize
= int(HeaderOut
[13], 16);
879 fseeknextfile
= HeaderOut
[25];
880 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-4].lower(), True, formatspecs
);
881 if(fcs
!=newfcs
and not skipchecksum
):
882 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
883 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
885 fhend
= fp
.tell() - 1;
886 fcontentstart
= fp
.tell();
887 fcontents
= BytesIO();
888 if(fsize
>0 and not listonly
):
889 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
890 fcontents
.write(fp
.read(fsize
));
892 fcontents
.write(fp
.read(fcsize
));
893 elif(fsize
>0 and listonly
):
894 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
898 fcontents
.seek(0, 0);
899 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
900 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
901 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
902 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
904 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
907 fcontents
.seek(0, 0);
909 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
910 fcontentend
= fp
.tell();
911 if(re
.findall(r
"^\+([0-9]+)", fseeknextfile
)):
912 fseeknextasnum
= int(fseeknextfile
.replace("+", ""));
913 if(abs(fseeknextasnum
)==0):
915 fp
.seek(fseeknextasnum
, 1);
916 elif(re
.findall(r
"^\-([0-9]+)", fseeknextfile
)):
917 fseeknextasnum
= int(fseeknextfile
);
918 if(abs(fseeknextasnum
)==0):
920 fp
.seek(fseeknextasnum
, 1);
921 elif(re
.findall(r
"^([0-9]+)", fseeknextfile
)):
922 fseeknextasnum
= int(fseeknextfile
);
923 if(abs(fseeknextasnum
)==0):
925 fp
.seek(fseeknextasnum
, 0);
928 HeaderOut
.append(fcontents
);
931 def ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
932 formatspecs
= FormatSpecsListToDict(formatspecs
);
933 delimiter
= formatspecs
['format_delimiter'];
934 fheaderstart
= fp
.tell();
935 if(formatspecs
['new_style']):
936 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
938 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
939 if(len(HeaderOut
)==0):
941 fheadsize
= int(HeaderOut
[0], 16);
942 fnumfields
= int(HeaderOut
[1], 16);
943 ftype
= int(HeaderOut
[2], 16);
944 if(re
.findall(r
"^[.|/]", HeaderOut
[3])):
945 fname
= HeaderOut
[3];
947 fname
= "./"+HeaderOut
[3];
948 fbasedir
= os
.path
.dirname(fname
);
949 flinkname
= HeaderOut
[4];
950 fsize
= int(HeaderOut
[5], 16);
951 fatime
= int(HeaderOut
[6], 16);
952 fmtime
= int(HeaderOut
[7], 16);
953 fctime
= int(HeaderOut
[8], 16);
954 fbtime
= int(HeaderOut
[9], 16);
955 fmode
= int(HeaderOut
[10], 16);
956 fchmode
= stat
.S_IMODE(fmode
);
957 ftypemod
= stat
.S_IFMT(fmode
);
958 fwinattributes
= int(HeaderOut
[11], 16);
959 fcompression
= HeaderOut
[12];
960 fcsize
= int(HeaderOut
[13], 16);
961 fuid
= int(HeaderOut
[14], 16);
962 funame
= HeaderOut
[15];
963 fgid
= int(HeaderOut
[16], 16);
964 fgname
= HeaderOut
[17];
965 fid
= int(HeaderOut
[18], 16);
966 finode
= int(HeaderOut
[19], 16);
967 flinkcount
= int(HeaderOut
[20], 16);
968 fdev_minor
= int(HeaderOut
[21], 16);
969 fdev_major
= int(HeaderOut
[22], 16);
970 frdev_minor
= int(HeaderOut
[23], 16);
971 frdev_major
= int(HeaderOut
[24], 16);
972 fseeknextfile
= HeaderOut
[25];
973 fextrasize
= int(HeaderOut
[26], 16);
974 fextrafields
= int(HeaderOut
[27], 16);
975 extrafieldslist
= [];
977 extraend
= extrastart
+ fextrafields
;
978 extrafieldslist
= [];
979 if(extrastart
<extraend
):
980 extrafieldslist
.append(HeaderOut
[extrastart
]);
981 extrastart
= extrastart
+ 1;
982 fcs
= HeaderOut
[-2].lower();
983 fccs
= HeaderOut
[-1].lower();
984 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-4].lower(), True, formatspecs
);
985 if(fcs
!=newfcs
and not skipchecksum
):
986 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
987 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
989 fhend
= fp
.tell() - 1;
990 fcontentstart
= fp
.tell();
991 fcontents
= BytesIO();
992 pyhascontents
= False;
993 if(fsize
>0 and not listonly
):
994 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
995 fcontents
.write(fp
.read(fsize
));
997 fcontents
.write(fp
.read(fcsize
));
998 pyhascontents
= True;
999 elif(fsize
>0 and listonly
):
1000 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
1004 pyhascontents
= False;
1005 fcontents
.seek(0, 0);
1006 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
1007 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
1008 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
1009 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
1011 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
1014 fcontents
.seek(0, 0);
1016 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
1017 fcontents
.seek(0, 0);
1018 fccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
1019 fcontentend
= fp
.tell() - 1;
1020 if(re
.findall(r
"^\+([0-9]+)", fseeknextfile
)):
1021 fseeknextasnum
= int(fseeknextfile
.replace("+", ""));
1022 if(abs(fseeknextasnum
)==0):
1024 fp
.seek(fseeknextasnum
, 1);
1025 elif(re
.findall(r
"^\-([0-9]+)", fseeknextfile
)):
1026 fseeknextasnum
= int(fseeknextfile
);
1027 if(abs(fseeknextasnum
)==0):
1029 fp
.seek(fseeknextasnum
, 1);
1030 elif(re
.findall(r
"^([0-9]+)", fseeknextfile
)):
1031 fseeknextasnum
= int(fseeknextfile
);
1032 if(abs(fseeknextasnum
)==0):
1034 fp
.seek(fseeknextasnum
, 0);
1037 fcontents
.seek(0, 0);
1038 catlist
= {'fheadersize': fheadsize
, 'fhstart': fheaderstart
, 'fhend': fhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': fseeknextfile
, 'fheaderchecksumtype': HeaderOut
[-4], 'fcontentchecksumtype': HeaderOut
[-3], 'fnumfields': fnumfields
+ 2, 'frawheader': HeaderOut
, 'fextrafields': fextrafields
, 'fextrafieldsize': fextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': fcs
, 'fcontentchecksum': fccs
, 'fhascontents': pyhascontents
, 'fcontentstart': fcontentstart
, 'fcontentend': fcontentend
, 'fcontents': fcontents
};
1041 def ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1042 formatspecs
= FormatSpecsListToDict(formatspecs
);
1043 delimiter
= formatspecs
['format_delimiter'];
1044 fheaderstart
= fp
.tell();
1045 if(formatspecs
['new_style']):
1046 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
1048 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
1049 if(len(HeaderOut
)==0):
1051 fheadsize
= int(HeaderOut
[0], 16);
1052 fnumfields
= int(HeaderOut
[1], 16);
1053 ftype
= int(HeaderOut
[2], 16);
1054 if(re
.findall(r
"^[.|/]", HeaderOut
[3])):
1055 fname
= HeaderOut
[3];
1057 fname
= "./"+HeaderOut
[3];
1058 fbasedir
= os
.path
.dirname(fname
);
1059 flinkname
= HeaderOut
[4];
1060 fsize
= int(HeaderOut
[5], 16);
1061 fatime
= int(HeaderOut
[6], 16);
1062 fmtime
= int(HeaderOut
[7], 16);
1063 fctime
= int(HeaderOut
[8], 16);
1064 fbtime
= int(HeaderOut
[9], 16);
1065 fmode
= int(HeaderOut
[10], 16);
1066 fchmode
= stat
.S_IMODE(fmode
);
1067 ftypemod
= stat
.S_IFMT(fmode
);
1068 fwinattributes
= int(HeaderOut
[11], 16);
1069 fcompression
= HeaderOut
[12];
1070 fcsize
= int(HeaderOut
[13], 16);
1071 fuid
= int(HeaderOut
[14], 16);
1072 funame
= HeaderOut
[15];
1073 fgid
= int(HeaderOut
[16], 16);
1074 fgname
= HeaderOut
[17];
1075 fid
= int(HeaderOut
[18], 16);
1076 finode
= int(HeaderOut
[19], 16);
1077 flinkcount
= int(HeaderOut
[20], 16);
1078 fdev_minor
= int(HeaderOut
[21], 16);
1079 fdev_major
= int(HeaderOut
[22], 16);
1080 frdev_minor
= int(HeaderOut
[23], 16);
1081 frdev_major
= int(HeaderOut
[24], 16);
1082 fseeknextfile
= HeaderOut
[25];
1083 fextrasize
= int(HeaderOut
[26], 16);
1084 fextrafields
= int(HeaderOut
[27], 16);
1085 extrafieldslist
= [];
1087 extraend
= extrastart
+ fextrafields
;
1088 extrafieldslist
= [];
1089 if(extrastart
<extraend
):
1090 extrafieldslist
.append(HeaderOut
[extrastart
]);
1091 extrastart
= extrastart
+ 1;
1092 fheaderchecksumtype
= [extrastart
].lower();
1093 fcontentchecksumtype
= [extrastart
+ 1].lower();
1094 fcs
= HeaderOut
[-2].lower();
1095 fccs
= HeaderOut
[-1].lower();
1096 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-4].lower(), True, formatspecs
);
1097 if(fcs
!=newfcs
and not skipchecksum
):
1098 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
1099 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
1101 fhend
= fp
.tell() - 1;
1102 fcontentstart
= fp
.tell();
1103 fcontents
= BytesIO();
1104 pyhascontents
= False;
1105 if(fsize
>0 and not listonly
):
1106 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
1107 fcontents
.write(fp
.read(fsize
));
1109 fcontents
.write(fp
.read(fcsize
));
1110 pyhascontents
= True;
1111 elif(fsize
>0 and listonly
):
1112 if(fcompression
=="none" or fcompression
=="" or fcompression
=="atuo"):
1116 pyhascontents
= False;
1117 fcontents
.seek(0, 0);
1118 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
1119 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
1120 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
1121 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
1123 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
1126 fcontents
.seek(0, 0);
1128 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
1129 fcontents
.seek(0, 0);
1130 fcontentend
= fp
.tell() - 1;
1131 if(re
.findall(r
"^\+([0-9]+)", fseeknextfile
)):
1132 fseeknextasnum
= int(fseeknextfile
.replace("+", ""));
1133 if(abs(fseeknextasnum
)==0):
1135 fp
.seek(fseeknextasnum
, 1);
1136 elif(re
.findall(r
"^\-([0-9]+)", fseeknextfile
)):
1137 fseeknextasnum
= int(fseeknextfile
);
1138 if(abs(fseeknextasnum
)==0):
1140 fp
.seek(fseeknextasnum
, 1);
1141 elif(re
.findall(r
"^([0-9]+)", fseeknextfile
)):
1142 fseeknextasnum
= int(fseeknextfile
);
1143 if(abs(fseeknextasnum
)==0):
1145 fp
.seek(fseeknextasnum
, 0);
1148 catlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, fseeknextfile
, extrafieldslist
, fheaderchecksumtype
, fcontentchecksumtype
, fcontents
];
1151 def ReadFileDataBySizeWithContent(fp
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1152 formatspecs
= FormatSpecsListToDict(formatspecs
);
1153 delimiter
= formatspecs
['format_delimiter'];
1157 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1160 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1161 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1162 if(not headercheck
and not skipchecksum
):
1163 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1164 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1166 fnumfiles
= int(catheader
[1], 16);
1169 while(countnum
< fnumfiles
):
1170 HeaderOut
= ReadFileHeaderDataBySizeWithContent(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
)
1171 if(len(HeaderOut
)==0):
1173 flist
.append(HeaderOut
);
1174 countnum
= countnum
+ 1;
1177 def ReadFileDataBySizeWithContentToArray(fp
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1178 formatspecs
= FormatSpecsListToDict(formatspecs
);
1179 delimiter
= formatspecs
['format_delimiter'];
1183 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1186 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1187 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1188 if(not headercheck
and not skipchecksum
):
1189 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1190 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1192 catstring
= catheader
[0];
1193 catversion
= re
.findall(r
"([\d]+)", catstring
);
1194 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
1195 fprenumfiles
= catheader
[1];
1196 fnumfiles
= int(fprenumfiles
, 16);
1197 fprechecksumtype
= catheader
[2];
1198 fprechecksum
= catheader
[3];
1199 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': []};
1200 if(seekstart
<0 and seekstart
>fnumfiles
):
1202 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
1203 seekend
= fnumfiles
;
1204 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
1205 seekend
= fnumfiles
- abs(seekend
);
1208 while(il
< seekstart
):
1209 prefhstart
= fp
.tell();
1210 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
1211 if(len(preheaderdata
)==0):
1213 prefsize
= int(preheaderdata
[5], 16);
1214 prefseeknextfile
= preheaderdata
[25];
1215 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
1216 prefcs
= preheaderdata
[-2];
1217 if(prefcs
!=prenewfcs
and not skipchecksum
):
1218 VVerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
1219 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
1221 valid_archive
= False;
1222 invalid_archive
= True;
1223 prefhend
= fp
.tell() - 1;
1224 prefcontentstart
= fp
.tell();
1225 prefcontents
= BytesIO();
1226 pyhascontents
= False;
1228 prefcontents
.write(fp
.read(prefsize
));
1229 prefcontents
.seek(0, 0);
1230 prenewfccs
= GetFileChecksum(prefcontents
.read(), preheaderdata
[-3].lower(), False, formatspecs
);
1231 prefccs
= preheaderdata
[-1];
1232 pyhascontents
= True;
1233 if(prefccs
!=prenewfccs
and not skipchecksum
):
1234 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
1235 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
1237 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
1238 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
1239 if(abs(fseeknextasnum
)==0):
1241 fp
.seek(fseeknextasnum
, 1);
1242 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
1243 fseeknextasnum
= int(prefseeknextfile
);
1244 if(abs(fseeknextasnum
)==0):
1246 fp
.seek(fseeknextasnum
, 1);
1247 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
1248 fseeknextasnum
= int(prefseeknextfile
);
1249 if(abs(fseeknextasnum
)==0):
1251 fp
.seek(fseeknextasnum
, 0);
1256 countnum
= seekstart
;
1257 while(countnum
< seekend
):
1258 HeaderOut
= ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1259 if(len(HeaderOut
)==0):
1261 HeaderOut
.update({'fid': realidnum
, 'fidalt': realidnum
});
1262 catlist
['ffilelist'].append(HeaderOut
);
1263 countnum
= countnum
+ 1;
1264 realidnum
= realidnum
+ 1;
1267 def ReadFileDataBySizeWithContentToList(fp
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1268 formatspecs
= FormatSpecsListToDict(formatspecs
);
1269 delimiter
= formatspecs
['format_delimiter'];
1273 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1276 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1277 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1278 if(not headercheck
and not skipchecksum
):
1279 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1280 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1282 catstring
= catheader
[0];
1283 catversion
= re
.findall(r
"([\d]+)", catstring
);
1284 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
1285 fprenumfiles
= catheader
[1];
1286 fnumfiles
= int(fprenumfiles
, 16);
1287 fprechecksumtype
= catheader
[2];
1288 fprechecksum
= catheader
[3];
1290 if(seekstart
<0 and seekstart
>fnumfiles
):
1292 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
1293 seekend
= fnumfiles
;
1294 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
1295 seekend
= fnumfiles
- abs(seekend
);
1298 while(il
< seekstart
):
1299 prefhstart
= fp
.tell();
1300 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
1301 if(len(preheaderdata
)==0):
1303 prefsize
= int(preheaderdata
[5], 16);
1304 prefcompression
= preheaderdata
[12];
1305 prefcsize
= int(preheaderdata
[13], 16);
1306 prefseeknextfile
= HeaderOut
[25];
1307 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
1308 prefcs
= preheaderdata
[-2];
1309 if(prefcs
!=prenewfcs
and not skipchecksum
):
1310 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
1311 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
1313 valid_archive
= False;
1314 invalid_archive
= True;
1315 prefhend
= fp
.tell() - 1;
1316 prefcontentstart
= fp
.tell();
1318 pyhascontents
= False;
1320 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
1321 prefcontents
= catfp
.read(prefsize
);
1323 prefcontents
= catfp
.read(prefcsize
);
1324 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
1325 prefccs
= preheaderdata
[-1];
1326 pyhascontents
= True;
1327 if(prefccs
!=prenewfccs
and not skipchecksum
):
1328 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
1329 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
1331 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
1332 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
1333 if(abs(fseeknextasnum
)==0):
1335 catfp
.seek(fseeknextasnum
, 1);
1336 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
1337 fseeknextasnum
= int(prefseeknextfile
);
1338 if(abs(fseeknextasnum
)==0):
1340 catfp
.seek(fseeknextasnum
, 1);
1341 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
1342 fseeknextasnum
= int(prefseeknextfile
);
1343 if(abs(fseeknextasnum
)==0):
1345 catfp
.seek(fseeknextasnum
, 0);
1350 countnum
= seekstart
;
1351 while(countnum
< seekend
):
1352 HeaderOut
= ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1353 if(len(HeaderOut
)==0):
1355 catlist
.append(HeaderOut
);
1356 countnum
= countnum
+ 1;
1357 realidnum
= realidnum
+ 1;
1360 def ReadInFileBySizeWithContentToArray(infile
, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1361 formatspecs
= FormatSpecsListToDict(formatspecs
);
1362 delimiter
= formatspecs
['format_delimiter'];
1363 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1366 fp
= UncompressArchiveFile(fp
, formatspecs
);
1367 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1368 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1375 if(hasattr(sys
.stdin
, "buffer")):
1376 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1378 shutil
.copyfileobj(sys
.stdin
, fp
);
1380 fp
= UncompressArchiveFile(fp
, formatspecs
);
1384 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1385 fp
= download_file_from_internet_file(infile
);
1386 fp
= UncompressArchiveFile(fp
, formatspecs
);
1392 infile
= RemoveWindowsPath(infile
);
1393 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1394 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1396 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1397 if(not compresscheck
):
1398 fextname
= os
.path
.splitext(infile
)[1];
1399 if(fextname
==".gz"):
1400 compresscheck
= "gzip";
1401 elif(fextname
==".bz2"):
1402 compresscheck
= "bzip2";
1403 elif(fextname
==".zst"):
1404 compresscheck
= "zstd";
1405 elif(fextname
==".lz4" or fextname
==".clz4"):
1406 compresscheck
= "lz4";
1407 elif(fextname
==".lzo" or fextname
==".lzop"):
1408 compresscheck
= "lzo";
1409 elif(fextname
==".lzma" or fextname
==".xz"):
1410 compresscheck
= "lzma";
1413 if(not compresscheck
):
1415 fp
= UncompressFile(infile
, formatspecs
, "rb");
1416 return ReadFileDataBySizeWithContentToArray(fp
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1418 def ReadInFileBySizeWithContentToList(infile
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1419 formatspecs
= FormatSpecsListToDict(formatspecs
);
1420 delimiter
= formatspecs
['format_delimiter'];
1421 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1424 fp
= UncompressArchiveFile(fp
, formatspecs
);
1425 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1426 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1433 if(hasattr(sys
.stdin
, "buffer")):
1434 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1436 shutil
.copyfileobj(sys
.stdin
, fp
);
1438 fp
= UncompressArchiveFile(fp
, formatspecs
);
1442 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1443 fp
= download_file_from_internet_file(infile
);
1444 fp
= UncompressArchiveFile(fp
, formatspecs
);
1450 infile
= RemoveWindowsPath(infile
);
1451 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1452 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1454 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1455 if(not compresscheck
):
1456 fextname
= os
.path
.splitext(infile
)[1];
1457 if(fextname
==".gz"):
1458 compresscheck
= "gzip";
1459 elif(fextname
==".bz2"):
1460 compresscheck
= "bzip2";
1461 elif(fextname
==".zst"):
1462 compresscheck
= "zstd";
1463 elif(fextname
==".lz4" or fextname
==".clz4"):
1464 compresscheck
= "lz4";
1465 elif(fextname
==".lzo" or fextname
==".lzop"):
1466 compresscheck
= "lzo";
1467 elif(fextname
==".lzma" or fextname
==".xz"):
1468 compresscheck
= "lzma";
1471 if(not compresscheck
):
1473 fp
= UncompressFile(infile
, formatspecs
, "rb");
1474 return ReadFileDataBySizeWithContentToList(fp
, seekstart
, seekend
, listonly
, uncompress
, skipchecksum
, formatspecs
);
1476 def AppendNullByte(indata
, delimiter
=__file_format_dict__
['format_delimiter']):
1477 outdata
= str(indata
) + delimiter
;
1480 def AppendNullBytes(indata
=[], delimiter
=__file_format_dict__
['format_delimiter']):
1485 outdata
= outdata
+ AppendNullByte(indata
[inum
], delimiter
);
1489 def AppendFileHeader(fp
, numfiles
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1490 formatspecs
= FormatSpecsListToDict(formatspecs
);
1491 delimiter
= formatspecs
['format_delimiter'];
1492 catver
= formatspecs
['format_ver'];
1493 fileheaderver
= str(int(catver
.replace(".", "")));
1494 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
1495 fp
.write(fileheader
.encode('UTF-8'));
1496 fnumfiles
= format(int(numfiles
), 'x').lower();
1497 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
['format_delimiter']);
1498 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
1499 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
1500 fp
.write(fnumfilesa
.encode('UTF-8'));
1503 os
.fsync(fp
.fileno());
1504 except io
.UnsupportedOperation
:
1506 except AttributeError:
1508 except OSError as e
:
1512 def MakeEmptyFilePointer(fp
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1513 formatspecs
= FormatSpecsListToDict(formatspecs
);
1514 AppendFileHeader(fp
, 0, checksumtype
, formatspecs
);
1517 def MakeEmptyFile(outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", formatspecs
=__file_format_dict__
, returnfp
=False):
1518 formatspecs
= FormatSpecsListToDict(formatspecs
);
1519 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1520 if(os
.path
.exists(outfile
)):
1523 except OSError as e
:
1527 catfpfp
= BytesIO();
1528 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1530 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1533 fbasename
= os
.path
.splitext(outfile
)[0];
1534 fextname
= os
.path
.splitext(outfile
)[1];
1535 catfp
= CompressOpenFile(outfile
, True, compressionlevel
);
1536 catfp
= AppendFileHeader(catfp
, 0, checksumtype
, formatspecs
);
1537 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1538 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1541 os
.fsync(catfp
.fileno());
1542 except io
.UnsupportedOperation
:
1544 except AttributeError:
1546 except OSError as e
:
1550 if(hasattr(sys
.stdout
, "buffer")):
1551 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1553 shutil
.copyfileobj(catfp
, sys
.stdout
);
1554 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1555 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1557 upload_file_to_internet_file(catfp
, outfile
);
1565 def AppendFileHeaderWithContent(fp
, filevalues
=[], extradata
=[], filecontent
="", checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1566 formatspecs
= FormatSpecsListToDict(formatspecs
);
1567 extrafields
= format(len(extradata
), 'x').lower();
1568 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
1569 if(len(extradata
)>0):
1570 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1571 extrasizelen
= format(len(extrasizestr
), 'x').lower();
1572 catoutlen
= len(filevalues
) + len(extradata
) + 5;
1573 catoutlenhex
= format(catoutlen
, 'x').lower();
1574 catoutlist
= filevalues
;
1575 catoutlist
.insert(0, catoutlenhex
);
1576 catoutlist
.append(extrasizelen
);
1577 catoutlist
.append(extrafields
);
1578 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
1579 if(len(extradata
)>0):
1580 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1581 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
1582 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1583 catfilecontentcshex
= GetFileChecksum(filecontent
, checksumtype
, False, formatspecs
);
1584 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1585 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
1586 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
1587 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1588 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1589 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
1590 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
1591 catfileout
= catfileoutstrecd
+ filecontent
+ nullstrecd
;
1592 fp
.write(catfileout
);
1595 os
.fsync(fp
.fileno());
1596 except io
.UnsupportedOperation
:
1598 except AttributeError:
1600 except OSError as e
:
1604 def AppendFilesWithContent(infiles
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1605 formatspecs
= FormatSpecsListToDict(formatspecs
);
1606 advancedlist
= formatspecs
['use_advanced_list'];
1607 altinode
= formatspecs
['use_alt_inode'];
1609 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1611 for line
in sys
.stdin
:
1612 infilelist
.append(line
.strip());
1613 infilelist
= list(filter(None, infilelist
));
1614 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
1615 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
1617 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
1618 for line
in finfile
:
1619 infilelist
.append(line
.strip());
1620 infilelist
= list(filter(None, infilelist
));
1622 if(isinstance(infiles
, (list, tuple, ))):
1623 infilelist
= list(filter(None, infiles
));
1624 elif(isinstance(infiles
, (str, ))):
1625 infilelist
= list(filter(None, [infiles
]));
1627 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
1629 GetDirList
= ListDir(infilelist
, followlink
, False);
1637 inodetocatinode
= {};
1638 numfiles
= int(len(GetDirList
));
1639 fnumfiles
= format(numfiles
, 'x').lower();
1640 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1641 for curfname
in GetDirList
:
1642 if(re
.findall(r
"^[.|/]", curfname
)):
1645 fname
= "./"+curfname
;
1647 VerbosePrintOut(fname
);
1648 if(not followlink
or followlink
is None):
1649 fstatinfo
= os
.lstat(fname
);
1651 fstatinfo
= os
.stat(fname
);
1652 fpremode
= fstatinfo
.st_mode
;
1653 finode
= fstatinfo
.st_ino
;
1654 flinkcount
= fstatinfo
.st_nlink
;
1656 if(stat
.S_ISREG(fpremode
)):
1658 elif(stat
.S_ISLNK(fpremode
)):
1660 elif(stat
.S_ISCHR(fpremode
)):
1662 elif(stat
.S_ISBLK(fpremode
)):
1664 elif(stat
.S_ISDIR(fpremode
)):
1666 elif(stat
.S_ISFIFO(fpremode
)):
1668 elif(stat
.S_ISSOCK(fpremode
)):
1670 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
1672 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
1674 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
1679 fcurfid
= format(int(curfid
), 'x').lower();
1680 if(not followlink
and finode
!=0):
1682 if(finode
in inodelist
):
1684 flinkname
= inodetofile
[finode
];
1686 fcurinode
= format(int(finode
), 'x').lower();
1688 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
1689 if(finode
not in inodelist
):
1690 inodelist
.append(finode
);
1691 inodetofile
.update({finode
: fname
});
1692 inodetocatinode
.update({finode
: curinode
});
1694 fcurinode
= format(int(finode
), 'x').lower();
1696 fcurinode
= format(int(curinode
), 'x').lower();
1697 curinode
= curinode
+ 1;
1699 fcurinode
= format(int(curinode
), 'x').lower();
1700 curinode
= curinode
+ 1;
1701 curfid
= curfid
+ 1;
1703 flinkname
= os
.readlink(fname
);
1704 fdev
= fstatinfo
.st_dev
;
1705 getfdev
= GetDevMajorMinor(fdev
);
1706 fdev_minor
= getfdev
[0];
1707 fdev_major
= getfdev
[1];
1708 frdev
= fstatinfo
.st_dev
;
1709 if(hasattr(fstatinfo
, "st_rdev")):
1710 frdev
= fstatinfo
.st_rdev
;
1712 frdev
= fstatinfo
.st_dev
;
1713 getfrdev
= GetDevMajorMinor(frdev
);
1714 frdev_minor
= getfrdev
[0];
1715 frdev_major
= getfrdev
[1];
1716 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
1717 fsize
= format(int("0"), 'x').lower();
1718 elif(ftype
==0 or ftype
==7):
1719 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
1721 fsize
= format(int(fstatinfo
.st_size
)).lower();
1722 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
1723 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
1724 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1725 if(hasattr(fstatinfo
, "st_birthtime")):
1726 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
1728 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1729 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
1730 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
1731 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
1732 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
1733 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
1738 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
1739 funame
= userinfo
.pw_name
;
1748 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
1749 fgname
= groupinfo
.gr_name
;
1754 fdev_minor
= format(int(fdev_minor
), 'x').lower();
1755 fdev_major
= format(int(fdev_major
), 'x').lower();
1756 frdev_minor
= format(int(frdev_minor
), 'x').lower();
1757 frdev_major
= format(int(frdev_major
), 'x').lower();
1758 finode
= format(int(finode
), 'x').lower();
1759 flinkcount
= format(int(flinkcount
), 'x').lower();
1760 if(hasattr(fstatinfo
, "st_file_attributes")):
1761 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
1763 fwinattributes
= format(int(0), 'x').lower();
1765 fcsize
= format(int(0), 'x').lower();
1766 fcontents
= BytesIO();
1768 if(ftype
==0 or ftype
==7):
1769 with
open(fname
, "rb") as fpc
:
1770 shutil
.copyfileobj(fpc
, fcontents
);
1771 if(not compresswholefile
):
1772 fcontents
.seek(0, 2);
1773 ucfsize
= fcontents
.tell();
1774 fcontents
.seek(0, 0);
1775 if(compression
=="auto"):
1776 ilsize
= len(compressionlistalt
);
1779 while(ilmin
< ilsize
):
1780 cfcontents
= BytesIO();
1781 shutil
.copyfileobj(fcontents
, cfcontents
);
1782 fcontents
.seek(0, 0);
1783 cfcontents
.seek(0, 0);
1784 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1786 cfcontents
.seek(0, 2);
1787 ilcsize
.append(cfcontents
.tell());
1791 ilcsize
.append(sys
.maxint
);
1792 except AttributeError:
1793 ilcsize
.append(sys
.maxsize
);
1795 ilcmin
= ilcsize
.index(min(ilcsize
));
1796 compression
= compressionlistalt
[ilcmin
];
1797 fcontents
.seek(0, 0);
1798 cfcontents
= BytesIO();
1799 shutil
.copyfileobj(fcontents
, cfcontents
);
1800 cfcontents
.seek(0, 0);
1801 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1802 cfcontents
.seek(0, 2);
1803 cfsize
= cfcontents
.tell();
1804 if(ucfsize
> cfsize
):
1805 fcsize
= format(int(cfsize
), 'x').lower();
1806 fcompression
= compression
;
1808 fcontents
= cfcontents
;
1809 if(followlink
and (ftype
==1 or ftype
==2)):
1810 flstatinfo
= os
.stat(flinkname
);
1811 with
open(flinkname
, "rb") as fpc
:
1812 shutil
.copyfileobj(fpc
, fcontents
);
1813 if(not compresswholefile
):
1814 fcontents
.seek(0, 2);
1815 ucfsize
= fcontents
.tell();
1816 fcontents
.seek(0, 0);
1817 if(compression
=="auto"):
1818 ilsize
= len(compressionlistalt
);
1821 while(ilmin
< ilsize
):
1822 cfcontents
= BytesIO();
1823 shutil
.copyfileobj(fcontents
, cfcontents
);
1824 fcontents
.seek(0, 0);
1825 cfcontents
.seek(0, 0);
1826 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1828 cfcontents
.seek(0, 2);
1829 ilcsize
.append(cfcontents
.tell());
1833 ilcsize
.append(sys
.maxint
);
1834 except AttributeError:
1835 ilcsize
.append(sys
.maxsize
);
1837 ilcmin
= ilcsize
.index(min(ilcsize
));
1838 compression
= compressionlistalt
[ilcmin
];
1839 fcontents
.seek(0, 0);
1840 cfcontents
= BytesIO();
1841 shutil
.copyfileobj(fcontents
, cfcontents
);
1842 cfcontents
.seek(0, 0);
1843 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1844 cfcontents
.seek(0, 2);
1845 cfsize
= cfcontents
.tell();
1846 if(ucfsize
> cfsize
):
1847 fcsize
= format(int(cfsize
), 'x').lower();
1848 fcompression
= compression
;
1850 fcontents
= cfcontents
;
1851 if(fcompression
=="none"):
1853 fcontents
.seek(0, 0);
1854 ftypehex
= format(ftype
, 'x').lower();
1855 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
1856 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1858 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1862 def AppendListsWithContent(inlist
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1863 formatspecs
= FormatSpecsListToDict(formatspecs
);
1865 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1866 GetDirList
= inlist
;
1874 inodetocatinode
= {};
1875 numfiles
= int(len(GetDirList
));
1876 fnumfiles
= format(numfiles
, 'x').lower();
1877 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1878 for curfname
in GetDirList
:
1879 ftype
= format(curfname
[0], 'x').lower();
1880 if(re
.findall(r
"^[.|/]", curfname
[1])):
1881 fname
= curfname
[1];
1883 fname
= "./"+curfname
[1];
1884 fbasedir
= os
.path
.dirname(fname
);
1885 flinkname
= curfname
[2];
1886 fsize
= format(curfname
[3], 'x').lower();
1887 fatime
= format(curfname
[4], 'x').lower();
1888 fmtime
= format(curfname
[5], 'x').lower();
1889 fctime
= format(curfname
[6], 'x').lower();
1890 fbtime
= format(curfname
[7], 'x').lower();
1891 fmode
= format(curfname
[8], 'x').lower();
1892 fwinattributes
= format(curfname
[9], 'x').lower();
1893 fcompression
= curfname
[10];
1894 fcsize
= format(curfname
[11], 'x').lower();
1895 fuid
= format(curfname
[12], 'x').lower();
1896 funame
= curfname
[13];
1897 fgid
= format(curfname
[14], 'x').lower();
1898 fgname
= curfname
[15];
1899 fid
= format(curfname
[16], 'x').lower();
1900 finode
= format(curfname
[17], 'x').lower();
1901 flinkcount
= format(curfname
[18], 'x').lower();
1902 fdev_minor
= format(curfname
[19], 'x').lower();
1903 fdev_major
= format(curfname
[20], 'x').lower();
1904 frdev_minor
= format(curfname
[21], 'x').lower();
1905 frdev_major
= format(curfname
[22], 'x').lower();
1906 fseeknextfile
= curfname
[23];
1907 extradata
= curfname
[24];
1908 fheaderchecksumtype
= curfname
[25];
1909 fcontentchecksumtype
= curfname
[26];
1910 fcontents
= curfname
[27];
1911 catoutlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, fseeknextfile
];
1912 fcontents
.seek(0, 0);
1913 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1915 fp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1918 def AppendInFileWithContent(infile
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1919 formatspecs
= FormatSpecsListToDict(formatspecs
);
1920 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, True, False, formatspecs
);
1921 return AppendListsWithContent(inlist
, fp
, dirlistfromtxt
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
);
1923 def AppendFilesWithContentToOutFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1924 formatspecs
= FormatSpecsListToDict(formatspecs
);
1925 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1926 if(os
.path
.exists(outfile
)):
1929 except OSError as e
:
1933 catfpfp
= BytesIO();
1934 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1936 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1939 fbasename
= os
.path
.splitext(outfile
)[0];
1940 fextname
= os
.path
.splitext(outfile
)[1];
1941 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1942 catfp
= AppendFilesWithContent(infiles
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1943 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1944 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1947 os
.fsync(catfp
.fileno());
1948 except io
.UnsupportedOperation
:
1950 except AttributeError:
1952 except OSError as e
:
1956 if(hasattr(sys
.stdout
, "buffer")):
1957 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1959 shutil
.copyfileobj(catfp
, sys
.stdout
);
1960 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1961 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1963 upload_file_to_internet_file(catfp
, outfile
);
1971 def AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1972 formatspecs
= FormatSpecsListToDict(formatspecs
);
1973 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1974 if(os
.path
.exists(outfile
)):
1977 except OSError as e
:
1981 catfpfp
= BytesIO();
1982 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1984 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1987 fbasename
= os
.path
.splitext(outfile
)[0];
1988 fextname
= os
.path
.splitext(outfile
)[1];
1989 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1990 catfp
= AppendListsWithContent(inlist
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1991 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1992 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1995 os
.fsync(catfp
.fileno());
1996 except io
.UnsupportedOperation
:
1998 except AttributeError:
2000 except OSError as e
:
2004 if(hasattr(sys
.stdout
, "buffer")):
2005 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
2007 shutil
.copyfileobj(catfp
, sys
.stdout
);
2008 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2009 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
2011 upload_file_to_internet_file(catfp
, outfile
);
2019 def AppendInFileWithContentToOutFile(infile
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2020 formatspecs
= FormatSpecsListToDict(formatspecs
);
2021 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, True, False, formatspecs
);
2022 return AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
, returnfp
);
2024 def PrintPermissionString(fchmode
, ftype
):
2025 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2027 for fmodval
in str(oct(fchmode
))[-3:]:
2028 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
2029 if(ftype
==0 or ftype
==7):
2030 permissionstr
= "-" + permissionstr
;
2032 permissionstr
= "h" + permissionstr
;
2034 permissionstr
= "l" + permissionstr
;
2036 permissionstr
= "c" + permissionstr
;
2038 permissionstr
= "b" + permissionstr
;
2040 permissionstr
= "d" + permissionstr
;
2042 permissionstr
= "f" + permissionstr
;
2044 permissionstr
= "D" + permissionstr
;
2046 permissionstr
= "p" + permissionstr
;
2048 permissionstr
= "w" + permissionstr
;
2050 permissionoutstr
= stat
.filemode(fchmode
);
2051 except AttributeError:
2052 permissionoutstr
= permissionstr
;
2054 permissionoutstr
= permissionstr
;
2055 return permissionoutstr
;
2057 def PrintPermissionStringAlt(fchmode
, ftype
):
2059 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2060 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2062 # Translate file mode into permission string
2063 permissionstr
= ''.join([permissions
[i
] for i
in str(oct(fchmode
))[-3:]]);
2064 # Append file type indicator
2066 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2067 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2069 file_type
= type_indicators
.get(ftype
, '-');
2070 permissionstr
= file_type
+ permissionstr
;
2072 permissionoutstr
= stat
.filemode(fchmode
);
2073 except AttributeError:
2074 permissionoutstr
= permissionstr
;
2075 return permissionoutstr
;
2077 def CheckCompressionType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
2078 formatspecs
= FormatSpecsListToDict(formatspecs
);
2079 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2083 catfp
= open(infile
, "rb");
2084 except FileNotFoundError
:
2087 prefp
= catfp
.read(2);
2089 if(prefp
==binascii
.unhexlify("1f8b")):
2092 prefp
= catfp
.read(3);
2093 if(prefp
==binascii
.unhexlify("425a68")):
2095 if(prefp
==binascii
.unhexlify("5d0000")):
2098 prefp
= catfp
.read(4);
2099 if(prefp
==binascii
.unhexlify("28b52ffd")):
2101 if(prefp
==binascii
.unhexlify("04224d18")):
2103 if(prefp
==binascii
.unhexlify("504B0304")):
2104 filetype
= "zipfile";
2106 prefp
= catfp
.read(5);
2107 if(prefp
==binascii
.unhexlify("7573746172")):
2108 filetype
= "tarfile";
2110 prefp
= catfp
.read(6);
2111 if(prefp
==binascii
.unhexlify("fd377a585a00")):
2113 if(prefp
==binascii
.unhexlify("377abcaf271c")):
2114 filetype
= "7zipfile";
2116 prefp
= catfp
.read(7);
2117 if(prefp
==binascii
.unhexlify("526172211a0700")):
2118 filetype
= "rarfile";
2119 if(prefp
==binascii
.unhexlify("43617446696c65")):
2120 filetype
= "catfile";
2122 prefp
= catfp
.read(8);
2123 if(prefp
==binascii
.unhexlify("526172211a070100")):
2124 filetype
= "rarfile";
2126 prefp
= catfp
.read(formatspecs
['format_len']);
2127 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
2128 filetype
= formatspecs
['format_lower'];
2130 prefp
= catfp
.read(9);
2131 if(prefp
==binascii
.unhexlify("894c5a4f000d0a1a0a")):
2134 prefp
= catfp
.read(10);
2135 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2136 filetype
= "tarfile";
2138 if(filetype
=="gzip" or filetype
=="bzip2" or filetype
=="lzma" or filetype
=="zstd" or filetype
=="lz4"):
2139 if(TarFileCheck(catfp
)):
2140 filetype
= "tarfile";
2142 if(TarFileCheck(catfp
)):
2143 filetype
= "tarfile";
2144 elif(zipfile
.is_zipfile(catfp
)):
2145 filetype
= "zipfile";
2146 elif(rarfile
.is_rarfile(catfp
) or rarfile
.is_rarfile_sfx(catfp
)):
2147 filetype
= "rarile";
2155 def CheckCompressionTypeFromString(instring
, formatspecs
=__file_format_dict__
, closefp
=True):
2156 formatspecs
= FormatSpecsListToDict(formatspecs
);
2158 instringsfile
= BytesIO(instring
);
2160 instringsfile
= BytesIO(instring
.encode("UTF-8"));
2161 return CheckCompressionType(instringsfile
, formatspecs
, closefp
);
2163 def GetCompressionMimeType(infile
, formatspecs
=__file_format_dict__
):
2164 formatspecs
= FormatSpecsListToDict(formatspecs
);
2165 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
2166 if(compresscheck
=="gzip" or compresscheck
=="gz"):
2167 return archivefile_gzip_mimetype
;
2168 if(compresscheck
=="bzip2" or compresscheck
=="bz2"):
2169 return archivefile_bzip2_mimetype
;
2170 if(compresscheck
=="zstd" or compresscheck
=="zstandard"):
2171 return archivefile_zstandard_mimetype
;
2172 if(compresscheck
=="lz4"):
2173 return archivefile_lz4_mimetype
;
2174 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
2175 return archivefile_lzop_mimetype
;
2176 if(compresscheck
=="lzma"):
2177 return archivefile_lzma_mimetype
;
2178 if(compresscheck
=="xz"):
2179 return archivefile_xz_mimetype
;
2180 if(compresscheck
=="catfile" or compresscheck
=="cat" or compresscheck
==formatspecs
['format_lower']):
2181 return archivefile_cat_mimetype
;
2182 if(not compresscheck
):
2186 def UncompressArchiveFile(fp
, formatspecs
=__file_format_dict__
):
2187 formatspecs
= FormatSpecsListToDict(formatspecs
);
2188 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2190 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
2191 if(compresscheck
=="gzip"):
2196 catfp
= gzip
.GzipFile(fileobj
=fp
, mode
="rb");
2197 if(compresscheck
=="bzip2"):
2203 catfp
.write(bz2
.decompress(fp
.read()));
2204 if(compresscheck
=="zstd"):
2210 catfp
.write(zstandard
.decompress(fp
.read()));
2211 if(compresscheck
=="lz4"):
2217 catfp
.write(lz4
.frame
.decompress(fp
.read()));
2218 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
2224 catfp
.write(lzo
.decompress(fp
.read()));
2225 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2230 from backports
import lzma
2234 catfp
.write(lzma
.decompress(fp
.read()));
2235 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2237 if(not compresscheck
):
2242 from backports
import lzma
2246 with fp
as fpcontent
:
2248 catfp
.write(lzma
.decompress(fp
.read()));
2249 except lzma
.LZMAError
:
2251 if(compresscheck
!="catfile" or compresscheck
!=formatspecs
['format_lower']):
2255 create_alias_function("Uncompress", __file_format_name__
, "", UncompressArchiveFile
);
2257 def UncompressFile(infile
, formatspecs
=__file_format_dict__
, mode
="rb"):
2258 formatspecs
= FormatSpecsListToDict(formatspecs
);
2259 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2260 if(sys
.version_info
[0]==2 and compresscheck
):
2266 if(compresscheck
=="gzip"):
2272 filefp
= gzip
.open(infile
, mode
, encoding
="UTF-8");
2273 except (ValueError, TypeError) as e
:
2274 filefp
= gzip
.open(infile
, mode
);
2275 if(compresscheck
=="bzip2"):
2281 filefp
= bz2
.open(infile
, mode
, encoding
="UTF-8");
2282 except (ValueError, TypeError) as e
:
2283 filefp
= bz2
.open(infile
, mode
);
2284 if(compresscheck
=="zstd"):
2290 filefp
= zstandard
.open(infile
, mode
, encoding
="UTF-8");
2291 except (ValueError, TypeError) as e
:
2292 filefp
= zstandard
.open(infile
, mode
);
2293 if(compresscheck
=="lz4"):
2299 filefp
= lz4
.frame
.open(infile
, mode
, encoding
="UTF-8");
2300 except (ValueError, TypeError) as e
:
2301 filefp
= lz4
.frame
.open(infile
, mode
);
2302 if(compresscheck
=="lzo"):
2308 filefp
= lzo
.open(infile
, mode
, encoding
="UTF-8");
2309 except (ValueError, TypeError) as e
:
2310 filefp
= lzo
.open(infile
, mode
);
2311 if(compresscheck
=="lzma"):
2316 from backports
import lzma
2320 filefp
= lzma
.open(infile
, mode
, encoding
="UTF-8");
2321 except (ValueError, TypeError) as e
:
2322 filefp
= lzma
.open(infile
, mode
);
2323 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2325 filefp
= open(infile
, mode
, encoding
="UTF-8");
2326 except (ValueError, TypeError) as e
:
2327 filefp
= open(infile
, mode
);
2328 if(not compresscheck
):
2330 filefp
= open(infile
, mode
, encoding
="UTF-8");
2331 except (ValueError, TypeError) as e
:
2332 filefp
= open(infile
, mode
);
2333 except FileNotFoundError
:
2337 def UncompressString(infile
):
2338 compresscheck
= CheckCompressionTypeFromString(infile
, formatspecs
, False);
2339 if(compresscheck
=="gzip"):
2344 fileuz
= gzip
.decompress(infile
);
2345 if(compresscheck
=="bzip2"):
2350 fileuz
= bz2
.decompress(infile
);
2351 if(compresscheck
=="zstd"):
2356 fileuz
= zstandard
.decompress(infile
);
2357 if(compresscheck
=="lz4"):
2362 fileuz
= lz4
.frame
.decompress(infile
);
2363 if(compresscheck
=="lzo"):
2368 fileuz
= lzo
.decompress(infile
);
2369 if(compresscheck
=="lzma"):
2374 from backports
import lzma
2377 fileuz
= lzma
.decompress(infile
);
2378 if(not compresscheck
):
2380 if(hasattr(fileuz
, 'decode')):
2381 fileuz
= fileuz
.decode("UTF-8");
2384 def UncompressStringAlt(infile
):
2385 filefp
= StringIO();
2386 outstring
= UncompressString(infile
);
2387 filefp
.write(outstring
);
2391 def CheckCompressionSubType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
2392 formatspecs
= FormatSpecsListToDict(formatspecs
);
2393 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2394 if(not compresscheck
):
2395 fextname
= os
.path
.splitext(infile
)[1];
2396 if(fextname
==".gz"):
2397 compresscheck
= "gzip";
2398 elif(fextname
==".bz2"):
2399 compresscheck
= "bzip2";
2400 elif(fextname
==".zst"):
2401 compresscheck
= "zstd";
2402 elif(fextname
==".lz4"):
2403 compresscheck
= "lz4";
2404 elif(fextname
==".lzo" or fextname
==".lzop"):
2405 compresscheck
= "lzo";
2406 elif(fextname
==".lzma" or fextname
==".xz"):
2407 compresscheck
= "lzma";
2410 if(compresscheck
=="gzip" or compresscheck
=="bzip2" or compresscheck
=="lzma" or compresscheck
=="zstd" or compresscheck
=="lz4"):
2411 if(TarFileCheck(infile
)):
2412 filetype
= "tarfile";
2413 if(not compresscheck
):
2414 if(TarFileCheck(infile
)):
2416 elif(zipfile
.is_zipfile(infile
)):
2418 elif(rarfile
.is_rarfile(infile
) or rarfile
.is_rarfile_sfx(infile
)):
2423 if(compresscheck
=="catfile"):
2425 if(compresscheck
==formatspecs
['format_lower']):
2426 return formatspecs
['format_lower'];
2427 if(compresscheck
=="tarfile"):
2429 if(compresscheck
=="zipfile"):
2431 if(rarfile_support
and compresscheck
=="rarfile"):
2433 if(py7zr_support
and compresscheck
=="7zipfile"):
2435 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2436 catfp
= UncompressArchiveFile(infile
, formatspecs
['format_lower']);
2439 if(compresscheck
=="gzip"):
2444 catfp
= gzip
.GzipFile(infile
, "rb");
2445 if(compresscheck
=="bzip2"):
2450 catfp
= bz2
.BZ2File(infile
, "rb");
2451 if(compresscheck
=="lz4"):
2456 catfp
= lz4
.frame
.open(infile
, "rb");
2457 if(compresscheck
=="zstd"):
2462 catfp
= zstandard
.open(infile
, "rb");
2463 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2468 from backports
import lzma
2471 catfp
= lzma
.open(infile
, "rb");
2472 except FileNotFoundError
:
2475 prefp
= catfp
.read(5);
2476 if(prefp
==binascii
.unhexlify("7573746172")):
2477 filetype
= "tarfile";
2479 prefp
= catfp
.read(7);
2480 if(prefp
==binascii
.unhexlify("43617446696c65")):
2481 filetype
= "catfile";
2483 prefp
= catfp
.read(formatspecs
['format_len']);
2484 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
2485 filetype
= formatspecs
['format_lower'];
2487 prefp
= catfp
.read(10);
2488 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2489 filetype
= "tarfile";
2495 def GZipCompress(data
, compresslevel
=9):
2500 tmpfp
= tempfile
.NamedTemporaryFile("wb", delete
=False);
2502 tmpfp
= gzip
.GzipFile(tmpfp
.name
, mode
="wb", compresslevel
=compresslevel
);
2506 catfp
= open(tmpfp
.name
, "rb");
2507 except FileNotFoundError
:
2509 catdata
= catfp
.read();
2513 def CompressArchiveFile(fp
, compression
="auto", compressionlevel
=None, formatspecs
=__file_format_dict__
):
2514 formatspecs
= FormatSpecsListToDict(formatspecs
);
2515 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2518 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2519 compression
= "auto";
2520 if(compression
not in compressionlist
and compression
is None):
2521 compression
= "auto";
2522 if(compression
=="gzip"):
2528 if(compressionlevel
is None):
2529 compressionlevel
= 9;
2531 compressionlevel
= int(compressionlevel
);
2532 catfp
.write(gzip
.compress(fp
.read(), compresslevel
=compressionlevel
));
2533 if(compression
=="bzip2"):
2539 if(compressionlevel
is None):
2540 compressionlevel
= 9;
2542 compressionlevel
= int(compressionlevel
);
2543 catfp
.write(bz2
.compress(fp
.read(), compresslevel
=compressionlevel
));
2544 if(compression
=="lz4"):
2550 if(compressionlevel
is None):
2551 compressionlevel
= 9;
2553 compressionlevel
= int(compressionlevel
);
2554 catfp
.write(lz4
.frame
.compress(fp
.read(), compression_level
=compressionlevel
));
2555 if(compression
=="lzo" or compression
=="lzop"):
2561 if(compressionlevel
is None):
2562 compressionlevel
= 9;
2564 compressionlevel
= int(compressionlevel
);
2565 catfp
.write(lzo
.compress(fp
.read(), compresslevel
=compressionlevel
));
2566 if(compression
=="zstd"):
2572 if(compressionlevel
is None):
2573 compressionlevel
= 10;
2575 compressionlevel
= int(compressionlevel
);
2576 catfp
.write(zstandard
.compress(fp
.read(), level
=compressionlevel
));
2577 if(compression
=="lzma"):
2582 from backports
import lzma
2586 if(compressionlevel
is None):
2587 compressionlevel
= 9;
2589 compressionlevel
= int(compressionlevel
);
2590 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]));
2591 if(compression
=="xz"):
2596 from backports
import lzma
2600 if(compressionlevel
is None):
2601 compressionlevel
= 9;
2603 compressionlevel
= int(compressionlevel
);
2604 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]));
2605 if(compression
=="auto" or compression
is None):
2610 create_alias_function("Compress", __file_format_name__
, "", CompressArchiveFile
);
2612 def CompressOpenFile(outfile
, compressionenable
=True, compressionlevel
=None):
2613 if(outfile
is None):
2615 fbasename
= os
.path
.splitext(outfile
)[0];
2616 fextname
= os
.path
.splitext(outfile
)[1];
2617 if(compressionlevel
is None and fextname
!=".zst"):
2618 compressionlevel
= 9;
2619 elif(compressionlevel
is None and fextname
==".zst"):
2620 compressionlevel
= 10;
2622 compressionlevel
= int(compressionlevel
);
2623 if(sys
.version_info
[0]==2):
2628 if(fextname
not in outextlistwd
or not compressionenable
):
2630 outfp
= open(outfile
, "wb", encoding
="UTF-8");
2631 except (ValueError, TypeError) as e
:
2632 outfp
= open(outfile
, "wb");
2633 elif(fextname
==".gz"):
2639 outfp
= gzip
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2640 except (ValueError, TypeError) as e
:
2641 outfp
= gzip
.open(outfile
, mode
, compressionlevel
);
2642 elif(fextname
==".bz2"):
2648 outfp
= bz2
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2649 except (ValueError, TypeError) as e
:
2650 outfp
= bz2
.open(outfile
, mode
, compressionlevel
);
2651 elif(fextname
==".zst"):
2657 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
), encoding
="UTF-8");
2658 except (ValueError, TypeError) as e
:
2659 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
));
2660 elif(fextname
==".xz"):
2665 from backports
import lzma
2669 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}], encoding
="UTF-8");
2670 except (ValueError, TypeError) as e
:
2671 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]);
2672 elif(fextname
==".lz4"):
2678 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
, encoding
="UTF-8");
2679 except (ValueError, TypeError) as e
:
2680 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
);
2681 elif(fextname
==".lzo"):
2687 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
, encoding
="UTF-8");
2688 except (ValueError, TypeError) as e
:
2689 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
);
2690 elif(fextname
==".lzma"):
2695 from backports
import lzma
2699 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}], encoding
="UTF-8");
2700 except (ValueError, TypeError) as e
:
2701 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]);
2702 except FileNotFoundError
:
2706 def GetDevMajorMinor(fdev
):
2708 if(hasattr(os
, "minor")):
2709 retdev
.append(os
.minor(fdev
));
2712 if(hasattr(os
, "major")):
2713 retdev
.append(os
.major(fdev
));
2718 def CheckSumSupport(checkfor
, guaranteed
=True):
2721 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2722 except AttributeError:
2723 hash_list
= sorted(list(hashlib
.algorithms
));
2726 hash_list
= sorted(list(hashlib
.algorithms_available
));
2727 except AttributeError:
2728 hash_list
= sorted(list(hashlib
.algorithms
));
2729 checklistout
= sorted(hash_list
+ ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2730 if(checkfor
in checklistout
):
2735 def CheckSumSupportAlt(checkfor
, guaranteed
=True):
2738 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2739 except AttributeError:
2740 hash_list
= sorted(list(hashlib
.algorithms
));
2743 hash_list
= sorted(list(hashlib
.algorithms_available
));
2744 except AttributeError:
2745 hash_list
= sorted(list(hashlib
.algorithms
));
2746 checklistout
= hash_list
;
2747 if(checkfor
in checklistout
):
2752 def PackArchiveFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2753 formatspecs
= FormatSpecsListToDict(formatspecs
);
2754 advancedlist
= formatspecs
['use_advanced_list'];
2755 altinode
= formatspecs
['use_alt_inode'];
2756 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2757 outfile
= RemoveWindowsPath(outfile
);
2758 checksumtype
= checksumtype
.lower();
2759 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2760 checksumtype
="crc32";
2761 if(checksumtype
=="none"):
2763 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2764 compression
= "auto";
2765 if(compression
not in compressionlist
and compression
is None):
2766 compression
= "auto";
2768 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2769 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2770 if(os
.path
.exists(outfile
)):
2773 except OSError as e
:
2778 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2780 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2783 fbasename
= os
.path
.splitext(outfile
)[0];
2784 fextname
= os
.path
.splitext(outfile
)[1];
2785 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
2786 catver
= formatspecs
['format_ver'];
2787 fileheaderver
= str(int(catver
.replace(".", "")));
2790 for line
in sys
.stdin
:
2791 infilelist
.append(line
.strip());
2792 infilelist
= list(filter(None, infilelist
));
2793 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
2794 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
2796 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
2797 for line
in finfile
:
2798 infilelist
.append(line
.strip());
2799 infilelist
= list(filter(None, infilelist
));
2801 if(isinstance(infiles
, (list, tuple, ))):
2802 infilelist
= list(filter(None, infiles
));
2803 elif(isinstance(infiles
, (str, ))):
2804 infilelist
= list(filter(None, [infiles
]));
2806 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
2808 GetDirList
= ListDir(infilelist
, followlink
, False);
2816 inodetocatinode
= {};
2817 numfiles
= int(len(GetDirList
));
2818 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
2819 for curfname
in GetDirList
:
2820 if(re
.findall(r
"^[.|/]", curfname
)):
2823 fname
= "./"+curfname
;
2825 VerbosePrintOut(fname
);
2826 if(not followlink
or followlink
is None):
2827 fstatinfo
= os
.lstat(fname
);
2829 fstatinfo
= os
.stat(fname
);
2830 fpremode
= fstatinfo
.st_mode
;
2831 finode
= fstatinfo
.st_ino
;
2832 flinkcount
= fstatinfo
.st_nlink
;
2834 if(stat
.S_ISREG(fpremode
)):
2836 elif(stat
.S_ISLNK(fpremode
)):
2838 elif(stat
.S_ISCHR(fpremode
)):
2840 elif(stat
.S_ISBLK(fpremode
)):
2842 elif(stat
.S_ISDIR(fpremode
)):
2844 elif(stat
.S_ISFIFO(fpremode
)):
2846 elif(stat
.S_ISSOCK(fpremode
)):
2848 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
2850 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
2852 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
2857 fcurfid
= format(int(curfid
), 'x').lower();
2858 if(not followlink
and finode
!=0):
2860 if(finode
in inodelist
):
2862 flinkname
= inodetofile
[finode
];
2864 fcurinode
= format(int(finode
), 'x').lower();
2866 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
2867 if(finode
not in inodelist
):
2868 inodelist
.append(finode
);
2869 inodetofile
.update({finode
: fname
});
2870 inodetocatinode
.update({finode
: curinode
});
2872 fcurinode
= format(int(finode
), 'x').lower();
2874 fcurinode
= format(int(curinode
), 'x').lower();
2875 curinode
= curinode
+ 1;
2877 fcurinode
= format(int(curinode
), 'x').lower();
2878 curinode
= curinode
+ 1;
2879 curfid
= curfid
+ 1;
2881 flinkname
= os
.readlink(fname
);
2882 fdev
= fstatinfo
.st_dev
;
2883 getfdev
= GetDevMajorMinor(fdev
);
2884 fdev_minor
= getfdev
[0];
2885 fdev_major
= getfdev
[1];
2886 frdev
= fstatinfo
.st_dev
;
2887 if(hasattr(fstatinfo
, "st_rdev")):
2888 frdev
= fstatinfo
.st_rdev
;
2890 frdev
= fstatinfo
.st_dev
;
2891 getfrdev
= GetDevMajorMinor(frdev
);
2892 frdev_minor
= getfrdev
[0];
2893 frdev_major
= getfrdev
[1];
2894 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
2895 fsize
= format(int("0"), 'x').lower();
2896 elif(ftype
==0 or ftype
==7):
2897 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
2899 fsize
= format(int(fstatinfo
.st_size
)).lower();
2900 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
2901 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
2902 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2903 if(hasattr(fstatinfo
, "st_birthtime")):
2904 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
2906 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2907 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
2908 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
2909 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
2910 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
2911 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
2916 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
2917 funame
= userinfo
.pw_name
;
2926 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
2927 fgname
= groupinfo
.gr_name
;
2932 fdev_minor
= format(int(fdev_minor
), 'x').lower();
2933 fdev_major
= format(int(fdev_major
), 'x').lower();
2934 frdev_minor
= format(int(frdev_minor
), 'x').lower();
2935 frdev_major
= format(int(frdev_major
), 'x').lower();
2936 finode
= format(int(finode
), 'x').lower();
2937 flinkcount
= format(int(flinkcount
), 'x').lower();
2938 if(hasattr(fstatinfo
, "st_file_attributes")):
2939 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
2941 fwinattributes
= format(int(0), 'x').lower();
2943 fcsize
= format(int(0), 'x').lower();
2944 fcontents
= BytesIO();
2945 if(ftype
==0 or ftype
==7):
2946 with
open(fname
, "rb") as fpc
:
2947 shutil
.copyfileobj(fpc
, fcontents
);
2948 if(not compresswholefile
):
2949 fcontents
.seek(0, 2);
2950 ucfsize
= fcontents
.tell();
2951 fcontents
.seek(0, 0);
2952 if(compression
=="auto"):
2953 ilsize
= len(compressionlistalt
);
2956 while(ilmin
< ilsize
):
2957 cfcontents
= BytesIO();
2958 shutil
.copyfileobj(fcontents
, cfcontents
);
2959 fcontents
.seek(0, 0);
2960 cfcontents
.seek(0, 0);
2961 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
2963 cfcontents
.seek(0, 2);
2964 ilcsize
.append(cfcontents
.tell());
2968 ilcsize
.append(sys
.maxint
);
2969 except AttributeError:
2970 ilcsize
.append(sys
.maxsize
);
2972 ilcmin
= ilcsize
.index(min(ilcsize
));
2973 compression
= compressionlistalt
[ilcmin
];
2974 fcontents
.seek(0, 0);
2975 cfcontents
= BytesIO();
2976 shutil
.copyfileobj(fcontents
, cfcontents
);
2977 cfcontents
.seek(0, 0);
2978 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
2979 cfcontents
.seek(0, 2);
2980 cfsize
= cfcontents
.tell();
2981 if(ucfsize
> cfsize
):
2982 fcsize
= format(int(cfsize
), 'x').lower();
2983 fcompression
= compression
;
2985 fcontents
= cfcontents
;
2986 if(fcompression
=="none"):
2988 if(followlink
and (ftype
==1 or ftype
==2)):
2989 flstatinfo
= os
.stat(flinkname
);
2990 with
open(flinkname
, "rb") as fpc
:
2991 shutil
.copyfileobj(fpc
, fcontents
);
2992 if(not compresswholefile
):
2993 fcontents
.seek(0, 2);
2994 ucfsize
= fcontents
.tell();
2995 fcontents
.seek(0, 0);
2996 if(compression
=="auto"):
2997 ilsize
= len(compressionlistalt
);
3000 while(ilmin
< ilsize
):
3001 cfcontents
= BytesIO();
3002 shutil
.copyfileobj(fcontents
, cfcontents
);
3003 fcontents
.seek(0, 0);
3004 cfcontents
.seek(0, 0);
3005 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3007 cfcontents
.seek(0, 2);
3008 ilcsize
.append(cfcontents
.tell());
3012 ilcsize
.append(sys
.maxint
);
3013 except AttributeError:
3014 ilcsize
.append(sys
.maxsize
);
3016 ilcmin
= ilcsize
.index(min(ilcsize
));
3017 compression
= compressionlistalt
[ilcmin
];
3018 fcontents
.seek(0, 0);
3019 cfcontents
= BytesIO();
3020 shutil
.copyfileobj(fcontents
, cfcontents
);
3021 cfcontents
.seek(0, 0);
3022 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3023 cfcontents
.seek(0, 2);
3024 cfsize
= cfcontents
.tell();
3025 if(ucfsize
> cfsize
):
3026 fcsize
= format(int(cfsize
), 'x').lower();
3027 fcompression
= compression
;
3029 fcontents
= cfcontents
;
3030 fcontents
.seek(0, 0);
3031 ftypehex
= format(ftype
, 'x').lower();
3032 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
3033 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3036 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3037 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3038 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3041 os
.fsync(catfp
.fileno());
3042 except io
.UnsupportedOperation
:
3044 except AttributeError:
3046 except OSError as e
:
3050 if(hasattr(sys
.stdout
, "buffer")):
3051 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3053 shutil
.copyfileobj(catfp
, sys
.stdout
);
3054 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3055 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3057 upload_file_to_internet_file(catfp
, outfile
);
3065 create_alias_function("Pack", __file_format_name__
, "", PackArchiveFile
);
3067 if(hasattr(shutil
, "register_archive_format")):
3068 def PackArchiveFileFunc(archive_name
, source_dir
, **kwargs
):
3069 return PackArchiveFile(source_dir
, archive_name
, False, "auto", True, None, False, "crc32", [], __file_format_dict__
['format_delimiter'], False, False);
3070 create_alias_function("Pack", __file_format_name__
, "Func", PackArchiveFileFunc
);
3072 def PackArchiveFileFromDirList(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3073 formatspecs
= FormatSpecsListToDict(formatspecs
);
3074 return PackArchiveFile(infiles
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3076 create_alias_function("Pack", __file_format_name__
, "FromDirList", PackArchiveFileFromDirList
);
3078 def PackArchiveFileFromTarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3079 formatspecs
= FormatSpecsListToDict(formatspecs
);
3080 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3081 outfile
= RemoveWindowsPath(outfile
);
3082 checksumtype
= checksumtype
.lower();
3083 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3084 checksumtype
="crc32";
3085 if(checksumtype
=="none"):
3087 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3088 compression
= "auto";
3089 if(compression
not in compressionlist
and compression
is None):
3090 compression
= "auto";
3092 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3093 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3094 if(os
.path
.exists(outfile
)):
3097 except OSError as e
:
3102 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3104 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3107 fbasename
= os
.path
.splitext(outfile
)[0];
3108 fextname
= os
.path
.splitext(outfile
)[1];
3109 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3110 catver
= formatspecs
['format_ver'];
3111 fileheaderver
= str(int(catver
.replace(".", "")));
3117 inodetocatinode
= {};
3120 if(hasattr(sys
.stdin
, "buffer")):
3121 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
3123 shutil
.copyfileobj(sys
.stdin
, infile
);
3128 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3129 infile
= download_file_from_internet_file(infile
);
3134 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3136 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
3138 if(not tarfile
.TarFileCheck(infile
)):
3140 except AttributeError:
3141 if(not TarFileCheck(infile
)):
3146 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
3147 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
3149 tarfp
= tarfile
.open(infile
, "r");
3150 except FileNotFoundError
:
3152 numfiles
= int(len(tarfp
.getmembers()));
3153 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3154 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
3155 if(re
.findall(r
"^[.|/]", member
.name
)):
3156 fname
= member
.name
;
3158 fname
= "./"+member
.name
;
3160 VerbosePrintOut(fname
);
3161 fpremode
= member
.mode
;
3162 ffullmode
= member
.mode
;
3166 ffullmode
= member
.mode
+ stat
.S_IFREG
;
3168 elif(member
.isdev()):
3169 ffullmode
= member
.mode
;
3171 elif(member
.islnk()):
3172 ffullmode
= member
.mode
+ stat
.S_IFREG
;
3174 elif(member
.issym()):
3175 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
3177 elif(member
.ischr()):
3178 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
3180 elif(member
.isblk()):
3181 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
3183 elif(member
.isdir()):
3184 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
3186 elif(member
.isfifo()):
3187 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
3189 elif(member
.issparse()):
3190 ffullmode
= member
.mode
;
3193 ffullmode
= member
.mode
;
3196 fcurfid
= format(int(curfid
), 'x').lower();
3197 fcurinode
= format(int(curfid
), 'x').lower();
3198 curfid
= curfid
+ 1;
3200 flinkname
= member
.linkname
;
3201 fdev_minor
= format(int(member
.devminor
), 'x').lower();
3202 fdev_major
= format(int(member
.devmajor
), 'x').lower();
3203 frdev_minor
= format(int(member
.devminor
), 'x').lower();
3204 frdev_major
= format(int(member
.devmajor
), 'x').lower();
3205 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
3206 fsize
= format(int("0"), 'x').lower();
3207 elif(ftype
==0 or ftype
==7):
3208 fsize
= format(int(member
.size
), 'x').lower();
3210 fsize
= format(int(member
.size
), 'x').lower();
3211 fatime
= format(int(member
.mtime
), 'x').lower();
3212 fmtime
= format(int(member
.mtime
), 'x').lower();
3213 fctime
= format(int(member
.mtime
), 'x').lower();
3214 fbtime
= format(int(member
.mtime
), 'x').lower();
3215 fmode
= format(int(ffullmode
), 'x').lower();
3216 fchmode
= format(int(stat
.S_IMODE(ffullmode
)), 'x').lower();
3217 ftypemod
= format(int(stat
.S_IFMT(ffullmode
)), 'x').lower();
3218 fuid
= format(int(member
.uid
), 'x').lower();
3219 fgid
= format(int(member
.gid
), 'x').lower();
3220 funame
= member
.uname
;
3221 fgname
= member
.gname
;
3222 flinkcount
= format(int(flinkcount
), 'x').lower();
3223 fwinattributes
= format(int(0), 'x').lower();
3225 fcsize
= format(int(0), 'x').lower();
3226 fcontents
= BytesIO();
3227 if(ftype
==0 or ftype
==7):
3228 with tarfp
.extractfile(member
) as fpc
:
3229 shutil
.copyfileobj(fpc
, fcontents
);
3230 if(not compresswholefile
):
3231 fcontents
.seek(0, 2);
3232 ucfsize
= fcontents
.tell();
3233 fcontents
.seek(0, 0);
3234 if(compression
=="auto"):
3235 ilsize
= len(compressionlistalt
);
3238 while(ilmin
< ilsize
):
3239 cfcontents
= BytesIO();
3240 shutil
.copyfileobj(fcontents
, cfcontents
);
3241 fcontents
.seek(0, 0);
3242 cfcontents
.seek(0, 0);
3243 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3245 cfcontents
.seek(0, 2);
3246 ilcsize
.append(cfcontents
.tell());
3250 ilcsize
.append(sys
.maxint
);
3251 except AttributeError:
3252 ilcsize
.append(sys
.maxsize
);
3254 ilcmin
= ilcsize
.index(min(ilcsize
));
3255 compression
= compressionlistalt
[ilcmin
];
3256 fcontents
.seek(0, 0);
3257 cfcontents
= BytesIO();
3258 shutil
.copyfileobj(fcontents
, cfcontents
);
3259 cfcontents
.seek(0, 0);
3260 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3261 cfcontents
.seek(0, 2);
3262 cfsize
= cfcontents
.tell();
3263 if(ucfsize
> cfsize
):
3264 fcsize
= format(int(cfsize
), 'x').lower();
3265 fcompression
= compression
;
3267 fcontents
= cfcontents
;
3268 if(fcompression
=="none"):
3270 fcontents
.seek(0, 0);
3271 ftypehex
= format(ftype
, 'x').lower();
3272 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
3273 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3276 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3277 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3278 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3281 os
.fsync(catfp
.fileno());
3282 except io
.UnsupportedOperation
:
3284 except AttributeError:
3286 except OSError as e
:
3290 if(hasattr(sys
.stdout
, "buffer")):
3291 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3293 shutil
.copyfileobj(catfp
, sys
.stdout
);
3294 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3295 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3297 upload_file_to_internet_file(catfp
, outfile
);
3305 create_alias_function("Pack", __file_format_name__
, "FromTarFile", PackArchiveFileFromTarFile
);
3307 def PackArchiveFileFromZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3308 formatspecs
= FormatSpecsListToDict(formatspecs
);
3309 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3310 outfile
= RemoveWindowsPath(outfile
);
3311 checksumtype
= checksumtype
.lower();
3312 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3313 checksumtype
="crc32";
3314 if(checksumtype
=="none"):
3316 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3317 compression
= "auto";
3318 if(compression
not in compressionlist
and compression
is None):
3319 compression
= "auto";
3321 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3322 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3323 if(os
.path
.exists(outfile
)):
3326 except OSError as e
:
3331 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3333 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3336 fbasename
= os
.path
.splitext(outfile
)[0];
3337 fextname
= os
.path
.splitext(outfile
)[1];
3338 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3339 catver
= formatspecs
['format_ver'];
3340 fileheaderver
= str(int(catver
.replace(".", "")));
3346 inodetocatinode
= {};
3349 if(hasattr(sys
.stdin
, "buffer")):
3350 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
3352 shutil
.copyfileobj(sys
.stdin
, infile
);
3357 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3358 infile
= download_file_from_internet_file(infile
);
3363 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3367 if(not zipfile
.is_zipfile(infile
)):
3370 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
3371 except FileNotFoundError
:
3373 ziptest
= zipfp
.testzip();
3375 VerbosePrintOut("Bad file found!");
3376 numfiles
= int(len(zipfp
.infolist()));
3377 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3378 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
3379 if(re
.findall(r
"^[.|/]", member
.filename
)):
3380 fname
= member
.filename
;
3382 fname
= "./"+member
.filename
;
3383 zipinfo
= zipfp
.getinfo(member
.filename
);
3385 VerbosePrintOut(fname
);
3386 if(not member
.is_dir()):
3387 fpremode
= int(stat
.S_IFREG
+ 438);
3388 elif(member
.is_dir()):
3389 fpremode
= int(stat
.S_IFDIR
+ 511);
3392 if(not member
.is_dir()):
3394 elif(member
.is_dir()):
3397 fcurfid
= format(int(curfid
), 'x').lower();
3398 fcurinode
= format(int(curfid
), 'x').lower();
3399 curfid
= curfid
+ 1;
3400 fdev_minor
= format(int(0), 'x').lower();
3401 fdev_major
= format(int(0), 'x').lower();
3402 frdev_minor
= format(int(0), 'x').lower();
3403 frdev_major
= format(int(0), 'x').lower();
3405 fsize
= format(int("0"), 'x').lower();
3407 fsize
= format(int(member
.file_size
), 'x').lower();
3409 fsize
= format(int(member
.file_size
), 'x').lower();
3410 fatime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3411 fmtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3412 fctime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3413 fbtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3414 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
3415 fwinattributes
= format(int(zipinfo
.external_attr
), 'x').lower();
3416 if(not member
.is_dir()):
3417 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3418 fchmode
= stat
.S_IMODE(fmode
);
3419 ftypemod
= stat
.S_IFMT(fmode
);
3420 elif(member
.is_dir()):
3421 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3422 fchmode
= stat
.S_IMODE(fmode
);
3423 ftypemod
= stat
.S_IFMT(fmode
);
3424 elif(zipinfo
.create_system
==3):
3425 fwinattributes
= format(int(0), 'x').lower();
3426 fmode
= format(int(zipinfo
.external_attr
), 'x').lower();
3427 fchmode
= stat
.S_IMODE(fmode
);
3428 ftypemod
= stat
.S_IFMT(fmode
);
3430 fwinattributes
= format(int(0), 'x').lower();
3431 if(not member
.is_dir()):
3432 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3433 fchmode
= stat
.S_IMODE(fmode
);
3434 ftypemod
= stat
.S_IFMT(fmode
);
3435 elif(member
.is_dir()):
3436 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3437 fchmode
= stat
.S_IMODE(fmode
);
3438 ftypemod
= stat
.S_IFMT(fmode
);
3440 fcsize
= format(int(0), 'x').lower();
3442 fuid
= format(int(os
.getuid()), 'x').lower();
3443 except AttributeError:
3444 fuid
= format(int(0), 'x').lower();
3446 fuid
= format(int(0), 'x').lower();
3448 fgid
= format(int(os
.getgid()), 'x').lower();
3449 except AttributeError:
3450 fgid
= format(int(0), 'x').lower();
3452 fgid
= format(int(0), 'x').lower();
3456 userinfo
= pwd
.getpwuid(os
.getuid());
3457 funame
= userinfo
.pw_name
;
3460 except AttributeError:
3468 groupinfo
= grp
.getgrgid(os
.getgid());
3469 fgname
= groupinfo
.gr_name
;
3472 except AttributeError:
3476 fcontents
= BytesIO();
3478 fcontents
.write(zipfp
.read(member
.filename
));
3479 if(not compresswholefile
):
3480 fcontents
.seek(0, 2);
3481 ucfsize
= fcontents
.tell();
3482 fcontents
.seek(0, 0);
3483 if(compression
=="auto"):
3484 ilsize
= len(compressionlistalt
);
3487 while(ilmin
< ilsize
):
3488 cfcontents
= BytesIO();
3489 shutil
.copyfileobj(fcontents
, cfcontents
);
3490 fcontents
.seek(0, 0);
3491 cfcontents
.seek(0, 0);
3492 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3493 cfcontents
.seek(0, 2);
3494 ilcsize
.append(cfcontents
.tell());
3497 ilcmin
= ilcsize
.index(min(ilcsize
));
3498 compression
= compressionlistalt
[ilcmin
];
3499 fcontents
.seek(0, 0);
3500 cfcontents
= BytesIO();
3501 shutil
.copyfileobj(fcontents
, cfcontents
);
3502 cfcontents
.seek(0, 0);
3503 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3504 cfcontents
.seek(0, 2);
3505 cfsize
= cfcontents
.tell();
3506 if(ucfsize
> cfsize
):
3507 fcsize
= format(int(cfsize
), 'x').lower();
3508 fcompression
= compression
;
3510 fcontents
= cfcontents
;
3511 if(fcompression
=="none"):
3513 fcontents
.seek(0, 0);
3514 ftypehex
= format(ftype
, 'x').lower();
3515 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
3516 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3519 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3520 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3521 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3524 os
.fsync(catfp
.fileno());
3525 except io
.UnsupportedOperation
:
3527 except AttributeError:
3529 except OSError as e
:
3533 if(hasattr(sys
.stdout
, "buffer")):
3534 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3536 shutil
.copyfileobj(catfp
, sys
.stdout
);
3537 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3538 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3540 upload_file_to_internet_file(catfp
, outfile
);
3548 create_alias_function("Pack", __file_format_name__
, "FromZipFile", PackArchiveFileFromZipFile
);
3550 if(not rarfile_support
):
3551 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3554 if(rarfile_support
):
3555 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3556 formatspecs
= FormatSpecsListToDict(formatspecs
);
3557 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3558 outfile
= RemoveWindowsPath(outfile
);
3559 checksumtype
= checksumtype
.lower();
3560 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3561 checksumtype
="crc32";
3562 if(checksumtype
=="none"):
3564 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3565 compression
= "auto";
3566 if(compression
not in compressionlist
and compression
is None):
3567 compression
= "auto";
3569 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3570 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3571 if(os
.path
.exists(outfile
)):
3574 except OSError as e
:
3579 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3581 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3584 fbasename
= os
.path
.splitext(outfile
)[0];
3585 fextname
= os
.path
.splitext(outfile
)[1];
3586 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3587 catver
= formatspecs
['format_ver'];
3588 fileheaderver
= str(int(catver
.replace(".", "")));
3594 inodetocatinode
= {};
3595 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3597 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
3599 rarfp
= rarfile
.RarFile(infile
, "r");
3600 rartest
= rarfp
.testrar();
3602 VerbosePrintOut("Bad file found!");
3603 numfiles
= int(len(rarfp
.infolist()));
3604 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3607 os
.fsync(catfp
.fileno());
3608 except io
.UnsupportedOperation
:
3610 except AttributeError:
3612 except OSError as e
:
3614 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
3617 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
3620 member
.external_attr
3622 except AttributeError:
3624 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
3627 member
.external_attr
3629 except AttributeError:
3634 if(re
.findall(r
"^[.|/]", member
.filename
)):
3635 fname
= member
.filename
;
3637 fname
= "./"+member
.filename
;
3638 rarinfo
= rarfp
.getinfo(member
.filename
);
3640 VerbosePrintOut(fname
);
3641 if(is_unix
and member
.external_attr
!=0):
3642 fpremode
= int(member
.external_attr
);
3643 elif(member
.is_file()):
3644 fpremode
= int(stat
.S_IFREG
+ 438);
3645 elif(member
.is_symlink()):
3646 fpremode
= int(stat
.S_IFLNK
+ 438);
3647 elif(member
.is_dir()):
3648 fpremode
= int(stat
.S_IFDIR
+ 511);
3649 if(is_windows
and member
.external_attr
!=0):
3650 fwinattributes
= format(int(member
.external_attr
), 'x').lower();
3652 fwinattributes
= format(int(0), 'x').lower();
3654 fcsize
= format(int(0), 'x').lower();
3657 if(member
.is_file()):
3659 elif(member
.is_symlink()):
3661 elif(member
.is_dir()):
3665 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
3666 fcurfid
= format(int(curfid
), 'x').lower();
3667 fcurinode
= format(int(curfid
), 'x').lower();
3668 curfid
= curfid
+ 1;
3669 fdev_minor
= format(int(0), 'x').lower();
3670 fdev_major
= format(int(0), 'x').lower();
3671 frdev_minor
= format(int(0), 'x').lower();
3672 frdev_major
= format(int(0), 'x').lower();
3674 fsize
= format(int("0"), 'x').lower();
3676 fsize
= format(int(member
.file_size
), 'x').lower();
3678 fsize
= format(int(member
.file_size
), 'x').lower();
3681 fatime
= format(int(member
.atime
.timestamp()), 'x').lower();
3683 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3684 except AttributeError:
3685 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3686 fmtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3689 fctime
= format(int(member
.ctime
.timestamp()), 'x').lower();
3691 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3692 except AttributeError:
3693 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3694 fbtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3695 if(is_unix
and member
.external_attr
!=0):
3696 fmode
= format(int(member
.external_attr
), 'x').lower();
3697 fchmode
= format(int(stat
.S_IMODE(member
.external_attr
)), 'x').lower();
3698 ftypemod
= format(int(stat
.S_IFMT(member
.external_attr
)), 'x').lower();
3699 elif(member
.is_file()):
3700 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3701 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3702 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3703 elif(member
.is_symlink()):
3704 fmode
= format(int(stat
.S_IFLNK
+ 438), 'x').lower();
3705 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3706 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3707 elif(member
.is_dir()):
3708 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3709 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3710 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3712 fuid
= format(int(os
.getuid()), 'x').lower();
3713 except AttributeError:
3714 fuid
= format(int(0), 'x').lower();
3716 fuid
= format(int(0), 'x').lower();
3718 fgid
= format(int(os
.getgid()), 'x').lower();
3719 except AttributeError:
3720 fgid
= format(int(0), 'x').lower();
3722 fgid
= format(int(0), 'x').lower();
3726 userinfo
= pwd
.getpwuid(os
.getuid());
3727 funame
= userinfo
.pw_name
;
3730 except AttributeError:
3738 groupinfo
= grp
.getgrgid(os
.getgid());
3739 fgname
= groupinfo
.gr_name
;
3742 except AttributeError:
3746 fcontents
= BytesIO();
3748 fcontents
.write(rarfp
.read(member
.filename
));
3749 if(not compresswholefile
):
3750 fcontents
.seek(0, 2);
3751 ucfsize
= fcontents
.tell();
3752 fcontents
.seek(0, 0);
3753 if(compression
=="auto"):
3754 ilsize
= len(compressionlistalt
);
3757 while(ilmin
< ilsize
):
3758 cfcontents
= BytesIO();
3759 shutil
.copyfileobj(fcontents
, cfcontents
);
3760 fcontents
.seek(0, 0);
3761 cfcontents
.seek(0, 0);
3762 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3764 cfcontents
.seek(0, 2);
3765 ilcsize
.append(cfcontents
.tell());
3769 ilcsize
.append(sys
.maxint
);
3770 except AttributeError:
3771 ilcsize
.append(sys
.maxsize
);
3773 ilcmin
= ilcsize
.index(min(ilcsize
));
3774 compression
= compressionlistalt
[ilcmin
];
3775 fcontents
.seek(0, 0);
3776 cfcontents
= BytesIO();
3777 shutil
.copyfileobj(fcontents
, cfcontents
);
3778 cfcontents
.seek(0, 0);
3779 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3780 cfcontents
.seek(0, 2);
3781 cfsize
= cfcontents
.tell();
3782 if(ucfsize
> cfsize
):
3783 fcsize
= format(int(cfsize
), 'x').lower();
3784 fcompression
= compression
;
3786 fcontents
= cfcontents
;
3787 if(fcompression
=="none"):
3789 fcontents
.seek(0, 0);
3790 ftypehex
= format(ftype
, 'x').lower();
3791 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
3792 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3795 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3796 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3797 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
3800 os
.fsync(catfp
.fileno());
3801 except io
.UnsupportedOperation
:
3803 except AttributeError:
3805 except OSError as e
:
3809 if(hasattr(sys
.stdout
, "buffer")):
3810 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3812 shutil
.copyfileobj(catfp
, sys
.stdout
);
3813 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3814 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3816 upload_file_to_internet_file(catfp
, outfile
);
3824 create_alias_function("Pack", __file_format_name__
, "FromRarFile", PackArchiveFileFromRarFile
);
3826 if(not py7zr_support
):
3827 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3831 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3832 formatspecs
= FormatSpecsListToDict(formatspecs
);
3833 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3834 outfile
= RemoveWindowsPath(outfile
);
3835 checksumtype
= checksumtype
.lower();
3836 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3837 checksumtype
="crc32";
3838 if(checksumtype
=="none"):
3840 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3841 compression
= "auto";
3842 if(compression
not in compressionlist
and compression
is None):
3843 compression
= "auto";
3845 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3846 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3847 if(os
.path
.exists(outfile
)):
3850 except OSError as e
:
3855 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3857 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3860 fbasename
= os
.path
.splitext(outfile
)[0];
3861 fextname
= os
.path
.splitext(outfile
)[1];
3862 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3863 catver
= formatspecs
['format_ver'];
3864 fileheaderver
= str(int(catver
.replace(".", "")));
3870 inodetocatinode
= {};
3871 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3873 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
3874 file_content
= szpfp
.readall();
3875 #sztest = szpfp.testzip();
3876 sztestalt
= szpfp
.test();
3878 VerbosePrintOut("Bad file found!");
3879 numfiles
= int(len(szpfp
.list()));
3880 AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3881 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
3882 if(re
.findall(r
"^[.|/]", member
.filename
)):
3883 fname
= member
.filename
;
3885 fname
= "./"+member
.filename
;
3887 VerbosePrintOut(fname
);
3888 if(not member
.is_directory
):
3889 fpremode
= int(stat
.S_IFREG
+ 438);
3890 elif(member
.is_directory
):
3891 fpremode
= int(stat
.S_IFDIR
+ 511);
3892 fwinattributes
= format(int(0), 'x').lower();
3894 fcsize
= format(int(0), 'x').lower();
3897 if(member
.is_directory
):
3902 fcurfid
= format(int(curfid
), 'x').lower();
3903 fcurinode
= format(int(curfid
), 'x').lower();
3904 curfid
= curfid
+ 1;
3905 fdev_minor
= format(int(0), 'x').lower();
3906 fdev_major
= format(int(0), 'x').lower();
3907 frdev_minor
= format(int(0), 'x').lower();
3908 frdev_major
= format(int(0), 'x').lower();
3910 fsize
= format(int("0"), 'x').lower();
3911 fatime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3912 fmtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3913 fctime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3914 fbtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3915 if(member
.is_directory
):
3916 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3917 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3918 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3920 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3921 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3922 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3924 fuid
= format(int(os
.getuid()), 'x').lower();
3925 except AttributeError:
3926 fuid
= format(int(0), 'x').lower();
3928 fuid
= format(int(0), 'x').lower();
3930 fgid
= format(int(os
.getgid()), 'x').lower();
3931 except AttributeError:
3932 fgid
= format(int(0), 'x').lower();
3934 fgid
= format(int(0), 'x').lower();
3938 userinfo
= pwd
.getpwuid(os
.getuid());
3939 funame
= userinfo
.pw_name
;
3942 except AttributeError:
3950 groupinfo
= grp
.getgrgid(os
.getgid());
3951 fgname
= groupinfo
.gr_name
;
3954 except AttributeError:
3958 fcontents
= BytesIO();
3960 fcontents
.write(file_content
[member
.filename
].read());
3961 fsize
= format(fcontents
.tell(), 'x').lower();
3962 file_content
[member
.filename
].close();
3963 if(not compresswholefile
):
3964 fcontents
.seek(0, 2);
3965 ucfsize
= fcontents
.tell();
3966 fcontents
.seek(0, 0);
3967 if(compression
=="auto"):
3968 ilsize
= len(compressionlistalt
);
3971 while(ilmin
< ilsize
):
3972 cfcontents
= BytesIO();
3973 shutil
.copyfileobj(fcontents
, cfcontents
);
3974 fcontents
.seek(0, 0);
3975 cfcontents
.seek(0, 0);
3976 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3978 cfcontents
.seek(0, 2);
3979 ilcsize
.append(cfcontents
.tell());
3983 ilcsize
.append(sys
.maxint
);
3984 except AttributeError:
3985 ilcsize
.append(sys
.maxsize
);
3987 ilcmin
= ilcsize
.index(min(ilcsize
));
3988 compression
= compressionlistalt
[ilcmin
];
3989 fcontents
.seek(0, 0);
3990 cfcontents
= BytesIO();
3991 shutil
.copyfileobj(fcontents
, cfcontents
);
3992 cfcontents
.seek(0, 0);
3993 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3994 cfcontents
.seek(0, 2);
3995 cfsize
= cfcontents
.tell();
3996 if(ucfsize
> cfsize
):
3997 fcsize
= format(int(cfsize
), 'x').lower();
3998 fcompression
= compression
;
4000 fcontents
= cfcontents
;
4001 if(fcompression
=="none"):
4003 fcontents
.seek(0, 0);
4004 ftypehex
= format(ftype
, 'x').lower();
4005 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, "+1"];
4006 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
4009 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
4010 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
4011 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
4014 os
.fsync(catfp
.fileno());
4015 except io
.UnsupportedOperation
:
4017 except AttributeError:
4019 except OSError as e
:
4023 if(hasattr(sys
.stdout
, "buffer")):
4024 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
4026 shutil
.copyfileobj(catfp
, sys
.stdout
);
4027 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
4028 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
4030 upload_file_to_internet_file(catfp
, outfile
);
4038 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFile", PackArchiveFileFromSevenZipFile
);
4040 def PackArchiveFileFromInFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
4041 formatspecs
= FormatSpecsListToDict(formatspecs
);
4042 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4044 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
4045 if(checkcompressfile
=="tarfile"):
4046 return PackArchiveFileFromTarFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4047 elif(checkcompressfile
=="zipfile"):
4048 return PackArchiveFileFromZipFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4049 elif(checkcompressfile
=="catfile"):
4050 return RePackArchiveFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, False, 0, 0, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
4051 elif(rarfile_support
and checkcompressfile
=="rarfile"):
4052 return PackArchiveFileFromRarFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4053 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
4054 return PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
4059 create_alias_function("Pack", __file_format_name__
, "FromInFile", PackArchiveFileFromInFile
);
4061 def ArchiveFileSeekToFileNum(infile
, seekto
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4062 formatspecs
= FormatSpecsListToDict(formatspecs
);
4063 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4066 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4067 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4068 if(checkcompressfile
=="tarfile"):
4069 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4070 if(checkcompressfile
=="zipfile"):
4071 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4072 if(rarfile_support
and checkcompressfile
=="rarfile"):
4073 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4074 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4075 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4076 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4083 if(hasattr(sys
.stdin
, "buffer")):
4084 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4086 shutil
.copyfileobj(sys
.stdin
, catfp
);
4088 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4092 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4093 catfp
= download_file_from_internet_file(infile
);
4095 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4100 infile
= RemoveWindowsPath(infile
);
4101 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4102 if(checkcompressfile
=="tarfile"):
4103 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4104 if(checkcompressfile
=="zipfile"):
4105 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4106 if(rarfile_support
and checkcompressfile
=="rarfile"):
4107 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4108 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4109 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4110 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4112 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4113 if(not compresscheck
):
4114 fextname
= os
.path
.splitext(infile
)[1];
4115 if(fextname
==".gz"):
4116 compresscheck
= "gzip";
4117 elif(fextname
==".bz2"):
4118 compresscheck
= "bzip2";
4119 elif(fextname
==".zst"):
4120 compresscheck
= "zstd";
4121 elif(fextname
==".lz4" or fextname
==".clz4"):
4122 compresscheck
= "lz4";
4123 elif(fextname
==".lzo" or fextname
==".lzop"):
4124 compresscheck
= "lzo";
4125 elif(fextname
==".lzma" or fextname
==".xz"):
4126 compresscheck
= "lzma";
4129 if(not compresscheck
):
4131 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4136 SeekToEndOfFile(catfp);
4138 SeekToEndOfFile(catfp);
4139 CatSize = catfp.tell();
4140 CatSizeEnd = CatSize;
4148 curloc
= catfp
.tell();
4151 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4153 catfp
.seek(curloc
, 0);
4154 catstring
= catheader
[0];
4155 catversion
= re
.findall(r
"([\d]+)", catstring
);
4156 fprenumfiles
= catheader
[1];
4157 fnumfiles
= int(fprenumfiles
, 16);
4158 fprechecksumtype
= catheader
[2];
4159 fprechecksum
= catheader
[3];
4160 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4161 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4162 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4163 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4164 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4165 fheadtell
= len(fileheader
);
4166 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4167 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4168 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4170 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
4171 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4172 if(seekto
>=fnumfiles
):
4173 seekto
= fnumfiles
- 1;
4179 prefhstart
= catfp
.tell();
4180 if(formatspecs
['new_style']):
4181 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4183 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4184 if(len(preheaderdata
)==0):
4186 prefheadsize
= int(preheaderdata
[0], 16);
4187 prefnumfields
= int(preheaderdata
[1], 16);
4188 preftype
= int(preheaderdata
[2], 16);
4189 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4190 prefname
= preheaderdata
[3];
4192 prefname
= "./"+preheaderdata
[3];
4193 prefbasedir
= os
.path
.dirname(prefname
);
4194 preflinkname
= preheaderdata
[4];
4195 prefsize
= int(preheaderdata
[5], 16);
4196 prefatime
= int(preheaderdata
[6], 16);
4197 prefmtime
= int(preheaderdata
[7], 16);
4198 prefctime
= int(preheaderdata
[8], 16);
4199 prefbtime
= int(preheaderdata
[9], 16);
4200 prefmode
= int(preheaderdata
[10], 16);
4201 prefchmode
= stat
.S_IMODE(prefmode
);
4202 preftypemod
= stat
.S_IFMT(prefmode
);
4203 prefwinattributes
= int(preheaderdata
[11], 16);
4204 prefcompression
= preheaderdata
[12];
4205 prefcsize
= int(preheaderdata
[13], 16);
4206 prefuid
= int(preheaderdata
[14], 16);
4207 prefuname
= preheaderdata
[15];
4208 prefgid
= int(preheaderdata
[16], 16);
4209 prefgname
= preheaderdata
[17];
4210 fid
= int(preheaderdata
[18], 16);
4211 finode
= int(preheaderdata
[19], 16);
4212 flinkcount
= int(preheaderdata
[20], 16);
4213 prefdev_minor
= int(preheaderdata
[21], 16);
4214 prefdev_major
= int(preheaderdata
[22], 16);
4215 prefrdev_minor
= int(preheaderdata
[23], 16);
4216 prefrdev_major
= int(preheaderdata
[24], 16);
4217 prefseeknextfile
= preheaderdata
[25];
4218 prefextrasize
= int(preheaderdata
[26], 16);
4219 prefextrafields
= int(preheaderdata
[27], 16);
4220 extrafieldslist
= [];
4222 extraend
= extrastart
+ prefextrafields
;
4223 extrafieldslist
= [];
4224 if(extrastart
<extraend
):
4225 extrafieldslist
.append(preheaderdata
[extrastart
]);
4226 extrastart
= extrastart
+ 1;
4227 prefcs
= preheaderdata
[-2].lower();
4228 prenewfcs
= preheaderdata
[-1].lower();
4229 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
4230 if(prefcs
!=prenewfcs
and not skipchecksum
):
4231 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4232 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4234 valid_archive
= False;
4235 invalid_archive
= True;
4236 prefhend
= catfp
.tell() - 1;
4237 prefcontentstart
= catfp
.tell();
4239 pyhascontents
= False;
4241 if(prefcompression
):
4242 prefcontents
= catfp
.read(prefsize
);
4244 prefcontents
= catfp
.read(prefcsize
);
4245 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4246 pyhascontents
= True;
4247 if(prefccs
!=prenewfccs
and not skipchecksum
):
4248 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4249 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4251 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
4252 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
4253 if(abs(fseeknextasnum
)==0):
4255 catfp
.seek(fseeknextasnum
, 1);
4256 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
4257 fseeknextasnum
= int(prefseeknextfile
);
4258 if(abs(fseeknextasnum
)==0):
4260 catfp
.seek(fseeknextasnum
, 1);
4261 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
4262 fseeknextasnum
= int(prefseeknextfile
);
4263 if(abs(fseeknextasnum
)==0):
4265 catfp
.seek(fseeknextasnum
, 0);
4269 catfp
.seek(seekstart
, 0);
4271 catfheadsize
= int(preheaderdata
[0], 16);
4272 catfnumfields
= int(preheaderdata
[1], 16);
4273 catftype
= int(preheaderdata
[2], 16);
4274 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4275 catfname
= preheaderdata
[3];
4277 catfname
= "./"+preheaderdata
[3];
4278 catflinkname
= preheaderdata
[4];
4279 catfsize
= int(preheaderdata
[5], 16);
4280 catfbasedir
= os
.path
.dirname(catfname
);
4281 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4283 catlist
.update({'catfp': catfp
});
4288 create_alias_function("", __file_format_name__
, "SeekToFileNum", ArchiveFileSeekToFileNum
);
4290 def ArchiveFileSeekToFileName(infile
, seekfile
=None, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4291 formatspecs
= FormatSpecsListToDict(formatspecs
);
4292 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4295 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4296 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4297 if(checkcompressfile
=="tarfile"):
4298 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4299 if(checkcompressfile
=="zipfile"):
4300 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4301 if(rarfile_support
and checkcompressfile
=="rarfile"):
4302 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4303 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4304 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4305 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4312 if(hasattr(sys
.stdin
, "buffer")):
4313 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4315 shutil
.copyfileobj(sys
.stdin
, catfp
);
4317 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4321 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4322 catfp
= download_file_from_internet_file(infile
);
4323 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4329 infile
= RemoveWindowsPath(infile
);
4330 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4331 if(checkcompressfile
=="tarfile"):
4332 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4333 if(checkcompressfile
=="zipfile"):
4334 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4335 if(rarfile_support
and checkcompressfile
=="rarfile"):
4336 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4337 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4338 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4339 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4341 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4342 if(not compresscheck
):
4343 fextname
= os
.path
.splitext(infile
)[1];
4344 if(fextname
==".gz"):
4345 compresscheck
= "gzip";
4346 elif(fextname
==".bz2"):
4347 compresscheck
= "bzip2";
4348 elif(fextname
==".zst"):
4349 compresscheck
= "zstd";
4350 elif(fextname
==".lz4" or fextname
==".clz4"):
4351 compresscheck
= "lz4";
4352 elif(fextname
==".lzo" or fextname
==".lzop"):
4353 compresscheck
= "lzo";
4354 elif(fextname
==".lzma" or fextname
==".xz"):
4355 compresscheck
= "lzma";
4358 if(not compresscheck
):
4360 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4365 SeekToEndOfFile(catfp);
4367 SeekToEndOfFile(catfp);
4368 CatSize = catfp.tell();
4369 CatSizeEnd = CatSize;
4377 curloc
= catfp
.tell();
4380 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4382 catfp
.seek(curloc
, 0);
4383 catstring
= catheader
[0];
4384 catversion
= re
.findall(r
"([\d]+)", catstring
);
4385 fprenumfiles
= catheader
[1];
4386 fnumfiles
= int(fprenumfiles
, 16);
4387 fprechecksumtype
= catheader
[2];
4388 fprechecksum
= catheader
[3];
4389 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4390 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4391 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4392 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4393 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4394 fheadtell
= len(fileheader
);
4395 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4396 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4397 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4399 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
4400 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4401 seekto
= fnumfiles
- 1
4406 prefhstart
= catfp
.tell();
4407 if(formatspecs
['new_style']):
4408 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4410 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4411 if(len(preheaderdata
)==0):
4413 prefheadsize
= int(preheaderdata
[0], 16);
4414 prefnumfields
= int(preheaderdata
[1], 16);
4415 preftype
= int(preheaderdata
[2], 16);
4416 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4417 prefname
= preheaderdata
[3];
4419 prefname
= "./"+preheaderdata
[3];
4420 prefbasedir
= os
.path
.dirname(prefname
);
4421 preflinkname
= preheaderdata
[4];
4422 prefsize
= int(preheaderdata
[5], 16);
4423 prefatime
= int(preheaderdata
[6], 16);
4424 prefmtime
= int(preheaderdata
[7], 16);
4425 prefctime
= int(preheaderdata
[8], 16);
4426 prefbtime
= int(preheaderdata
[9], 16);
4427 prefmode
= int(preheaderdata
[10], 16);
4428 prefchmode
= stat
.S_IMODE(prefmode
);
4429 preftypemod
= stat
.S_IFMT(prefmode
);
4430 prefwinattributes
= int(preheaderdata
[11], 16);
4431 prefcompression
= preheaderdata
[12];
4432 prefcsize
= int(preheaderdata
[13], 16);
4433 prefuid
= int(preheaderdata
[14], 16);
4434 prefuname
= preheaderdata
[15];
4435 prefgid
= int(preheaderdata
[16], 16);
4436 prefgname
= preheaderdata
[17];
4437 fid
= int(preheaderdata
[18], 16);
4438 finode
= int(preheaderdata
[19], 16);
4439 flinkcount
= int(preheaderdata
[20], 16);
4440 prefdev_minor
= int(preheaderdata
[21], 16);
4441 prefdev_major
= int(preheaderdata
[22], 16);
4442 prefrdev_minor
= int(preheaderdata
[23], 16);
4443 prefrdev_major
= int(preheaderdata
[24], 16);
4444 prefseeknextfile
= preheaderdata
[25];
4445 prefextrasize
= int(preheaderdata
[26], 16);
4446 prefextrafields
= int(preheaderdata
[27], 16);
4447 extrafieldslist
= [];
4449 extraend
= extrastart
+ prefextrafields
;
4450 extrafieldslist
= [];
4451 if(extrastart
<extraend
):
4452 extrafieldslist
.append(preheaderdata
[extrastart
]);
4453 extrastart
= extrastart
+ 1;
4454 prefcs
= preheaderdata
[-2].lower();
4455 prenewfcs
= preheaderdata
[-1].lower();
4456 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
4457 if(prefcs
!=prenewfcs
and not skipchecksum
):
4458 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4459 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4461 valid_archive
= False;
4462 invalid_archive
= True;
4463 prefhend
= catfp
.tell() - 1;
4464 prefcontentstart
= catfp
.tell();
4466 pyhascontents
= False;
4468 if(prefcompression
):
4469 prefcontents
= catfp
.read(prefsize
);
4471 prefcontents
= catfp
.read(prefcsize
);
4472 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4473 pyhascontents
= True;
4474 if(prefccs
!=prenewfccs
and not skipchecksum
):
4475 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4476 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4478 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
4479 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
4480 if(abs(fseeknextasnum
)==0):
4482 catfp
.seek(fseeknextasnum
, 1);
4483 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
4484 fseeknextasnum
= int(prefseeknextfile
);
4485 if(abs(fseeknextasnum
)==0):
4487 catfp
.seek(fseeknextasnum
, 1);
4488 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
4489 fseeknextasnum
= int(prefseeknextfile
);
4490 if(abs(fseeknextasnum
)==0):
4492 catfp
.seek(fseeknextasnum
, 0);
4497 prefname
= preheaderdata
[2];
4498 if(re
.findall(r
"^[.|/]", preheaderdata
[2])):
4499 prefname
= preheaderdata
[2];
4501 prefname
= "./"+preheaderdata
[2];
4502 if(prefname
==seekfile
):
4505 catfp
.seek(seekstart
, 0);
4507 catfheadsize
= int(preheaderdata
[0], 16);
4508 catfnumfields
= int(preheaderdata
[1], 16);
4509 catftype
= int(preheaderdata
[2], 16);
4510 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4511 catfname
= preheaderdata
[3];
4513 catfname
= "./"+preheaderdata
[3];
4514 catflinkname
= preheaderdata
[4];
4515 catfsize
= int(preheaderdata
[5], 16);
4516 catfbasedir
= os
.path
.dirname(catfname
);
4518 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4522 catlist
.update({'catfp': catfp
});
4527 create_alias_function("", __file_format_name__
, "SeekToFileName", ArchiveFileSeekToFileName
);
4529 def ArchiveFileValidate(infile
, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
4530 formatspecs
= FormatSpecsListToDict(formatspecs
);
4532 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
4533 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4536 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4537 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4538 if(checkcompressfile
=="tarfile"):
4539 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4540 if(checkcompressfile
=="zipfile"):
4541 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4542 if(rarfile_support
and checkcompressfile
=="rarfile"):
4543 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4544 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4545 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4546 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4553 if(hasattr(sys
.stdin
, "buffer")):
4554 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4556 shutil
.copyfileobj(sys
.stdin
, catfp
);
4558 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4562 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4563 catfp
= download_file_from_internet_file(infile
);
4564 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4570 infile
= RemoveWindowsPath(infile
);
4571 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4572 if(checkcompressfile
=="tarfile"):
4573 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4574 if(checkcompressfile
=="zipfile"):
4575 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4576 if(rarfile_support
and checkcompressfile
=="rarfile"):
4577 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4578 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4579 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4580 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4582 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4583 if(not compresscheck
):
4584 fextname
= os
.path
.splitext(infile
)[1];
4585 if(fextname
==".gz"):
4586 compresscheck
= "gzip";
4587 elif(fextname
==".bz2"):
4588 compresscheck
= "bzip2";
4589 elif(fextname
==".zst"):
4590 compresscheck
= "zstd";
4591 elif(fextname
==".lz4" or fextname
==".clz4"):
4592 compresscheck
= "lz4";
4593 elif(fextname
==".lzo" or fextname
==".lzop"):
4594 compresscheck
= "lzo";
4595 elif(fextname
==".lzma" or fextname
==".xz"):
4596 compresscheck
= "lzma";
4599 if(not compresscheck
):
4601 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4606 SeekToEndOfFile(catfp);
4608 SeekToEndOfFile(catfp);
4609 CatSize = catfp.tell();
4610 CatSizeEnd = CatSize;
4618 curloc
= catfp
.tell();
4621 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4623 catfp
.seek(curloc
, 0);
4624 catstring
= catheader
[0];
4625 catversion
= re
.findall(r
"([\d]+)", catstring
);
4626 fprenumfiles
= catheader
[1];
4627 fnumfiles
= int(fprenumfiles
, 16);
4628 fprechecksumtype
= catheader
[2];
4629 fprechecksum
= catheader
[3];
4631 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4632 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4633 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4634 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4635 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4636 valid_archive
= True;
4637 invalid_archive
= False;
4639 VerbosePrintOut(infile
);
4640 VerbosePrintOut("Number of Records " + str(fnumfiles
));
4641 if(fprechecksum
==catfileheadercshex
):
4643 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4644 VerbosePrintOut("'" + str(fprechecksum
) + "' == " + "'" + str(catfileheadercshex
) + "'");
4647 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4648 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4649 valid_archive
= False;
4650 invalid_archive
= True;
4652 VerbosePrintOut("");
4653 while(il
<fnumfiles
):
4654 catfhstart
= catfp
.tell();
4655 if(formatspecs
['new_style']):
4656 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4658 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4659 if(len(catheaderdata
)==0):
4661 catfheadsize
= int(catheaderdata
[0], 16);
4662 catfnumfields
= int(catheaderdata
[1], 16);
4663 catftype
= int(catheaderdata
[2], 16);
4664 if(re
.findall(r
"^[.|/]", catheaderdata
[3])):
4665 catfname
= catheaderdata
[3];
4667 catfname
= "./"+catheaderdata
[3];
4668 catfbasedir
= os
.path
.dirname(catfname
);
4669 catflinkname
= catheaderdata
[4];
4670 catfsize
= int(catheaderdata
[5], 16);
4671 catfatime
= int(catheaderdata
[6], 16);
4672 catfmtime
= int(catheaderdata
[7], 16);
4673 catfctime
= int(catheaderdata
[8], 16);
4674 catfbtime
= int(catheaderdata
[9], 16);
4675 catfmode
= int(catheaderdata
[10], 16);
4676 catfchmode
= stat
.S_IMODE(catfmode
);
4677 catftypemod
= stat
.S_IFMT(catfmode
);
4678 catfwinattributes
= int(catheaderdata
[11], 16);
4679 catfcompression
= catheaderdata
[12];
4680 catfcsize
= int(catheaderdata
[13], 16);
4681 catfuid
= int(catheaderdata
[14], 16);
4682 catfuname
= catheaderdata
[15];
4683 catfgid
= int(catheaderdata
[16], 16);
4684 catfgname
= catheaderdata
[17];
4685 fid
= int(catheaderdata
[18], 16);
4686 finode
= int(catheaderdata
[19], 16);
4687 flinkcount
= int(catheaderdata
[20], 16);
4688 catfdev_minor
= int(catheaderdata
[21], 16);
4689 catfdev_major
= int(catheaderdata
[22], 16);
4690 catfrdev_minor
= int(catheaderdata
[23], 16);
4691 catfrdev_major
= int(catheaderdata
[24], 16);
4692 catfseeknextfile
= catheaderdata
[25];
4693 catfextrasize
= int(catheaderdata
[26], 16);
4694 catfextrafields
= int(catheaderdata
[27], 16);
4695 extrafieldslist
= [];
4697 extraend
= extrastart
+ catfextrafields
;
4698 extrafieldslist
= [];
4699 if(extrastart
<extraend
):
4700 extrafieldslist
.append(catheaderdata
[extrastart
]);
4701 extrastart
= extrastart
+ 1;
4702 catfcs
= catheaderdata
[-2].lower();
4703 catfccs
= catheaderdata
[-1].lower();
4704 catnewfcs
= GetHeaderChecksum(catheaderdata
[:-2], catheaderdata
[-4].lower(), True, formatspecs
);
4706 VerbosePrintOut(catfname
);
4707 VerbosePrintOut("Record Number " + str(il
) + "; File ID " + str(fid
) + "; iNode Number " + str(finode
));
4708 if(catfcs
==catnewfcs
):
4710 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart
));
4711 VerbosePrintOut("'" + str(catfcs
) + "' == " + "'" + str(catnewfcs
) + "'");
4714 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart
));
4715 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
4716 valid_archive
= False;
4717 invalid_archive
= True;
4718 catfhend
= catfp
.tell() - 1;
4719 catfcontentstart
= catfp
.tell();
4721 pyhascontents
= False;
4723 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4724 catfcontents
= catfp
.read(catfsize
);
4726 catfcontents
= catfp
.read(catfcsize
);
4727 catnewfccs
= GetFileChecksum(catfcontents
, catheaderdata
[-3].lower(), False, formatspecs
);
4728 pyhascontents
= True;
4729 if(catfccs
==catnewfccs
):
4731 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart
));
4732 VerbosePrintOut("'" + str(catfccs
) + "' == " + "'" + str(catnewfccs
) + "'");
4735 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart
));
4736 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
4737 valid_archive
= False;
4738 invalid_archive
= True;
4740 VerbosePrintOut("");
4741 if(re
.findall(r
"^\+([0-9]+)", catfseeknextfile
)):
4742 fseeknextasnum
= int(catfseeknextfile
.replace("+", ""));
4743 if(abs(fseeknextasnum
)==0):
4745 catfp
.seek(fseeknextasnum
, 1);
4746 elif(re
.findall(r
"^\-([0-9]+)", catfseeknextfile
)):
4747 fseeknextasnum
= int(catfseeknextfile
);
4748 if(abs(fseeknextasnum
)==0):
4750 catfp
.seek(fseeknextasnum
, 1);
4751 elif(re
.findall(r
"^([0-9]+)", catfseeknextfile
)):
4752 fseeknextasnum
= int(catfseeknextfile
);
4753 if(abs(fseeknextasnum
)==0):
4755 catfp
.seek(fseeknextasnum
, 0);
4769 create_alias_function("", __file_format_name__
, "Validate", ArchiveFileValidate
);
4771 def ArchiveFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4772 formatspecs
= FormatSpecsListToDict(formatspecs
);
4773 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4776 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4777 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4778 if(checkcompressfile
=="tarfile"):
4779 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4780 if(checkcompressfile
=="zipfile"):
4781 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4782 if(rarfile_support
and checkcompressfile
=="rarfile"):
4783 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4784 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4785 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4786 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4793 if(hasattr(sys
.stdin
, "buffer")):
4794 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4796 shutil
.copyfileobj(sys
.stdin
, catfp
);
4798 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4802 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4803 catfp
= download_file_from_internet_file(infile
);
4804 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4810 infile
= RemoveWindowsPath(infile
);
4811 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4812 if(checkcompressfile
=="tarfile"):
4813 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4814 if(checkcompressfile
=="zipfile"):
4815 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4816 if(rarfile_support
and checkcompressfile
=="rarfile"):
4817 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4818 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4819 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4820 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4822 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4823 if(not compresscheck
):
4824 fextname
= os
.path
.splitext(infile
)[1];
4825 if(fextname
==".gz"):
4826 compresscheck
= "gzip";
4827 elif(fextname
==".bz2"):
4828 compresscheck
= "bzip2";
4829 elif(fextname
==".zst"):
4830 compresscheck
= "zstd";
4831 elif(fextname
==".lz4" or fextname
==".clz4"):
4832 compresscheck
= "lz4";
4833 elif(fextname
==".lzo" or fextname
==".lzop"):
4834 compresscheck
= "lzo";
4835 elif(fextname
==".lzma" or fextname
==".xz"):
4836 compresscheck
= "lzma";
4839 if(not compresscheck
):
4841 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4846 SeekToEndOfFile(catfp);
4848 SeekToEndOfFile(catfp);
4849 CatSize = catfp.tell();
4850 CatSizeEnd = CatSize;
4858 curloc
= catfp
.tell();
4861 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4863 catfp
.seek(curloc
, 0);
4864 catstring
= catheader
[0];
4865 catversion
= re
.findall(r
"([\d]+)", catstring
);
4866 fprenumfiles
= catheader
[1];
4867 fnumfiles
= int(fprenumfiles
, 16);
4868 fprechecksumtype
= catheader
[2];
4869 fprechecksum
= catheader
[3];
4870 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4871 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4872 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4873 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4874 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4875 fheadtell
= len(fileheader
);
4876 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4877 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4878 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4880 catversions
= re
.search(r
'(.*?)(\d+)', catstring
).groups();
4881 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': []};
4882 if(seekstart
<0 and seekstart
>fnumfiles
):
4884 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
4885 seekend
= fnumfiles
;
4886 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
4887 seekend
= fnumfiles
- abs(seekend
);
4890 while(il
< seekstart
):
4891 prefhstart
= catfp
.tell();
4892 if(formatspecs
['new_style']):
4893 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4895 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4896 if(len(preheaderdata
)==0):
4898 prefheadsize
= int(preheaderdata
[0], 16);
4899 prefnumfields
= int(preheaderdata
[1], 16);
4900 if(re
.findall(r
"^[.|/]", preheaderdata
[3])):
4901 prefname
= preheaderdata
[3];
4903 prefname
= "./"+preheaderdata
[3];
4904 prefsize
= int(preheaderdata
[5], 16);
4905 prefcompression
= preheaderdata
[12];
4906 prefcsize
= int(preheaderdata
[13], 16);
4907 prefseeknextfile
= preheaderdata
[25];
4908 prefextrasize
= int(preheaderdata
[26], 16);
4909 prefextrafields
= int(preheaderdata
[27], 16);
4910 extrafieldslist
= [];
4912 extraend
= extrastart
+ prefextrafields
;
4913 extrafieldslist
= [];
4914 if(extrastart
<extraend
):
4915 extrafieldslist
.append(preheaderdata
[extrastart
]);
4916 extrastart
= extrastart
+ 1;
4917 prefcs
= preheaderdata
[-2].lower();
4918 prenewfcs
= preheaderdata
[-1].lower();
4919 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-4].lower(), True, formatspecs
);
4920 if(prefcs
!=prenewfcs
and not skipchecksum
):
4921 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4922 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4924 valid_archive
= False;
4925 invalid_archive
= True;
4926 prefhend
= catfp
.tell() - 1;
4927 prefcontentstart
= catfp
.tell();
4929 pyhascontents
= False;
4931 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
4932 prefcontents
= catfp
.read(prefsize
);
4934 prefcontents
= catfp
.read(prefcsize
);
4935 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4936 pyhascontents
= True;
4937 if(prefccs
!=prenewfccs
and not skipchecksum
):
4938 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4939 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4941 if(re
.findall(r
"^\+([0-9]+)", prefseeknextfile
)):
4942 fseeknextasnum
= int(prefseeknextfile
.replace("+", ""));
4943 if(abs(fseeknextasnum
)==0):
4945 catfp
.seek(fseeknextasnum
, 1);
4946 elif(re
.findall(r
"^\-([0-9]+)", prefseeknextfile
)):
4947 fseeknextasnum
= int(prefseeknextfile
);
4948 if(abs(fseeknextasnum
)==0):
4950 catfp
.seek(fseeknextasnum
, 1);
4951 elif(re
.findall(r
"^([0-9]+)", prefseeknextfile
)):
4952 fseeknextasnum
= int(prefseeknextfile
);
4953 if(abs(fseeknextasnum
)==0):
4955 catfp
.seek(fseeknextasnum
, 0);
4959 fileidnum
= seekstart
;
4961 while(fileidnum
<seekend
):
4962 catfhstart
= catfp
.tell();
4963 if(formatspecs
['new_style']):
4964 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4966 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4967 if(len(catheaderdata
)==0):
4969 catfheadsize
= int(catheaderdata
[0], 16);
4970 catfnumfields
= int(catheaderdata
[1], 16);
4971 catftype
= int(catheaderdata
[2], 16);
4972 if(re
.findall(r
"^[.|/]", catheaderdata
[3])):
4973 catfname
= catheaderdata
[3];
4975 catfname
= "./"+catheaderdata
[3];
4976 catfbasedir
= os
.path
.dirname(catfname
);
4977 catflinkname
= catheaderdata
[4];
4978 catfsize
= int(catheaderdata
[5], 16);
4979 catfatime
= int(catheaderdata
[6], 16);
4980 catfmtime
= int(catheaderdata
[7], 16);
4981 catfctime
= int(catheaderdata
[8], 16);
4982 catfbtime
= int(catheaderdata
[9], 16);
4983 catfmode
= int(catheaderdata
[10], 16);
4984 catfchmode
= stat
.S_IMODE(catfmode
);
4985 catftypemod
= stat
.S_IFMT(catfmode
);
4986 catfwinattributes
= int(catheaderdata
[11], 16);
4987 catfcompression
= catheaderdata
[12];
4988 catfcsize
= int(catheaderdata
[13], 16);
4989 catfuid
= int(catheaderdata
[14], 16);
4990 catfuname
= catheaderdata
[15];
4991 catfgid
= int(catheaderdata
[16], 16);
4992 catfgname
= catheaderdata
[17];
4993 catfid
= int(catheaderdata
[18], 16);
4994 catfinode
= int(catheaderdata
[19], 16);
4995 catflinkcount
= int(catheaderdata
[20], 16);
4996 catfdev_minor
= int(catheaderdata
[21], 16);
4997 catfdev_major
= int(catheaderdata
[22], 16);
4998 catfrdev_minor
= int(catheaderdata
[23], 16);
4999 catfrdev_major
= int(catheaderdata
[24], 16);
5000 catfseeknextfile
= catheaderdata
[25];
5001 catfextrasize
= int(catheaderdata
[26], 16);
5002 catfextrafields
= int(catheaderdata
[27], 16);
5003 extrafieldslist
= [];
5005 extraend
= extrastart
+ catfextrafields
;
5006 extrafieldslist
= [];
5007 if(extrastart
<extraend
):
5008 extrafieldslist
.append(catheaderdata
[extrastart
]);
5009 extrastart
= extrastart
+ 1;
5010 catfcs
= catheaderdata
[-2].lower();
5011 catfccs
= catheaderdata
[-1].lower();
5012 catnewfcs
= GetHeaderChecksum(catheaderdata
[:-2], catheaderdata
[-4].lower(), True, formatspecs
);
5013 if(catfcs
!=catnewfcs
and not skipchecksum
):
5014 VerbosePrintOut("File Header Checksum Error with file " + catfname
+ " at offset " + str(catfhstart
));
5015 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
5017 catfhend
= catfp
.tell() - 1;
5018 catfcontentstart
= catfp
.tell();
5019 catfcontents
= BytesIO();
5020 pyhascontents
= False;
5021 if(catfsize
>0 and not listonly
):
5022 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
5023 catfcontents
.write(catfp
.read(catfsize
));
5025 catfcontents
.write(catfp
.read(catfcsize
));
5026 catfcontents
.seek(0, 0);
5027 catnewfccs
= GetFileChecksum(catfcontents
.read(), catheaderdata
[-3].lower(), False, formatspecs
);
5028 pyhascontents
= True;
5029 if(catfccs
!=catnewfccs
and skipchecksum
):
5030 VerbosePrintOut("File Content Checksum Error with file " + catfname
+ " at offset " + str(catfcontentstart
));
5031 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
5033 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
5036 catfcontents
.seek(0, 0);
5038 catfcontents
= UncompressArchiveFile(catfcontents
, formatspecs
);
5039 catfcontents
.seek(0, 0);
5040 catfccs
= GetFileChecksum(catfcontents
.read(), catheaderdata
[-3].lower(), False, formatspecs
);
5041 if(catfsize
>0 and listonly
):
5042 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
5043 catfp
.seek(catfsize
, 1);
5045 catfp
.seek(catfcsize
, 1);
5046 pyhascontents
= False;
5047 catfcontentend
= catfp
.tell();
5048 if(re
.findall(r
"^\+([0-9]+)", catfseeknextfile
)):
5049 fseeknextasnum
= int(catfseeknextfile
.replace("+", ""));
5050 if(abs(fseeknextasnum
)==0):
5052 catfp
.seek(fseeknextasnum
, 1);
5053 elif(re
.findall(r
"^\-([0-9]+)", catfseeknextfile
)):
5054 fseeknextasnum
= int(catfseeknextfile
);
5055 if(abs(fseeknextasnum
)==0):
5057 catfp
.seek(fseeknextasnum
, 1);
5058 elif(re
.findall(r
"^([0-9]+)", catfseeknextfile
)):
5059 fseeknextasnum
= int(catfseeknextfile
);
5060 if(abs(fseeknextasnum
)==0):
5062 catfp
.seek(fseeknextasnum
, 0);
5065 catfcontents
.seek(0, 0);
5066 catlist
['ffilelist'].append({'fid': realidnum
, 'fidalt': fileidnum
, 'fheadersize': catfheadsize
, 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
, 'fatime': catfatime
, 'fmtime': catfmtime
, 'fctime': catfctime
, 'fbtime': catfbtime
, 'fmode': catfmode
, 'fchmode': catfchmode
, 'ftypemod': catftypemod
, 'fwinattributes': catfwinattributes
, 'fcompression': catfcompression
, 'fcsize': catfcsize
, 'fuid': catfuid
, 'funame': catfuname
, 'fgid': catfgid
, 'fgname': catfgname
, 'finode': catfinode
, 'flinkcount': catflinkcount
, 'fminor': catfdev_minor
, 'fmajor': catfdev_major
, 'frminor': catfrdev_minor
, 'frmajor': catfrdev_major
, 'fseeknextfile': catfseeknextfile
, 'fheaderchecksumtype': catheaderdata
[-4], 'fcontentchecksumtype': catheaderdata
[-3], 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': catfextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': catfcs
, 'fcontentchecksum': catfccs
, 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': catfcontents
});
5067 fileidnum
= fileidnum
+ 1;
5068 realidnum
= realidnum
+ 1;
5070 catlist
.update({'catfp': catfp
});
5075 create_alias_function("", __file_format_name__
, "ToArray", ArchiveFileToArray
);
5077 def ArchiveFileStringToArray(catstr
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5078 formatspecs
= FormatSpecsListToDict(formatspecs
);
5079 catfp
= BytesIO(catstr
);
5080 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5081 return listcatfiles
;
5083 create_alias_function("", __file_format_name__
, "StringToArray", ArchiveFileStringToArray
);
5085 def TarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5086 formatspecs
= FormatSpecsListToDict(formatspecs
);
5088 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5089 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5090 return listcatfiles
;
5092 def ZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5093 formatspecs
= FormatSpecsListToDict(formatspecs
);
5095 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5096 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5097 return listcatfiles
;
5099 if(not rarfile_support
):
5100 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5103 if(rarfile_support
):
5104 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5105 formatspecs
= FormatSpecsListToDict(formatspecs
);
5107 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5108 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5109 return listcatfiles
;
5111 if(not py7zr_support
):
5112 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5116 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5117 formatspecs
= FormatSpecsListToDict(formatspecs
);
5119 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", True, None, "crc32", [], formatspecs
, False, True);
5120 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5121 return listcatfiles
;
5123 def InFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5124 formatspecs
= FormatSpecsListToDict(formatspecs
);
5125 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
5126 if(checkcompressfile
=="tarfile"):
5127 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5128 elif(checkcompressfile
=="zipfile"):
5129 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5130 elif(checkcompressfile
=="catfile"):
5131 return ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
5132 elif(rarfile_support
and checkcompressfile
=="rarfile"):
5133 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5134 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
5135 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5140 def ListDirToArrayAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5141 formatspecs
= FormatSpecsListToDict(formatspecs
);
5142 catver
= formatspecs
['format_ver'];
5143 fileheaderver
= str(int(catver
.replace(".", "")));
5144 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5145 advancedlist
= formatspecs
['use_advanced_list'];
5146 altinode
= formatspecs
['use_alt_inode'];
5149 for line
in sys
.stdin
:
5150 infilelist
.append(line
.strip());
5151 infilelist
= list(filter(None, infilelist
));
5152 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
5153 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
5155 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
5156 for line
in finfile
:
5157 infilelist
.append(line
.strip());
5158 infilelist
= list(filter(None, infilelist
));
5160 if(isinstance(infiles
, (list, tuple, ))):
5161 infilelist
= list(filter(None, infiles
));
5162 elif(isinstance(infiles
, (str, ))):
5163 infilelist
= list(filter(None, [infiles
]));
5165 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
5167 GetDirList
= ListDir(infilelist
, followlink
, False);
5175 inodetocatinode
= {};
5177 fnumfiles
= int(len(GetDirList
));
5178 catver
= formatspecs
['format_ver'];
5179 fileheaderver
= str(int(catver
.replace(".", "")));
5180 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5181 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5182 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5183 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5184 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5185 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5186 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5187 fheadtell
= len(fileheader
);
5188 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5189 for curfname
in GetDirList
:
5190 catfhstart
= fheadtell
;
5191 if(re
.findall(r
"^[.|/]", curfname
)):
5194 fname
= "./"+curfname
;
5196 VerbosePrintOut(fname
);
5197 if(not followlink
or followlink
is None):
5198 fstatinfo
= os
.lstat(fname
);
5200 fstatinfo
= os
.stat(fname
);
5201 fpremode
= fstatinfo
.st_mode
;
5202 finode
= fstatinfo
.st_ino
;
5203 flinkcount
= fstatinfo
.st_nlink
;
5205 if(stat
.S_ISREG(fpremode
)):
5207 elif(stat
.S_ISLNK(fpremode
)):
5209 elif(stat
.S_ISCHR(fpremode
)):
5211 elif(stat
.S_ISBLK(fpremode
)):
5213 elif(stat
.S_ISDIR(fpremode
)):
5215 elif(stat
.S_ISFIFO(fpremode
)):
5217 elif(stat
.S_ISSOCK(fpremode
)):
5219 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
5221 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
5223 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
5228 fbasedir
= os
.path
.dirname(fname
);
5230 if(not followlink
and finode
!=0):
5232 if(finode
in inodelist
):
5234 flinkname
= inodetofile
[finode
];
5238 fcurinode
= inodetocatinode
[finode
];
5239 if(finode
not in inodelist
):
5240 inodelist
.append(finode
);
5241 inodetofile
.update({finode
: fname
});
5242 inodetocatinode
.update({finode
: curinode
});
5246 fcurinode
= curinode
;
5247 curinode
= curinode
+ 1;
5249 fcurinode
= curinode
;
5250 curinode
= curinode
+ 1;
5251 curfid
= curfid
+ 1;
5253 flinkname
= os
.readlink(fname
);
5254 fdev
= fstatinfo
.st_dev
;
5255 getfdev
= GetDevMajorMinor(fdev
);
5256 fdev_minor
= getfdev
[0];
5257 fdev_major
= getfdev
[1];
5258 frdev
= fstatinfo
.st_dev
;
5259 if(hasattr(fstatinfo
, "st_rdev")):
5260 frdev
= fstatinfo
.st_rdev
;
5262 frdev
= fstatinfo
.st_dev
;
5263 getfrdev
= GetDevMajorMinor(frdev
);
5264 frdev_minor
= getfrdev
[0];
5265 frdev_major
= getfrdev
[1];
5266 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
5268 if(ftype
==0 or ftype
==7):
5269 fsize
= fstatinfo
.st_size
;
5270 fatime
= fstatinfo
.st_atime
;
5271 fmtime
= fstatinfo
.st_mtime
;
5272 fctime
= fstatinfo
.st_ctime
;
5273 if(hasattr(fstatinfo
, "st_birthtime")):
5274 fbtime
= fstatinfo
.st_birthtime
;
5276 fbtime
= fstatinfo
.st_ctime
;
5277 fmode
= fstatinfo
.st_mode
;
5278 fchmode
= stat
.S_IMODE(fstatinfo
.st_mode
);
5279 ftypemod
= stat
.S_IFMT(fstatinfo
.st_mode
);
5280 fuid
= fstatinfo
.st_uid
;
5281 fgid
= fstatinfo
.st_gid
;
5286 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
5287 funame
= userinfo
.pw_name
;
5296 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
5297 fgname
= groupinfo
.gr_name
;
5302 fdev_minor
= fdev_minor
;
5303 fdev_major
= fdev_major
;
5304 frdev_minor
= frdev_minor
;
5305 frdev_major
= frdev_major
;
5306 flinkcount
= flinkcount
;
5307 if(hasattr(fstatinfo
, "st_file_attributes")):
5308 fwinattributes
= fstatinfo
.st_file_attributes
;
5313 fcontents
= BytesIO();
5314 if(ftype
==0 or ftype
==7):
5315 with
open(fname
, "rb") as fpc
:
5316 shutil
.copyfileobj(fpc
, fcontents
);
5317 if(followlink
and (ftype
==1 or ftype
==2)):
5318 flstatinfo
= os
.stat(flinkname
);
5319 with
open(flinkname
, "rb") as fpc
:
5320 shutil
.copyfileobj(fpc
, fcontents
);
5321 fcontents
.seek(0, 0);
5322 ftypehex
= format(ftype
, 'x').lower();
5323 extrafields
= len(extradata
);
5324 extrafieldslist
= extradata
;
5325 catfextrafields
= extrafields
;
5326 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5327 if(len(extradata
)>0):
5328 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5329 extrasizelen
= len(extrasizestr
);
5330 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5331 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5332 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5333 catoutlenhex
= format(catoutlen
, 'x').lower();
5334 catoutlist
.insert(0, catoutlenhex
);
5335 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5336 catheaderdata
= catoutlist
;
5337 if(len(extradata
)>0):
5338 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5339 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
5340 catfnumfields
= catoutlen
;
5341 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5342 fcontents
.seek(0, 0);
5343 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5344 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5345 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5346 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5347 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5348 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5349 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5350 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5351 catfcontentstart
= fheadtell
;
5352 fheadtell
+= len(catfileoutstr
) + 1;
5353 catfcontentend
= fheadtell
- 1;
5354 catfhend
= catfcontentend
;
5355 fcontents
.seek(0, 0);
5356 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5357 pyhascontents
= False;
5358 if(int(fsize
)>0 and not listonly
):
5359 pyhascontents
= True;
5360 if(int(fsize
)>0 and listonly
):
5361 fcontents
= BytesIO();
5362 pyhascontents
= False;
5363 fcontents
.seek(0, 0);
5364 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
5365 fileidnum
= fileidnum
+ 1;
5368 def TarFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5369 formatspecs
= FormatSpecsListToDict(formatspecs
);
5375 inodetocatinode
= {};
5379 if(hasattr(sys
.stdin
, "buffer")):
5380 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5382 shutil
.copyfileobj(sys
.stdin
, infile
);
5387 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5388 infile
= download_file_from_internet_file(infile
);
5393 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5395 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
5397 if(not tarfile
.TarFileCheck(infile
)):
5399 except AttributeError:
5400 if(not TarFileCheck(infile
)):
5405 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
5406 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
5408 tarfp
= tarfile
.open(infile
, "r");
5409 except FileNotFoundError
:
5411 fnumfiles
= int(len(tarfp
.getmembers()));
5412 catver
= formatspecs
['format_ver'];
5413 fileheaderver
= str(int(catver
.replace(".", "")));
5414 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5415 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5416 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5417 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5418 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5419 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5420 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5421 fheadtell
= len(fileheader
);
5422 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5423 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
5424 catfhstart
= fheadtell
;
5425 if(re
.findall(r
"^[.|/]", member
.name
)):
5426 fname
= member
.name
;
5428 fname
= "./"+member
.name
;
5430 VerbosePrintOut(fname
);
5431 fpremode
= member
.mode
;
5432 ffullmode
= member
.mode
;
5436 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5438 elif(member
.isdev()):
5439 ffullmode
= member
.mode
;
5441 elif(member
.islnk()):
5442 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5444 elif(member
.issym()):
5445 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
5447 elif(member
.ischr()):
5448 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
5450 elif(member
.isblk()):
5451 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
5453 elif(member
.isdir()):
5454 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
5456 elif(member
.isfifo()):
5457 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
5459 elif(member
.issparse()):
5460 ffullmode
= member
.mode
;
5463 ffullmode
= member
.mode
;
5466 fbasedir
= os
.path
.dirname(fname
);
5470 curfid
= curfid
+ 1;
5472 flinkname
= member
.linkname
;
5473 fdev_minor
= member
.devminor
;
5474 fdev_major
= member
.devmajor
;
5475 frdev_minor
= member
.devminor
;
5476 frdev_major
= member
.devmajor
;
5477 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
5479 elif(ftype
==0 or ftype
==7):
5480 fsize
= member
.size
;
5482 fsize
= member
.size
;
5483 fatime
= member
.mtime
;
5484 fmtime
= member
.mtime
;
5485 fctime
= member
.mtime
;
5486 fbtime
= member
.mtime
;
5488 fchmode
= stat
.S_IMODE(ffullmode
);
5489 ftypemod
= stat
.S_IFMT(ffullmode
);
5492 funame
= member
.uname
;
5493 fgname
= member
.gname
;
5494 flinkcount
= flinkcount
;
5495 fwinattributes
= int(0);
5498 fcontents
= BytesIO();
5499 if(ftype
==0 or ftype
==7):
5500 with tarfp
.extractfile(member
) as fpc
:
5501 shutil
.copyfileobj(fpc
, fcontents
);
5502 fcontents
.seek(0, 0);
5503 ftypehex
= format(ftype
, 'x').lower();
5504 extrafields
= len(extradata
);
5505 extrafieldslist
= extradata
;
5506 catfextrafields
= extrafields
;
5507 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5508 if(len(extradata
)>0):
5509 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5510 extrasizelen
= len(extrasizestr
);
5511 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5512 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5513 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5514 catoutlenhex
= format(catoutlen
, 'x').lower();
5515 catoutlist
.insert(0, catoutlenhex
);
5516 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5517 catheaderdata
= catoutlist
;
5518 if(len(extradata
)>0):
5519 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5520 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
5521 catfnumfields
= catoutlen
;
5522 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5523 fcontents
.seek(0, 0);
5524 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5525 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5526 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5527 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5528 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5529 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5530 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5531 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5532 catfcontentstart
= fheadtell
;
5533 fheadtell
+= len(catfileoutstr
) + 1;
5534 catfcontentend
= fheadtell
- 1;
5535 catfhend
= catfcontentend
;
5536 fcontents
.seek(0, 0);
5537 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5538 pyhascontents
= False;
5539 if(int(fsize
)>0 and not listonly
):
5540 pyhascontents
= True;
5541 if(int(fsize
)>0 and listonly
):
5542 fcontents
= BytesIO();
5543 pyhascontents
= False;
5544 fcontents
.seek(0, 0);
5545 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
5546 fileidnum
= fileidnum
+ 1;
5549 def ZipFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5550 formatspecs
= FormatSpecsListToDict(formatspecs
);
5556 inodetocatinode
= {};
5560 if(hasattr(sys
.stdin
, "buffer")):
5561 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5563 shutil
.copyfileobj(sys
.stdin
, infile
);
5568 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5569 infile
= download_file_from_internet_file(infile
);
5574 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5578 if(not zipfile
.is_zipfile(infile
)):
5581 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
5582 except FileNotFoundError
:
5584 ziptest
= zipfp
.testzip();
5586 VerbosePrintOut("Bad file found!");
5587 fnumfiles
= int(len(zipfp
.infolist()));
5588 catver
= formatspecs
['format_ver'];
5589 fileheaderver
= str(int(catver
.replace(".", "")));
5590 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5591 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5592 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5593 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5594 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5595 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5596 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5597 fheadtell
= len(fileheader
);
5598 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5599 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
5600 catfhstart
= fheadtell
;
5601 if(re
.findall(r
"^[.|/]", member
.filename
)):
5602 fname
= member
.filename
;
5604 fname
= "./"+member
.filename
;
5605 zipinfo
= zipfp
.getinfo(member
.filename
);
5607 VerbosePrintOut(fname
);
5608 if(not member
.is_dir()):
5609 fpremode
= stat
.S_IFREG
+ 438;
5610 elif(member
.is_dir()):
5611 fpremode
= stat
.S_IFDIR
+ 511;
5614 if(not member
.is_dir()):
5616 elif(member
.is_dir()):
5619 fbasedir
= os
.path
.dirname(fname
);
5623 curfid
= curfid
+ 1;
5631 fsize
= member
.file_size
;
5633 fsize
= member
.file_size
;
5634 fatime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5635 fmtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5636 fctime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5637 fbtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5638 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
5639 fwinattributes
= int(zipinfo
.external_attr
);
5640 if(not member
.is_dir()):
5641 fmode
= int(stat
.S_IFREG
+ 438);
5642 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5643 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5644 elif(member
.is_dir()):
5645 fmode
= int(stat
.S_IFDIR
+ 511);
5646 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5647 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5648 elif(zipinfo
.create_system
==3):
5649 fwinattributes
= int(0);
5650 fmode
= int(zipinfo
.external_attr
);
5652 fwinattributes
= int(0);
5653 if(not member
.is_dir()):
5654 fmode
= int(stat
.S_IFREG
+ 438);
5655 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5656 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5657 elif(member
.is_dir()):
5658 fmode
= int(stat
.S_IFDIR
+ 511);
5659 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5660 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5665 except AttributeError:
5671 except AttributeError:
5678 userinfo
= pwd
.getpwuid(os
.getuid());
5679 funame
= userinfo
.pw_name
;
5682 except AttributeError:
5690 groupinfo
= grp
.getgrgid(os
.getgid());
5691 fgname
= groupinfo
.gr_name
;
5694 except AttributeError:
5698 fcontents
= BytesIO();
5700 fcontents
.write(zipfp
.read(member
.filename
));
5701 fcontents
.seek(0, 0);
5702 ftypehex
= format(ftype
, 'x').lower();
5703 extrafields
= len(extradata
);
5704 extrafieldslist
= extradata
;
5705 catfextrafields
= extrafields
;
5706 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5707 if(len(extradata
)>0):
5708 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5709 extrasizelen
= len(extrasizestr
);
5710 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5711 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5712 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5713 catoutlenhex
= format(catoutlen
, 'x').lower();
5714 catoutlist
.insert(0, catoutlenhex
);
5715 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5716 catheaderdata
= catoutlist
;
5717 if(len(extradata
)>0):
5718 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5719 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
5720 catfnumfields
= catoutlen
;
5721 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5722 fcontents
.seek(0, 0);
5723 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5724 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5725 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5726 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5727 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5728 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5729 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5730 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5731 catfcontentstart
= fheadtell
;
5732 fheadtell
+= len(catfileoutstr
) + 1;
5733 catfcontentend
= fheadtell
- 1;
5734 catfhend
= catfcontentend
;
5735 fcontents
.seek(0, 0);
5736 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5737 pyhascontents
= False;
5738 if(int(fsize
)>0 and not listonly
):
5739 pyhascontents
= True;
5740 if(int(fsize
)>0 and listonly
):
5741 fcontents
= BytesIO();
5742 pyhascontents
= False;
5743 fcontents
.seek(0, 0);
5744 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
5745 fileidnum
= fileidnum
+ 1;
5748 if(not rarfile_support
):
5749 def RarFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5752 if(rarfile_support
):
5753 def RarFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5754 formatspecs
= FormatSpecsListToDict(formatspecs
);
5760 inodetocatinode
= {};
5762 if(not os
.path
.exists(infile
,) or not os
.path
.isfile(infile
,)):
5764 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
5766 rarfp
= rarfile
.RarFile(infile
, "r");
5767 rartest
= rarfp
.testrar();
5769 VerbosePrintOut("Bad file found!");
5770 fnumfiles
= int(len(rarfp
.infolist()));
5771 catver
= formatspecs
['format_ver'];
5772 fileheaderver
= str(int(catver
.replace(".", "")));
5773 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5774 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5775 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5776 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5777 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5778 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5779 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5780 fheadtell
= len(fileheader
);
5781 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5782 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
5783 catfhstart
= fheadtell
;
5786 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
5789 member
.external_attr
5791 except AttributeError:
5793 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
5796 member
.external_attr
5798 except AttributeError:
5803 if(re
.findall(r
"^[.|/]", member
.filename
)):
5804 fname
= member
.filename
;
5806 fname
= "./"+member
.filename
;
5807 rarinfo
= rarfp
.getinfo(member
.filename
);
5809 VerbosePrintOut(fname
);
5810 if(is_unix
and member
.external_attr
!=0):
5811 fpremode
= int(member
.external_attr
);
5812 elif(member
.is_file()):
5813 fpremode
= stat
.S_IFREG
+ 438;
5814 elif(member
.is_symlink()):
5815 fpremode
= stat
.S_IFLNK
+ 438;
5816 elif(member
.is_dir()):
5817 fpremode
= stat
.S_IFDIR
+ 511;
5818 if(is_windows
and member
.external_attr
!=0):
5819 fwinattributes
= int(member
.external_attr
);
5821 fwinattributes
= int(0);
5826 if(member
.is_file()):
5828 elif(member
.is_symlink()):
5830 elif(member
.is_dir()):
5834 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
5835 fbasedir
= os
.path
.dirname(fname
);
5839 curfid
= curfid
+ 1;
5847 fsize
= member
.file_size
;
5850 fatime
= int(member
.atime
.timestamp());
5852 fatime
= int(member
.mtime
.timestamp());
5853 except AttributeError:
5854 fatime
= int(member
.mtime
.timestamp());
5855 fmtime
= int(member
.mtime
.timestamp());
5858 fctime
= int(member
.ctime
.timestamp());
5860 fctime
= int(member
.mtime
.timestamp());
5861 except AttributeError:
5862 fctime
= int(member
.mtime
.timestamp());
5863 fbtime
= int(member
.mtime
.timestamp());
5864 if(is_unix
and member
.external_attr
!=0):
5865 fmode
= int(member
.external_attr
);
5866 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
5867 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
5868 elif(member
.is_file()):
5869 fmode
= int(stat
.S_IFREG
+ 438)
5870 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5871 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5872 elif(member
.is_symlink()):
5873 fmode
= int(stat
.S_IFLNK
+ 438)
5874 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5875 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5876 elif(member
.is_dir()):
5877 fmode
= int(stat
.S_IFDIR
+ 511)
5878 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5879 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5882 except AttributeError:
5888 except AttributeError:
5895 userinfo
= pwd
.getpwuid(os
.getuid());
5896 funame
= userinfo
.pw_name
;
5899 except AttributeError:
5907 groupinfo
= grp
.getgrgid(os
.getgid());
5908 fgname
= groupinfo
.gr_name
;
5911 except AttributeError:
5915 fcontents
= BytesIO();
5917 fcontents
.write(rarfp
.read(member
.filename
));
5918 fcontents
.seek(0, 0);
5919 ftypehex
= format(ftype
, 'x').lower();
5920 extrafields
= len(extradata
);
5921 extrafieldslist
= extradata
;
5922 catfextrafields
= extrafields
;
5923 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5924 if(len(extradata
)>0):
5925 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5926 extrasizelen
= len(extrasizestr
);
5927 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5928 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5929 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5930 catoutlenhex
= format(catoutlen
, 'x').lower();
5931 catoutlist
.insert(0, catoutlenhex
);
5932 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5933 if(len(extradata
)>0):
5934 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5935 ccatfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
5936 catfnumfields
= 24 + catfextrafields
;
5937 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5938 fcontents
.seek(0, 0);
5939 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5940 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5941 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5942 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5943 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5944 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5945 catheaderdata
= catoutlist
;
5946 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5947 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5948 catfcontentstart
= fheadtell
;
5949 fheadtell
+= len(catfileoutstr
) + 1;
5950 catfcontentend
= fheadtell
- 1;
5951 catfhend
= catfcontentend
;
5952 fcontents
.seek(0, 0);
5953 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5954 pyhascontents
= False;
5955 if(int(fsize
)>0 and not listonly
):
5956 pyhascontents
= True;
5957 if(int(fsize
)>0 and listonly
):
5958 fcontents
= BytesIO();
5959 pyhascontents
= False;
5960 fcontents
.seek(0, 0);
5961 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
5962 fileidnum
= fileidnum
+ 1;
5965 if(not py7zr_support
):
5966 def SevenZipFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5970 def SevenZipFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5971 formatspecs
= FormatSpecsListToDict(formatspecs
);
5977 inodetocatinode
= {};
5979 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
5980 file_content
= szpfp
.readall();
5981 #sztest = szpfp.testzip();
5982 sztestalt
= szpfp
.test();
5984 VerbosePrintOut("Bad file found!");
5985 numfiles
= int(len(szpfp
.list()));
5986 catver
= formatspecs
['format_ver'];
5987 fileheaderver
= str(int(catver
.replace(".", "")));
5988 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5989 catversion
= re
.findall(r
"([\d]+)", fileheader
);
5990 catversions
= re
.search(r
'(.*?)(\d+)', fileheader
).groups();
5991 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5992 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5993 catfileheadercshex
= GetFileChecksum(fileheader
, checksumtype
, True, formatspecs
);
5994 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5995 fheadtell
= len(fileheader
);
5996 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': []};
5997 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
5998 catfhstart
= fheadtell
;
5999 if(re
.findall(r
"^[.|/]", member
.filename
)):
6000 fname
= member
.filename
;
6002 fname
= "./"+member
.filename
;
6003 if(not member
.is_directory
):
6004 fpremode
= int(stat
.S_IFREG
+ 438);
6005 elif(member
.is_directory
):
6006 fpremode
= int(stat
.S_IFDIR
+ 511);
6007 fwinattributes
= int(0);
6012 if(member
.is_directory
):
6017 fbasedir
= os
.path
.dirname(fname
);
6021 curfid
= curfid
+ 1;
6028 fatime
= int(member
.creationtime
.timestamp());
6029 fmtime
= int(member
.creationtime
.timestamp());
6030 fctime
= int(member
.creationtime
.timestamp());
6031 fbtime
= int(member
.creationtime
.timestamp());
6032 if(member
.is_directory
):
6033 fmode
= int(stat
.S_IFDIR
+ 511)
6034 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
6035 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
6037 fmode
= int(stat
.S_IFLNK
+ 438)
6038 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
6039 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
6042 except AttributeError:
6048 except AttributeError:
6055 userinfo
= pwd
.getpwuid(os
.getuid());
6056 funame
= userinfo
.pw_name
;
6059 except AttributeError:
6067 groupinfo
= grp
.getgrgid(os
.getgid());
6068 fgname
= groupinfo
.gr_name
;
6071 except AttributeError:
6075 fcontents
= BytesIO();
6077 fcontents
.write(file_content
[member
.filename
].read());
6078 fsize
= format(fcontents
.tell(), 'x').lower();
6080 fcontents
.seek(0, 0);
6081 ftypehex
= format(ftype
, 'x').lower();
6082 extrafields
= len(extradata
);
6083 extrafieldslist
= extradata
;
6084 catfextrafields
= extrafields
;
6085 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
6086 if(len(extradata
)>0):
6087 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
6088 extrasizelen
= len(extrasizestr
);
6089 extrasizelenhex
= format(extrasizelen
, 'x').lower();
6090 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), "+1", extrasizelenhex
, format(catfextrafields
, 'x').lower()];
6091 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
6092 catoutlenhex
= format(catoutlen
, 'x').lower();
6093 catoutlist
.insert(0, catoutlenhex
);
6094 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
6095 catheaderdata
= catoutlist
;
6096 if(len(extradata
)>0):
6097 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
6098 catfileoutstr
= catfileoutstr
+ AppendNullBytes([checksumtype
, checksumtype
], formatspecs
['format_delimiter']);
6099 catfnumfields
= 24 + catfextrafields
;
6100 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
6101 fcontents
.seek(0, 0);
6102 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
6103 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
6104 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
6105 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
6106 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
6107 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
6108 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
6109 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
6110 catfcontentstart
= fheadtell
;
6111 fheadtell
+= len(catfileoutstr
) + 1;
6112 catfcontentend
= fheadtell
- 1;
6113 catfhend
= catfcontentend
;
6114 fcontents
.seek(0, 0);
6115 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
6116 pyhascontents
= False;
6117 if(int(fsize
)>0 and not listonly
):
6118 pyhascontents
= True;
6119 if(int(fsize
)>0 and listonly
):
6120 fcontents
= BytesIO();
6121 pyhascontents
= False;
6122 fcontents
.seek(0, 0);
6123 catlist
['ffilelist'].append({'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype
, 'fcontentchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
});
6124 fileidnum
= fileidnum
+ 1;
6127 def InFileToArrayAlt(infile
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6128 formatspecs
= FormatSpecsListToDict(formatspecs
);
6129 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
6130 if(checkcompressfile
=="tarfile"):
6131 return TarFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6132 elif(checkcompressfile
=="zipfile"):
6133 return ZipFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6134 elif(checkcompressfile
=="catfile"):
6135 return ArchiveFileToArray(infile
, 0, 0, listonly
, True, False, formatspecs
, False);
6136 elif(rarfile_support
and checkcompressfile
=="rarfile"):
6137 return RarFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6138 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
6139 return SevenZipFileToArrayAlt(infile
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6144 def ListDirToArray(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6145 formatspecs
= FormatSpecsListToDict(formatspecs
);
6146 outarray
= BytesIO();
6147 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
6148 listcatfiles
= ArchiveFileToArray(outarray
, seekstart
, seekend
, listonly
, True, skipchecksum
, formatspecs
, returnfp
);
6149 return listcatfiles
;
6151 def ArchiveFileArrayToArrayIndex(inarray
, seekstart
=0, seekend
=0, listonly
=False, uncompress
=True, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6152 formatspecs
= FormatSpecsListToDict(formatspecs
);
6153 if(isinstance(inarray
, dict)):
6154 listcatfiles
= inarray
;
6157 if(not listcatfiles
):
6159 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6161 catarray
.update({'catfp': listcatfiles
['catfp']});
6162 lenlist
= len(listcatfiles
['ffilelist']);
6164 lcfx
= int(listcatfiles
['fnumfiles']);
6165 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6166 lcfx
= int(lenlist
);
6168 lcfx
= int(listcatfiles
['fnumfiles']);
6170 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6171 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6172 catarray
['filetoid'].update(filetoidarray
);
6173 catarray
['idtofile'].update(idtofilearray
);
6174 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6175 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6176 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6177 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6178 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6179 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6180 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6181 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6182 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6183 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6184 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6185 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6186 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6187 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6188 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6189 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6190 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6191 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6192 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6193 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6194 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6195 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6196 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6197 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6198 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6199 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6200 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6201 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6202 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6203 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6204 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6208 create_alias_function("", __file_format_name__
, "ArrayToArrayIndex", ArchiveFileArrayToArrayIndex
);
6210 def RePackArchiveFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6211 formatspecs
= FormatSpecsListToDict(formatspecs
);
6212 if(isinstance(infile
, dict)):
6213 listcatfiles
= infile
;
6215 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6216 infile
= RemoveWindowsPath(infile
);
6217 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6218 if(outfile
!="-" and not hasattr(infile
, "read") and not hasattr(outfile
, "write")):
6219 outfile
= RemoveWindowsPath(outfile
);
6220 checksumtype
= checksumtype
.lower();
6221 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
6222 checksumtype
="crc32";
6223 if(checksumtype
=="none"):
6225 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
6226 compression
= "auto";
6227 if(compression
not in compressionlist
and compression
is None):
6228 compression
= "auto";
6230 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6231 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
6232 if(os
.path
.exists(outfile
)):
6235 except OSError as e
:
6237 if(not listcatfiles
):
6242 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6244 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6247 fbasename
= os
.path
.splitext(outfile
)[0];
6248 fextname
= os
.path
.splitext(outfile
)[1];
6249 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
6250 catver
= formatspecs
['format_ver'];
6251 fileheaderver
= str(int(catver
.replace(".", "")));
6252 lenlist
= len(listcatfiles
['ffilelist']);
6253 fnumfiles
= int(listcatfiles
['fnumfiles']);
6254 if(lenlist
>fnumfiles
or lenlist
<fnumfiles
):
6255 fnumfiles
= lenlist
;
6256 AppendFileHeader(catfp
, fnumfiles
, checksumtype
, formatspecs
);
6257 lenlist
= len(listcatfiles
['ffilelist']);
6258 fnumfiles
= int(listcatfiles
['fnumfiles']);
6260 lcfx
= int(listcatfiles
['fnumfiles']);
6261 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6262 lcfx
= int(lenlist
);
6264 lcfx
= int(listcatfiles
['fnumfiles']);
6272 if(re
.findall(r
"^[.|/]", listcatfiles
['ffilelist'][reallcfi
]['fname'])):
6273 fname
= listcatfiles
['ffilelist'][reallcfi
]['fname'];
6275 fname
= "./"+listcatfiles
['ffilelist'][reallcfi
]['fname'];
6277 VerbosePrintOut(fname
);
6278 fheadersize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fheadersize']), 'x').lower();
6279 fsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fsize']), 'x').lower();
6280 flinkname
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6281 fatime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fatime']), 'x').lower();
6282 fmtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmtime']), 'x').lower();
6283 fctime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fctime']), 'x').lower();
6284 fbtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fbtime']), 'x').lower();
6285 fmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmode']), 'x').lower();
6286 fchmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fchmode']), 'x').lower();
6287 fuid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fuid']), 'x').lower();
6288 funame
= listcatfiles
['ffilelist'][reallcfi
]['funame'];
6289 fgid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fgid']), 'x').lower();
6290 fgname
= listcatfiles
['ffilelist'][reallcfi
]['fgname'];
6291 finode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['finode']), 'x').lower();
6292 flinkcount
= format(int(listcatfiles
['ffilelist'][reallcfi
]['flinkcount']), 'x').lower();
6293 fwinattributes
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fwinattributes']), 'x').lower();
6294 fcompression
= listcatfiles
['ffilelist'][reallcfi
]['fcompression'];
6295 fcsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fcsize']), 'x').lower();
6296 fdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fminor']), 'x').lower();
6297 fdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmajor']), 'x').lower();
6298 frdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frminor']), 'x').lower();
6299 frdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frmajor']), 'x').lower();
6300 fseeknextfile
= listcatfiles
['ffilelist'][reallcfi
]['fseeknextfile'];
6301 if(len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] and len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>0):
6302 listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] = len(listcatfiles
['ffilelist'][reallcfi
]['fextralist']);
6303 if(not followlink
and len(extradata
)<0):
6304 extradata
= listcatfiles
['ffilelist'][reallcfi
]['fextralist'];
6305 fcontents
= listcatfiles
['ffilelist'][reallcfi
]['fcontents'];
6307 fcsize
= format(int(0), 'x').lower();
6308 if(not compresswholefile
):
6309 fcontents
.seek(0, 2);
6310 ucfsize
= fcontents
.tell();
6311 fcontents
.seek(0, 0);
6312 if(compression
=="auto"):
6313 ilsize
= len(compressionlistalt
);
6316 while(ilmin
< ilsize
):
6317 cfcontents
= BytesIO();
6318 shutil
.copyfileobj(fcontents
, cfcontents
);
6319 fcontents
.seek(0, 0);
6320 cfcontents
.seek(0, 0);
6321 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
6323 cfcontents
.seek(0, 2);
6324 ilcsize
.append(cfcontents
.tell());
6328 ilcsize
.append(sys
.maxint
);
6329 except AttributeError:
6330 ilcsize
.append(sys
.maxsize
);
6332 ilcmin
= ilcsize
.index(min(ilcsize
));
6333 compression
= compressionlistalt
[ilcmin
];
6334 fcontents
.seek(0, 0);
6335 cfcontents
= BytesIO();
6336 shutil
.copyfileobj(fcontents
, cfcontents
);
6337 cfcontents
.seek(0, 0);
6338 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
6339 cfcontents
.seek(0, 2);
6340 cfsize
= cfcontents
.tell();
6341 if(ucfsize
> cfsize
):
6342 fcsize
= format(int(cfsize
), 'x').lower();
6343 fcompression
= compression
;
6345 fcontents
= cfcontents
;
6347 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']==1 or listcatfiles
['ffilelist'][reallcfi
]['ftype']==2):
6348 getflinkpath
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6349 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6350 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6351 fheadersize
= format(int(flinkinfo
['fheadersize']), 'x').lower();
6352 fsize
= format(int(flinkinfo
['fsize']), 'x').lower();
6353 flinkname
= flinkinfo
['flinkname'];
6354 fatime
= format(int(flinkinfo
['fatime']), 'x').lower();
6355 fmtime
= format(int(flinkinfo
['fmtime']), 'x').lower();
6356 fctime
= format(int(flinkinfo
['fctime']), 'x').lower();
6357 fbtime
= format(int(flinkinfo
['fbtime']), 'x').lower();
6358 fmode
= format(int(flinkinfo
['fmode']), 'x').lower();
6359 fchmode
= format(int(flinkinfo
['fchmode']), 'x').lower();
6360 fuid
= format(int(flinkinfo
['fuid']), 'x').lower();
6361 funame
= flinkinfo
['funame'];
6362 fgid
= format(int(flinkinfo
['fgid']), 'x').lower();
6363 fgname
= flinkinfo
['fgname'];
6364 finode
= format(int(flinkinfo
['finode']), 'x').lower();
6365 flinkcount
= format(int(flinkinfo
['flinkcount']), 'x').lower();
6366 fwinattributes
= format(int(flinkinfo
['fwinattributes']), 'x').lower();
6367 fcompression
= flinkinfo
['fcompression'];
6368 fcsize
= format(int(flinkinfo
['fcsize']), 'x').lower();
6369 fdev_minor
= format(int(flinkinfo
['fminor']), 'x').lower();
6370 fdev_major
= format(int(flinkinfo
['fmajor']), 'x').lower();
6371 frdev_minor
= format(int(flinkinfo
['frminor']), 'x').lower();
6372 frdev_major
= format(int(flinkinfo
['frmajor']), 'x').lower();
6373 fseeknextfile
= flinkinfo
['fseeknextfile'];
6374 if(len(flinkinfo
['fextralist'])>flinkinfo
['fextrafields'] and len(flinkinfo
['fextralist'])>0):
6375 flinkinfo
['fextrafields'] = len(flinkinfo
['fextralist']);
6376 if(len(extradata
)<0):
6377 extradata
= flinkinfo
['fextralist'];
6378 fcontents
= flinkinfo
['fcontents'];
6379 ftypehex
= format(flinkinfo
['ftype'], 'x').lower();
6381 ftypehex
= format(listcatfiles
['ffilelist'][reallcfi
]['ftype'], 'x').lower();
6382 fcurfid
= format(curfid
, 'x').lower();
6383 if(not followlink
and finode
!=0):
6384 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']!=1):
6385 fcurinode
= format(int(curinode
), 'x').lower();
6386 inodetofile
.update({curinode
: fname
});
6387 filetoinode
.update({fname
: curinode
});
6388 curinode
= curinode
+ 1;
6390 fcurinode
= format(int(filetoinode
[flinkname
]), 'x').lower();
6392 fcurinode
= format(int(curinode
), 'x').lower();
6393 curinode
= curinode
+ 1;
6394 curfid
= curfid
+ 1;
6395 if(fcompression
=="none"):
6397 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, fseeknextfile
];
6398 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
6401 reallcfi
= reallcfi
+ 1;
6403 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
6404 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6405 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6408 os
.fsync(catfp
.fileno());
6409 except io
.UnsupportedOperation
:
6411 except AttributeError:
6413 except OSError as e
:
6417 if(hasattr(sys
.stdout
, "buffer")):
6418 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
6420 shutil
.copyfileobj(catfp
, sys
.stdout
);
6421 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6422 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6424 upload_file_to_internet_file(catfp
, outfile
);
6432 create_alias_function("RePack", __file_format_name__
, "", RePackArchiveFile
);
6434 def RePackArchiveFileFromString(catstr
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6435 formatspecs
= FormatSpecsListToDict(formatspecs
);
6436 catfp
= BytesIO(catstr
);
6437 listcatfiles
= RePackArchiveFile(catfp
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6438 return listcatfiles
;
6440 create_alias_function("RePack", __file_format_name__
, "FromString", RePackArchiveFileFromString
);
6442 def PackArchiveFileFromListDir(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6443 formatspecs
= FormatSpecsListToDict(formatspecs
);
6444 outarray
= BytesIO();
6445 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
6446 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6447 return listcatfiles
;
6449 create_alias_function("Pack", __file_format_name__
, "FromListDir", PackArchiveFileFromListDir
);
6451 def UnPackArchiveFile(infile
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, preservepermissions
=True, preservetime
=True, verbose
=False, returnfp
=False):
6452 formatspecs
= FormatSpecsListToDict(formatspecs
);
6453 if(outdir
is not None):
6454 outdir
= RemoveWindowsPath(outdir
);
6456 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6457 if(isinstance(infile
, dict)):
6458 listcatfiles
= infile
;
6460 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6461 infile
= RemoveWindowsPath(infile
);
6462 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, True, skipchecksum
, formatspecs
, returnfp
);
6463 if(not listcatfiles
):
6465 lenlist
= len(listcatfiles
['ffilelist']);
6466 fnumfiles
= int(listcatfiles
['fnumfiles']);
6468 lcfx
= int(listcatfiles
['fnumfiles']);
6469 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6470 lcfx
= int(lenlist
);
6472 lcfx
= int(listcatfiles
['fnumfiles']);
6478 userinfo
= pwd
.getpwuid(listcatfiles
['ffilelist'][lcfi
]['fuid']);
6479 funame
= userinfo
.pw_name
;
6488 groupinfo
= grp
.getgrgid(listcatfiles
['ffilelist'][lcfi
]['fgid']);
6489 fgname
= groupinfo
.gr_name
;
6495 VerbosePrintOut(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6496 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6497 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6498 listcatfiles
['ffilelist'][lcfi
]['fcontents'].seek(0, 0);
6499 shutil
.copyfileobj(listcatfiles
['ffilelist'][lcfi
]['fcontents'], fpc
);
6502 os
.fsync(fpc
.fileno());
6503 except io
.UnsupportedOperation
:
6505 except AttributeError:
6507 except OSError as e
:
6509 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6510 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6511 if(preservepermissions
):
6512 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6514 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6515 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6517 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6518 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6519 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6524 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6525 funame
= userinfo
.pw_name
;
6534 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6535 fgname
= groupinfo
.gr_name
;
6540 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6541 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6542 flinkinfo
['fcontents'].seek(0, 0);
6543 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6546 os
.fsync(fpc
.fileno());
6547 except io
.UnsupportedOperation
:
6549 except AttributeError:
6551 except OSError as e
:
6553 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6554 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6555 if(preservepermissions
):
6556 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6558 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6559 if(flinkinfo
['ftype']==1):
6560 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6561 if(flinkinfo
['ftype']==2):
6562 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6563 if(flinkinfo
['ftype']==5):
6564 if(preservepermissions
):
6565 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6567 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6568 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6569 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6570 if(preservepermissions
):
6571 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6573 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6574 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6575 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6577 os
.link(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6578 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6580 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6581 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6582 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6587 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6588 funame
= userinfo
.pw_name
;
6597 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6598 fgname
= groupinfo
.gr_name
;
6603 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6604 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6605 flinkinfo
['fcontents'].seek(0, 0);
6606 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6609 os
.fsync(fpc
.fileno());
6610 except io
.UnsupportedOperation
:
6612 except AttributeError:
6614 except OSError as e
:
6616 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6617 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6618 if(preservepermissions
):
6619 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6621 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6622 if(flinkinfo
['ftype']==1):
6623 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6624 if(flinkinfo
['ftype']==2):
6625 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6626 if(flinkinfo
['ftype']==5):
6627 if(preservepermissions
):
6628 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6630 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6631 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6632 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6633 if(preservepermissions
):
6634 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6636 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6637 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6638 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6640 os
.symlink(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6641 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6642 if(preservepermissions
):
6643 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6645 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6646 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6647 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6648 if(preservepermissions
):
6649 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6651 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6652 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6 and hasattr(os
, "mkfifo")):
6653 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6656 return listcatfiles
['ffilelist']['catfp'];
6660 create_alias_function("UnPack", __file_format_name__
, "", UnPackArchiveFile
);
6662 if(hasattr(shutil
, "register_unpack_format")):
6663 def UnPackArchiveFileFunc(archive_name
, extract_dir
=None, **kwargs
):
6664 return UnPackArchiveFile(archive_name
, extract_dir
, False, 0, 0, False, __file_format_dict__
['format_delimiter'], False, False);
6665 create_alias_function("UnPack", __file_format_name__
, "Func", UnPackArchiveFileFunc
);
6667 def UnPackArchiveFileString(catstr
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6668 formatspecs
= FormatSpecsListToDict(formatspecs
);
6669 catfp
= BytesIO(catstr
);
6670 listcatfiles
= UnPackArchiveFile(catfp
, outdir
, followlink
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6671 return listcatfiles
;
6673 create_alias_function("UnPack", __file_format_name__
, "String", UnPackArchiveFileString
);
6675 def ArchiveFileListFiles(infile
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6676 formatspecs
= FormatSpecsListToDict(formatspecs
);
6677 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6678 if(isinstance(infile
, dict)):
6679 listcatfiles
= infile
;
6681 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6682 infile
= RemoveWindowsPath(infile
);
6683 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, True, False, skipchecksum
, formatspecs
, returnfp
);
6684 if(not listcatfiles
):
6686 lenlist
= len(listcatfiles
['ffilelist']);
6687 fnumfiles
= int(listcatfiles
['fnumfiles']);
6689 lcfx
= int(listcatfiles
['fnumfiles']);
6690 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6691 lcfx
= int(lenlist
);
6693 lcfx
= int(listcatfiles
['fnumfiles']);
6696 returnval
.update({lcfi
: listcatfiles
['ffilelist'][lcfi
]['fname']});
6698 VerbosePrintOut(listcatfiles
['ffilelist'][lcfi
]['fname']);
6700 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6701 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'];
6702 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6703 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " link to " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6704 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6705 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " -> " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6706 fuprint
= listcatfiles
['ffilelist'][lcfi
]['funame'];
6707 if(len(fuprint
)<=0):
6708 fuprint
= listcatfiles
['ffilelist'][lcfi
]['fuid'];
6709 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgname'];
6710 if(len(fgprint
)<=0):
6711 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgid'];
6712 VerbosePrintOut(PrintPermissionString(listcatfiles
['ffilelist'][lcfi
]['fmode'], listcatfiles
['ffilelist'][lcfi
]['ftype']) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(listcatfiles
['ffilelist'][lcfi
]['fsize']).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(listcatfiles
['ffilelist'][lcfi
]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6715 return listcatfiles
['catfp'];
6719 create_alias_function("", __file_format_name__
, "ListFiles", ArchiveFileListFiles
);
6721 def ArchiveFileStringListFiles(catstr
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6722 formatspecs
= FormatSpecsListToDict(formatspecs
);
6723 catfp
= BytesIO(catstr
);
6724 listcatfiles
= ArchiveFileListFiles(catstr
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6725 return listcatfiles
;
6727 create_alias_function("", __file_format_name__
, "StringListFiles", ArchiveFileStringListFiles
);
6729 def TarFileListFiles(infile
, verbose
=False, returnfp
=False):
6730 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6733 if(hasattr(sys
.stdin
, "buffer")):
6734 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6736 shutil
.copyfileobj(sys
.stdin
, infile
);
6741 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6742 infile
= download_file_from_internet_file(infile
);
6747 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6749 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
6751 if(not tarfile
.TarFileCheck(infile
)):
6753 except AttributeError:
6754 if(not TarFileCheck(infile
)):
6759 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
6760 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
6762 tarfp
= tarfile
.open(infile
, "r");
6763 except FileNotFoundError
:
6767 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
6768 returnval
.update({lcfi
: member
.name
});
6769 fpremode
= member
.mode
;
6770 ffullmode
= member
.mode
;
6774 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6776 elif(member
.isdev()):
6777 ffullmode
= member
.mode
;
6779 elif(member
.islnk()):
6780 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6782 elif(member
.issym()):
6783 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
6785 elif(member
.ischr()):
6786 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
6788 elif(member
.isblk()):
6789 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
6791 elif(member
.isdir()):
6792 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
6794 elif(member
.isfifo()):
6795 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
6797 elif(member
.issparse()):
6798 ffullmode
= member
.mode
;
6801 VerbosePrintOut(member
.name
);
6803 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6804 printfname
= member
.name
;
6806 printfname
= member
.name
+ " link to " + member
.linkname
;
6807 elif(member
.issym()):
6808 printfname
= member
.name
+ " -> " + member
.linkname
;
6809 fuprint
= member
.uname
;
6810 if(len(fuprint
)<=0):
6811 fuprint
= member
.uid
;
6812 fgprint
= member
.gname
;
6813 if(len(fgprint
)<=0):
6814 fgprint
= member
.gid
;
6815 VerbosePrintOut(PrintPermissionString(ffullmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(member
.mtime
).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6818 return listcatfiles
['catfp'];
6822 def ZipFileListFiles(infile
, verbose
=False, returnfp
=False):
6823 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6826 if(hasattr(sys
.stdin
, "buffer")):
6827 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6829 shutil
.copyfileobj(sys
.stdin
, infile
);
6834 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6835 infile
= download_file_from_internet_file(infile
);
6840 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6844 if(not zipfile
.is_zipfile(infile
)):
6847 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
6848 except FileNotFoundError
:
6852 ziptest
= zipfp
.testzip();
6854 VerbosePrintOut("Bad file found!");
6855 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
6856 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
6857 fwinattributes
= int(zipinfo
.external_attr
);
6858 if(not member
.is_dir()):
6859 fmode
= int(stat
.S_IFREG
+ 438);
6860 fchmode
= int(stat
.S_IMODE(fmode
));
6861 ftypemod
= int(stat
.S_IFMT(fmode
));
6862 elif(member
.is_dir()):
6863 fmode
= int(stat
.S_IFDIR
+ 511);
6864 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6865 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6866 elif(zipinfo
.create_system
==3):
6867 fwinattributes
=int(0);
6868 fmode
= int(zipinfo
.external_attr
);
6869 fchmode
= int(stat
.S_IMODE(fmode
));
6870 ftypemod
= int(stat
.S_IFMT(fmode
));
6872 fwinattributes
= int(0);
6873 if(not member
.is_dir()):
6874 fmode
= int(stat
.S_IFREG
+ 438);
6875 fchmode
= int(stat
.S_IMODE(fmode
));
6876 ftypemod
= int(stat
.S_IFMT(fmode
));
6877 elif(member
.is_dir()):
6878 fmode
= int(stat
.S_IFDIR
+ 511);
6879 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6880 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6881 returnval
.update({lcfi
: member
.filename
});
6883 VerbosePrintOut(member
.filename
);
6885 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6887 for fmodval
in str(oct(fmode
))[-3:]:
6888 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
6889 if(not member
.is_dir()):
6891 permissionstr
= "-" + permissionstr
;
6892 elif(member
.is_dir()):
6894 permissionstr
= "d" + permissionstr
;
6895 printfname
= member
.filename
;
6897 fuid
= int(os
.getuid());
6898 except AttributeError:
6903 fgid
= int(os
.getgid());
6904 except AttributeError:
6911 userinfo
= pwd
.getpwuid(os
.getuid());
6912 funame
= userinfo
.pw_name
;
6915 except AttributeError:
6923 groupinfo
= grp
.getgrgid(os
.getgid());
6924 fgname
= groupinfo
.gr_name
;
6927 except AttributeError:
6932 if(len(fuprint
)<=0):
6933 fuprint
= str(fuid
);
6935 if(len(fgprint
)<=0):
6936 fgprint
= str(fgid
);
6937 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(int(time
.mktime(member
.date_time
+ (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6940 return listcatfiles
['catfp'];
6944 if(not rarfile_support
):
6945 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
6946 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6947 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6950 if(rarfile_support
):
6951 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
6952 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6953 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6955 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
6959 rarfp
= rarfile
.RarFile(infile
, "r");
6960 rartest
= rarfp
.testrar();
6962 VerbosePrintOut("Bad file found!");
6963 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
6966 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
6969 member
.external_attr
6971 except AttributeError:
6973 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
6976 member
.external_attr
6978 except AttributeError:
6983 if(is_unix
and member
.external_attr
!=0):
6984 fpremode
= int(member
.external_attr
);
6985 elif(member
.is_file()):
6986 fpremode
= int(stat
.S_IFREG
+ 438);
6987 elif(member
.is_symlink()):
6988 fpremode
= int(stat
.S_IFLNK
+ 438);
6989 elif(member
.is_dir()):
6990 fpremode
= int(stat
.S_IFDIR
+ 511);
6991 if(is_windows
and member
.external_attr
!=0):
6992 fwinattributes
= int(member
.external_attr
);
6994 fwinattributes
= int(0);
6995 if(is_unix
and member
.external_attr
!=0):
6996 fmode
= int(member
.external_attr
);
6997 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
6998 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
6999 elif(member
.is_file()):
7000 fmode
= int(stat
.S_IFREG
+ 438);
7001 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
7002 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
7003 elif(member
.is_symlink()):
7004 fmode
= int(stat
.S_IFLNK
+ 438);
7005 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
7006 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
7007 elif(member
.is_dir()):
7008 fmode
= int(stat
.S_IFDIR
+ 511);
7009 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7010 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7011 returnval
.update({lcfi
: member
.filename
});
7013 VerbosePrintOut(member
.filename
);
7015 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7017 for fmodval
in str(oct(fmode
))[-3:]:
7018 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7019 if(member
.is_file()):
7021 permissionstr
= "-" + permissionstr
;
7022 printfname
= member
.filename
;
7023 elif(member
.is_symlink()):
7025 permissionstr
= "l" + permissionstr
;
7026 printfname
= member
.name
+ " -> " + member
.read().decode("UTF-8");
7027 elif(member
.is_dir()):
7029 permissionstr
= "d" + permissionstr
;
7030 printfname
= member
.filename
;
7032 fuid
= int(os
.getuid());
7033 except AttributeError:
7038 fgid
= int(os
.getgid());
7039 except AttributeError:
7046 userinfo
= pwd
.getpwuid(os
.getuid());
7047 funame
= userinfo
.pw_name
;
7050 except AttributeError:
7058 groupinfo
= grp
.getgrgid(os
.getgid());
7059 fgname
= groupinfo
.gr_name
;
7062 except AttributeError:
7067 if(len(fuprint
)<=0):
7068 fuprint
= str(fuid
);
7070 if(len(fgprint
)<=0):
7071 fgprint
= str(fgid
);
7072 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + member
.mtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7075 return listcatfiles
['catfp'];
7079 if(not py7zr_support
):
7080 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7081 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7082 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7086 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7087 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7088 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7092 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
7093 file_content
= szpfp
.readall();
7094 #sztest = szpfp.testzip();
7095 sztestalt
= szpfp
.test();
7097 VerbosePrintOut("Bad file found!");
7098 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
7099 if(re
.findall(r
"^[.|/]", member
.filename
)):
7100 fname
= member
.filename
;
7102 fname
= "./"+member
.filename
;
7103 if(not member
.is_directory
):
7104 fpremode
= int(stat
.S_IFREG
+ 438);
7105 elif(member
.is_directory
):
7106 fpremode
= int(stat
.S_IFDIR
+ 511);
7107 fwinattributes
= int(0);
7108 if(member
.is_directory
):
7109 fmode
= int(stat
.S_IFDIR
+ 511);
7110 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7111 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7113 fmode
= int(stat
.S_IFLNK
+ 438);
7114 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
7115 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
7116 returnval
.update({lcfi
: member
.filename
});
7118 VerbosePrintOut(member
.filename
);
7120 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7122 for fmodval
in str(oct(fmode
))[-3:]:
7123 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7125 if(not member
.is_directory
):
7127 permissionstr
= "-" + permissionstr
;
7128 printfname
= member
.filename
;
7129 elif(member
.is_directory
):
7131 permissionstr
= "d" + permissionstr
;
7132 printfname
= member
.filename
;
7134 fsize
= len(file_content
[member
.filename
].read());
7135 file_content
[member
.filename
].close();
7137 fuid
= int(os
.getuid());
7138 except AttributeError:
7143 fgid
= int(os
.getgid());
7144 except AttributeError:
7151 userinfo
= pwd
.getpwuid(os
.getuid());
7152 funame
= userinfo
.pw_name
;
7155 except AttributeError:
7163 groupinfo
= grp
.getgrgid(os
.getgid());
7164 fgname
= groupinfo
.gr_name
;
7167 except AttributeError:
7172 if(len(fuprint
)<=0):
7173 fuprint
= str(fuid
);
7175 if(len(fgprint
)<=0):
7176 fgprint
= str(fgid
);
7177 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(fsize
).rjust(15) + " " + member
.creationtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7180 return listcatfiles
['catfp'];
7184 def InFileListFiles(infile
, verbose
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
7185 formatspecs
= FormatSpecsListToDict(formatspecs
);
7186 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7187 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
7188 if(checkcompressfile
=="tarfile"):
7189 return TarFileListFiles(infile
, verbose
, returnfp
);
7190 elif(checkcompressfile
=="zipfile"):
7191 return ZipFileListFiles(infile
, verbose
, returnfp
);
7192 elif(checkcompressfile
=="catfile"):
7193 return ArchiveFileListFiles(infile
, 0, 0, False, formatspecs
, verbose
, returnfp
);
7194 elif(rarfile_support
and checkcompressfile
=="rarfile"):
7195 return RarFileListFiles(infile
, verbose
, returnfp
);
7196 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
7197 return SevenZipFileListFiles(infile
, verbose
, returnfp
);
7202 def ListDirListFiles(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7203 formatspecs
= FormatSpecsListToDict(formatspecs
);
7204 outarray
= BytesIO();
7205 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, False, True);
7206 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7207 return listcatfiles
;
7209 def ListDirListFilesAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7210 formatspecs
= FormatSpecsListToDict(formatspecs
);
7211 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, formatspecs
, verbose
);
7212 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7213 return listcatfiles
;
7215 def PackArchiveFileFromListDirAlt(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7216 formatspecs
= FormatSpecsListToDict(formatspecs
);
7217 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, False, checksumtype
, extradata
, formatspecs
, False);
7218 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
7219 return listcatfiles
;
7221 create_alias_function("Pack", __file_format_name__
, "FromListDirAlt", PackArchiveFileFromListDirAlt
);
7223 def PackArchiveFileFromTarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7224 formatspecs
= FormatSpecsListToDict(formatspecs
);
7225 outarray
= TarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7226 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7227 return listcatfiles
;
7229 create_alias_function("Pack", __file_format_name__
, "FromTarFileAlt", PackArchiveFileFromTarFileAlt
);
7231 def PackArchiveFileFromZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7232 formatspecs
= FormatSpecsListToDict(formatspecs
);
7233 outarray
= ZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7234 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7235 return listcatfiles
;
7237 create_alias_function("Pack", __file_format_name__
, "FromZipFileAlt", PackArchiveFileFromZipFileAlt
);
7239 if(not rarfile_support
):
7240 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7243 if(rarfile_support
):
7244 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7245 formatspecs
= FormatSpecsListToDict(formatspecs
);
7246 outarray
= RarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7247 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7248 return listcatfiles
;
7250 create_alias_function("Pack", __file_format_name__
, "FromRarFileAlt", PackArchiveFileFromRarFileAlt
);
7252 if(not py7zr_support
):
7253 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7257 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7258 formatspecs
= FormatSpecsListToDict(formatspecs
);
7259 outarray
= SevenZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7260 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7261 return listcatfiles
;
7263 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt
);
7265 def download_file_from_ftp_file(url
):
7266 urlparts
= urlparse(url
);
7267 file_name
= os
.path
.basename(urlparts
.path
);
7268 file_dir
= os
.path
.dirname(urlparts
.path
);
7269 if(urlparts
.username
is not None):
7270 ftp_username
= urlparts
.username
;
7272 ftp_username
= "anonymous";
7273 if(urlparts
.password
is not None):
7274 ftp_password
= urlparts
.password
;
7275 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7276 ftp_password
= "anonymous";
7279 if(urlparts
.scheme
=="ftp"):
7281 elif(urlparts
.scheme
=="ftps" and ftpssl
):
7285 if(urlparts
.scheme
=="sftp"):
7287 return download_file_from_pysftp_file(url
);
7289 return download_file_from_sftp_file(url
);
7290 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7291 return download_file_from_http_file(url
);
7292 ftp_port
= urlparts
.port
;
7293 if(urlparts
.port
is None):
7296 ftp
.connect(urlparts
.hostname
, ftp_port
);
7297 except socket
.gaierror
:
7298 log
.info("Error With URL "+url
);
7300 except socket
.timeout
:
7301 log
.info("Error With URL "+url
);
7303 ftp
.login(urlparts
.username
, urlparts
.password
);
7304 if(urlparts
.scheme
=="ftps"):
7306 ftpfile
= BytesIO();
7307 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
7308 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7313 def download_file_from_ftp_string(url
):
7314 ftpfile
= download_file_from_ftp_file(url
);
7315 return ftpfile
.read();
7317 def upload_file_to_ftp_file(ftpfile
, url
):
7318 urlparts
= urlparse(url
);
7319 file_name
= os
.path
.basename(urlparts
.path
);
7320 file_dir
= os
.path
.dirname(urlparts
.path
);
7321 if(urlparts
.username
is not None):
7322 ftp_username
= urlparts
.username
;
7324 ftp_username
= "anonymous";
7325 if(urlparts
.password
is not None):
7326 ftp_password
= urlparts
.password
;
7327 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7328 ftp_password
= "anonymous";
7331 if(urlparts
.scheme
=="ftp"):
7333 elif(urlparts
.scheme
=="ftps" and ftpssl
):
7337 if(urlparts
.scheme
=="sftp"):
7339 return upload_file_to_pysftp_file(url
);
7341 return upload_file_to_sftp_file(url
);
7342 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7344 ftp_port
= urlparts
.port
;
7345 if(urlparts
.port
is None):
7348 ftp
.connect(urlparts
.hostname
, ftp_port
);
7349 except socket
.gaierror
:
7350 log
.info("Error With URL "+url
);
7352 except socket
.timeout
:
7353 log
.info("Error With URL "+url
);
7355 ftp
.login(urlparts
.username
, urlparts
.password
);
7356 if(urlparts
.scheme
=="ftps"):
7358 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
7363 def upload_file_to_ftp_string(ftpstring
, url
):
7364 ftpfileo
= BytesIO(ftpstring
);
7365 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
7369 def download_file_from_http_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7370 # Parse the URL to extract username and password if present
7371 urlparts
= urlparse(url
);
7372 username
= urlparts
.username
;
7373 password
= urlparts
.password
;
7374 # Rebuild the URL without the username and password
7375 netloc
= urlparts
.hostname
;
7376 if(urlparts
.scheme
=="sftp"):
7378 return download_file_from_pysftp_file(url
);
7380 return download_file_from_sftp_file(url
);
7381 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7382 return download_file_from_ftp_file(url
);
7384 netloc
+= ':' + str(urlparts
.port
);
7385 rebuilt_url
= urlunparse((urlparts
.scheme
, netloc
, urlparts
.path
, urlparts
.params
, urlparts
.query
, urlparts
.fragment
));
7386 # Create a temporary file object
7387 httpfile
= BytesIO();
7389 # Use the requests library if available
7390 if username
and password
:
7391 response
= requests
.get(rebuilt_url
, headers
=headers
, auth
=(username
, password
), stream
=True);
7393 response
= requests
.get(rebuilt_url
, headers
=headers
, stream
=True);
7394 response
.raw
.decode_content
= True
7395 shutil
.copyfileobj(response
.raw
, httpfile
);
7397 # Build a Request object for urllib
7398 request
= Request(rebuilt_url
, headers
=headers
);
7399 # Create an opener object for handling URLs
7400 if username
and password
:
7401 # Create a password manager
7402 password_mgr
= HTTPPasswordMgrWithDefaultRealm();
7403 # Add the username and password
7404 password_mgr
.add_password(None, rebuilt_url
, username
, password
);
7405 # Create an authentication handler using the password manager
7406 auth_handler
= HTTPBasicAuthHandler(password_mgr
);
7407 # Build the opener with the authentication handler
7408 opener
= build_opener(auth_handler
);
7410 opener
= build_opener();
7411 with opener
.open(request
) as response
:
7412 shutil
.copyfileobj(response
, httpfile
);
7413 # Reset file pointer to the start
7414 httpfile
.seek(0, 0);
7415 # Return the temporary file object
7418 def download_file_from_http_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7419 httpfile
= download_file_from_http_file(url
, headers
);
7420 return ftpfile
.read();
7423 def download_file_from_sftp_file(url
):
7424 urlparts
= urlparse(url
);
7425 file_name
= os
.path
.basename(urlparts
.path
);
7426 file_dir
= os
.path
.dirname(urlparts
.path
);
7427 sftp_port
= urlparts
.port
;
7428 if(urlparts
.port
is None):
7431 sftp_port
= urlparts
.port
;
7432 if(urlparts
.username
is not None):
7433 sftp_username
= urlparts
.username
;
7435 sftp_username
= "anonymous";
7436 if(urlparts
.password
is not None):
7437 sftp_password
= urlparts
.password
;
7438 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7439 sftp_password
= "anonymous";
7442 if(urlparts
.scheme
=="ftp"):
7443 return download_file_from_ftp_file(url
);
7444 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7445 return download_file_from_http_file(url
);
7446 if(urlparts
.scheme
!="sftp"):
7448 ssh
= paramiko
.SSHClient();
7449 ssh
.load_system_host_keys();
7450 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7452 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7453 except paramiko
.ssh_exception
.SSHException
:
7455 except socket
.gaierror
:
7456 log
.info("Error With URL "+url
);
7458 except socket
.timeout
:
7459 log
.info("Error With URL "+url
);
7461 sftp
= ssh
.open_sftp();
7462 sftpfile
= BytesIO();
7463 sftp
.getfo(urlparts
.path
, sftpfile
);
7466 sftpfile
.seek(0, 0);
7469 def download_file_from_sftp_file(url
):
7473 def download_file_from_sftp_string(url
):
7474 sftpfile
= download_file_from_sftp_file(url
);
7475 return sftpfile
.read();
7477 def download_file_from_ftp_string(url
):
7481 def upload_file_to_sftp_file(sftpfile
, url
):
7482 urlparts
= urlparse(url
);
7483 file_name
= os
.path
.basename(urlparts
.path
);
7484 file_dir
= os
.path
.dirname(urlparts
.path
);
7485 sftp_port
= urlparts
.port
;
7486 if(urlparts
.port
is None):
7489 sftp_port
= urlparts
.port
;
7490 if(urlparts
.username
is not None):
7491 sftp_username
= urlparts
.username
;
7493 sftp_username
= "anonymous";
7494 if(urlparts
.password
is not None):
7495 sftp_password
= urlparts
.password
;
7496 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7497 sftp_password
= "anonymous";
7500 if(urlparts
.scheme
=="ftp"):
7501 return upload_file_to_ftp_file(url
);
7502 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7504 if(urlparts
.scheme
!="sftp"):
7506 ssh
= paramiko
.SSHClient();
7507 ssh
.load_system_host_keys();
7508 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7510 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7511 except paramiko
.ssh_exception
.SSHException
:
7513 except socket
.gaierror
:
7514 log
.info("Error With URL "+url
);
7516 except socket
.timeout
:
7517 log
.info("Error With URL "+url
);
7519 sftp
= ssh
.open_sftp();
7520 sftp
.putfo(sftpfile
, urlparts
.path
);
7523 sftpfile
.seek(0, 0);
7526 def upload_file_to_sftp_file(sftpfile
, url
):
7530 def upload_file_to_sftp_string(sftpstring
, url
):
7531 sftpfileo
= BytesIO(sftpstring
);
7532 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
7536 def upload_file_to_sftp_string(url
):
7540 def download_file_from_pysftp_file(url
):
7541 urlparts
= urlparse(url
);
7542 file_name
= os
.path
.basename(urlparts
.path
);
7543 file_dir
= os
.path
.dirname(urlparts
.path
);
7544 sftp_port
= urlparts
.port
;
7545 if(urlparts
.port
is None):
7548 sftp_port
= urlparts
.port
;
7549 if(urlparts
.username
is not None):
7550 sftp_username
= urlparts
.username
;
7552 sftp_username
= "anonymous";
7553 if(urlparts
.password
is not None):
7554 sftp_password
= urlparts
.password
;
7555 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7556 sftp_password
= "anonymous";
7559 if(urlparts
.scheme
=="ftp"):
7560 return download_file_from_ftp_file(url
);
7561 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7562 return download_file_from_http_file(url
);
7563 if(urlparts
.scheme
!="sftp"):
7566 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7567 except paramiko
.ssh_exception
.SSHException
:
7569 except socket
.gaierror
:
7570 log
.info("Error With URL "+url
);
7572 except socket
.timeout
:
7573 log
.info("Error With URL "+url
);
7575 sftp
= ssh
.open_sftp();
7576 sftpfile
= BytesIO();
7577 sftp
.getfo(urlparts
.path
, sftpfile
);
7580 sftpfile
.seek(0, 0);
7583 def download_file_from_pysftp_file(url
):
7587 def download_file_from_pysftp_string(url
):
7588 sftpfile
= download_file_from_pysftp_file(url
);
7589 return sftpfile
.read();
7591 def download_file_from_ftp_string(url
):
7595 def upload_file_to_pysftp_file(sftpfile
, url
):
7596 urlparts
= urlparse(url
);
7597 file_name
= os
.path
.basename(urlparts
.path
);
7598 file_dir
= os
.path
.dirname(urlparts
.path
);
7599 sftp_port
= urlparts
.port
;
7600 if(urlparts
.port
is None):
7603 sftp_port
= urlparts
.port
;
7604 if(urlparts
.username
is not None):
7605 sftp_username
= urlparts
.username
;
7607 sftp_username
= "anonymous";
7608 if(urlparts
.password
is not None):
7609 sftp_password
= urlparts
.password
;
7610 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7611 sftp_password
= "anonymous";
7614 if(urlparts
.scheme
=="ftp"):
7615 return upload_file_to_ftp_file(url
);
7616 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7618 if(urlparts
.scheme
!="sftp"):
7621 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7622 except paramiko
.ssh_exception
.SSHException
:
7624 except socket
.gaierror
:
7625 log
.info("Error With URL "+url
);
7627 except socket
.timeout
:
7628 log
.info("Error With URL "+url
);
7630 sftp
= ssh
.open_sftp();
7631 sftp
.putfo(sftpfile
, urlparts
.path
);
7634 sftpfile
.seek(0, 0);
7637 def upload_file_to_pysftp_file(sftpfile
, url
):
7641 def upload_file_to_pysftp_string(sftpstring
, url
):
7642 sftpfileo
= BytesIO(sftpstring
);
7643 sftpfile
= upload_file_to_pysftp_files(ftpfileo
, url
);
7647 def upload_file_to_pysftp_string(url
):
7650 def download_file_from_internet_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7651 urlparts
= urlparse(url
);
7652 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7653 return download_file_from_http_file(url
, headers
);
7654 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7655 return download_file_from_ftp_file(url
);
7656 elif(urlparts
.scheme
=="sftp"):
7657 if(__use_pysftp__
and havepysftp
):
7658 return download_file_from_pysftp_file(url
);
7660 return download_file_from_sftp_file(url
);
7665 def download_file_from_internet_uncompress_file(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7666 formatspecs
= FormatSpecsListToDict(formatspecs
);
7667 fp
= download_file_from_internet_file(url
);
7668 fp
= UncompressArchiveFile(fp
, formatspecs
);
7674 def download_file_from_internet_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7675 urlparts
= urlparse(url
);
7676 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7677 return download_file_from_http_string(url
, headers
);
7678 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7679 return download_file_from_ftp_string(url
);
7680 elif(urlparts
.scheme
=="sftp"):
7681 if(__use_pysftp__
and havepysftp
):
7682 return download_file_from_pysftp_string(url
);
7684 return download_file_from_sftp_string(url
);
7689 def download_file_from_internet_uncompress_string(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7690 formatspecs
= FormatSpecsListToDict(formatspecs
);
7691 fp
= download_file_from_internet_string(url
);
7692 fp
= UncompressArchiveFile(fp
, formatspecs
);
7698 def upload_file_to_internet_file(ifp
, url
):
7699 urlparts
= urlparse(url
);
7700 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7702 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7703 return upload_file_to_ftp_file(ifp
, url
);
7704 elif(urlparts
.scheme
=="sftp"):
7705 if(__use_pysftp__
and havepysftp
):
7706 return upload_file_to_pysftp_file(ifp
, url
);
7708 return upload_file_to_sftp_file(ifp
, url
);
7713 def upload_file_to_internet_compress_file(ifp
, url
, formatspecs
=__file_format_dict__
):
7714 formatspecs
= FormatSpecsListToDict(formatspecs
);
7715 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
7719 upload_file_to_internet_file(catfp
, outfile
);
7722 def upload_file_to_internet_string(ifp
, url
):
7723 urlparts
= urlparse(url
);
7724 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7726 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7727 return upload_file_to_ftp_string(ifp
, url
);
7728 elif(urlparts
.scheme
=="sftp"):
7729 if(__use_pysftp__
and havepysftp
):
7730 return upload_file_to_pysftp_string(ifp
, url
);
7732 return upload_file_to_sftp_string(ifp
, url
);
7737 def upload_file_to_internet_compress_string(ifp
, url
, formatspecs
=__file_format_dict__
):
7738 formatspecs
= FormatSpecsListToDict(formatspecs
);
7739 catfp
= CompressArchiveFile(BytesIO(ifp
), compression
, compressionlevel
, formatspecs
);
7743 upload_file_to_internet_file(catfp
, outfile
);
7747 if(hasattr(shutil
, "register_archive_format")):
7748 # Register the packing format
7749 shutil
.register_archive_format(__file_format_name__
, PackArchiveFileFunc
, description
='Pack concatenated files');
7750 except shutil
.RegistryError
:
7754 if(hasattr(shutil
, "register_unpack_format")):
7755 # Register the unpacking format
7756 shutil
.register_unpack_format(__file_format_name__
, archivefile_extensions
, UnPackArchiveFileFunc
, description
='UnPack concatenated files');
7757 except shutil
.RegistryError
: