2 # -*- coding: utf-8 -*-
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/5/2024 Ver. 0.10.2 RC 1 - Author: cooldude2k $
20 from __future__
import absolute_import
, division
, print_function
, unicode_literals
;
21 import io
, os
, re
, sys
, time
, stat
, zlib
, base64
, shutil
, socket
, hashlib
, datetime
, logging
, binascii
, tempfile
, zipfile
, platform
;
22 from ftplib
import FTP
, FTP_TLS
;
23 if(sys
.version
[0]=="2"):
24 from urlparse
import urlparse
, urlunparse
;
25 elif(sys
.version
[0]>="3"):
26 from urllib
.parse
import urlunparse
;
27 from urllib
.parse
import urlparse
;
29 if os
.name
== 'nt': # Only modify if on Windows
30 if sys
.version
[0] == "2":
32 sys
.stdout
= codecs
.getwriter('utf-8')(sys
.stdout
);
33 sys
.stderr
= codecs
.getwriter('utf-8')(sys
.stderr
);
36 sys
.stdout
= io
.TextIOWrapper(sys
.stdout
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
37 sys
.stderr
= io
.TextIOWrapper(sys
.stderr
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
39 hashlib_guaranteed
= False;
40 os
.environ
["PYTHONIOENCODING"] = "UTF-8";
41 os
.environ
["LC_CTYPE"] = "UTF-8";
45 sys
.setdefaultencoding('UTF-8');
48 except AttributeError:
52 except AttributeError:
56 from zlib
import crc32
;
58 from binascii
import crc32
;
60 if(sys
.version_info
[0]==2):
61 FileNotFoundError
= IOError;
63 rarfile_support
= False;
66 rarfile_support
= True;
68 rarfile_support
= False;
70 py7zr_support
= False;
75 py7zr_support
= False;
78 from xtarfile
import is_tarfile
;
81 from safetar
import is_tarfile
;
83 from tarfile
import is_tarfile
;
86 import xtarfile
as tarfile
;
89 import safetar
as tarfile
;
107 haverequests
= False;
112 haverequests
= False;
116 logging
.getLogger("urllib3").setLevel(logging
.WARNING
);
120 from urllib
.request
import Request
, build_opener
, HTTPBasicAuthHandler
;
121 from urllib
.parse
import urlparse
;
124 from urllib2
import Request
, build_opener
, HTTPBasicAuthHandler
;
125 from urlparse
import urlparse
;
127 if(sys
.version
[0]=="2"):
129 from io
import StringIO
, BytesIO
;
132 from cStringIO
import StringIO
;
133 from cStringIO
import StringIO
as BytesIO
;
135 from StringIO
import StringIO
;
136 from StringIO
import StringIO
as BytesIO
;
137 elif(sys
.version
[0]>="3"):
138 from io
import StringIO
, BytesIO
;
143 from cStringIO
import StringIO
as BytesIO
;
149 from StringIO
import StringIO
as BytesIO
;
155 from io
import BytesIO
;
160 __use_pysftp__
= False;
162 __use_pysftp__
= False;
163 __file_format_name__
= "CatFile";
164 __program_name__
= "Py"+__file_format_name__
;
165 __file_format_lower__
= __file_format_name__
.lower();
166 __file_format_magic__
= __file_format_name__
;
167 __file_format_len__
= len(__file_format_magic__
);
168 __file_format_hex__
= binascii
.hexlify(__file_format_magic__
.encode("UTF-8")).decode("UTF-8");
169 __file_format_delimiter__
= "\x00";
170 __file_format_ver__
= "001";
171 __use_new_style__
= True;
172 __use_advanced_list__
= True;
173 __use_alt_inode__
= False;
174 __file_format_list__
= [__file_format_name__
, __file_format_magic__
, __file_format_lower__
, __file_format_len__
, __file_format_hex__
, __file_format_delimiter__
, __file_format_ver__
, __use_new_style__
, __use_advanced_list__
, __use_alt_inode__
];
175 __file_format_dict__
= {'format_name': __file_format_name__
, 'format_magic': __file_format_magic__
, 'format_lower': __file_format_lower__
, 'format_len': __file_format_len__
, 'format_hex': __file_format_hex__
, 'format_delimiter': __file_format_delimiter__
, 'format_ver': __file_format_ver__
, 'new_style': __use_new_style__
, 'use_advanced_list': __use_advanced_list__
, 'use_alt_inode': __use_alt_inode__
};
176 __project__
= __program_name__
;
177 __project_url__
= "https://github.com/GameMaker2k/PyCatFile";
178 __version_info__
= (0, 10, 2, "RC 1", 1);
179 __version_date_info__
= (2024, 5, 5, "RC 1", 1);
180 __version_date__
= str(__version_date_info__
[0]) + "." + str(__version_date_info__
[1]).zfill(2) + "." + str(__version_date_info__
[2]).zfill(2);
181 __revision__
= __version_info__
[3];
182 __revision_id__
= "$Id$";
183 if(__version_info__
[4] is not None):
184 __version_date_plusrc__
= __version_date__
+ "-" + str(__version_date_info__
[4]);
185 if(__version_info__
[4] is None):
186 __version_date_plusrc__
= __version_date__
;
187 if(__version_info__
[3] is not None):
188 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]) + " " + str(__version_info__
[3]);
189 if(__version_info__
[3] is None):
190 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]);
192 PyBitness
= platform
.architecture();
193 if(PyBitness
=="32bit" or PyBitness
=="32"):
195 elif(PyBitness
=="64bit" or PyBitness
=="64"):
200 geturls_ua_pycatfile_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
201 if(platform
.python_implementation()!=""):
202 py_implementation
= platform
.python_implementation();
203 if(platform
.python_implementation()==""):
204 py_implementation
= "Python";
205 geturls_ua_pycatfile_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
206 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
207 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
208 geturls_headers_pycatfile_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
209 geturls_headers_pycatfile_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
210 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
211 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 compressionlist
= ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
214 compressionlistalt
= ['gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzma', 'xz'];
215 outextlist
= ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
216 outextlistwd
= ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
218 tarfile_mimetype
= "application/tar";
219 tarfile_tar_mimetype
= tarfile_mimetype
;
220 zipfile_mimetype
= "application/zip";
221 zipfile_zip_mimetype
= zipfile_mimetype
;
222 rarfile_mimetype
= "application/rar";
223 rarfile_rar_mimetype
= rarfile_mimetype
;
224 archivefile_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"";
225 archivefile_cat_mimetype
= archivefile_mimetype
;
226 archivefile_gzip_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+gzip";
227 archivefile_gz_mimetype
= archivefile_gzip_mimetype
;
228 archivefile_bzip2_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+bzip2";
229 archivefile_bz2_mimetype
= archivefile_bzip2_mimetype
;
230 archivefile_lz4_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lz4";
231 archivefile_lzop_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzop";
232 archivefile_lzo_mimetype
= archivefile_lzop_mimetype
;
233 archivefile_zstandard_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+zstandard";
234 archivefile_zstd_mimetype
= archivefile_zstandard_mimetype
;
235 archivefile_lzma_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+lzma";
236 archivefile_xz_mimetype
= "application/x-"+__file_format_dict__
['format_lower']+"+xz";
237 archivefile_extensions
= ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
239 if __name__
== "__main__":
241 curscrpath
= os
.path
.dirname(sys
.argv
[0]);
245 curscrpath
= curscrpath
.replace(os
.sep
, "/");
246 curscrpath
= curscrpath
+ "/";
247 scrfile
= curscrpath
+ "catfile.py";
248 if(os
.path
.exists(scrfile
) and os
.path
.isfile(scrfile
)):
249 scrcmd
= subprocess
.Popen([sys
.executable
, scrfile
] + sys
.argv
[1:]);
252 def VerbosePrintOut(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
258 "warning": logging
.warning
,
259 "error": logging
.error
,
260 "critical": logging
.critical
,
261 "exception": logging
.exception
,
262 "logalt": lambda x
: logging
.log(dgblevel
, x
),
263 "debug": logging
.debug
265 log_function
= log_functions
.get(outtype
);
267 log_function(dbgtxt
);
271 def VerbosePrintOutReturn(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
272 VerbosePrintOut(dbgtxt
, outtype
, dbgenable
, dgblevel
);
275 def RemoveWindowsPath(dpath
):
279 dpath
= dpath
.replace(os
.path
.sep
, "/");
280 dpath
= dpath
.rstrip("/");
281 if(dpath
=="." or dpath
==".."):
285 def NormalizeRelativePath(inpath
):
286 inpath
= RemoveWindowsPath(inpath
);
287 if(os
.path
.isabs(inpath
)):
290 if(inpath
.startswith("./") or inpath
.startswith("../")):
293 outpath
= "./" + inpath
;
296 def PrependPath(base_dir
, child_path
):
297 # Check if base_dir is None or empty, if so, return child_path as is
300 # Ensure base_dir ends with exactly one slash
301 if not base_dir
.endswith('/'):
303 # Check if child_path starts with ./ or ../ (indicating a relative path)
304 if child_path
.startswith('./') or child_path
.startswith('../'):
305 # For relative paths, we don't alter the child_path
306 return base_dir
+ child_path
;
308 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
309 return base_dir
+ child_path
.lstrip('/');
311 def ListDir(dirpath
, followlink
=False, duplicates
=False):
312 if(isinstance(dirpath
, (list, tuple, ))):
313 dirpath
= list(filter(None, dirpath
));
314 elif(isinstance(dirpath
, (str, ))):
315 dirpath
= list(filter(None, [dirpath
]));
317 for mydirfile
in dirpath
:
318 if(not os
.path
.exists(mydirfile
)):
320 mydirfile
= NormalizeRelativePath(mydirfile
);
321 if(os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
)):
322 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
));
323 if(os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
)):
324 for root
, dirs
, filenames
in os
.walk(mydirfile
):
326 dpath
= RemoveWindowsPath(dpath
);
327 if(dpath
not in retlist
and not duplicates
):
328 retlist
.append(dpath
);
330 retlist
.append(dpath
);
331 for file in filenames
:
332 fpath
= os
.path
.join(root
, file);
333 fpath
= RemoveWindowsPath(fpath
);
334 if(fpath
not in retlist
and not duplicates
):
335 retlist
.append(fpath
);
337 retlist
.append(fpath
);
339 retlist
.append(RemoveWindowsPath(mydirfile
));
342 def ListDirAdvanced(dirpath
, followlink
=False, duplicates
=False):
343 if isinstance(dirpath
, (list, tuple)):
344 dirpath
= list(filter(None, dirpath
));
345 elif isinstance(dirpath
, str):
346 dirpath
= list(filter(None, [dirpath
]));
348 for mydirfile
in dirpath
:
349 if not os
.path
.exists(mydirfile
):
351 mydirfile
= NormalizeRelativePath(mydirfile
);
352 if os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
) and followlink
:
353 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
))
354 if os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
):
355 for root
, dirs
, filenames
in os
.walk(mydirfile
):
356 # Sort dirs and filenames alphabetically in place
357 dirs
.sort(key
=lambda x
: x
.lower());
358 filenames
.sort(key
=lambda x
: x
.lower());
359 dpath
= RemoveWindowsPath(root
);
360 if not duplicates
and dpath
not in retlist
:
361 retlist
.append(dpath
);
363 retlist
.append(dpath
);
364 for file in filenames
:
365 fpath
= os
.path
.join(root
, file);
366 fpath
= RemoveWindowsPath(fpath
);
367 if not duplicates
and fpath
not in retlist
:
368 retlist
.append(fpath
);
370 retlist
.append(fpath
);
372 retlist
.append(RemoveWindowsPath(mydirfile
));
375 def create_alias_function(prefix
, base_name
, suffix
, target_function
):
376 # Define a new function that wraps the target function
377 def alias_function(*args
, **kwargs
):
378 return target_function(*args
, **kwargs
);
380 # Create the function name by combining the prefix, base name, and the suffix
381 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
);
383 # Add the new function to the global namespace
384 globals()[function_name
] = alias_function
;
386 def create_alias_function_alt(prefix
, base_name
, suffix
, target_function
):
387 # Create the function name by combining the prefix, base name, and the suffix
388 # Use the format method for string formatting, compatible with Python 2 and 3
389 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
)
390 # Add the new function (alias of the target_function) to the global namespace
391 # This line is compatible as-is with both Python 2 and 3
392 globals()[function_name
] = target_function
394 # initial_value can be 0xFFFF or 0x0000
395 def crc16_ansi(msg
, initial_value
=0xFFFF):
396 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
397 poly
= 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
398 crc
= initial_value
; # Initial value
400 crc ^
= b
<< 8; # XOR byte into CRC top byte
401 for _
in range(8): # Process each bit
402 if crc
& 0x8000: # If the top bit is set
403 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
405 crc
= crc
<< 1; # Just shift left
406 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
409 # initial_value can be 0xFFFF or 0x0000
410 def crc16_ibm(msg
, initial_value
=0xFFFF):
411 return crc16_ansi(msg
, initial_value
);
413 # initial_value is 0xFFFF
415 return crc16_ansi(msg
, 0xFFFF);
417 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
418 def crc16_ccitt(msg
, initial_value
=0xFFFF):
419 # CRC-16-CCITT polynomial
420 poly
= 0x1021; # Polynomial for CRC-16-CCITT
421 # Use the specified initial value
424 crc ^
= b
<< 8; # XOR byte into CRC top byte
425 for _
in range(8): # Process each bit
426 if crc
& 0x8000: # If the top bit is set
427 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
429 crc
= crc
<< 1; # Just shift left
430 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
433 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
434 def crc64_ecma(msg
, initial_value
=0x0000000000000000):
435 # CRC-64-ECMA polynomial and initial value
436 poly
= 0x42F0E1EBA9EA3693;
437 crc
= initial_value
; # Initial value for CRC-64-ECMA
439 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
440 for _
in range(8): # Process each bit
441 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
442 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
444 crc
<<= 1; # Just shift left if the MSB is 0
445 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
448 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
449 def crc64_iso(msg
, initial_value
=0xFFFFFFFFFFFFFFFF):
450 # CRC-64-ISO polynomial and initial value
451 poly
= 0x000000000000001B;
452 crc
= initial_value
; # Common initial value for CRC-64-ISO
454 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
455 for _
in range(8): # Process each bit
456 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
457 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
459 crc
<<= 1; # Just shift left if the MSB is 0
460 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
463 def GetDataFromArray(data
, path
, default
=None):
467 element
= element
[key
];
469 except (KeyError, TypeError, IndexError):
472 def GetDataFromArrayAlt(structure
, path
, default
=None):
475 if isinstance(element
, dict) and key
in element
:
476 element
= element
[key
];
477 elif isinstance(element
, list) and isinstance(key
, int) and -len(element
) <= key
< len(element
):
478 element
= element
[key
];
483 def GetHeaderChecksum(inlist
=[], checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
484 if isinstance(inlist
, list):
485 fileheader
= AppendNullBytes(inlist
, formatspecs
['format_delimiter']);
487 fileheader
= AppendNullByte(inlist
, formatspecs
['format_delimiter']);
489 fileheader
= fileheader
.encode('UTF-8');
490 if(checksumtype
=="none" or checksumtype
==""):
491 catfileheadercshex
= format(0, 'x').lower();
492 elif(checksumtype
=="crc16" or checksumtype
=="crc16_ansi" or checksumtype
=="crc16_ibm"):
493 catfileheadercshex
= format(crc16(fileheader
) & 0xffff, '04x').lower();
494 elif(checksumtype
=="crc16_ccitt"):
495 catfileheadercshex
= format(crc16_ccitt(fileheader
) & 0xffff, '04x').lower();
496 elif(checksumtype
=="adler32"):
497 catfileheadercshex
= format(zlib
.adler32(fileheader
) & 0xffffffff, '08x').lower();
498 elif(checksumtype
=="crc32"):
499 catfileheadercshex
= format(crc32(fileheader
) & 0xffffffff, '08x').lower();
500 elif(checksumtype
=="crc64_ecma"):
501 catfileheadercshex
= format(crc64_ecma(fileheader
) & 0xffffffffffffffff, '016x').lower();
502 elif(checksumtype
=="crc64" or checksumtype
=="crc64_iso"):
503 catfileheadercshex
= format(crc64_iso(fileheader
) & 0xffffffffffffffff, '016x').lower();
504 elif(CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
)):
505 checksumoutstr
= hashlib
.new(checksumtype
);
506 checksumoutstr
.update(fileheader
);
507 catfileheadercshex
= checksumoutstr
.hexdigest().lower();
509 catfileheadercshex
= format(0, 'x').lower();
510 return catfileheadercshex
;
512 def GetFileChecksum(instr
, checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_dict__
):
514 instr
= instr
.encode('UTF-8');
515 if(checksumtype
=="none" or checksumtype
==""):
516 catinstrcshex
= format(0, 'x').lower();
517 elif(checksumtype
=="crc16" or checksumtype
=="crc16_ansi" or checksumtype
=="crc16_ibm"):
518 catinstrcshex
= format(crc16(instr
) & 0xffff, '04x').lower();
519 elif(checksumtype
=="crc16_ccitt"):
520 catinstrcshex
= format(crc16_ccitt(instr
) & 0xffff, '04x').lower();
521 elif(checksumtype
=="adler32"):
522 catinstrcshex
= format(zlib
.adler32(instr
) & 0xffffffff, '08x').lower();
523 elif(checksumtype
=="crc32"):
524 catinstrcshex
= format(crc32(instr
) & 0xffffffff, '08x').lower();
525 elif(checksumtype
=="crc64_ecma"):
526 catinstrcshex
= format(crc64_ecma(instr
) & 0xffffffffffffffff, '016x').lower();
527 elif(checksumtype
=="crc64" or checksumtype
=="crc64_iso"):
528 catinstrcshex
= format(crc64_iso(instr
) & 0xffffffffffffffff, '016x').lower();
529 elif(CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
)):
530 checksumoutstr
= hashlib
.new(checksumtype
);
531 checksumoutstr
.update(instr
);
532 catinstrcshex
= checksumoutstr
.hexdigest().lower();
534 catinstrcshex
= format(0, 'x').lower();
535 return catinstrcshex
;
537 def ValidateHeaderChecksum(inlist
=[], checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
538 catfileheadercshex
= GetHeaderChecksum(inlist
, checksumtype
, True, formatspecs
);
539 inchecksum
= inchecksum
.lower();
540 catfileheadercshex
= catfileheadercshex
.lower();
541 if(inchecksum
==catfileheadercshex
):
546 def ValidateFileChecksum(infile
, checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_dict__
):
547 catinfilecshex
= GetFileChecksum(infile
, checksumtype
, True, formatspecs
);
548 inchecksum
= inchecksum
.lower();
549 catinfilecshex
= catinfilecshex
.lower();
550 if(inchecksum
==catinfilecshex
):
555 def ReadTillNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
558 nullbyte
= delimiter
.encode("UTF-8");
560 curbyte
= fp
.read(1);
561 if(curbyte
==nullbyte
or not curbyte
):
563 curfullbyte
= curfullbyte
+ curbyte
;
564 return curfullbyte
.decode('UTF-8');
566 def ReadUntilNullByteOld(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
567 return ReadTillNullByteOld(fp
, delimiter
);
569 def ReadTillNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
570 curfullbyte
= bytearray();
571 nullbyte
= delimiter
.encode("UTF-8");
572 total_read
= 0; # Track the total number of bytes read
574 curbyte
= fp
.read(1);
575 if curbyte
== nullbyte
or not curbyte
:
577 curfullbyte
.extend(curbyte
);
579 if total_read
>= max_read
:
580 raise MemoryError("Maximum read limit reached without finding the delimiter.");
581 # Decode the full byte array to string once out of the loop
583 return curfullbyte
.decode('UTF-8');
584 except UnicodeDecodeError:
585 # Handle potential partial UTF-8 characters
586 for i
in range(1, 4):
588 return curfullbyte
[:-i
].decode('UTF-8');
589 except UnicodeDecodeError:
591 raise; # Re-raise if decoding fails even after trimming
593 def ReadUntilNullByte(fp
, delimiter
=__file_format_dict__
['format_delimiter'], max_read
=1024000):
594 return ReadTillNullByte(fp
, delimiter
, max_read
);
596 def SeekToEndOfFile(fp
):
600 if(lasttell
==fp
.tell()):
602 lasttell
= fp
.tell();
605 def ReadFileHeaderData(fp
, rounds
=0, delimiter
=__file_format_dict__
['format_delimiter']):
609 while(rocount
<roend
):
610 HeaderOut
.append(ReadTillNullByte(fp
, delimiter
));
611 rocount
= rocount
+ 1;
614 def ReadFileHeaderDataBySize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
615 headerpresize
= ReadTillNullByte(fp
, delimiter
);
616 headersize
= int(headerpresize
, 16);
619 headercontent
= str(fp
.read(headersize
).decode('UTF-8')).split(delimiter
);
622 roend
= int(len(headercontent
));
623 HeaderOut
= [headerpresize
];
624 while(rocount
<roend
):
625 HeaderOut
.append(headercontent
[rocount
]);
626 rocount
= rocount
+ 1;
629 def ReadFileHeaderDataWoSize(fp
, delimiter
=__file_format_dict__
['format_delimiter']):
630 preheaderdata
= ReadFileHeaderData(fp
, 2, delimiter
);
631 headersize
= int(preheaderdata
[0], 16);
632 headernumfields
= int(preheaderdata
[1], 16);
633 if(headersize
<=0 or headernumfields
<=0):
635 headerdata
= ReadFileHeaderData(fp
, headernumfields
, delimiter
);
636 HeaderOut
= preheaderdata
+ headerdata
;
639 def ReadFileHeaderDataBySizeWithContent(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
640 delimiter
= formatspecs
['format_delimiter'];
641 fheaderstart
= fp
.tell();
642 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
643 if(len(HeaderOut
)==0):
645 if(re
.findall("^[.|/]", HeaderOut
[3])):
646 fname
= HeaderOut
[3];
648 fname
= "./"+HeaderOut
[3];
649 fchecksumtype
= HeaderOut
[-3].lower();
650 fcs
= HeaderOut
[-2].lower();
651 fccs
= HeaderOut
[-1].lower();
652 fsize
= int(HeaderOut
[5], 16);
653 fcompression
= HeaderOut
[12];
654 fcsize
= int(HeaderOut
[13], 16);
655 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
656 if(fcs
!=newfcs
and not skipchecksum
):
657 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
658 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
660 fhend
= fp
.tell() - 1;
661 fcontentstart
= fp
.tell();
662 fcontents
= BytesIO();
663 if(fsize
>0 and not listonly
):
664 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
665 fcontents
.write(fp
.read(fsize
));
667 fcontents
.write(fp
.read(fcsize
));
668 elif(fsize
>0 and listonly
):
669 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
673 fcontents
.seek(0, 0);
674 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
675 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
676 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
677 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
679 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
682 fcontents
.seek(0, 0);
683 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
685 fcontentend
= fp
.tell() - 1;
686 HeaderOut
.append(fcontents
);
689 def ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
690 delimiter
= formatspecs
['format_delimiter'];
691 fheaderstart
= fp
.tell();
692 if(formatspecs
['new_style']):
693 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
695 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
696 if(len(HeaderOut
)==0):
698 fheadsize
= int(HeaderOut
[0], 16);
699 fnumfields
= int(HeaderOut
[1], 16);
700 ftype
= int(HeaderOut
[2], 16);
701 if(re
.findall("^[.|/]", HeaderOut
[3])):
702 fname
= HeaderOut
[3];
704 fname
= "./"+HeaderOut
[3];
705 fbasedir
= os
.path
.dirname(fname
);
706 flinkname
= HeaderOut
[4];
707 fsize
= int(HeaderOut
[5], 16);
708 fatime
= int(HeaderOut
[6], 16);
709 fmtime
= int(HeaderOut
[7], 16);
710 fctime
= int(HeaderOut
[8], 16);
711 fbtime
= int(HeaderOut
[9], 16);
712 fmode
= int(HeaderOut
[10], 16);
713 fchmode
= stat
.S_IMODE(fmode
);
714 ftypemod
= stat
.S_IFMT(fmode
);
715 fwinattributes
= int(HeaderOut
[11], 16);
716 fcompression
= HeaderOut
[12];
717 fcsize
= int(HeaderOut
[13], 16);
718 fuid
= int(HeaderOut
[14], 16);
719 funame
= HeaderOut
[15];
720 fgid
= int(HeaderOut
[16], 16);
721 fgname
= HeaderOut
[17];
722 fid
= int(HeaderOut
[18], 16);
723 finode
= int(HeaderOut
[19], 16);
724 flinkcount
= int(HeaderOut
[20], 16);
725 fdev_minor
= int(HeaderOut
[21], 16);
726 fdev_major
= int(HeaderOut
[22], 16);
727 frdev_minor
= int(HeaderOut
[23], 16);
728 frdev_major
= int(HeaderOut
[24], 16);
729 fextrasize
= int(HeaderOut
[25], 16);
730 fextrafields
= int(HeaderOut
[26], 16);
731 extrafieldslist
= [];
733 extraend
= extrastart
+ fextrafields
;
734 extrafieldslist
= [];
735 if(extrastart
<extraend
):
736 extrafieldslist
.append(HeaderOut
[extrastart
]);
737 extrastart
= extrastart
+ 1;
738 fchecksumtype
= HeaderOut
[extrastart
].lower();
739 fcs
= HeaderOut
[extrastart
+ 1].lower();
740 fccs
= HeaderOut
[extrastart
+ 2].lower();
741 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
742 if(fcs
!=newfcs
and not skipchecksum
):
743 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
744 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
746 fhend
= fp
.tell() - 1;
747 fcontentstart
= fp
.tell();
748 fcontents
= BytesIO();
749 pyhascontents
= False;
750 if(fsize
>0 and not listonly
):
751 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
752 fcontents
.write(fp
.read(fsize
));
754 fcontents
.write(fp
.read(fcsize
));
755 pyhascontents
= True;
756 elif(fsize
>0 and listonly
):
757 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
761 pyhascontents
= False;
762 fcontents
.seek(0, 0);
763 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
764 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
765 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
766 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
768 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
771 fcontents
.seek(0, 0);
772 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
774 fcontentend
= fp
.tell() - 1;
775 fcontents
.seek(0, 0);
776 catlist
= {'fheadersize': fheadsize
, 'fhstart': fheaderstart
, 'fhend': fhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': fchecksumtype
, 'fnumfields': fnumfields
+ 2, 'frawheader': HeaderOut
, 'fextrafields': fextrafields
, 'fextrafieldsize': fextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': fcs
, 'fcontentchecksum': fccs
, 'fhascontents': pyhascontents
, 'fcontentstart': fcontentstart
, 'fcontentend': fcontentend
, 'fcontents': fcontents
};
779 def ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
780 delimiter
= formatspecs
['format_delimiter'];
781 fheaderstart
= fp
.tell();
782 if(formatspecs
['new_style']):
783 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
785 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
786 if(len(HeaderOut
)==0):
788 fheadsize
= int(HeaderOut
[0], 16);
789 fnumfields
= int(HeaderOut
[1], 16);
790 ftype
= int(HeaderOut
[2], 16);
791 if(re
.findall("^[.|/]", HeaderOut
[3])):
792 fname
= HeaderOut
[3];
794 fname
= "./"+HeaderOut
[3];
795 fbasedir
= os
.path
.dirname(fname
);
796 flinkname
= HeaderOut
[4];
797 fsize
= int(HeaderOut
[5], 16);
798 fatime
= int(HeaderOut
[6], 16);
799 fmtime
= int(HeaderOut
[7], 16);
800 fctime
= int(HeaderOut
[8], 16);
801 fbtime
= int(HeaderOut
[9], 16);
802 fmode
= int(HeaderOut
[10], 16);
803 fchmode
= stat
.S_IMODE(fmode
);
804 ftypemod
= stat
.S_IFMT(fmode
);
805 fwinattributes
= int(HeaderOut
[11], 16);
806 fcompression
= HeaderOut
[12];
807 fcsize
= int(HeaderOut
[13], 16);
808 fuid
= int(HeaderOut
[14], 16);
809 funame
= HeaderOut
[15];
810 fgid
= int(HeaderOut
[16], 16);
811 fgname
= HeaderOut
[17];
812 fid
= int(HeaderOut
[18], 16);
813 finode
= int(HeaderOut
[19], 16);
814 flinkcount
= int(HeaderOut
[20], 16);
815 fdev_minor
= int(HeaderOut
[21], 16);
816 fdev_major
= int(HeaderOut
[22], 16);
817 frdev_minor
= int(HeaderOut
[23], 16);
818 frdev_major
= int(HeaderOut
[24], 16);
819 fextrasize
= int(HeaderOut
[25], 16);
820 fextrafields
= int(HeaderOut
[26], 16);
821 extrafieldslist
= [];
823 extraend
= extrastart
+ fextrafields
;
824 extrafieldslist
= [];
825 if(extrastart
<extraend
):
826 extrafieldslist
.append(HeaderOut
[extrastart
]);
827 extrastart
= extrastart
+ 1;
828 fchecksumtype
= HeaderOut
[extrastart
].lower();
829 fcs
= HeaderOut
[extrastart
+ 1].lower();
830 fccs
= HeaderOut
[extrastart
+ 2].lower();
831 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
832 if(fcs
!=newfcs
and not skipchecksum
):
833 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
834 VerbosePrintOut("'" + str(fcs
) + "' != " + "'" + str(newfcs
) + "'");
836 fhend
= fp
.tell() - 1;
837 fcontentstart
= fp
.tell();
838 fcontents
= BytesIO();
839 pyhascontents
= False;
840 if(fsize
>0 and not listonly
):
841 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
842 fcontents
.write(fp
.read(fsize
));
844 fcontents
.write(fp
.read(fcsize
));
845 pyhascontents
= True;
846 elif(fsize
>0 and listonly
):
847 if(fcompression
=="none" or fcompression
=="" or fcompression
=="atuo"):
851 pyhascontents
= False;
852 fcontents
.seek(0, 0);
853 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
854 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
855 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
856 VerbosePrintOut("'" + str(fccs
) + "' != " + "'" + str(newfccs
) + "'");
858 if(fcompression
=="none" or fcompression
=="" or fcompression
=="auto"):
861 fcontents
.seek(0, 0);
862 fcontents
= UncompressArchiveFile(fcontents
, formatspecs
);
864 fcontentend
= fp
.tell() - 1;
865 catlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrafieldslist
, fchecksumtype
, fcontents
];
868 def ReadFileDataBySizeWithContent(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
869 delimiter
= formatspecs
['format_delimiter'];
873 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
876 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
877 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
878 if(not headercheck
and not skipchecksum
):
879 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
880 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
882 fnumfiles
= int(catheader
[1], 16);
885 while(countnum
< fnumfiles
):
886 HeaderOut
= ReadFileHeaderDataBySizeWithContent(fp
, listonly
, skipchecksum
, formatspecs
)
887 if(len(HeaderOut
)==0):
889 flist
.append(HeaderOut
);
890 countnum
= countnum
+ 1;
893 def ReadFileDataBySizeWithContentToArray(fp
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
894 delimiter
= formatspecs
['format_delimiter'];
898 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
901 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
902 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
903 if(not headercheck
and not skipchecksum
):
904 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
905 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
907 catstring
= catheader
[0];
908 catversion
= re
.findall(r
"([\d]+)$", catstring
);
909 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
910 fprenumfiles
= catheader
[1];
911 fnumfiles
= int(fprenumfiles
, 16);
912 fprechecksumtype
= catheader
[2];
913 fprechecksum
= catheader
[3];
914 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
915 if(seekstart
<0 and seekstart
>fnumfiles
):
917 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
919 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
920 seekend
= fnumfiles
- abs(seekend
);
923 while(il
< seekstart
):
924 prefhstart
= fp
.tell();
925 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
926 if(len(preheaderdata
)==0):
928 prefsize
= int(preheaderdata
[5], 16);
929 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-3].lower(), True, formatspecs
);
930 prefcs
= preheaderdata
[-2];
931 if(prefcs
!=prenewfcs
and not skipchecksum
):
932 VVerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
933 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
935 valid_archive
= False;
936 invalid_archive
= True;
937 prefhend
= fp
.tell() - 1;
938 prefcontentstart
= fp
.tell();
939 prefcontents
= BytesIO();
940 pyhascontents
= False;
942 prefcontents
.write(fp
.read(prefsize
));
943 prefcontents
.seek(0, 0);
944 prenewfccs
= GetFileChecksum(prefcontents
.read(), preheaderdata
[-3].lower(), False, formatspecs
);
945 prefccs
= preheaderdata
[-1];
946 pyhascontents
= True;
947 if(prefccs
!=prenewfccs
and not skipchecksum
):
948 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
949 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
954 countnum
= seekstart
;
955 while(countnum
< seekend
):
956 catlist
['ffilelist'].update({realidnum
: {'fid': realidnum
, 'fidalt': realidnum
}});
957 HeaderOut
= ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
, skipchecksum
, formatspecs
);
958 if(len(HeaderOut
)==0):
960 catlist
['ffilelist'][realidnum
].update(HeaderOut
);
961 countnum
= countnum
+ 1;
962 realidnum
= realidnum
+ 1;
965 def ReadFileDataBySizeWithContentToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
966 if(isinstance(infile
, dict)):
967 listcatfiles
= infile
;
969 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
971 listcatfiles
= ReadFileDataBySizeWithContentToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
);
972 if(not listcatfiles
):
974 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
975 lenlist
= len(listcatfiles
['ffilelist']);
977 lcfx
= int(listcatfiles
['fnumfiles']);
978 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
981 lcfx
= int(listcatfiles
['fnumfiles']);
983 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
984 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
985 catarray
['filetoid'].update(filetoidarray
);
986 catarray
['idtofile'].update(idtofilearray
);
987 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
988 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
989 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
990 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
991 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
992 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
993 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
994 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
995 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
996 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
997 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
998 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
999 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1000 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
1001 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
1002 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
1003 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1004 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1005 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
1006 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
1007 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
1008 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1009 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1010 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
1011 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
1012 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
1013 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
1014 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1015 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1016 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1017 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1021 def ReadFileDataBySizeWithContentToList(fp
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1022 delimiter
= formatspecs
['format_delimiter'];
1026 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
1029 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
1030 newfcs
= GetHeaderChecksum(catheader
[:-2], catheader
[2], True, formatspecs
);
1031 if(not headercheck
and not skipchecksum
):
1032 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1033 VerbosePrintOut("'" + str(newfcs
) + "' != " + "'" + str(catheader
[3]) + "'");
1035 catstring
= catheader
[0];
1036 catversion
= re
.findall(r
"([\d]+)$", catstring
);
1037 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
1038 fprenumfiles
= catheader
[1];
1039 fnumfiles
= int(fprenumfiles
, 16);
1040 fprechecksumtype
= catheader
[2];
1041 fprechecksum
= catheader
[3];
1043 if(seekstart
<0 and seekstart
>fnumfiles
):
1045 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
1046 seekend
= fnumfiles
;
1047 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
1048 seekend
= fnumfiles
- abs(seekend
);
1051 while(il
< seekstart
):
1052 prefhstart
= fp
.tell();
1053 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
['format_delimiter']);
1054 if(len(preheaderdata
)==0):
1056 prefsize
= int(preheaderdata
[5], 16);
1057 prefcompression
= preheaderdata
[12];
1058 prefcsize
= int(preheaderdata
[13], 16);
1059 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-3].lower(), True, formatspecs
);
1060 prefcs
= preheaderdata
[-2];
1061 if(prefcs
!=prenewfcs
and not skipchecksum
):
1062 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
1063 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
1065 valid_archive
= False;
1066 invalid_archive
= True;
1067 prefhend
= fp
.tell() - 1;
1068 prefcontentstart
= fp
.tell();
1070 pyhascontents
= False;
1072 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
1073 prefcontents
= catfp
.read(prefsize
);
1075 prefcontents
= catfp
.read(prefcsize
);
1076 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
1077 prefccs
= preheaderdata
[-1];
1078 pyhascontents
= True;
1079 if(prefccs
!=prenewfccs
and not skipchecksum
):
1080 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
1081 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
1086 countnum
= seekstart
;
1087 while(countnum
< seekend
):
1088 HeaderOut
= ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
, skipchecksum
, formatspecs
);
1089 if(len(HeaderOut
)==0):
1091 catlist
.append(HeaderOut
);
1092 countnum
= countnum
+ 1;
1093 realidnum
= realidnum
+ 1;
1096 def ReadInFileBySizeWithContentToArray(infile
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1097 delimiter
= formatspecs
['format_delimiter'];
1098 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1101 fp
= UncompressArchiveFile(fp
, formatspecs
);
1102 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1103 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1110 if(hasattr(sys
.stdin
, "buffer")):
1111 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1113 shutil
.copyfileobj(sys
.stdin
, fp
);
1115 fp
= UncompressArchiveFile(fp
, formatspecs
);
1119 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1120 fp
= download_file_from_internet_file(infile
);
1121 fp
= UncompressArchiveFile(fp
, formatspecs
);
1127 infile
= RemoveWindowsPath(infile
);
1128 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1129 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1131 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1132 if(not compresscheck
):
1133 fextname
= os
.path
.splitext(infile
)[1];
1134 if(fextname
==".gz"):
1135 compresscheck
= "gzip";
1136 elif(fextname
==".bz2"):
1137 compresscheck
= "bzip2";
1138 elif(fextname
==".zst"):
1139 compresscheck
= "zstd";
1140 elif(fextname
==".lz4" or fextname
==".clz4"):
1141 compresscheck
= "lz4";
1142 elif(fextname
==".lzo" or fextname
==".lzop"):
1143 compresscheck
= "lzo";
1144 elif(fextname
==".lzma" or fextname
==".xz"):
1145 compresscheck
= "lzma";
1148 if(not compresscheck
):
1150 fp
= UncompressFile(infile
, formatspecs
, "rb");
1151 return ReadFileDataBySizeWithContentToArray(fp
, listonly
, skipchecksum
, formatspecs
);
1153 def ReadInFileBySizeWithContentToList(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1154 delimiter
= formatspecs
['format_delimiter'];
1155 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1158 fp
= UncompressArchiveFile(fp
, formatspecs
);
1159 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1160 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1167 if(hasattr(sys
.stdin
, "buffer")):
1168 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1170 shutil
.copyfileobj(sys
.stdin
, fp
);
1172 fp
= UncompressArchiveFile(fp
, formatspecs
);
1176 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1177 fp
= download_file_from_internet_file(infile
);
1178 fp
= UncompressArchiveFile(fp
, formatspecs
);
1184 infile
= RemoveWindowsPath(infile
);
1185 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1186 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
1188 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1189 if(not compresscheck
):
1190 fextname
= os
.path
.splitext(infile
)[1];
1191 if(fextname
==".gz"):
1192 compresscheck
= "gzip";
1193 elif(fextname
==".bz2"):
1194 compresscheck
= "bzip2";
1195 elif(fextname
==".zst"):
1196 compresscheck
= "zstd";
1197 elif(fextname
==".lz4" or fextname
==".clz4"):
1198 compresscheck
= "lz4";
1199 elif(fextname
==".lzo" or fextname
==".lzop"):
1200 compresscheck
= "lzo";
1201 elif(fextname
==".lzma" or fextname
==".xz"):
1202 compresscheck
= "lzma";
1205 if(not compresscheck
):
1207 fp
= UncompressFile(infile
, formatspecs
, "rb");
1208 return ReadFileDataBySizeWithContentToList(fp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
);
1210 def ReadInFileBySizeWithContentToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
):
1211 if(isinstance(infile
, dict)):
1212 listcatfiles
= infile
;
1214 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
1215 infile
= RemoveWindowsPath(infile
);
1216 listcatfiles
= ReadInFileBySizeWithContentToArray(infile
, listonly
, skipchecksum
, formatspecs
);
1217 if(not listcatfiles
):
1219 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1220 lenlist
= len(listcatfiles
['ffilelist']);
1222 lcfx
= int(listcatfiles
['fnumfiles']);
1223 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
1224 lcfx
= int(lenlist
);
1226 lcfx
= int(listcatfiles
['fnumfiles']);
1228 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
1229 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
1230 catarray
['filetoid'].update(filetoidarray
);
1231 catarray
['idtofile'].update(idtofilearray
);
1232 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
1233 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
1234 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
1235 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
1236 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
1237 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
1238 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1239 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1240 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
1241 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1242 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1243 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1244 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1245 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
1246 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
1247 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
1248 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1249 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1250 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
1251 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
1252 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
1253 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1254 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1255 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
1256 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
1257 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
1258 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
1259 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1260 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1261 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1262 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1266 def AppendNullByte(indata
, delimiter
=__file_format_dict__
['format_delimiter']):
1267 outdata
= str(indata
) + delimiter
;
1270 def AppendNullBytes(indata
=[], delimiter
=__file_format_dict__
['format_delimiter']):
1275 outdata
= outdata
+ AppendNullByte(indata
[inum
], delimiter
);
1279 def AppendFileHeader(fp
, numfiles
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1280 delimiter
= formatspecs
['format_delimiter'];
1281 catver
= formatspecs
['format_ver'];
1282 fileheaderver
= str(int(catver
.replace(".", "")));
1283 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
1284 fp
.write(fileheader
.encode('UTF-8'));
1285 fnumfiles
= format(int(numfiles
), 'x').lower();
1286 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
['format_delimiter']);
1287 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
1288 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
1289 fp
.write(fnumfilesa
.encode('UTF-8'));
1292 os
.fsync(fp
.fileno());
1293 except io
.UnsupportedOperation
:
1295 except AttributeError:
1297 except OSError as e
:
1301 def MakeEmptyFilePointer(fp
, checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1302 AppendFileHeader(fp
, 0, checksumtype
, formatspecs
);
1305 def MakeEmptyFile(outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", formatspecs
=__file_format_dict__
, returnfp
=False):
1306 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1307 if(os
.path
.exists(outfile
)):
1310 except OSError as e
:
1314 catfpfp
= BytesIO();
1315 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1317 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1320 fbasename
= os
.path
.splitext(outfile
)[0];
1321 fextname
= os
.path
.splitext(outfile
)[1];
1322 catfp
= CompressOpenFile(outfile
, True, compressionlevel
);
1323 catfp
= AppendFileHeader(catfp
, 0, checksumtype
, formatspecs
);
1324 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1325 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1328 os
.fsync(catfp
.fileno());
1329 except io
.UnsupportedOperation
:
1331 except AttributeError:
1333 except OSError as e
:
1337 if(hasattr(sys
.stdout
, "buffer")):
1338 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1340 shutil
.copyfileobj(catfp
, sys
.stdout
);
1341 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1342 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1344 upload_file_to_internet_file(catfp
, outfile
);
1352 def AppendFileHeaderWithContent(fp
, filevalues
=[], extradata
=[], filecontent
="", checksumtype
="crc32", formatspecs
=__file_format_dict__
):
1353 extrafields
= format(len(extradata
), 'x').lower();
1354 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
1355 if(len(extradata
)>0):
1356 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1357 extrasizelen
= format(len(extrasizestr
), 'x').lower();
1358 catoutlen
= len(filevalues
) + len(extradata
) + 5;
1359 catoutlenhex
= format(catoutlen
, 'x').lower();
1360 catoutlist
= filevalues
;
1361 catoutlist
.insert(0, catoutlenhex
);
1362 catoutlist
.append(extrasizelen
);
1363 catoutlist
.append(extrafields
);
1364 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
1365 if(len(extradata
)>0):
1366 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
1367 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
1368 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1369 catfilecontentcshex
= GetFileChecksum(filecontent
, checksumtype
, False, formatspecs
);
1370 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1371 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
1372 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
1373 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1374 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
1375 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
1376 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
1377 catfileout
= catfileoutstrecd
+ filecontent
+ nullstrecd
;
1378 fp
.write(catfileout
);
1381 os
.fsync(fp
.fileno());
1382 except io
.UnsupportedOperation
:
1384 except AttributeError:
1386 except OSError as e
:
1390 def AppendFilesWithContent(infiles
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1391 advancedlist
= formatspecs
['use_advanced_list'];
1392 altinode
= formatspecs
['use_alt_inode'];
1394 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1396 for line
in sys
.stdin
:
1397 infilelist
.append(line
.strip());
1398 infilelist
= list(filter(None, infilelist
));
1399 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
1400 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
1402 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
1403 for line
in finfile
:
1404 infilelist
.append(line
.strip());
1405 infilelist
= list(filter(None, infilelist
));
1407 if(isinstance(infiles
, (list, tuple, ))):
1408 infilelist
= list(filter(None, infiles
));
1409 elif(isinstance(infiles
, (str, ))):
1410 infilelist
= list(filter(None, [infiles
]));
1412 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
1414 GetDirList
= ListDir(infilelist
, followlink
, False);
1422 inodetocatinode
= {};
1423 numfiles
= int(len(GetDirList
));
1424 fnumfiles
= format(numfiles
, 'x').lower();
1425 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1426 for curfname
in GetDirList
:
1427 if(re
.findall("^[.|/]", curfname
)):
1430 fname
= "./"+curfname
;
1432 VerbosePrintOut(fname
);
1433 if(not followlink
or followlink
is None):
1434 fstatinfo
= os
.lstat(fname
);
1436 fstatinfo
= os
.stat(fname
);
1437 fpremode
= fstatinfo
.st_mode
;
1438 finode
= fstatinfo
.st_ino
;
1439 flinkcount
= fstatinfo
.st_nlink
;
1441 if(stat
.S_ISREG(fpremode
)):
1443 elif(stat
.S_ISLNK(fpremode
)):
1445 elif(stat
.S_ISCHR(fpremode
)):
1447 elif(stat
.S_ISBLK(fpremode
)):
1449 elif(stat
.S_ISDIR(fpremode
)):
1451 elif(stat
.S_ISFIFO(fpremode
)):
1453 elif(stat
.S_ISSOCK(fpremode
)):
1455 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
1457 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
1459 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
1464 fcurfid
= format(int(curfid
), 'x').lower();
1465 if(not followlink
and finode
!=0):
1467 if(finode
in inodelist
):
1469 flinkname
= inodetofile
[finode
];
1471 fcurinode
= format(int(finode
), 'x').lower();
1473 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
1474 if(finode
not in inodelist
):
1475 inodelist
.append(finode
);
1476 inodetofile
.update({finode
: fname
});
1477 inodetocatinode
.update({finode
: curinode
});
1479 fcurinode
= format(int(finode
), 'x').lower();
1481 fcurinode
= format(int(curinode
), 'x').lower();
1482 curinode
= curinode
+ 1;
1484 fcurinode
= format(int(curinode
), 'x').lower();
1485 curinode
= curinode
+ 1;
1486 curfid
= curfid
+ 1;
1488 flinkname
= os
.readlink(fname
);
1489 fdev
= fstatinfo
.st_dev
;
1490 getfdev
= GetDevMajorMinor(fdev
);
1491 fdev_minor
= getfdev
[0];
1492 fdev_major
= getfdev
[1];
1493 frdev
= fstatinfo
.st_dev
;
1494 if(hasattr(fstatinfo
, "st_rdev")):
1495 frdev
= fstatinfo
.st_rdev
;
1497 frdev
= fstatinfo
.st_dev
;
1498 getfrdev
= GetDevMajorMinor(frdev
);
1499 frdev_minor
= getfrdev
[0];
1500 frdev_major
= getfrdev
[1];
1501 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
1502 fsize
= format(int("0"), 'x').lower();
1503 elif(ftype
==0 or ftype
==7):
1504 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
1506 fsize
= format(int(fstatinfo
.st_size
)).lower();
1507 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
1508 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
1509 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1510 if(hasattr(fstatinfo
, "st_birthtime")):
1511 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
1513 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1514 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
1515 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
1516 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
1517 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
1518 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
1523 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
1524 funame
= userinfo
.pw_name
;
1533 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
1534 fgname
= groupinfo
.gr_name
;
1539 fdev_minor
= format(int(fdev_minor
), 'x').lower();
1540 fdev_major
= format(int(fdev_major
), 'x').lower();
1541 frdev_minor
= format(int(frdev_minor
), 'x').lower();
1542 frdev_major
= format(int(frdev_major
), 'x').lower();
1543 finode
= format(int(finode
), 'x').lower();
1544 flinkcount
= format(int(flinkcount
), 'x').lower();
1545 if(hasattr(fstatinfo
, "st_file_attributes")):
1546 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
1548 fwinattributes
= format(int(0), 'x').lower();
1550 fcsize
= format(int(0), 'x').lower();
1551 fcontents
= BytesIO();
1553 if(ftype
==0 or ftype
==7):
1554 with
open(fname
, "rb") as fpc
:
1555 shutil
.copyfileobj(fpc
, fcontents
);
1556 if(not compresswholefile
):
1557 fcontents
.seek(0, 2);
1558 ucfsize
= fcontents
.tell();
1559 fcontents
.seek(0, 0);
1560 if(compression
=="auto"):
1561 ilsize
= len(compressionlistalt
);
1564 while(ilmin
< ilsize
):
1565 cfcontents
= BytesIO();
1566 shutil
.copyfileobj(fcontents
, cfcontents
);
1567 fcontents
.seek(0, 0);
1568 cfcontents
.seek(0, 0);
1569 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1571 cfcontents
.seek(0, 2);
1572 ilcsize
.append(cfcontents
.tell());
1576 ilcsize
.append(sys
.maxint
);
1577 except AttributeError:
1578 ilcsize
.append(sys
.maxsize
);
1580 ilcmin
= ilcsize
.index(min(ilcsize
));
1581 compression
= compressionlistalt
[ilcmin
];
1582 fcontents
.seek(0, 0);
1583 cfcontents
= BytesIO();
1584 shutil
.copyfileobj(fcontents
, cfcontents
);
1585 cfcontents
.seek(0, 0);
1586 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1587 cfcontents
.seek(0, 2);
1588 cfsize
= cfcontents
.tell();
1589 if(ucfsize
> cfsize
):
1590 fcsize
= format(int(cfsize
), 'x').lower();
1591 fcompression
= compression
;
1593 fcontents
= cfcontents
;
1594 if(followlink
and (ftype
==1 or ftype
==2)):
1595 flstatinfo
= os
.stat(flinkname
);
1596 with
open(flinkname
, "rb") as fpc
:
1597 shutil
.copyfileobj(fpc
, fcontents
);
1598 if(not compresswholefile
):
1599 fcontents
.seek(0, 2);
1600 ucfsize
= fcontents
.tell();
1601 fcontents
.seek(0, 0);
1602 if(compression
=="auto"):
1603 ilsize
= len(compressionlistalt
);
1606 while(ilmin
< ilsize
):
1607 cfcontents
= BytesIO();
1608 shutil
.copyfileobj(fcontents
, cfcontents
);
1609 fcontents
.seek(0, 0);
1610 cfcontents
.seek(0, 0);
1611 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
1613 cfcontents
.seek(0, 2);
1614 ilcsize
.append(cfcontents
.tell());
1618 ilcsize
.append(sys
.maxint
);
1619 except AttributeError:
1620 ilcsize
.append(sys
.maxsize
);
1622 ilcmin
= ilcsize
.index(min(ilcsize
));
1623 compression
= compressionlistalt
[ilcmin
];
1624 fcontents
.seek(0, 0);
1625 cfcontents
= BytesIO();
1626 shutil
.copyfileobj(fcontents
, cfcontents
);
1627 cfcontents
.seek(0, 0);
1628 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
1629 cfcontents
.seek(0, 2);
1630 cfsize
= cfcontents
.tell();
1631 if(ucfsize
> cfsize
):
1632 fcsize
= format(int(cfsize
), 'x').lower();
1633 fcompression
= compression
;
1635 fcontents
= cfcontents
;
1636 if(fcompression
=="none"):
1638 fcontents
.seek(0, 0);
1639 ftypehex
= format(ftype
, 'x').lower();
1640 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
1641 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1643 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1647 def AppendListsWithContent(inlist
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1649 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1650 GetDirList
= inlist
;
1658 inodetocatinode
= {};
1659 numfiles
= int(len(GetDirList
));
1660 fnumfiles
= format(numfiles
, 'x').lower();
1661 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1662 for curfname
in GetDirList
:
1663 ftype
= format(curfname
[0], 'x').lower();
1664 if(re
.findall("^[.|/]", curfname
[1])):
1665 fname
= curfname
[1];
1667 fname
= "./"+curfname
[1];
1668 fbasedir
= os
.path
.dirname(fname
);
1669 flinkname
= curfname
[2];
1670 fsize
= format(curfname
[3], 'x').lower();
1671 fatime
= format(curfname
[4], 'x').lower();
1672 fmtime
= format(curfname
[5], 'x').lower();
1673 fctime
= format(curfname
[6], 'x').lower();
1674 fbtime
= format(curfname
[7], 'x').lower();
1675 fmode
= format(curfname
[8], 'x').lower();
1676 fwinattributes
= format(curfname
[9], 'x').lower();
1677 fcompression
= curfname
[10];
1678 fcsize
= format(curfname
[11], 'x').lower();
1679 fuid
= format(curfname
[12], 'x').lower();
1680 funame
= curfname
[13];
1681 fgid
= format(curfname
[14], 'x').lower();
1682 fgname
= curfname
[15];
1683 fid
= format(curfname
[16], 'x').lower();
1684 finode
= format(curfname
[17], 'x').lower();
1685 flinkcount
= format(curfname
[18], 'x').lower();
1686 fdev_minor
= format(curfname
[19], 'x').lower();
1687 fdev_major
= format(curfname
[20], 'x').lower();
1688 frdev_minor
= format(curfname
[21], 'x').lower();
1689 frdev_major
= format(curfname
[22], 'x').lower();
1690 extradata
= curfname
[23];
1691 fchecksumtype
= curfname
[24];
1692 fcontents
= curfname
[25];
1693 catoutlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
1694 fcontents
.seek(0, 0);
1695 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1697 fp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
1700 def AppendInFileWithContent(infile
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False):
1701 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, False, formatspecs
);
1702 return AppendListsWithContent(inlist
, fp
, dirlistfromtxt
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
);
1704 def AppendFilesWithContentToOutFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1705 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1706 if(os
.path
.exists(outfile
)):
1709 except OSError as e
:
1713 catfpfp
= BytesIO();
1714 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1716 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1719 fbasename
= os
.path
.splitext(outfile
)[0];
1720 fextname
= os
.path
.splitext(outfile
)[1];
1721 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1722 catfp
= AppendFilesWithContent(infiles
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1723 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1724 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1727 os
.fsync(catfp
.fileno());
1728 except io
.UnsupportedOperation
:
1730 except AttributeError:
1732 except OSError as e
:
1736 if(hasattr(sys
.stdout
, "buffer")):
1737 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1739 shutil
.copyfileobj(catfp
, sys
.stdout
);
1740 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1741 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1743 upload_file_to_internet_file(catfp
, outfile
);
1751 def AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1752 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1753 if(os
.path
.exists(outfile
)):
1756 except OSError as e
:
1760 catfpfp
= BytesIO();
1761 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1763 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1766 fbasename
= os
.path
.splitext(outfile
)[0];
1767 fextname
= os
.path
.splitext(outfile
)[1];
1768 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
1769 catfp
= AppendListsWithContent(inlist
, catfp
, dirlistfromtxt
, filevalues
, extradata
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
);
1770 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1771 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1774 os
.fsync(catfp
.fileno());
1775 except io
.UnsupportedOperation
:
1777 except AttributeError:
1779 except OSError as e
:
1783 if(hasattr(sys
.stdout
, "buffer")):
1784 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1786 shutil
.copyfileobj(catfp
, sys
.stdout
);
1787 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1788 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
1790 upload_file_to_internet_file(catfp
, outfile
);
1798 def AppendInFileWithContentToOutFile(infile
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
1799 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, False, formatspecs
);
1800 return AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
, returnfp
);
1802 def PrintPermissionString(fchmode
, ftype
):
1803 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1805 for fmodval
in str(oct(fchmode
))[-3:]:
1806 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
1807 if(ftype
==0 or ftype
==7):
1808 permissionstr
= "-" + permissionstr
;
1810 permissionstr
= "h" + permissionstr
;
1812 permissionstr
= "l" + permissionstr
;
1814 permissionstr
= "c" + permissionstr
;
1816 permissionstr
= "b" + permissionstr
;
1818 permissionstr
= "d" + permissionstr
;
1820 permissionstr
= "f" + permissionstr
;
1822 permissionstr
= "D" + permissionstr
;
1824 permissionstr
= "p" + permissionstr
;
1826 permissionstr
= "w" + permissionstr
;
1828 permissionoutstr
= stat
.filemode(fchmode
);
1829 except AttributeError:
1830 permissionoutstr
= permissionstr
;
1832 permissionoutstr
= permissionstr
;
1833 return permissionoutstr
;
1835 def PrintPermissionStringAlt(fchmode
, ftype
):
1837 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1838 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1840 # Translate file mode into permission string
1841 permissionstr
= ''.join([permissions
[i
] for i
in str(oct(fchmode
))[-3:]]);
1842 # Append file type indicator
1844 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1845 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1847 file_type
= type_indicators
.get(ftype
, '-');
1848 permissionstr
= file_type
+ permissionstr
;
1850 permissionoutstr
= stat
.filemode(fchmode
);
1851 except AttributeError:
1852 permissionoutstr
= permissionstr
;
1853 return permissionoutstr
;
1855 def CompressionSupport():
1856 compression_list
= [];
1859 compression_list
.append("gz");
1860 compression_list
.append("gzip");
1865 compression_list
.append("bz2");
1866 compression_list
.append("bzip2");
1871 compression_list
.append("lz4");
1876 compression_list
.append("lzo");
1877 compression_list
.append("lzop");
1882 compression_list
.append("zstd");
1883 compression_list
.append("zstandard");
1888 compression_list
.append("lzma");
1889 compression_list
.append("xz");
1892 return compression_list
;
1894 def CheckCompressionType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
1895 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1899 catfp
= open(infile
, "rb");
1900 except FileNotFoundError
:
1903 prefp
= catfp
.read(2);
1905 if(prefp
==binascii
.unhexlify("1f8b")):
1908 prefp
= catfp
.read(3);
1909 if(prefp
==binascii
.unhexlify("425a68")):
1911 if(prefp
==binascii
.unhexlify("5d0000")):
1914 prefp
= catfp
.read(4);
1915 if(prefp
==binascii
.unhexlify("28b52ffd")):
1917 if(prefp
==binascii
.unhexlify("04224d18")):
1919 if(prefp
==binascii
.unhexlify("504B0304")):
1920 filetype
= "zipfile";
1922 prefp
= catfp
.read(5);
1923 if(prefp
==binascii
.unhexlify("7573746172")):
1924 filetype
= "tarfile";
1926 prefp
= catfp
.read(6);
1927 if(prefp
==binascii
.unhexlify("fd377a585a00")):
1929 if(prefp
==binascii
.unhexlify("377abcaf271c")):
1930 filetype
= "7zipfile";
1932 prefp
= catfp
.read(7);
1933 if(prefp
==binascii
.unhexlify("526172211a0700")):
1934 filetype
= "rarfile";
1935 if(prefp
==binascii
.unhexlify("43617446696c65")):
1936 filetype
= "catfile";
1938 prefp
= catfp
.read(8);
1939 if(prefp
==binascii
.unhexlify("526172211a070100")):
1940 filetype
= "rarfile";
1942 prefp
= catfp
.read(formatspecs
['format_len']);
1943 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
1944 filetype
= formatspecs
['format_lower'];
1946 prefp
= catfp
.read(9);
1947 if(prefp
==binascii
.unhexlify("894c5a4f000d0a1a0a")):
1950 prefp
= catfp
.read(10);
1951 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
1952 filetype
= "tarfile";
1954 if(filetype
=="gzip" or filetype
=="bzip2" or filetype
=="lzma" or filetype
=="zstd" or filetype
=="lz4"):
1955 if(is_tarfile(catfp
)):
1956 filetype
= "tarfile";
1958 if(is_tarfile(catfp
)):
1959 filetype
= "tarfile";
1960 elif(zipfile
.is_zipfile(catfp
)):
1961 filetype
= "zipfile";
1962 elif(rarfile
.is_rarfile(catfp
) or rarfile
.is_rarfile_sfx(catfp
)):
1963 filetype
= "rarile";
1971 def CheckCompressionTypeFromString(instring
, formatspecs
=__file_format_dict__
, closefp
=True):
1973 instringsfile
= BytesIO(instring
);
1975 instringsfile
= BytesIO(instring
.encode("UTF-8"));
1976 return CheckCompressionType(instringsfile
, formatspecs
, closefp
);
1978 def GetCompressionMimeType(infile
, formatspecs
=__file_format_dict__
):
1979 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
1980 if(compresscheck
=="gzip" or compresscheck
=="gz"):
1981 return archivefile_gzip_mimetype
;
1982 if(compresscheck
=="bzip2" or compresscheck
=="bz2"):
1983 return archivefile_bzip2_mimetype
;
1984 if(compresscheck
=="zstd" or compresscheck
=="zstandard"):
1985 return archivefile_zstandard_mimetype
;
1986 if(compresscheck
=="lz4"):
1987 return archivefile_lz4_mimetype
;
1988 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
1989 return archivefile_lzop_mimetype
;
1990 if(compresscheck
=="lzma"):
1991 return archivefile_lzma_mimetype
;
1992 if(compresscheck
=="xz"):
1993 return archivefile_xz_mimetype
;
1994 if(compresscheck
=="catfile" or compresscheck
=="cat" or compresscheck
==formatspecs
['format_lower']):
1995 return archivefile_cat_mimetype
;
1996 if(not compresscheck
):
2000 def UncompressArchiveFile(fp
, formatspecs
=__file_format_dict__
):
2001 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2003 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
2004 if(compresscheck
=="gzip"):
2009 catfp
= gzip
.GzipFile(fileobj
=fp
, mode
="rb");
2010 if(compresscheck
=="bzip2"):
2016 catfp
.write(bz2
.decompress(fp
.read()));
2017 if(compresscheck
=="zstd"):
2023 catfp
.write(zstandard
.decompress(fp
.read()));
2024 if(compresscheck
=="lz4"):
2030 catfp
.write(lz4
.frame
.decompress(fp
.read()));
2031 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
2037 catfp
.write(lzo
.decompress(fp
.read()));
2038 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2044 catfp
.write(lzma
.decompress(fp
.read()));
2045 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2047 if(not compresscheck
):
2053 with fp
as fpcontent
:
2055 catfp
.write(lzma
.decompress(fp
.read()));
2056 except lzma
.LZMAError
:
2058 if(compresscheck
!="catfile" or compresscheck
!=formatspecs
['format_lower']):
2062 create_alias_function("Uncompress", __file_format_name__
, "", UncompressArchiveFile
);
2064 def UncompressFile(infile
, formatspecs
=__file_format_dict__
, mode
="rb"):
2065 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2066 if(sys
.version_info
[0]==2 and compresscheck
):
2072 if(compresscheck
=="gzip"):
2078 filefp
= gzip
.open(infile
, mode
, encoding
="UTF-8");
2079 except (ValueError, TypeError) as e
:
2080 filefp
= gzip
.open(infile
, mode
);
2081 if(compresscheck
=="bzip2"):
2087 filefp
= bz2
.open(infile
, mode
, encoding
="UTF-8");
2088 except (ValueError, TypeError) as e
:
2089 filefp
= bz2
.open(infile
, mode
);
2090 if(compresscheck
=="zstd"):
2096 filefp
= zstandard
.open(infile
, mode
, encoding
="UTF-8");
2097 except (ValueError, TypeError) as e
:
2098 filefp
= zstandard
.open(infile
, mode
);
2099 if(compresscheck
=="lz4"):
2105 filefp
= lz4
.frame
.open(infile
, mode
, encoding
="UTF-8");
2106 except (ValueError, TypeError) as e
:
2107 filefp
= lz4
.frame
.open(infile
, mode
);
2108 if(compresscheck
=="lzo"):
2114 filefp
= lzo
.open(infile
, mode
, encoding
="UTF-8");
2115 except (ValueError, TypeError) as e
:
2116 filefp
= lzo
.open(infile
, mode
);
2117 if(compresscheck
=="lzma"):
2123 filefp
= lzma
.open(infile
, mode
, encoding
="UTF-8");
2124 except (ValueError, TypeError) as e
:
2125 filefp
= lzma
.open(infile
, mode
);
2126 if(compresscheck
=="catfile" or compresscheck
==formatspecs
['format_lower']):
2128 filefp
= open(infile
, mode
, encoding
="UTF-8");
2129 except (ValueError, TypeError) as e
:
2130 filefp
= open(infile
, mode
);
2131 if(not compresscheck
):
2133 filefp
= open(infile
, mode
, encoding
="UTF-8");
2134 except (ValueError, TypeError) as e
:
2135 filefp
= open(infile
, mode
);
2136 except FileNotFoundError
:
2140 def UncompressString(infile
):
2141 compresscheck
= CheckCompressionTypeFromString(infile
, formatspecs
, False);
2142 if(compresscheck
=="gzip"):
2147 fileuz
= gzip
.decompress(infile
);
2148 if(compresscheck
=="bzip2"):
2153 fileuz
= bz2
.decompress(infile
);
2154 if(compresscheck
=="zstd"):
2159 fileuz
= zstandard
.decompress(infile
);
2160 if(compresscheck
=="lz4"):
2165 fileuz
= lz4
.frame
.decompress(infile
);
2166 if(compresscheck
=="lzo"):
2171 fileuz
= lzo
.decompress(infile
);
2172 if(compresscheck
=="lzma"):
2177 fileuz
= lzma
.decompress(infile
);
2178 if(not compresscheck
):
2180 if(hasattr(fileuz
, 'decode')):
2181 fileuz
= fileuz
.decode("UTF-8");
2184 def UncompressStringAlt(infile
):
2185 filefp
= StringIO();
2186 outstring
= UncompressString(infile
);
2187 filefp
.write(outstring
);
2191 def CheckCompressionSubType(infile
, formatspecs
=__file_format_dict__
, closefp
=True):
2192 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2193 if(not compresscheck
):
2194 fextname
= os
.path
.splitext(infile
)[1];
2195 if(fextname
==".gz"):
2196 compresscheck
= "gzip";
2197 elif(fextname
==".bz2"):
2198 compresscheck
= "bzip2";
2199 elif(fextname
==".zst"):
2200 compresscheck
= "zstd";
2201 elif(fextname
==".lz4"):
2202 compresscheck
= "lz4";
2203 elif(fextname
==".lzo" or fextname
==".lzop"):
2204 compresscheck
= "lzo";
2205 elif(fextname
==".lzma" or fextname
==".xz"):
2206 compresscheck
= "lzma";
2209 if(compresscheck
=="gzip" or compresscheck
=="bzip2" or compresscheck
=="lzma" or compresscheck
=="zstd" or compresscheck
=="lz4"):
2210 if(is_tarfile(infile
)):
2211 filetype
= "tarfile";
2212 if(not compresscheck
):
2213 if(is_tarfile(infile
)):
2215 elif(zipfile
.is_zipfile(infile
)):
2217 elif(rarfile
.is_rarfile(infile
) or rarfile
.is_rarfile_sfx(infile
)):
2222 if(compresscheck
=="catfile"):
2224 if(compresscheck
==formatspecs
['format_lower']):
2225 return formatspecs
['format_lower'];
2226 if(compresscheck
=="tarfile"):
2228 if(compresscheck
=="zipfile"):
2230 if(rarfile_support
and compresscheck
=="rarfile"):
2232 if(py7zr_support
and compresscheck
=="7zipfile"):
2234 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2235 catfp
= UncompressArchiveFile(infile
, formatspecs
['format_lower']);
2238 if(compresscheck
=="gzip"):
2243 catfp
= gzip
.GzipFile(infile
, "rb");
2244 if(compresscheck
=="bzip2"):
2249 catfp
= bz2
.BZ2File(infile
, "rb");
2250 if(compresscheck
=="lz4"):
2255 catfp
= lz4
.frame
.open(infile
, "rb");
2256 if(compresscheck
=="zstd"):
2261 catfp
= zstandard
.open(infile
, "rb");
2262 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2267 catfp
= lzma
.open(infile
, "rb");
2268 except FileNotFoundError
:
2271 prefp
= catfp
.read(5);
2272 if(prefp
==binascii
.unhexlify("7573746172")):
2273 filetype
= "tarfile";
2275 prefp
= catfp
.read(7);
2276 if(prefp
==binascii
.unhexlify("43617446696c65")):
2277 filetype
= "catfile";
2279 prefp
= catfp
.read(formatspecs
['format_len']);
2280 if(prefp
==binascii
.unhexlify(formatspecs
['format_hex'])):
2281 filetype
= formatspecs
['format_lower'];
2283 prefp
= catfp
.read(10);
2284 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2285 filetype
= "tarfile";
2291 def GZipCompress(data
, compresslevel
=9):
2296 tmpfp
= tempfile
.NamedTemporaryFile("wb", delete
=False);
2298 tmpfp
= gzip
.GzipFile(tmpfp
.name
, mode
="wb", compresslevel
=compresslevel
);
2302 catfp
= open(tmpfp
.name
, "rb");
2303 except FileNotFoundError
:
2305 catdata
= catfp
.read();
2309 def CompressArchiveFile(fp
, compression
="auto", compressionlevel
=None, formatspecs
=__file_format_dict__
):
2310 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2313 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2314 compression
= "auto";
2315 if(compression
not in compressionlist
and compression
is None):
2316 compression
= "auto";
2317 if(compression
=="gzip"):
2323 if(compressionlevel
is None):
2324 compressionlevel
= 9;
2326 compressionlevel
= int(compressionlevel
);
2327 catfp
.write(gzip
.compress(fp
.read(), compresslevel
=compressionlevel
));
2328 if(compression
=="bzip2"):
2334 if(compressionlevel
is None):
2335 compressionlevel
= 9;
2337 compressionlevel
= int(compressionlevel
);
2338 catfp
.write(bz2
.compress(fp
.read(), compresslevel
=compressionlevel
));
2339 if(compression
=="lz4"):
2345 if(compressionlevel
is None):
2346 compressionlevel
= 9;
2348 compressionlevel
= int(compressionlevel
);
2349 catfp
.write(lz4
.frame
.compress(fp
.read(), compression_level
=compressionlevel
));
2350 if(compression
=="lzo" or compression
=="lzop"):
2356 if(compressionlevel
is None):
2357 compressionlevel
= 9;
2359 compressionlevel
= int(compressionlevel
);
2360 catfp
.write(lzo
.compress(fp
.read(), compresslevel
=compressionlevel
));
2361 if(compression
=="zstd"):
2367 if(compressionlevel
is None):
2368 compressionlevel
= 10;
2370 compressionlevel
= int(compressionlevel
);
2371 catfp
.write(zstandard
.compress(fp
.read(), level
=compressionlevel
));
2372 if(compression
=="lzma"):
2378 if(compressionlevel
is None):
2379 compressionlevel
= 9;
2381 compressionlevel
= int(compressionlevel
);
2382 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]));
2383 if(compression
=="xz"):
2389 if(compressionlevel
is None):
2390 compressionlevel
= 9;
2392 compressionlevel
= int(compressionlevel
);
2393 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]));
2394 if(compression
=="auto" or compression
is None):
2399 create_alias_function("Compress", __file_format_name__
, "", CompressArchiveFile
);
2401 def CompressOpenFile(outfile
, compressionenable
=True, compressionlevel
=None):
2402 if(outfile
is None):
2404 fbasename
= os
.path
.splitext(outfile
)[0];
2405 fextname
= os
.path
.splitext(outfile
)[1];
2406 if(compressionlevel
is None and fextname
!=".zst"):
2407 compressionlevel
= 9;
2408 elif(compressionlevel
is None and fextname
==".zst"):
2409 compressionlevel
= 10;
2411 compressionlevel
= int(compressionlevel
);
2412 if(sys
.version_info
[0]==2):
2417 if(fextname
not in outextlistwd
or not compressionenable
):
2419 outfp
= open(outfile
, "wb", encoding
="UTF-8");
2420 except (ValueError, TypeError) as e
:
2421 outfp
= open(outfile
, "wb");
2422 elif(fextname
==".gz"):
2428 outfp
= gzip
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2429 except (ValueError, TypeError) as e
:
2430 outfp
= gzip
.open(outfile
, mode
, compressionlevel
);
2431 elif(fextname
==".bz2"):
2437 outfp
= bz2
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2438 except (ValueError, TypeError) as e
:
2439 outfp
= bz2
.open(outfile
, mode
, compressionlevel
);
2440 elif(fextname
==".zst"):
2446 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
), encoding
="UTF-8");
2447 except (ValueError, TypeError) as e
:
2448 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
));
2449 elif(fextname
==".xz"):
2455 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}], encoding
="UTF-8");
2456 except (ValueError, TypeError) as e
:
2457 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]);
2458 elif(fextname
==".lz4"):
2464 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
, encoding
="UTF-8");
2465 except (ValueError, TypeError) as e
:
2466 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
);
2467 elif(fextname
==".lzo"):
2473 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
, encoding
="UTF-8");
2474 except (ValueError, TypeError) as e
:
2475 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
);
2476 elif(fextname
==".lzma"):
2482 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}], encoding
="UTF-8");
2483 except (ValueError, TypeError) as e
:
2484 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]);
2485 except FileNotFoundError
:
2489 def GetDevMajorMinor(fdev
):
2491 if(hasattr(os
, "minor")):
2492 retdev
.append(os
.minor(fdev
));
2495 if(hasattr(os
, "major")):
2496 retdev
.append(os
.major(fdev
));
2501 def CheckSumSupport(checkfor
, guaranteed
=True):
2503 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2505 hash_list
= sorted(list(hashlib
.algorithms_available
));
2506 checklistout
= sorted(hash_list
+ ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2507 if(checkfor
in checklistout
):
2512 def CheckSumSupportAlt(checkfor
, guaranteed
=True):
2514 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2516 hash_list
= sorted(list(hashlib
.algorithms_available
));
2517 checklistout
= hash_list
;
2518 if(checkfor
in checklistout
):
2523 def PackArchiveFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2524 advancedlist
= formatspecs
['use_advanced_list'];
2525 altinode
= formatspecs
['use_alt_inode'];
2526 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2527 outfile
= RemoveWindowsPath(outfile
);
2528 checksumtype
= checksumtype
.lower();
2529 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2530 checksumtype
="crc32";
2531 if(checksumtype
=="none"):
2533 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2534 compression
= "auto";
2535 if(compression
not in compressionlist
and compression
is None):
2536 compression
= "auto";
2538 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2539 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2540 if(os
.path
.exists(outfile
)):
2543 except OSError as e
:
2548 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2550 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2553 fbasename
= os
.path
.splitext(outfile
)[0];
2554 fextname
= os
.path
.splitext(outfile
)[1];
2555 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
2556 catver
= formatspecs
['format_ver'];
2557 fileheaderver
= str(int(catver
.replace(".", "")));
2560 for line
in sys
.stdin
:
2561 infilelist
.append(line
.strip());
2562 infilelist
= list(filter(None, infilelist
));
2563 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
2564 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
2566 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
2567 for line
in finfile
:
2568 infilelist
.append(line
.strip());
2569 infilelist
= list(filter(None, infilelist
));
2571 if(isinstance(infiles
, (list, tuple, ))):
2572 infilelist
= list(filter(None, infiles
));
2573 elif(isinstance(infiles
, (str, ))):
2574 infilelist
= list(filter(None, [infiles
]));
2576 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
2578 GetDirList
= ListDir(infilelist
, followlink
, False);
2586 inodetocatinode
= {};
2587 numfiles
= int(len(GetDirList
));
2588 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
2589 for curfname
in GetDirList
:
2590 if(re
.findall("^[.|/]", curfname
)):
2593 fname
= "./"+curfname
;
2595 VerbosePrintOut(fname
);
2596 if(not followlink
or followlink
is None):
2597 fstatinfo
= os
.lstat(fname
);
2599 fstatinfo
= os
.stat(fname
);
2600 fpremode
= fstatinfo
.st_mode
;
2601 finode
= fstatinfo
.st_ino
;
2602 flinkcount
= fstatinfo
.st_nlink
;
2604 if(stat
.S_ISREG(fpremode
)):
2606 elif(stat
.S_ISLNK(fpremode
)):
2608 elif(stat
.S_ISCHR(fpremode
)):
2610 elif(stat
.S_ISBLK(fpremode
)):
2612 elif(stat
.S_ISDIR(fpremode
)):
2614 elif(stat
.S_ISFIFO(fpremode
)):
2616 elif(stat
.S_ISSOCK(fpremode
)):
2618 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
2620 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
2622 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
2627 fcurfid
= format(int(curfid
), 'x').lower();
2628 if(not followlink
and finode
!=0):
2630 if(finode
in inodelist
):
2632 flinkname
= inodetofile
[finode
];
2634 fcurinode
= format(int(finode
), 'x').lower();
2636 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
2637 if(finode
not in inodelist
):
2638 inodelist
.append(finode
);
2639 inodetofile
.update({finode
: fname
});
2640 inodetocatinode
.update({finode
: curinode
});
2642 fcurinode
= format(int(finode
), 'x').lower();
2644 fcurinode
= format(int(curinode
), 'x').lower();
2645 curinode
= curinode
+ 1;
2647 fcurinode
= format(int(curinode
), 'x').lower();
2648 curinode
= curinode
+ 1;
2649 curfid
= curfid
+ 1;
2651 flinkname
= os
.readlink(fname
);
2652 fdev
= fstatinfo
.st_dev
;
2653 getfdev
= GetDevMajorMinor(fdev
);
2654 fdev_minor
= getfdev
[0];
2655 fdev_major
= getfdev
[1];
2656 frdev
= fstatinfo
.st_dev
;
2657 if(hasattr(fstatinfo
, "st_rdev")):
2658 frdev
= fstatinfo
.st_rdev
;
2660 frdev
= fstatinfo
.st_dev
;
2661 getfrdev
= GetDevMajorMinor(frdev
);
2662 frdev_minor
= getfrdev
[0];
2663 frdev_major
= getfrdev
[1];
2664 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
2665 fsize
= format(int("0"), 'x').lower();
2666 elif(ftype
==0 or ftype
==7):
2667 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
2669 fsize
= format(int(fstatinfo
.st_size
)).lower();
2670 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
2671 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
2672 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2673 if(hasattr(fstatinfo
, "st_birthtime")):
2674 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
2676 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2677 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
2678 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
2679 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
2680 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
2681 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
2686 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
2687 funame
= userinfo
.pw_name
;
2696 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
2697 fgname
= groupinfo
.gr_name
;
2702 fdev_minor
= format(int(fdev_minor
), 'x').lower();
2703 fdev_major
= format(int(fdev_major
), 'x').lower();
2704 frdev_minor
= format(int(frdev_minor
), 'x').lower();
2705 frdev_major
= format(int(frdev_major
), 'x').lower();
2706 finode
= format(int(finode
), 'x').lower();
2707 flinkcount
= format(int(flinkcount
), 'x').lower();
2708 if(hasattr(fstatinfo
, "st_file_attributes")):
2709 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
2711 fwinattributes
= format(int(0), 'x').lower();
2713 fcsize
= format(int(0), 'x').lower();
2714 fcontents
= BytesIO();
2715 if(ftype
==0 or ftype
==7):
2716 with
open(fname
, "rb") as fpc
:
2717 shutil
.copyfileobj(fpc
, fcontents
);
2718 if(not compresswholefile
):
2719 fcontents
.seek(0, 2);
2720 ucfsize
= fcontents
.tell();
2721 fcontents
.seek(0, 0);
2722 if(compression
=="auto"):
2723 ilsize
= len(compressionlistalt
);
2726 while(ilmin
< ilsize
):
2727 cfcontents
= BytesIO();
2728 shutil
.copyfileobj(fcontents
, cfcontents
);
2729 fcontents
.seek(0, 0);
2730 cfcontents
.seek(0, 0);
2731 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
2733 cfcontents
.seek(0, 2);
2734 ilcsize
.append(cfcontents
.tell());
2738 ilcsize
.append(sys
.maxint
);
2739 except AttributeError:
2740 ilcsize
.append(sys
.maxsize
);
2742 ilcmin
= ilcsize
.index(min(ilcsize
));
2743 compression
= compressionlistalt
[ilcmin
];
2744 fcontents
.seek(0, 0);
2745 cfcontents
= BytesIO();
2746 shutil
.copyfileobj(fcontents
, cfcontents
);
2747 cfcontents
.seek(0, 0);
2748 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
2749 cfcontents
.seek(0, 2);
2750 cfsize
= cfcontents
.tell();
2751 if(ucfsize
> cfsize
):
2752 fcsize
= format(int(cfsize
), 'x').lower();
2753 fcompression
= compression
;
2755 fcontents
= cfcontents
;
2756 if(fcompression
=="none"):
2758 if(followlink
and (ftype
==1 or ftype
==2)):
2759 flstatinfo
= os
.stat(flinkname
);
2760 with
open(flinkname
, "rb") as fpc
:
2761 shutil
.copyfileobj(fpc
, fcontents
);
2762 if(not compresswholefile
):
2763 fcontents
.seek(0, 2);
2764 ucfsize
= fcontents
.tell();
2765 fcontents
.seek(0, 0);
2766 if(compression
=="auto"):
2767 ilsize
= len(compressionlistalt
);
2770 while(ilmin
< ilsize
):
2771 cfcontents
= BytesIO();
2772 shutil
.copyfileobj(fcontents
, cfcontents
);
2773 fcontents
.seek(0, 0);
2774 cfcontents
.seek(0, 0);
2775 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
2777 cfcontents
.seek(0, 2);
2778 ilcsize
.append(cfcontents
.tell());
2782 ilcsize
.append(sys
.maxint
);
2783 except AttributeError:
2784 ilcsize
.append(sys
.maxsize
);
2786 ilcmin
= ilcsize
.index(min(ilcsize
));
2787 compression
= compressionlistalt
[ilcmin
];
2788 fcontents
.seek(0, 0);
2789 cfcontents
= BytesIO();
2790 shutil
.copyfileobj(fcontents
, cfcontents
);
2791 cfcontents
.seek(0, 0);
2792 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
2793 cfcontents
.seek(0, 2);
2794 cfsize
= cfcontents
.tell();
2795 if(ucfsize
> cfsize
):
2796 fcsize
= format(int(cfsize
), 'x').lower();
2797 fcompression
= compression
;
2799 fcontents
= cfcontents
;
2800 fcontents
.seek(0, 0);
2801 ftypehex
= format(ftype
, 'x').lower();
2802 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
2803 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
2806 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
2807 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2808 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
2811 os
.fsync(catfp
.fileno());
2812 except io
.UnsupportedOperation
:
2814 except AttributeError:
2816 except OSError as e
:
2820 if(hasattr(sys
.stdout
, "buffer")):
2821 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
2823 shutil
.copyfileobj(catfp
, sys
.stdout
);
2824 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2825 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
2827 upload_file_to_internet_file(catfp
, outfile
);
2835 create_alias_function("Pack", __file_format_name__
, "", PackArchiveFile
);
2837 if(hasattr(shutil
, "register_archive_format")):
2838 def PackArchiveFileFunc(archive_name
, source_dir
, **kwargs
):
2839 return PackArchiveFile(source_dir
, archive_name
, False, "auto", True, None, False, "crc32", [], __file_format_dict__
['format_delimiter'], False, False);
2840 create_alias_function("Pack", __file_format_name__
, "Func", PackArchiveFileFunc
);
2842 def PackArchiveFileFromDirList(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2843 return PackArchiveFile(infiles
, outfile
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
2845 create_alias_function("Pack", __file_format_name__
, "FromDirList", PackArchiveFileFromDirList
);
2847 def PackArchiveFileFromTarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
2848 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2849 outfile
= RemoveWindowsPath(outfile
);
2850 checksumtype
= checksumtype
.lower();
2851 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2852 checksumtype
="crc32";
2853 if(checksumtype
=="none"):
2855 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
2856 compression
= "auto";
2857 if(compression
not in compressionlist
and compression
is None):
2858 compression
= "auto";
2860 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2861 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2862 if(os
.path
.exists(outfile
)):
2865 except OSError as e
:
2870 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2872 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2875 fbasename
= os
.path
.splitext(outfile
)[0];
2876 fextname
= os
.path
.splitext(outfile
)[1];
2877 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
2878 catver
= formatspecs
['format_ver'];
2879 fileheaderver
= str(int(catver
.replace(".", "")));
2885 inodetocatinode
= {};
2888 if(hasattr(sys
.stdin
, "buffer")):
2889 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
2891 shutil
.copyfileobj(sys
.stdin
, infile
);
2896 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
2897 infile
= download_file_from_internet_file(infile
);
2902 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
2904 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
2906 if(not tarfile
.is_tarfile(infile
)):
2908 except AttributeError:
2909 if(not is_tarfile(infile
)):
2914 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2915 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
2917 tarfp
= tarfile
.open(infile
, "r");
2918 except FileNotFoundError
:
2920 numfiles
= int(len(tarfp
.getmembers()));
2921 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
2922 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
2923 if(re
.findall("^[.|/]", member
.name
)):
2924 fname
= member
.name
;
2926 fname
= "./"+member
.name
;
2928 VerbosePrintOut(fname
);
2929 fpremode
= member
.mode
;
2930 ffullmode
= member
.mode
;
2934 ffullmode
= member
.mode
+ stat
.S_IFREG
;
2936 elif(member
.isdev()):
2937 ffullmode
= member
.mode
;
2939 elif(member
.islnk()):
2940 ffullmode
= member
.mode
+ stat
.S_IFREG
;
2942 elif(member
.issym()):
2943 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
2945 elif(member
.ischr()):
2946 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
2948 elif(member
.isblk()):
2949 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
2951 elif(member
.isdir()):
2952 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
2954 elif(member
.isfifo()):
2955 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
2957 elif(member
.issparse()):
2958 ffullmode
= member
.mode
;
2961 ffullmode
= member
.mode
;
2964 fcurfid
= format(int(curfid
), 'x').lower();
2965 fcurinode
= format(int(curfid
), 'x').lower();
2966 curfid
= curfid
+ 1;
2968 flinkname
= member
.linkname
;
2969 fdev_minor
= format(int(member
.devminor
), 'x').lower();
2970 fdev_major
= format(int(member
.devmajor
), 'x').lower();
2971 frdev_minor
= format(int(member
.devminor
), 'x').lower();
2972 frdev_major
= format(int(member
.devmajor
), 'x').lower();
2973 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
2974 fsize
= format(int("0"), 'x').lower();
2975 elif(ftype
==0 or ftype
==7):
2976 fsize
= format(int(member
.size
), 'x').lower();
2978 fsize
= format(int(member
.size
), 'x').lower();
2979 fatime
= format(int(member
.mtime
), 'x').lower();
2980 fmtime
= format(int(member
.mtime
), 'x').lower();
2981 fctime
= format(int(member
.mtime
), 'x').lower();
2982 fbtime
= format(int(member
.mtime
), 'x').lower();
2983 fmode
= format(int(ffullmode
), 'x').lower();
2984 fchmode
= format(int(stat
.S_IMODE(ffullmode
)), 'x').lower();
2985 ftypemod
= format(int(stat
.S_IFMT(ffullmode
)), 'x').lower();
2986 fuid
= format(int(member
.uid
), 'x').lower();
2987 fgid
= format(int(member
.gid
), 'x').lower();
2988 funame
= member
.uname
;
2989 fgname
= member
.gname
;
2990 flinkcount
= format(int(flinkcount
), 'x').lower();
2991 fwinattributes
= format(int(0), 'x').lower();
2993 fcsize
= format(int(0), 'x').lower();
2994 fcontents
= BytesIO();
2995 if(ftype
==0 or ftype
==7):
2996 with tarfp
.extractfile(member
) as fpc
:
2997 shutil
.copyfileobj(fpc
, fcontents
);
2998 if(not compresswholefile
):
2999 fcontents
.seek(0, 2);
3000 ucfsize
= fcontents
.tell();
3001 fcontents
.seek(0, 0);
3002 if(compression
=="auto"):
3003 ilsize
= len(compressionlistalt
);
3006 while(ilmin
< ilsize
):
3007 cfcontents
= BytesIO();
3008 shutil
.copyfileobj(fcontents
, cfcontents
);
3009 fcontents
.seek(0, 0);
3010 cfcontents
.seek(0, 0);
3011 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3013 cfcontents
.seek(0, 2);
3014 ilcsize
.append(cfcontents
.tell());
3018 ilcsize
.append(sys
.maxint
);
3019 except AttributeError:
3020 ilcsize
.append(sys
.maxsize
);
3022 ilcmin
= ilcsize
.index(min(ilcsize
));
3023 compression
= compressionlistalt
[ilcmin
];
3024 fcontents
.seek(0, 0);
3025 cfcontents
= BytesIO();
3026 shutil
.copyfileobj(fcontents
, cfcontents
);
3027 cfcontents
.seek(0, 0);
3028 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3029 cfcontents
.seek(0, 2);
3030 cfsize
= cfcontents
.tell();
3031 if(ucfsize
> cfsize
):
3032 fcsize
= format(int(cfsize
), 'x').lower();
3033 fcompression
= compression
;
3035 fcontents
= cfcontents
;
3036 if(fcompression
=="none"):
3038 fcontents
.seek(0, 0);
3039 ftypehex
= format(ftype
, 'x').lower();
3040 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3041 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3044 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3045 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3046 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3049 os
.fsync(catfp
.fileno());
3050 except io
.UnsupportedOperation
:
3052 except AttributeError:
3054 except OSError as e
:
3058 if(hasattr(sys
.stdout
, "buffer")):
3059 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3061 shutil
.copyfileobj(catfp
, sys
.stdout
);
3062 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3063 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3065 upload_file_to_internet_file(catfp
, outfile
);
3073 create_alias_function("Pack", __file_format_name__
, "FromTarFile", PackArchiveFileFromTarFile
);
3075 def PackArchiveFileFromZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3076 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3077 outfile
= RemoveWindowsPath(outfile
);
3078 checksumtype
= checksumtype
.lower();
3079 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3080 checksumtype
="crc32";
3081 if(checksumtype
=="none"):
3083 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3084 compression
= "auto";
3085 if(compression
not in compressionlist
and compression
is None):
3086 compression
= "auto";
3088 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3089 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3090 if(os
.path
.exists(outfile
)):
3093 except OSError as e
:
3098 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3100 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3103 fbasename
= os
.path
.splitext(outfile
)[0];
3104 fextname
= os
.path
.splitext(outfile
)[1];
3105 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3106 catver
= formatspecs
['format_ver'];
3107 fileheaderver
= str(int(catver
.replace(".", "")));
3113 inodetocatinode
= {};
3116 if(hasattr(sys
.stdin
, "buffer")):
3117 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
3119 shutil
.copyfileobj(sys
.stdin
, infile
);
3124 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3125 infile
= download_file_from_internet_file(infile
);
3130 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3134 if(not zipfile
.is_zipfile(infile
)):
3137 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
3138 except FileNotFoundError
:
3140 ziptest
= zipfp
.testzip();
3142 VerbosePrintOut("Bad file found!");
3143 numfiles
= int(len(zipfp
.infolist()));
3144 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3145 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
3146 if(re
.findall("^[.|/]", member
.filename
)):
3147 fname
= member
.filename
;
3149 fname
= "./"+member
.filename
;
3150 zipinfo
= zipfp
.getinfo(member
.filename
);
3152 VerbosePrintOut(fname
);
3153 if(not member
.is_dir()):
3154 fpremode
= int(stat
.S_IFREG
+ 438);
3155 elif(member
.is_dir()):
3156 fpremode
= int(stat
.S_IFDIR
+ 511);
3159 if(not member
.is_dir()):
3161 elif(member
.is_dir()):
3164 fcurfid
= format(int(curfid
), 'x').lower();
3165 fcurinode
= format(int(curfid
), 'x').lower();
3166 curfid
= curfid
+ 1;
3167 fdev_minor
= format(int(0), 'x').lower();
3168 fdev_major
= format(int(0), 'x').lower();
3169 frdev_minor
= format(int(0), 'x').lower();
3170 frdev_major
= format(int(0), 'x').lower();
3172 fsize
= format(int("0"), 'x').lower();
3174 fsize
= format(int(member
.file_size
), 'x').lower();
3176 fsize
= format(int(member
.file_size
), 'x').lower();
3177 fatime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3178 fmtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3179 fctime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3180 fbtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
3181 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
3182 fwinattributes
= format(int(zipinfo
.external_attr
), 'x').lower();
3183 if(not member
.is_dir()):
3184 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3185 fchmode
= stat
.S_IMODE(fmode
);
3186 ftypemod
= stat
.S_IFMT(fmode
);
3187 elif(member
.is_dir()):
3188 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3189 fchmode
= stat
.S_IMODE(fmode
);
3190 ftypemod
= stat
.S_IFMT(fmode
);
3191 elif(zipinfo
.create_system
==3):
3192 fwinattributes
= format(int(0), 'x').lower();
3193 fmode
= format(int(zipinfo
.external_attr
), 'x').lower();
3194 fchmode
= stat
.S_IMODE(fmode
);
3195 ftypemod
= stat
.S_IFMT(fmode
);
3197 fwinattributes
= format(int(0), 'x').lower();
3198 if(not member
.is_dir()):
3199 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3200 fchmode
= stat
.S_IMODE(fmode
);
3201 ftypemod
= stat
.S_IFMT(fmode
);
3202 elif(member
.is_dir()):
3203 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3204 fchmode
= stat
.S_IMODE(fmode
);
3205 ftypemod
= stat
.S_IFMT(fmode
);
3207 fcsize
= format(int(0), 'x').lower();
3209 fuid
= format(int(os
.getuid()), 'x').lower();
3210 except AttributeError:
3211 fuid
= format(int(0), 'x').lower();
3213 fuid
= format(int(0), 'x').lower();
3215 fgid
= format(int(os
.getgid()), 'x').lower();
3216 except AttributeError:
3217 fgid
= format(int(0), 'x').lower();
3219 fgid
= format(int(0), 'x').lower();
3223 userinfo
= pwd
.getpwuid(os
.getuid());
3224 funame
= userinfo
.pw_name
;
3227 except AttributeError:
3235 groupinfo
= grp
.getgrgid(os
.getgid());
3236 fgname
= groupinfo
.gr_name
;
3239 except AttributeError:
3243 fcontents
= BytesIO();
3245 fcontents
.write(zipfp
.read(member
.filename
));
3246 if(not compresswholefile
):
3247 fcontents
.seek(0, 2);
3248 ucfsize
= fcontents
.tell();
3249 fcontents
.seek(0, 0);
3250 if(compression
=="auto"):
3251 ilsize
= len(compressionlistalt
);
3254 while(ilmin
< ilsize
):
3255 cfcontents
= BytesIO();
3256 shutil
.copyfileobj(fcontents
, cfcontents
);
3257 fcontents
.seek(0, 0);
3258 cfcontents
.seek(0, 0);
3259 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3260 cfcontents
.seek(0, 2);
3261 ilcsize
.append(cfcontents
.tell());
3264 ilcmin
= ilcsize
.index(min(ilcsize
));
3265 compression
= compressionlistalt
[ilcmin
];
3266 fcontents
.seek(0, 0);
3267 cfcontents
= BytesIO();
3268 shutil
.copyfileobj(fcontents
, cfcontents
);
3269 cfcontents
.seek(0, 0);
3270 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3271 cfcontents
.seek(0, 2);
3272 cfsize
= cfcontents
.tell();
3273 if(ucfsize
> cfsize
):
3274 fcsize
= format(int(cfsize
), 'x').lower();
3275 fcompression
= compression
;
3277 fcontents
= cfcontents
;
3278 if(fcompression
=="none"):
3280 fcontents
.seek(0, 0);
3281 ftypehex
= format(ftype
, 'x').lower();
3282 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3283 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3286 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3287 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3288 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3291 os
.fsync(catfp
.fileno());
3292 except io
.UnsupportedOperation
:
3294 except AttributeError:
3296 except OSError as e
:
3300 if(hasattr(sys
.stdout
, "buffer")):
3301 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3303 shutil
.copyfileobj(catfp
, sys
.stdout
);
3304 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3305 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3307 upload_file_to_internet_file(catfp
, outfile
);
3315 create_alias_function("Pack", __file_format_name__
, "FromZipFile", PackArchiveFileFromZipFile
);
3317 if(not rarfile_support
):
3318 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3321 if(rarfile_support
):
3322 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3323 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3324 outfile
= RemoveWindowsPath(outfile
);
3325 checksumtype
= checksumtype
.lower();
3326 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3327 checksumtype
="crc32";
3328 if(checksumtype
=="none"):
3330 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3331 compression
= "auto";
3332 if(compression
not in compressionlist
and compression
is None):
3333 compression
= "auto";
3335 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3336 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3337 if(os
.path
.exists(outfile
)):
3340 except OSError as e
:
3345 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3347 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3350 fbasename
= os
.path
.splitext(outfile
)[0];
3351 fextname
= os
.path
.splitext(outfile
)[1];
3352 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3353 catver
= formatspecs
['format_ver'];
3354 fileheaderver
= str(int(catver
.replace(".", "")));
3360 inodetocatinode
= {};
3361 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3363 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
3365 rarfp
= rarfile
.RarFile(infile
, "r");
3366 rartest
= rarfp
.testrar();
3368 VerbosePrintOut("Bad file found!");
3369 numfiles
= int(len(rarfp
.infolist()));
3370 catfp
= AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3373 os
.fsync(catfp
.fileno());
3374 except io
.UnsupportedOperation
:
3376 except AttributeError:
3378 except OSError as e
:
3380 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
3383 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
3386 member
.external_attr
3388 except AttributeError:
3390 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
3393 member
.external_attr
3395 except AttributeError:
3400 if(re
.findall("^[.|/]", member
.filename
)):
3401 fname
= member
.filename
;
3403 fname
= "./"+member
.filename
;
3404 rarinfo
= rarfp
.getinfo(member
.filename
);
3406 VerbosePrintOut(fname
);
3407 if(is_unix
and member
.external_attr
!=0):
3408 fpremode
= int(member
.external_attr
);
3409 elif(member
.is_file()):
3410 fpremode
= int(stat
.S_IFREG
+ 438);
3411 elif(member
.is_symlink()):
3412 fpremode
= int(stat
.S_IFLNK
+ 438);
3413 elif(member
.is_dir()):
3414 fpremode
= int(stat
.S_IFDIR
+ 511);
3415 if(is_windows
and member
.external_attr
!=0):
3416 fwinattributes
= format(int(member
.external_attr
), 'x').lower();
3418 fwinattributes
= format(int(0), 'x').lower();
3420 fcsize
= format(int(0), 'x').lower();
3423 if(member
.is_file()):
3425 elif(member
.is_symlink()):
3427 elif(member
.is_dir()):
3431 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
3432 fcurfid
= format(int(curfid
), 'x').lower();
3433 fcurinode
= format(int(curfid
), 'x').lower();
3434 curfid
= curfid
+ 1;
3435 fdev_minor
= format(int(0), 'x').lower();
3436 fdev_major
= format(int(0), 'x').lower();
3437 frdev_minor
= format(int(0), 'x').lower();
3438 frdev_major
= format(int(0), 'x').lower();
3440 fsize
= format(int("0"), 'x').lower();
3442 fsize
= format(int(member
.file_size
), 'x').lower();
3444 fsize
= format(int(member
.file_size
), 'x').lower();
3447 fatime
= format(int(member
.atime
.timestamp()), 'x').lower();
3449 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3450 except AttributeError:
3451 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3452 fmtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3455 fctime
= format(int(member
.ctime
.timestamp()), 'x').lower();
3457 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3458 except AttributeError:
3459 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3460 fbtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3461 if(is_unix
and member
.external_attr
!=0):
3462 fmode
= format(int(member
.external_attr
), 'x').lower();
3463 fchmode
= format(int(stat
.S_IMODE(member
.external_attr
)), 'x').lower();
3464 ftypemod
= format(int(stat
.S_IFMT(member
.external_attr
)), 'x').lower();
3465 elif(member
.is_file()):
3466 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3467 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3468 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3469 elif(member
.is_symlink()):
3470 fmode
= format(int(stat
.S_IFLNK
+ 438), 'x').lower();
3471 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3472 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3473 elif(member
.is_dir()):
3474 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3475 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3476 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3478 fuid
= format(int(os
.getuid()), 'x').lower();
3479 except AttributeError:
3480 fuid
= format(int(0), 'x').lower();
3482 fuid
= format(int(0), 'x').lower();
3484 fgid
= format(int(os
.getgid()), 'x').lower();
3485 except AttributeError:
3486 fgid
= format(int(0), 'x').lower();
3488 fgid
= format(int(0), 'x').lower();
3492 userinfo
= pwd
.getpwuid(os
.getuid());
3493 funame
= userinfo
.pw_name
;
3496 except AttributeError:
3504 groupinfo
= grp
.getgrgid(os
.getgid());
3505 fgname
= groupinfo
.gr_name
;
3508 except AttributeError:
3512 fcontents
= BytesIO();
3514 fcontents
.write(rarfp
.read(member
.filename
));
3515 if(not compresswholefile
):
3516 fcontents
.seek(0, 2);
3517 ucfsize
= fcontents
.tell();
3518 fcontents
.seek(0, 0);
3519 if(compression
=="auto"):
3520 ilsize
= len(compressionlistalt
);
3523 while(ilmin
< ilsize
):
3524 cfcontents
= BytesIO();
3525 shutil
.copyfileobj(fcontents
, cfcontents
);
3526 fcontents
.seek(0, 0);
3527 cfcontents
.seek(0, 0);
3528 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3530 cfcontents
.seek(0, 2);
3531 ilcsize
.append(cfcontents
.tell());
3535 ilcsize
.append(sys
.maxint
);
3536 except AttributeError:
3537 ilcsize
.append(sys
.maxsize
);
3539 ilcmin
= ilcsize
.index(min(ilcsize
));
3540 compression
= compressionlistalt
[ilcmin
];
3541 fcontents
.seek(0, 0);
3542 cfcontents
= BytesIO();
3543 shutil
.copyfileobj(fcontents
, cfcontents
);
3544 cfcontents
.seek(0, 0);
3545 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3546 cfcontents
.seek(0, 2);
3547 cfsize
= cfcontents
.tell();
3548 if(ucfsize
> cfsize
):
3549 fcsize
= format(int(cfsize
), 'x').lower();
3550 fcompression
= compression
;
3552 fcontents
= cfcontents
;
3553 if(fcompression
=="none"):
3555 fcontents
.seek(0, 0);
3556 ftypehex
= format(ftype
, 'x').lower();
3557 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3558 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3561 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3562 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3563 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
3566 os
.fsync(catfp
.fileno());
3567 except io
.UnsupportedOperation
:
3569 except AttributeError:
3571 except OSError as e
:
3575 if(hasattr(sys
.stdout
, "buffer")):
3576 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3578 shutil
.copyfileobj(catfp
, sys
.stdout
);
3579 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3580 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3582 upload_file_to_internet_file(catfp
, outfile
);
3590 create_alias_function("Pack", __file_format_name__
, "FromRarFile", PackArchiveFileFromRarFile
);
3592 if(not py7zr_support
):
3593 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3597 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3598 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3599 outfile
= RemoveWindowsPath(outfile
);
3600 checksumtype
= checksumtype
.lower();
3601 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3602 checksumtype
="crc32";
3603 if(checksumtype
=="none"):
3605 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
3606 compression
= "auto";
3607 if(compression
not in compressionlist
and compression
is None):
3608 compression
= "auto";
3610 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3611 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3612 if(os
.path
.exists(outfile
)):
3615 except OSError as e
:
3620 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3622 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3625 fbasename
= os
.path
.splitext(outfile
)[0];
3626 fextname
= os
.path
.splitext(outfile
)[1];
3627 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
3628 catver
= formatspecs
['format_ver'];
3629 fileheaderver
= str(int(catver
.replace(".", "")));
3635 inodetocatinode
= {};
3636 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3638 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
3639 file_content
= szpfp
.readall();
3640 #sztest = szpfp.testzip();
3641 sztestalt
= szpfp
.test();
3643 VerbosePrintOut("Bad file found!");
3644 numfiles
= int(len(szpfp
.list()));
3645 AppendFileHeader(catfp
, numfiles
, checksumtype
, formatspecs
);
3646 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
3647 if(re
.findall("^[.|/]", member
.filename
)):
3648 fname
= member
.filename
;
3650 fname
= "./"+member
.filename
;
3652 VerbosePrintOut(fname
);
3653 if(not member
.is_directory
):
3654 fpremode
= int(stat
.S_IFREG
+ 438);
3655 elif(member
.is_directory
):
3656 fpremode
= int(stat
.S_IFDIR
+ 511);
3657 fwinattributes
= format(int(0), 'x').lower();
3659 fcsize
= format(int(0), 'x').lower();
3662 if(member
.is_directory
):
3667 fcurfid
= format(int(curfid
), 'x').lower();
3668 fcurinode
= format(int(curfid
), 'x').lower();
3669 curfid
= curfid
+ 1;
3670 fdev_minor
= format(int(0), 'x').lower();
3671 fdev_major
= format(int(0), 'x').lower();
3672 frdev_minor
= format(int(0), 'x').lower();
3673 frdev_major
= format(int(0), 'x').lower();
3675 fsize
= format(int("0"), 'x').lower();
3676 fatime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3677 fmtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3678 fctime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3679 fbtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3680 if(member
.is_directory
):
3681 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3682 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3683 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3685 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3686 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3687 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3689 fuid
= format(int(os
.getuid()), 'x').lower();
3690 except AttributeError:
3691 fuid
= format(int(0), 'x').lower();
3693 fuid
= format(int(0), 'x').lower();
3695 fgid
= format(int(os
.getgid()), 'x').lower();
3696 except AttributeError:
3697 fgid
= format(int(0), 'x').lower();
3699 fgid
= format(int(0), 'x').lower();
3703 userinfo
= pwd
.getpwuid(os
.getuid());
3704 funame
= userinfo
.pw_name
;
3707 except AttributeError:
3715 groupinfo
= grp
.getgrgid(os
.getgid());
3716 fgname
= groupinfo
.gr_name
;
3719 except AttributeError:
3723 fcontents
= BytesIO();
3725 fcontents
.write(file_content
[member
.filename
].read());
3726 fsize
= format(fcontents
.tell(), 'x').lower();
3727 file_content
[member
.filename
].close();
3728 if(not compresswholefile
):
3729 fcontents
.seek(0, 2);
3730 ucfsize
= fcontents
.tell();
3731 fcontents
.seek(0, 0);
3732 if(compression
=="auto"):
3733 ilsize
= len(compressionlistalt
);
3736 while(ilmin
< ilsize
):
3737 cfcontents
= BytesIO();
3738 shutil
.copyfileobj(fcontents
, cfcontents
);
3739 fcontents
.seek(0, 0);
3740 cfcontents
.seek(0, 0);
3741 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
3743 cfcontents
.seek(0, 2);
3744 ilcsize
.append(cfcontents
.tell());
3748 ilcsize
.append(sys
.maxint
);
3749 except AttributeError:
3750 ilcsize
.append(sys
.maxsize
);
3752 ilcmin
= ilcsize
.index(min(ilcsize
));
3753 compression
= compressionlistalt
[ilcmin
];
3754 fcontents
.seek(0, 0);
3755 cfcontents
= BytesIO();
3756 shutil
.copyfileobj(fcontents
, cfcontents
);
3757 cfcontents
.seek(0, 0);
3758 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
3759 cfcontents
.seek(0, 2);
3760 cfsize
= cfcontents
.tell();
3761 if(ucfsize
> cfsize
):
3762 fcsize
= format(int(cfsize
), 'x').lower();
3763 fcompression
= compression
;
3765 fcontents
= cfcontents
;
3766 if(fcompression
=="none"):
3768 fcontents
.seek(0, 0);
3769 ftypehex
= format(ftype
, 'x').lower();
3770 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
3771 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
3774 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
3775 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3776 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
)
3779 os
.fsync(catfp
.fileno());
3780 except io
.UnsupportedOperation
:
3782 except AttributeError:
3784 except OSError as e
:
3788 if(hasattr(sys
.stdout
, "buffer")):
3789 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3791 shutil
.copyfileobj(catfp
, sys
.stdout
);
3792 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3793 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
3795 upload_file_to_internet_file(catfp
, outfile
);
3803 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFile", PackArchiveFileFromSevenZipFile
);
3805 def PackArchiveFileFromInFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
3806 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
3808 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3809 if(checkcompressfile
=="tarfile"):
3810 return PackArchiveFileFromTarFile(infile
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3811 elif(checkcompressfile
=="zipfile"):
3812 return PackArchiveFileFromZipFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3813 elif(checkcompressfile
=="catfile"):
3814 return RePackArchiveFile(infile
, outfile
, compression
, compressionlevel
, False, 0, 0, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
3815 elif(rarfile_support
and checkcompressfile
=="rarfile"):
3816 return PackArchiveFileFromRarFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3817 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
3818 return PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3823 create_alias_function("Pack", __file_format_name__
, "FromInFile", PackArchiveFileFromInFile
);
3825 def ArchiveFileSeekToFileNum(infile
, seekto
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
3826 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
3829 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3830 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
3831 if(checkcompressfile
=="tarfile"):
3832 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3833 if(checkcompressfile
=="zipfile"):
3834 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3835 if(rarfile_support
and checkcompressfile
=="rarfile"):
3836 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3837 if(py7zr_support
and checkcompressfile
=="7zipfile"):
3838 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3839 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
3846 if(hasattr(sys
.stdin
, "buffer")):
3847 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
3849 shutil
.copyfileobj(sys
.stdin
, catfp
);
3851 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3855 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3856 catfp
= download_file_from_internet_file(infile
);
3858 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3863 infile
= RemoveWindowsPath(infile
);
3864 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
3865 if(checkcompressfile
=="tarfile"):
3866 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3867 if(checkcompressfile
=="zipfile"):
3868 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3869 if(rarfile_support
and checkcompressfile
=="rarfile"):
3870 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3871 if(py7zr_support
and checkcompressfile
=="7zipfile"):
3872 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3873 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
3875 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
3876 if(not compresscheck
):
3877 fextname
= os
.path
.splitext(infile
)[1];
3878 if(fextname
==".gz"):
3879 compresscheck
= "gzip";
3880 elif(fextname
==".bz2"):
3881 compresscheck
= "bzip2";
3882 elif(fextname
==".zst"):
3883 compresscheck
= "zstd";
3884 elif(fextname
==".lz4" or fextname
==".clz4"):
3885 compresscheck
= "lz4";
3886 elif(fextname
==".lzo" or fextname
==".lzop"):
3887 compresscheck
= "lzo";
3888 elif(fextname
==".lzma" or fextname
==".xz"):
3889 compresscheck
= "lzma";
3892 if(not compresscheck
):
3894 catfp
= UncompressFile(infile
, formatspecs
, "rb");
3899 SeekToEndOfFile(catfp);
3901 SeekToEndOfFile(catfp);
3902 CatSize = catfp.tell();
3903 CatSizeEnd = CatSize;
3911 curloc
= catfp
.tell();
3914 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
3916 catfp
.seek(curloc
, 0);
3917 catstring
= catheader
[0];
3918 catversion
= re
.findall(r
"([\d]+)$", catstring
);
3919 fprenumfiles
= catheader
[1];
3920 fnumfiles
= int(fprenumfiles
, 16);
3921 fprechecksumtype
= catheader
[2];
3922 fprechecksum
= catheader
[3];
3923 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
3924 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
3925 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
3926 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
3927 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
3928 fheadtell
= len(fileheader
);
3929 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
3930 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
3931 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
3933 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
3934 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
3935 if(seekto
>=fnumfiles
):
3936 seekto
= fnumfiles
- 1;
3942 prefhstart
= catfp
.tell();
3943 if(formatspecs
['new_style']):
3944 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
3946 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
3947 if(len(preheaderdata
)==0):
3949 prefheadsize
= int(preheaderdata
[0], 16);
3950 prefnumfields
= int(preheaderdata
[1], 16);
3951 preftype
= int(preheaderdata
[2], 16);
3952 if(re
.findall("^[.|/]", preheaderdata
[3])):
3953 prefname
= preheaderdata
[3];
3955 prefname
= "./"+preheaderdata
[3];
3956 prefbasedir
= os
.path
.dirname(prefname
);
3957 preflinkname
= preheaderdata
[4];
3958 prefsize
= int(preheaderdata
[5], 16);
3959 prefatime
= int(preheaderdata
[6], 16);
3960 prefmtime
= int(preheaderdata
[7], 16);
3961 prefctime
= int(preheaderdata
[8], 16);
3962 prefbtime
= int(preheaderdata
[9], 16);
3963 prefmode
= int(preheaderdata
[10], 16);
3964 prefchmode
= stat
.S_IMODE(prefmode
);
3965 preftypemod
= stat
.S_IFMT(prefmode
);
3966 prefwinattributes
= int(preheaderdata
[11], 16);
3967 prefcompression
= preheaderdata
[12];
3968 prefcsize
= int(preheaderdata
[13], 16);
3969 prefuid
= int(preheaderdata
[14], 16);
3970 prefuname
= preheaderdata
[15];
3971 prefgid
= int(preheaderdata
[16], 16);
3972 prefgname
= preheaderdata
[17];
3973 fid
= int(preheaderdata
[18], 16);
3974 finode
= int(preheaderdata
[19], 16);
3975 flinkcount
= int(preheaderdata
[20], 16);
3976 prefdev_minor
= int(preheaderdata
[21], 16);
3977 prefdev_major
= int(preheaderdata
[22], 16);
3978 prefrdev_minor
= int(preheaderdata
[23], 16);
3979 prefrdev_major
= int(preheaderdata
[24], 16);
3980 prefextrasize
= int(preheaderdata
[25], 16);
3981 prefextrafields
= int(preheaderdata
[26], 16);
3982 extrafieldslist
= [];
3984 extraend
= extrastart
+ prefextrafields
;
3985 extrafieldslist
= [];
3986 if(extrastart
<extraend
):
3987 extrafieldslist
.append(preheaderdata
[extrastart
]);
3988 extrastart
= extrastart
+ 1;
3989 prefchecksumtype
= preheaderdata
[extrastart
].lower();
3990 prefcs
= preheaderdata
[extrastart
+ 1].lower();
3991 prefccs
= preheaderdata
[extrastart
+ 2].lower();
3993 hcmax
= len(preheaderdata
) - 2;
3996 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
['format_delimiter']);
3998 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
3999 if(prefcs
!=prenewfcs
and not skipchecksum
):
4000 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4001 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4003 valid_archive
= False;
4004 invalid_archive
= True;
4005 prefhend
= catfp
.tell() - 1;
4006 prefcontentstart
= catfp
.tell();
4008 pyhascontents
= False;
4010 if(prefcompression
):
4011 prefcontents
= catfp
.read(prefsize
);
4013 prefcontents
= catfp
.read(prefcsize
);
4014 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4015 pyhascontents
= True;
4016 if(prefccs
!=prenewfccs
and not skipchecksum
):
4017 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4018 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4022 catfp
.seek(seekstart
, 0);
4024 catfheadsize
= int(preheaderdata
[0], 16);
4025 catfnumfields
= int(preheaderdata
[1], 16);
4026 catftype
= int(preheaderdata
[2], 16);
4027 if(re
.findall("^[.|/]", preheaderdata
[3])):
4028 catfname
= preheaderdata
[3];
4030 catfname
= "./"+preheaderdata
[3];
4031 catflinkname
= preheaderdata
[4];
4032 catfsize
= int(preheaderdata
[5], 16);
4033 catfbasedir
= os
.path
.dirname(catfname
);
4034 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4036 catlist
.update({'catfp': catfp
});
4041 create_alias_function("", __file_format_name__
, "SeekToFileNum", ArchiveFileSeekToFileNum
);
4043 def ArchiveFileSeekToFileName(infile
, seekfile
=None, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4044 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4047 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4048 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4049 if(checkcompressfile
=="tarfile"):
4050 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4051 if(checkcompressfile
=="zipfile"):
4052 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4053 if(rarfile_support
and checkcompressfile
=="rarfile"):
4054 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4055 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4056 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4057 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4064 if(hasattr(sys
.stdin
, "buffer")):
4065 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4067 shutil
.copyfileobj(sys
.stdin
, catfp
);
4069 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4073 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4074 catfp
= download_file_from_internet_file(infile
);
4075 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4081 infile
= RemoveWindowsPath(infile
);
4082 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4083 if(checkcompressfile
=="tarfile"):
4084 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4085 if(checkcompressfile
=="zipfile"):
4086 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4087 if(rarfile_support
and checkcompressfile
=="rarfile"):
4088 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4089 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4090 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
4091 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4093 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4094 if(not compresscheck
):
4095 fextname
= os
.path
.splitext(infile
)[1];
4096 if(fextname
==".gz"):
4097 compresscheck
= "gzip";
4098 elif(fextname
==".bz2"):
4099 compresscheck
= "bzip2";
4100 elif(fextname
==".zst"):
4101 compresscheck
= "zstd";
4102 elif(fextname
==".lz4" or fextname
==".clz4"):
4103 compresscheck
= "lz4";
4104 elif(fextname
==".lzo" or fextname
==".lzop"):
4105 compresscheck
= "lzo";
4106 elif(fextname
==".lzma" or fextname
==".xz"):
4107 compresscheck
= "lzma";
4110 if(not compresscheck
):
4112 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4117 SeekToEndOfFile(catfp);
4119 SeekToEndOfFile(catfp);
4120 CatSize = catfp.tell();
4121 CatSizeEnd = CatSize;
4129 curloc
= catfp
.tell();
4132 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4134 catfp
.seek(curloc
, 0);
4135 catstring
= catheader
[0];
4136 catversion
= re
.findall(r
"([\d]+)$", catstring
);
4137 fprenumfiles
= catheader
[1];
4138 fnumfiles
= int(fprenumfiles
, 16);
4139 fprechecksumtype
= catheader
[2];
4140 fprechecksum
= catheader
[3];
4141 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4142 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4143 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4144 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4145 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4146 fheadtell
= len(fileheader
);
4147 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4148 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4149 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4151 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4152 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4153 seekto
= fnumfiles
- 1
4158 prefhstart
= catfp
.tell();
4159 if(formatspecs
['new_style']):
4160 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4162 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4163 if(len(preheaderdata
)==0):
4165 prefheadsize
= int(preheaderdata
[0], 16);
4166 prefnumfields
= int(preheaderdata
[1], 16);
4167 preftype
= int(preheaderdata
[2], 16);
4168 if(re
.findall("^[.|/]", preheaderdata
[3])):
4169 prefname
= preheaderdata
[3];
4171 prefname
= "./"+preheaderdata
[3];
4172 prefbasedir
= os
.path
.dirname(prefname
);
4173 preflinkname
= preheaderdata
[4];
4174 prefsize
= int(preheaderdata
[5], 16);
4175 prefatime
= int(preheaderdata
[6], 16);
4176 prefmtime
= int(preheaderdata
[7], 16);
4177 prefctime
= int(preheaderdata
[8], 16);
4178 prefbtime
= int(preheaderdata
[9], 16);
4179 prefmode
= int(preheaderdata
[10], 16);
4180 prefchmode
= stat
.S_IMODE(prefmode
);
4181 preftypemod
= stat
.S_IFMT(prefmode
);
4182 prefwinattributes
= int(preheaderdata
[11], 16);
4183 prefcompression
= preheaderdata
[12];
4184 prefcsize
= int(preheaderdata
[13], 16);
4185 prefuid
= int(preheaderdata
[14], 16);
4186 prefuname
= preheaderdata
[15];
4187 prefgid
= int(preheaderdata
[16], 16);
4188 prefgname
= preheaderdata
[17];
4189 fid
= int(preheaderdata
[18], 16);
4190 finode
= int(preheaderdata
[19], 16);
4191 flinkcount
= int(preheaderdata
[20], 16);
4192 prefdev_minor
= int(preheaderdata
[21], 16);
4193 prefdev_major
= int(preheaderdata
[22], 16);
4194 prefrdev_minor
= int(preheaderdata
[23], 16);
4195 prefrdev_major
= int(preheaderdata
[24], 16);
4196 prefextrasize
= int(preheaderdata
[25], 16);
4197 prefextrafields
= int(preheaderdata
[26], 16);
4198 extrafieldslist
= [];
4200 extraend
= extrastart
+ prefextrafields
;
4201 extrafieldslist
= [];
4202 if(extrastart
<extraend
):
4203 extrafieldslist
.append(preheaderdata
[extrastart
]);
4204 extrastart
= extrastart
+ 1;
4205 prefchecksumtype
= preheaderdata
[extrastart
].lower();
4206 prefcs
= preheaderdata
[extrastart
+ 1].lower();
4207 prefccs
= preheaderdata
[extrastart
+ 2].lower();
4209 hcmax
= len(preheaderdata
) - 2;
4212 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
['format_delimiter']);
4214 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
4215 if(prefcs
!=prenewfcs
and not skipchecksum
):
4216 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4217 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4219 valid_archive
= False;
4220 invalid_archive
= True;
4221 prefhend
= catfp
.tell() - 1;
4222 prefcontentstart
= catfp
.tell();
4224 pyhascontents
= False;
4226 if(prefcompression
):
4227 prefcontents
= catfp
.read(prefsize
);
4229 prefcontents
= catfp
.read(prefcsize
);
4230 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4231 pyhascontents
= True;
4232 if(prefccs
!=prenewfccs
and not skipchecksum
):
4233 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4234 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4239 prefname
= preheaderdata
[2];
4240 if(re
.findall("^[.|/]", preheaderdata
[2])):
4241 prefname
= preheaderdata
[2];
4243 prefname
= "./"+preheaderdata
[2];
4244 if(prefname
==seekfile
):
4247 catfp
.seek(seekstart
, 0);
4249 catfheadsize
= int(preheaderdata
[0], 16);
4250 catfnumfields
= int(preheaderdata
[1], 16);
4251 catftype
= int(preheaderdata
[2], 16);
4252 if(re
.findall("^[.|/]", preheaderdata
[3])):
4253 catfname
= preheaderdata
[3];
4255 catfname
= "./"+preheaderdata
[3];
4256 catflinkname
= preheaderdata
[4];
4257 catfsize
= int(preheaderdata
[5], 16);
4258 catfbasedir
= os
.path
.dirname(catfname
);
4260 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4264 catlist
.update({'catfp': catfp
});
4269 create_alias_function("", __file_format_name__
, "SeekToFileName", ArchiveFileSeekToFileName
);
4271 def ArchiveFileValidate(infile
, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
4273 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
4274 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4277 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4278 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4279 if(checkcompressfile
=="tarfile"):
4280 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4281 if(checkcompressfile
=="zipfile"):
4282 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4283 if(rarfile_support
and checkcompressfile
=="rarfile"):
4284 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4285 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4286 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4287 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4294 if(hasattr(sys
.stdin
, "buffer")):
4295 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4297 shutil
.copyfileobj(sys
.stdin
, catfp
);
4299 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4303 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4304 catfp
= download_file_from_internet_file(infile
);
4305 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4311 infile
= RemoveWindowsPath(infile
);
4312 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4313 if(checkcompressfile
=="tarfile"):
4314 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4315 if(checkcompressfile
=="zipfile"):
4316 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4317 if(rarfile_support
and checkcompressfile
=="rarfile"):
4318 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4319 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4320 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4321 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4323 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4324 if(not compresscheck
):
4325 fextname
= os
.path
.splitext(infile
)[1];
4326 if(fextname
==".gz"):
4327 compresscheck
= "gzip";
4328 elif(fextname
==".bz2"):
4329 compresscheck
= "bzip2";
4330 elif(fextname
==".zst"):
4331 compresscheck
= "zstd";
4332 elif(fextname
==".lz4" or fextname
==".clz4"):
4333 compresscheck
= "lz4";
4334 elif(fextname
==".lzo" or fextname
==".lzop"):
4335 compresscheck
= "lzo";
4336 elif(fextname
==".lzma" or fextname
==".xz"):
4337 compresscheck
= "lzma";
4340 if(not compresscheck
):
4342 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4347 SeekToEndOfFile(catfp);
4349 SeekToEndOfFile(catfp);
4350 CatSize = catfp.tell();
4351 CatSizeEnd = CatSize;
4359 curloc
= catfp
.tell();
4362 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4364 catfp
.seek(curloc
, 0);
4365 catstring
= catheader
[0];
4366 catversion
= re
.findall(r
"([\d]+)$", catstring
);
4367 fprenumfiles
= catheader
[1];
4368 fnumfiles
= int(fprenumfiles
, 16);
4369 fprechecksumtype
= catheader
[2];
4370 fprechecksum
= catheader
[3];
4372 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4373 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4374 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4375 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4376 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4377 valid_archive
= True;
4378 invalid_archive
= False;
4380 VerbosePrintOut(infile
);
4381 VerbosePrintOut("Number of Records " + str(fnumfiles
));
4382 if(fprechecksum
==catfileheadercshex
):
4384 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4385 VerbosePrintOut("'" + str(fprechecksum
) + "' == " + "'" + str(catfileheadercshex
) + "'");
4388 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4389 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4390 valid_archive
= False;
4391 invalid_archive
= True;
4393 VerbosePrintOut("");
4394 while(il
<fnumfiles
):
4395 catfhstart
= catfp
.tell();
4396 if(formatspecs
['new_style']):
4397 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4399 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4400 if(len(catheaderdata
)==0):
4402 catfheadsize
= int(catheaderdata
[0], 16);
4403 catfnumfields
= int(catheaderdata
[1], 16);
4404 catftype
= int(catheaderdata
[2], 16);
4405 if(re
.findall("^[.|/]", catheaderdata
[3])):
4406 catfname
= catheaderdata
[3];
4408 catfname
= "./"+catheaderdata
[3];
4409 catfbasedir
= os
.path
.dirname(catfname
);
4410 catflinkname
= catheaderdata
[4];
4411 catfsize
= int(catheaderdata
[5], 16);
4412 catfatime
= int(catheaderdata
[6], 16);
4413 catfmtime
= int(catheaderdata
[7], 16);
4414 catfctime
= int(catheaderdata
[8], 16);
4415 catfbtime
= int(catheaderdata
[9], 16);
4416 catfmode
= int(catheaderdata
[10], 16);
4417 catfchmode
= stat
.S_IMODE(catfmode
);
4418 catftypemod
= stat
.S_IFMT(catfmode
);
4419 catfwinattributes
= int(catheaderdata
[11], 16);
4420 catfcompression
= catheaderdata
[12];
4421 catfcsize
= int(catheaderdata
[13], 16);
4422 catfuid
= int(catheaderdata
[14], 16);
4423 catfuname
= catheaderdata
[15];
4424 catfgid
= int(catheaderdata
[16], 16);
4425 catfgname
= catheaderdata
[17];
4426 fid
= int(catheaderdata
[18], 16);
4427 finode
= int(catheaderdata
[19], 16);
4428 flinkcount
= int(catheaderdata
[20], 16);
4429 catfdev_minor
= int(catheaderdata
[21], 16);
4430 catfdev_major
= int(catheaderdata
[22], 16);
4431 catfrdev_minor
= int(catheaderdata
[23], 16);
4432 catfrdev_major
= int(catheaderdata
[24], 16);
4433 catfextrasize
= int(catheaderdata
[25], 16);
4434 catfextrafields
= int(catheaderdata
[26], 16);
4435 extrafieldslist
= [];
4437 extraend
= extrastart
+ catfextrafields
;
4438 extrafieldslist
= [];
4439 if(extrastart
<extraend
):
4440 extrafieldslist
.append(catheaderdata
[extrastart
]);
4441 extrastart
= extrastart
+ 1;
4442 catfchecksumtype
= catheaderdata
[extrastart
].lower();
4443 catfcs
= catheaderdata
[extrastart
+ 1].lower();
4444 catfccs
= catheaderdata
[extrastart
+ 2].lower();
4446 hcmax
= len(catheaderdata
) - 2;
4449 hout
= hout
+ AppendNullByte(catheaderdata
[hc
], formatspecs
['format_delimiter']);
4451 catnewfcs
= GetFileChecksum(hout
, catheaderdata
[-3].lower(), True, formatspecs
);
4453 VerbosePrintOut(catfname
);
4454 VerbosePrintOut("Record Number " + str(il
) + "; File ID " + str(fid
) + "; iNode Number " + str(finode
));
4455 if(catfcs
==catnewfcs
):
4457 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart
));
4458 VerbosePrintOut("'" + str(catfcs
) + "' == " + "'" + str(catnewfcs
) + "'");
4461 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart
));
4462 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
4463 valid_archive
= False;
4464 invalid_archive
= True;
4465 catfhend
= catfp
.tell() - 1;
4466 catfcontentstart
= catfp
.tell();
4468 pyhascontents
= False;
4470 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4471 catfcontents
= catfp
.read(catfsize
);
4473 catfcontents
= catfp
.read(catfcsize
);
4474 catnewfccs
= GetFileChecksum(catfcontents
, catheaderdata
[-3].lower(), False, formatspecs
);
4475 pyhascontents
= True;
4476 if(catfccs
==catnewfccs
):
4478 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart
));
4479 VerbosePrintOut("'" + str(catfccs
) + "' == " + "'" + str(catnewfccs
) + "'");
4482 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart
));
4483 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
4484 valid_archive
= False;
4485 invalid_archive
= True;
4487 VerbosePrintOut("");
4500 create_alias_function("", __file_format_name__
, "Validate", ArchiveFileValidate
);
4502 def ArchiveFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4503 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4506 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4507 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4508 if(checkcompressfile
=="tarfile"):
4509 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4510 if(checkcompressfile
=="zipfile"):
4511 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4512 if(rarfile_support
and checkcompressfile
=="rarfile"):
4513 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4514 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4515 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4516 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4523 if(hasattr(sys
.stdin
, "buffer")):
4524 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4526 shutil
.copyfileobj(sys
.stdin
, catfp
);
4528 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4532 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4533 catfp
= download_file_from_internet_file(infile
);
4534 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4540 infile
= RemoveWindowsPath(infile
);
4541 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4542 if(checkcompressfile
=="tarfile"):
4543 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4544 if(checkcompressfile
=="zipfile"):
4545 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4546 if(rarfile_support
and checkcompressfile
=="rarfile"):
4547 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4548 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4549 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4550 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
['format_lower']):
4552 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4553 if(not compresscheck
):
4554 fextname
= os
.path
.splitext(infile
)[1];
4555 if(fextname
==".gz"):
4556 compresscheck
= "gzip";
4557 elif(fextname
==".bz2"):
4558 compresscheck
= "bzip2";
4559 elif(fextname
==".zst"):
4560 compresscheck
= "zstd";
4561 elif(fextname
==".lz4" or fextname
==".clz4"):
4562 compresscheck
= "lz4";
4563 elif(fextname
==".lzo" or fextname
==".lzop"):
4564 compresscheck
= "lzo";
4565 elif(fextname
==".lzma" or fextname
==".xz"):
4566 compresscheck
= "lzma";
4569 if(not compresscheck
):
4571 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4576 SeekToEndOfFile(catfp);
4578 SeekToEndOfFile(catfp);
4579 CatSize = catfp.tell();
4580 CatSizeEnd = CatSize;
4588 curloc
= catfp
.tell();
4591 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
['format_delimiter']);
4593 catfp
.seek(curloc
, 0);
4594 catstring
= catheader
[0];
4595 catversion
= re
.findall(r
"([\d]+)$", catstring
);
4596 fprenumfiles
= catheader
[1];
4597 fnumfiles
= int(fprenumfiles
, 16);
4598 fprechecksumtype
= catheader
[2];
4599 fprechecksum
= catheader
[3];
4600 fileheader
= AppendNullByte(catstring
, formatspecs
['format_delimiter']);
4601 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4602 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
['format_delimiter']);
4603 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4604 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4605 fheadtell
= len(fileheader
);
4606 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4607 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4608 VerbosePrintOut("'" + str(fprechecksum
) + "' != " + "'" + str(catfileheadercshex
) + "'");
4610 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4611 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4612 if(seekstart
<0 and seekstart
>fnumfiles
):
4614 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
4615 seekend
= fnumfiles
;
4616 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
4617 seekend
= fnumfiles
- abs(seekend
);
4620 while(il
< seekstart
):
4621 prefhstart
= catfp
.tell();
4622 if(formatspecs
['new_style']):
4623 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4625 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4626 if(len(preheaderdata
)==0):
4628 prefheadsize
= int(preheaderdata
[0], 16);
4629 prefnumfields
= int(preheaderdata
[1], 16);
4630 if(re
.findall("^[.|/]", preheaderdata
[3])):
4631 prefname
= preheaderdata
[3];
4633 prefname
= "./"+preheaderdata
[3];
4634 prefsize
= int(preheaderdata
[5], 16);
4635 prefcompression
= preheaderdata
[12];
4636 prefcsize
= int(preheaderdata
[13], 16);
4637 prefextrasize
= int(preheaderdata
[24], 16);
4638 prefextrafields
= int(preheaderdata
[25], 16);
4639 extrafieldslist
= [];
4641 extraend
= extrastart
+ prefextrafields
;
4642 extrafieldslist
= [];
4643 if(extrastart
<extraend
):
4644 extrafieldslist
.append(preheaderdata
[extrastart
]);
4645 extrastart
= extrastart
+ 1;
4646 prefchecksumtype
= preheaderdata
[extrastart
].lower();
4647 prefcs
= preheaderdata
[extrastart
+ 1].lower();
4648 prefccs
= preheaderdata
[extrastart
+ 2].lower();
4650 hcmax
= len(preheaderdata
) - 2;
4653 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
['format_delimiter']);
4655 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
4656 if(prefcs
!=prenewfcs
and not skipchecksum
):
4657 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4658 VerbosePrintOut("'" + str(prefcs
) + "' != " + "'" + str(prenewfcs
) + "'");
4660 valid_archive
= False;
4661 invalid_archive
= True;
4662 prefhend
= catfp
.tell() - 1;
4663 prefcontentstart
= catfp
.tell();
4665 pyhascontents
= False;
4667 if(prefcompression
=="none" or prefcompression
=="" or prefcompression
=="auto"):
4668 prefcontents
= catfp
.read(prefsize
);
4670 prefcontents
= catfp
.read(prefcsize
);
4671 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4672 pyhascontents
= True;
4673 if(prefccs
!=prenewfccs
and not skipchecksum
):
4674 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4675 VerbosePrintOut("'" + str(prefccs
) + "' != " + "'" + str(prenewfccs
) + "'");
4679 fileidnum
= seekstart
;
4681 while(fileidnum
<seekend
):
4682 catfhstart
= catfp
.tell();
4683 if(formatspecs
['new_style']):
4684 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
['format_delimiter']);
4686 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
['format_delimiter']);
4687 if(len(catheaderdata
)==0):
4689 catfheadsize
= int(catheaderdata
[0], 16);
4690 catfnumfields
= int(catheaderdata
[1], 16);
4691 catftype
= int(catheaderdata
[2], 16);
4692 if(re
.findall("^[.|/]", catheaderdata
[3])):
4693 catfname
= catheaderdata
[3];
4695 catfname
= "./"+catheaderdata
[3];
4696 catfbasedir
= os
.path
.dirname(catfname
);
4697 catflinkname
= catheaderdata
[4];
4698 catfsize
= int(catheaderdata
[5], 16);
4699 catfatime
= int(catheaderdata
[6], 16);
4700 catfmtime
= int(catheaderdata
[7], 16);
4701 catfctime
= int(catheaderdata
[8], 16);
4702 catfbtime
= int(catheaderdata
[9], 16);
4703 catfmode
= int(catheaderdata
[10], 16);
4704 catfchmode
= stat
.S_IMODE(catfmode
);
4705 catftypemod
= stat
.S_IFMT(catfmode
);
4706 catfwinattributes
= int(catheaderdata
[11], 16);
4707 catfcompression
= catheaderdata
[12];
4708 catfcsize
= int(catheaderdata
[13], 16);
4709 catfuid
= int(catheaderdata
[14], 16);
4710 catfuname
= catheaderdata
[15];
4711 catfgid
= int(catheaderdata
[16], 16);
4712 catfgname
= catheaderdata
[17];
4713 catfid
= int(catheaderdata
[18], 16);
4714 catfinode
= int(catheaderdata
[19], 16);
4715 catflinkcount
= int(catheaderdata
[20], 16);
4716 catfdev_minor
= int(catheaderdata
[21], 16);
4717 catfdev_major
= int(catheaderdata
[22], 16);
4718 catfrdev_minor
= int(catheaderdata
[23], 16);
4719 catfrdev_major
= int(catheaderdata
[24], 16);
4720 catfextrasize
= int(catheaderdata
[25], 16);
4721 catfextrafields
= int(catheaderdata
[26], 16);
4722 extrafieldslist
= [];
4724 extraend
= extrastart
+ catfextrafields
;
4725 extrafieldslist
= [];
4726 if(extrastart
<extraend
):
4727 extrafieldslist
.append(catheaderdata
[extrastart
]);
4728 extrastart
= extrastart
+ 1;
4729 catfchecksumtype
= catheaderdata
[extrastart
].lower();
4730 catfcs
= catheaderdata
[extrastart
+ 1].lower();
4731 catfccs
= catheaderdata
[extrastart
+ 2].lower();
4733 hcmax
= len(catheaderdata
) - 2;
4736 hout
= hout
+ AppendNullByte(catheaderdata
[hc
], formatspecs
['format_delimiter']);
4738 catnewfcs
= GetFileChecksum(hout
, catheaderdata
[-3].lower(), True, formatspecs
);
4739 if(catfcs
!=catnewfcs
and not skipchecksum
):
4740 VerbosePrintOut("File Header Checksum Error with file " + catfname
+ " at offset " + str(catfhstart
));
4741 VerbosePrintOut("'" + str(catfcs
) + "' != " + "'" + str(catnewfcs
) + "'");
4743 catfhend
= catfp
.tell() - 1;
4744 catfcontentstart
= catfp
.tell();
4745 catfcontents
= BytesIO();
4746 pyhascontents
= False;
4747 if(catfsize
>0 and not listonly
):
4748 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4749 catfcontents
.write(catfp
.read(catfsize
));
4751 catfcontents
.write(catfp
.read(catfcsize
));
4752 catfcontents
.seek(0, 0);
4753 catnewfccs
= GetFileChecksum(catfcontents
.read(), catheaderdata
[-3].lower(), False, formatspecs
);
4754 pyhascontents
= True;
4755 if(catfccs
!=catnewfccs
and skipchecksum
):
4756 VerbosePrintOut("File Content Checksum Error with file " + catfname
+ " at offset " + str(catfcontentstart
));
4757 VerbosePrintOut("'" + str(catfccs
) + "' != " + "'" + str(catnewfccs
) + "'");
4759 if(catfcompression
=="none" or catfcompression
=="" or catfcompression
=="auto"):
4762 catfcontents
.seek(0, 0);
4763 catfcontents
= UncompressArchiveFile(catfcontents
, formatspecs
);
4764 if(catfsize
>0 and listonly
):
4765 catfp
.seek(catfsize
, 1);
4766 pyhascontents
= False;
4768 catfcontentend
= catfp
.tell() - 1;
4769 catfcontents
.seek(0, 0);
4770 catlist
['ffilelist'].update({realidnum
: {'fid': realidnum
, 'fidalt': fileidnum
, 'fheadersize': catfheadsize
, 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
, 'fatime': catfatime
, 'fmtime': catfmtime
, 'fctime': catfctime
, 'fbtime': catfbtime
, 'fmode': catfmode
, 'fchmode': catfchmode
, 'ftypemod': catftypemod
, 'fwinattributes': catfwinattributes
, 'fcompression': catfcompression
, 'fcsize': catfcsize
, 'fuid': catfuid
, 'funame': catfuname
, 'fgid': catfgid
, 'fgname': catfgname
, 'finode': catfinode
, 'flinkcount': catflinkcount
, 'fminor': catfdev_minor
, 'fmajor': catfdev_major
, 'frminor': catfrdev_minor
, 'frmajor': catfrdev_major
, 'fchecksumtype': catfchecksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': catfextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': catfcs
, 'fcontentchecksum': catfccs
, 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': catfcontents
} });
4771 fileidnum
= fileidnum
+ 1;
4772 realidnum
= realidnum
+ 1;
4774 catlist
.update({'catfp': catfp
});
4779 create_alias_function("", __file_format_name__
, "ToArray", ArchiveFileToArray
);
4781 def ArchiveFileStringToArray(catstr
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4782 catfp
= BytesIO(catstr
);
4783 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4784 return listcatfiles
;
4786 create_alias_function("", __file_format_name__
, "StringToArray", ArchiveFileStringToArray
);
4788 def TarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4790 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4791 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4792 return listcatfiles
;
4794 def ZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4796 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4797 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4798 return listcatfiles
;
4800 if(not rarfile_support
):
4801 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4804 if(rarfile_support
):
4805 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4807 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4808 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4809 return listcatfiles
;
4811 if(not py7zr_support
):
4812 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4816 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4818 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4819 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4820 return listcatfiles
;
4822 def InFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
4823 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4824 if(checkcompressfile
=="tarfile"):
4825 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4826 elif(checkcompressfile
=="zipfile"):
4827 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4828 elif(checkcompressfile
=="catfile"):
4829 return ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4830 elif(rarfile_support
and checkcompressfile
=="rarfile"):
4831 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4832 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
4833 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4838 def ListDirToArrayAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
4839 catver
= formatspecs
['format_ver'];
4840 fileheaderver
= str(int(catver
.replace(".", "")));
4841 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
4842 advancedlist
= formatspecs
['use_advanced_list'];
4843 altinode
= formatspecs
['use_alt_inode'];
4846 for line
in sys
.stdin
:
4847 infilelist
.append(line
.strip());
4848 infilelist
= list(filter(None, infilelist
));
4849 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
4850 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
4852 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
4853 for line
in finfile
:
4854 infilelist
.append(line
.strip());
4855 infilelist
= list(filter(None, infilelist
));
4857 if(isinstance(infiles
, (list, tuple, ))):
4858 infilelist
= list(filter(None, infiles
));
4859 elif(isinstance(infiles
, (str, ))):
4860 infilelist
= list(filter(None, [infiles
]));
4862 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
4864 GetDirList
= ListDir(infilelist
, followlink
, False);
4872 inodetocatinode
= {};
4874 fnumfiles
= int(len(GetDirList
));
4875 catver
= formatspecs
['format_ver'];
4876 fileheaderver
= str(int(catver
.replace(".", "")));
4877 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
4878 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4879 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
4880 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
4881 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4882 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4883 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
4884 fheadtell
= len(fileheader
);
4885 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
4886 for curfname
in GetDirList
:
4887 if(re
.findall("^[.|/]", curfname
)):
4890 fname
= "./"+curfname
;
4892 VerbosePrintOut(fname
);
4893 if(not followlink
or followlink
is None):
4894 fstatinfo
= os
.lstat(fname
);
4896 fstatinfo
= os
.stat(fname
);
4897 fpremode
= fstatinfo
.st_mode
;
4898 finode
= fstatinfo
.st_ino
;
4899 flinkcount
= fstatinfo
.st_nlink
;
4901 if(stat
.S_ISREG(fpremode
)):
4903 elif(stat
.S_ISLNK(fpremode
)):
4905 elif(stat
.S_ISCHR(fpremode
)):
4907 elif(stat
.S_ISBLK(fpremode
)):
4909 elif(stat
.S_ISDIR(fpremode
)):
4911 elif(stat
.S_ISFIFO(fpremode
)):
4913 elif(stat
.S_ISSOCK(fpremode
)):
4915 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
4917 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
4919 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
4924 fbasedir
= os
.path
.dirname(fname
);
4926 if(not followlink
and finode
!=0):
4928 if(finode
in inodelist
):
4930 flinkname
= inodetofile
[finode
];
4934 fcurinode
= inodetocatinode
[finode
];
4935 if(finode
not in inodelist
):
4936 inodelist
.append(finode
);
4937 inodetofile
.update({finode
: fname
});
4938 inodetocatinode
.update({finode
: curinode
});
4942 fcurinode
= curinode
;
4943 curinode
= curinode
+ 1;
4945 fcurinode
= curinode
;
4946 curinode
= curinode
+ 1;
4947 curfid
= curfid
+ 1;
4949 flinkname
= os
.readlink(fname
);
4950 fdev
= fstatinfo
.st_dev
;
4951 getfdev
= GetDevMajorMinor(fdev
);
4952 fdev_minor
= getfdev
[0];
4953 fdev_major
= getfdev
[1];
4954 frdev
= fstatinfo
.st_dev
;
4955 if(hasattr(fstatinfo
, "st_rdev")):
4956 frdev
= fstatinfo
.st_rdev
;
4958 frdev
= fstatinfo
.st_dev
;
4959 getfrdev
= GetDevMajorMinor(frdev
);
4960 frdev_minor
= getfrdev
[0];
4961 frdev_major
= getfrdev
[1];
4962 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
4964 if(ftype
==0 or ftype
==7):
4965 fsize
= fstatinfo
.st_size
;
4966 fatime
= fstatinfo
.st_atime
;
4967 fmtime
= fstatinfo
.st_mtime
;
4968 fctime
= fstatinfo
.st_ctime
;
4969 if(hasattr(fstatinfo
, "st_birthtime")):
4970 fbtime
= fstatinfo
.st_birthtime
;
4972 fbtime
= fstatinfo
.st_ctime
;
4973 fmode
= fstatinfo
.st_mode
;
4974 fchmode
= stat
.S_IMODE(fstatinfo
.st_mode
);
4975 ftypemod
= stat
.S_IFMT(fstatinfo
.st_mode
);
4976 fuid
= fstatinfo
.st_uid
;
4977 fgid
= fstatinfo
.st_gid
;
4982 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
4983 funame
= userinfo
.pw_name
;
4992 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
4993 fgname
= groupinfo
.gr_name
;
4998 fdev_minor
= fdev_minor
;
4999 fdev_major
= fdev_major
;
5000 frdev_minor
= frdev_minor
;
5001 frdev_major
= frdev_major
;
5002 flinkcount
= flinkcount
;
5003 if(hasattr(fstatinfo
, "st_file_attributes")):
5004 fwinattributes
= fstatinfo
.st_file_attributes
;
5009 fcontents
= BytesIO();
5010 if(ftype
==0 or ftype
==7):
5011 with
open(fname
, "rb") as fpc
:
5012 shutil
.copyfileobj(fpc
, fcontents
);
5013 if(followlink
and (ftype
==1 or ftype
==2)):
5014 flstatinfo
= os
.stat(flinkname
);
5015 with
open(flinkname
, "rb") as fpc
:
5016 shutil
.copyfileobj(fpc
, fcontents
);
5017 fcontents
.seek(0, 0);
5018 ftypehex
= format(ftype
, 'x').lower();
5019 extrafields
= len(extradata
);
5020 extrafieldslist
= extradata
;
5021 catfextrafields
= extrafields
;
5022 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5023 if(len(extradata
)>0):
5024 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5025 extrasizelen
= len(extrasizestr
);
5026 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5027 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5028 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5029 catoutlenhex
= format(catoutlen
, 'x').lower();
5030 catoutlist
.insert(0, catoutlenhex
);
5031 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5032 if(len(extradata
)>0):
5033 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5034 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5035 catfnumfields
= catoutlen
;
5036 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5037 fcontents
.seek(0, 0);
5038 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5039 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5040 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5041 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5042 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5043 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5044 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5045 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5046 fheadtell
+= len(catfileoutstr
) + 1;
5047 catfcontentend
= fheadtell
- 1;
5048 fcontents
.seek(0, 0);
5049 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5050 pyhascontents
= False;
5051 if(int(fsize
)>0 and not listonly
):
5052 pyhascontents
= True;
5053 if(int(fsize
)>0 and listonly
):
5054 fcontents
= BytesIO();
5055 pyhascontents
= False;
5056 fcontents
.seek(0, 0);
5057 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5058 fileidnum
= fileidnum
+ 1;
5061 def TarFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5067 inodetocatinode
= {};
5071 if(hasattr(sys
.stdin
, "buffer")):
5072 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5074 shutil
.copyfileobj(sys
.stdin
, infile
);
5079 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5080 infile
= download_file_from_internet_file(infile
);
5085 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5087 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
5089 if(not tarfile
.is_tarfile(infile
)):
5091 except AttributeError:
5092 if(not is_tarfile(infile
)):
5097 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
5098 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
5100 tarfp
= tarfile
.open(infile
, "r");
5101 except FileNotFoundError
:
5103 fnumfiles
= int(len(tarfp
.getmembers()));
5104 catver
= formatspecs
['format_ver'];
5105 fileheaderver
= str(int(catver
.replace(".", "")));
5106 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5107 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5108 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5109 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5110 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5111 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5112 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5113 fheadtell
= len(fileheader
);
5114 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5115 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
5116 if(re
.findall("^[.|/]", member
.name
)):
5117 fname
= member
.name
;
5119 fname
= "./"+member
.name
;
5121 VerbosePrintOut(fname
);
5122 fpremode
= member
.mode
;
5123 ffullmode
= member
.mode
;
5127 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5129 elif(member
.isdev()):
5130 ffullmode
= member
.mode
;
5132 elif(member
.islnk()):
5133 ffullmode
= member
.mode
+ stat
.S_IFREG
;
5135 elif(member
.issym()):
5136 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
5138 elif(member
.ischr()):
5139 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
5141 elif(member
.isblk()):
5142 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
5144 elif(member
.isdir()):
5145 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
5147 elif(member
.isfifo()):
5148 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
5150 elif(member
.issparse()):
5151 ffullmode
= member
.mode
;
5154 ffullmode
= member
.mode
;
5157 fbasedir
= os
.path
.dirname(fname
);
5161 curfid
= curfid
+ 1;
5163 flinkname
= member
.linkname
;
5164 fdev_minor
= member
.devminor
;
5165 fdev_major
= member
.devmajor
;
5166 frdev_minor
= member
.devminor
;
5167 frdev_major
= member
.devmajor
;
5168 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
5170 elif(ftype
==0 or ftype
==7):
5171 fsize
= member
.size
;
5173 fsize
= member
.size
;
5174 fatime
= member
.mtime
;
5175 fmtime
= member
.mtime
;
5176 fctime
= member
.mtime
;
5177 fbtime
= member
.mtime
;
5179 fchmode
= stat
.S_IMODE(ffullmode
);
5180 ftypemod
= stat
.S_IFMT(ffullmode
);
5183 funame
= member
.uname
;
5184 fgname
= member
.gname
;
5185 flinkcount
= flinkcount
;
5186 fwinattributes
= int(0);
5189 fcontents
= BytesIO();
5190 if(ftype
==0 or ftype
==7):
5191 with tarfp
.extractfile(member
) as fpc
:
5192 shutil
.copyfileobj(fpc
, fcontents
);
5193 fcontents
.seek(0, 0);
5194 ftypehex
= format(ftype
, 'x').lower();
5195 extrafields
= len(extradata
);
5196 extrafieldslist
= extradata
;
5197 catfextrafields
= extrafields
;
5198 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5199 if(len(extradata
)>0):
5200 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5201 extrasizelen
= len(extrasizestr
);
5202 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5203 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5204 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5205 catoutlenhex
= format(catoutlen
, 'x').lower();
5206 catoutlist
.insert(0, catoutlenhex
);
5207 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5208 if(len(extradata
)>0):
5209 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5210 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5211 catfnumfields
= catoutlen
;
5212 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5213 fcontents
.seek(0, 0);
5214 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5215 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5216 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5217 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5218 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5219 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5220 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5221 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5222 fheadtell
+= len(catfileoutstr
) + 1;
5223 catfcontentend
= fheadtell
- 1;
5224 fcontents
.seek(0, 0);
5225 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5226 pyhascontents
= False;
5227 if(int(fsize
)>0 and not listonly
):
5228 pyhascontents
= True;
5229 if(int(fsize
)>0 and listonly
):
5231 pyhascontents
= False;
5232 fcontents
.seek(0, 0);
5233 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5234 fileidnum
= fileidnum
+ 1;
5237 def ZipFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5243 inodetocatinode
= {};
5247 if(hasattr(sys
.stdin
, "buffer")):
5248 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
5250 shutil
.copyfileobj(sys
.stdin
, infile
);
5255 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
5256 infile
= download_file_from_internet_file(infile
);
5261 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
5265 if(not zipfile
.is_zipfile(infile
)):
5268 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
5269 except FileNotFoundError
:
5271 ziptest
= zipfp
.testzip();
5273 VerbosePrintOut("Bad file found!");
5274 fnumfiles
= int(len(zipfp
.infolist()));
5275 catver
= formatspecs
['format_ver'];
5276 fileheaderver
= str(int(catver
.replace(".", "")));
5277 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5278 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5279 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5280 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5281 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5282 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5283 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5284 fheadtell
= len(fileheader
);
5285 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5286 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
5287 if(re
.findall("^[.|/]", member
.filename
)):
5288 fname
= member
.filename
;
5290 fname
= "./"+member
.filename
;
5291 zipinfo
= zipfp
.getinfo(member
.filename
);
5293 VerbosePrintOut(fname
);
5294 if(not member
.is_dir()):
5295 fpremode
= stat
.S_IFREG
+ 438;
5296 elif(member
.is_dir()):
5297 fpremode
= stat
.S_IFDIR
+ 511;
5300 if(not member
.is_dir()):
5302 elif(member
.is_dir()):
5305 fbasedir
= os
.path
.dirname(fname
);
5309 curfid
= curfid
+ 1;
5317 fsize
= member
.file_size
;
5319 fsize
= member
.file_size
;
5320 fatime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5321 fmtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5322 fctime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5323 fbtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5324 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
5325 fwinattributes
= int(zipinfo
.external_attr
);
5326 if(not member
.is_dir()):
5327 fmode
= int(stat
.S_IFREG
+ 438);
5328 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5329 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5330 elif(member
.is_dir()):
5331 fmode
= int(stat
.S_IFDIR
+ 511);
5332 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5333 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5334 elif(zipinfo
.create_system
==3):
5335 fwinattributes
= int(0);
5336 fmode
= int(zipinfo
.external_attr
);
5338 fwinattributes
= int(0);
5339 if(not member
.is_dir()):
5340 fmode
= int(stat
.S_IFREG
+ 438);
5341 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5342 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5343 elif(member
.is_dir()):
5344 fmode
= int(stat
.S_IFDIR
+ 511);
5345 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5346 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5351 except AttributeError:
5357 except AttributeError:
5364 userinfo
= pwd
.getpwuid(os
.getuid());
5365 funame
= userinfo
.pw_name
;
5368 except AttributeError:
5376 groupinfo
= grp
.getgrgid(os
.getgid());
5377 fgname
= groupinfo
.gr_name
;
5380 except AttributeError:
5384 fcontents
= BytesIO();
5386 fcontents
.write(zipfp
.read(member
.filename
));
5387 fcontents
.seek(0, 0);
5388 ftypehex
= format(ftype
, 'x').lower();
5389 extrafields
= len(extradata
);
5390 extrafieldslist
= extradata
;
5391 catfextrafields
= extrafields
;
5392 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5393 if(len(extradata
)>0):
5394 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5395 extrasizelen
= len(extrasizestr
);
5396 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5397 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5398 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5399 catoutlenhex
= format(catoutlen
, 'x').lower();
5400 catoutlist
.insert(0, catoutlenhex
);
5401 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5402 if(len(extradata
)>0):
5403 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5404 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5405 catfnumfields
= catoutlen
;
5406 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5407 fcontents
.seek(0, 0);
5408 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5409 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5410 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5411 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5412 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5413 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5414 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5415 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5416 fheadtell
+= len(catfileoutstr
) + 1;
5417 catfcontentend
= fheadtell
- 1;
5418 fcontents
.seek(0, 0);
5419 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5420 pyhascontents
= False;
5421 if(int(fsize
)>0 and not listonly
):
5422 pyhascontents
= True;
5423 if(int(fsize
)>0 and listonly
):
5424 fcontents
= BytesIO();
5425 pyhascontents
= False;
5426 fcontents
.seek(0, 0);
5427 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5428 fileidnum
= fileidnum
+ 1;
5431 if(not rarfile_support
):
5432 def RarFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5435 if(rarfile_support
):
5436 def RarFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5442 inodetocatinode
= {};
5444 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
5446 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
5448 rarfp
= rarfile
.RarFile(infile
, "r");
5449 rartest
= rarfp
.testrar();
5451 VerbosePrintOut("Bad file found!");
5452 fnumfiles
= int(len(rarfp
.infolist()));
5453 catver
= formatspecs
['format_ver'];
5454 fileheaderver
= str(int(catver
.replace(".", "")));
5455 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5456 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5457 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5458 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5459 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5460 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5461 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5462 fheadtell
= len(fileheader
);
5463 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5464 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
5467 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
5470 member
.external_attr
5472 except AttributeError:
5474 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
5477 member
.external_attr
5479 except AttributeError:
5484 if(re
.findall("^[.|/]", member
.filename
)):
5485 fname
= member
.filename
;
5487 fname
= "./"+member
.filename
;
5488 rarinfo
= rarfp
.getinfo(member
.filename
);
5490 VerbosePrintOut(fname
);
5491 if(is_unix
and member
.external_attr
!=0):
5492 fpremode
= int(member
.external_attr
);
5493 elif(member
.is_file()):
5494 fpremode
= stat
.S_IFREG
+ 438;
5495 elif(member
.is_symlink()):
5496 fpremode
= stat
.S_IFLNK
+ 438;
5497 elif(member
.is_dir()):
5498 fpremode
= stat
.S_IFDIR
+ 511;
5499 if(is_windows
and member
.external_attr
!=0):
5500 fwinattributes
= int(member
.external_attr
);
5502 fwinattributes
= int(0);
5507 if(member
.is_file()):
5509 elif(member
.is_symlink()):
5511 elif(member
.is_dir()):
5515 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
5516 fbasedir
= os
.path
.dirname(fname
);
5520 curfid
= curfid
+ 1;
5528 fsize
= member
.file_size
;
5531 fatime
= int(member
.atime
.timestamp());
5533 fatime
= int(member
.mtime
.timestamp());
5534 except AttributeError:
5535 fatime
= int(member
.mtime
.timestamp());
5536 fmtime
= int(member
.mtime
.timestamp());
5539 fctime
= int(member
.ctime
.timestamp());
5541 fctime
= int(member
.mtime
.timestamp());
5542 except AttributeError:
5543 fctime
= int(member
.mtime
.timestamp());
5544 fbtime
= int(member
.mtime
.timestamp());
5545 if(is_unix
and member
.external_attr
!=0):
5546 fmode
= int(member
.external_attr
);
5547 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
5548 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
5549 elif(member
.is_file()):
5550 fmode
= int(stat
.S_IFREG
+ 438)
5551 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5552 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5553 elif(member
.is_symlink()):
5554 fmode
= int(stat
.S_IFLNK
+ 438)
5555 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5556 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5557 elif(member
.is_dir()):
5558 fmode
= int(stat
.S_IFDIR
+ 511)
5559 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5560 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5563 except AttributeError:
5569 except AttributeError:
5576 userinfo
= pwd
.getpwuid(os
.getuid());
5577 funame
= userinfo
.pw_name
;
5580 except AttributeError:
5588 groupinfo
= grp
.getgrgid(os
.getgid());
5589 fgname
= groupinfo
.gr_name
;
5592 except AttributeError:
5596 fcontents
= BytesIO();
5598 fcontents
.write(rarfp
.read(member
.filename
));
5599 fcontents
.seek(0, 0);
5600 ftypehex
= format(ftype
, 'x').lower();
5601 extrafields
= len(extradata
);
5602 extrafieldslist
= extradata
;
5603 catfextrafields
= extrafields
;
5604 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5605 if(len(extradata
)>0):
5606 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5607 extrasizelen
= len(extrasizestr
);
5608 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5609 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5610 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5611 catoutlenhex
= format(catoutlen
, 'x').lower();
5612 catoutlist
.insert(0, catoutlenhex
);
5613 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5614 if(len(extradata
)>0):
5615 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5616 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5617 catfnumfields
= 24 + catfextrafields
;
5618 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5619 fcontents
.seek(0, 0);
5620 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5621 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5622 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5623 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5624 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5625 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5626 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5627 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5628 fheadtell
+= len(catfileoutstr
) + 1;
5629 catfcontentend
= fheadtell
- 1;
5630 fcontents
.seek(0, 0);
5631 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5632 pyhascontents
= False;
5633 if(int(fsize
)>0 and not listonly
):
5634 pyhascontents
= True;
5635 if(int(fsize
)>0 and listonly
):
5636 fcontents
= BytesIO();
5637 pyhascontents
= False;
5638 fcontents
.seek(0, 0);
5639 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5640 fileidnum
= fileidnum
+ 1;
5643 if(not py7zr_support
):
5644 def SevenZipFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5648 def SevenZipFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5654 inodetocatinode
= {};
5656 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
5657 file_content
= szpfp
.readall();
5658 #sztest = szpfp.testzip();
5659 sztestalt
= szpfp
.test();
5661 VerbosePrintOut("Bad file found!");
5662 numfiles
= int(len(szpfp
.list()));
5663 catver
= formatspecs
['format_ver'];
5664 fileheaderver
= str(int(catver
.replace(".", "")));
5665 fileheader
= AppendNullByte(formatspecs
['format_magic'] + fileheaderver
, formatspecs
['format_delimiter']);
5666 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5667 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5668 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5669 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
['format_delimiter']);
5670 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5671 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
['format_delimiter']);
5672 fheadtell
= len(fileheader
);
5673 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5674 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
5675 if(re
.findall("^[.|/]", member
.filename
)):
5676 fname
= member
.filename
;
5678 fname
= "./"+member
.filename
;
5679 if(not member
.is_directory
):
5680 fpremode
= int(stat
.S_IFREG
+ 438);
5681 elif(member
.is_directory
):
5682 fpremode
= int(stat
.S_IFDIR
+ 511);
5683 fwinattributes
= int(0);
5688 if(member
.is_directory
):
5693 fbasedir
= os
.path
.dirname(fname
);
5697 curfid
= curfid
+ 1;
5704 fatime
= int(member
.creationtime
.timestamp());
5705 fmtime
= int(member
.creationtime
.timestamp());
5706 fctime
= int(member
.creationtime
.timestamp());
5707 fbtime
= int(member
.creationtime
.timestamp());
5708 if(member
.is_directory
):
5709 fmode
= int(stat
.S_IFDIR
+ 511)
5710 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5711 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5713 fmode
= int(stat
.S_IFLNK
+ 438)
5714 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5715 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5718 except AttributeError:
5724 except AttributeError:
5731 userinfo
= pwd
.getpwuid(os
.getuid());
5732 funame
= userinfo
.pw_name
;
5735 except AttributeError:
5743 groupinfo
= grp
.getgrgid(os
.getgid());
5744 fgname
= groupinfo
.gr_name
;
5747 except AttributeError:
5751 fcontents
= BytesIO();
5753 fcontents
.write(file_content
[member
.filename
].read());
5754 fsize
= format(fcontents
.tell(), 'x').lower();
5756 fcontents
.seek(0, 0);
5757 ftypehex
= format(ftype
, 'x').lower();
5758 extrafields
= len(extradata
);
5759 extrafieldslist
= extradata
;
5760 catfextrafields
= extrafields
;
5761 extrasizestr
= AppendNullByte(extrafields
, formatspecs
['format_delimiter']);
5762 if(len(extradata
)>0):
5763 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5764 extrasizelen
= len(extrasizestr
);
5765 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5766 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), fcompression
, format(int(fcsize
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5767 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5768 catoutlenhex
= format(catoutlen
, 'x').lower();
5769 catoutlist
.insert(0, catoutlenhex
);
5770 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
['format_delimiter']);
5771 if(len(extradata
)>0):
5772 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
['format_delimiter']);
5773 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
['format_delimiter']);
5774 catfnumfields
= 24 + catfextrafields
;
5775 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5776 fcontents
.seek(0, 0);
5777 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5778 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5779 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5780 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
['format_delimiter']) + catfileoutstr
;
5781 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5782 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
['format_delimiter']);
5783 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5784 nullstrecd
= formatspecs
['format_delimiter'].encode('UTF-8');
5785 fheadtell
+= len(catfileoutstr
) + 1;
5786 catfcontentend
= fheadtell
- 1;
5787 fcontents
.seek(0, 0);
5788 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5789 pyhascontents
= False;
5790 if(int(fsize
)>0 and not listonly
):
5791 pyhascontents
= True;
5792 if(int(fsize
)>0 and listonly
):
5794 pyhascontents
= False;
5795 fcontents
.seek(0, 0);
5796 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fcompression': fcompression
, 'fcsize': fcsize
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5797 fileidnum
= fileidnum
+ 1;
5800 def InFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5801 checkcompressfile
= CheckCompressionSubType(infiles
, formatspecs
, True);
5802 if(checkcompressfile
=="tarfile"):
5803 return TarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5804 elif(checkcompressfile
=="zipfile"):
5805 return ZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5806 elif(checkcompressfile
=="catfile"):
5807 return ArchiveFileToArray(infile
, 0, 0, listonly
, False, formatspecs
, False);
5808 elif(rarfile_support
and checkcompressfile
=="rarfile"):
5809 return RarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5810 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
5811 return SevenZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5816 def ListDirToArray(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
5817 outarray
= BytesIO();
5818 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
5819 listcatfiles
= ArchiveFileToArray(outarray
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5820 return listcatfiles
;
5822 def ArchiveFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
5823 if(isinstance(infile
, dict)):
5824 listcatfiles
= infile
;
5826 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
5827 infile
= RemoveWindowsPath(infile
);
5828 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5829 if(not listcatfiles
):
5831 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5833 catarray
.update({'catfp': listcatfiles
['catfp']});
5834 lenlist
= len(listcatfiles
['ffilelist']);
5836 lcfx
= int(listcatfiles
['fnumfiles']);
5837 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5838 lcfx
= int(lenlist
);
5840 lcfx
= int(listcatfiles
['fnumfiles']);
5842 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5843 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5844 catarray
['filetoid'].update(filetoidarray
);
5845 catarray
['idtofile'].update(idtofilearray
);
5846 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5847 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5848 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5849 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5850 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5851 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5852 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5853 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5854 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5855 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5856 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5857 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5858 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5859 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5860 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5861 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5862 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5863 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5864 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5865 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5866 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5867 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5868 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5869 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5870 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5871 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5872 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5873 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5874 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5875 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5876 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5880 create_alias_function("", __file_format_name__
, "ToArrayIndex", ArchiveFileToArrayIndex
);
5882 def ListDirToArrayIndexAlt(infiles
, dirlistfromtxt
=False, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5883 listcatfiles
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5884 if(not listcatfiles
):
5886 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5887 lenlist
= len(listcatfiles
['ffilelist']);
5889 lcfx
= int(listcatfiles
['fnumfiles']);
5890 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5891 lcfx
= int(lenlist
);
5893 lcfx
= int(listcatfiles
['fnumfiles']);
5895 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5896 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5897 catarray
['filetoid'].update(filetoidarray
);
5898 catarray
['idtofile'].update(idtofilearray
);
5899 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5900 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5901 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5902 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5903 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5904 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5905 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5906 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5907 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5908 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5909 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5910 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5911 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5912 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5913 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5914 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5915 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5916 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5917 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5918 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5919 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5920 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5921 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5922 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5923 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5924 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5925 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5926 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5927 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5928 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5929 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5933 def TarFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5934 listcatfiles
= TarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5935 if(not listcatfiles
):
5937 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5938 lenlist
= len(listcatfiles
['ffilelist']);
5940 lcfx
= int(listcatfiles
['fnumfiles']);
5941 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5942 lcfx
= int(lenlist
);
5944 lcfx
= int(listcatfiles
['fnumfiles']);
5946 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5947 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5948 catarray
['filetoid'].update(filetoidarray
);
5949 catarray
['idtofile'].update(idtofilearray
);
5950 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5951 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5952 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5953 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5954 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5955 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5956 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5957 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5958 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5959 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5960 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5961 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5962 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5963 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5964 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5965 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5966 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5967 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5968 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5969 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5970 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5971 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5972 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5973 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5974 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5975 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5976 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5977 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5978 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5979 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5980 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5984 def ZipFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
5985 listcatfiles
= ZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5986 if(not listcatfiles
):
5988 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5989 lenlist
= len(listcatfiles
['ffilelist']);
5991 lcfx
= int(listcatfiles
['fnumfiles']);
5992 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5993 lcfx
= int(lenlist
);
5995 lcfx
= int(listcatfiles
['fnumfiles']);
5997 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5998 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5999 catarray
['filetoid'].update(filetoidarray
);
6000 catarray
['idtofile'].update(idtofilearray
);
6001 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6002 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6003 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6004 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6005 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6006 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6007 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6008 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6009 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6010 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6011 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6012 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6013 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6014 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6015 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6016 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6017 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6018 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6019 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6020 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6021 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6022 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6023 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6024 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6025 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6026 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6027 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6028 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6029 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6030 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6031 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6035 if(not rarfile_support
):
6036 def RarFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6039 if(rarfile_support
):
6040 def RarFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6041 listcatfiles
= RarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6042 if(not listcatfiles
):
6044 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6045 lenlist
= len(listcatfiles
['ffilelist']);
6047 lcfx
= int(listcatfiles
['fnumfiles']);
6048 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6049 lcfx
= int(lenlist
);
6051 lcfx
= int(listcatfiles
['fnumfiles']);
6053 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6054 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6055 catarray
['filetoid'].update(filetoidarray
);
6056 catarray
['idtofile'].update(idtofilearray
);
6057 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6058 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6059 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6060 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6061 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6062 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6063 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6064 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6065 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6066 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6067 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6068 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6069 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6070 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6071 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6072 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6073 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6074 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6075 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6076 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6077 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6078 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6079 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6080 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6081 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6082 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6083 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6084 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6085 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6086 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6087 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6091 if(not py7zr_support
):
6092 def SevenZipFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6096 def SevenZipFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False):
6097 listcatfiles
= SevenZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
6098 if(not listcatfiles
):
6100 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6101 lenlist
= len(listcatfiles
['ffilelist']);
6103 lcfx
= int(listcatfiles
['fnumfiles']);
6104 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6105 lcfx
= int(lenlist
);
6107 lcfx
= int(listcatfiles
['fnumfiles']);
6109 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
6110 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
6111 catarray
['filetoid'].update(filetoidarray
);
6112 catarray
['idtofile'].update(idtofilearray
);
6113 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6114 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
6115 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
6116 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6117 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
6118 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
6119 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6120 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6121 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6122 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6123 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6124 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
6125 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
6126 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
6127 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
6128 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
6129 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6130 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6131 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
6132 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
6133 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
6134 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6135 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6136 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6137 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
6138 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
6139 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
6140 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
6141 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
6142 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
6143 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
6147 def ArchiveFileStringToArrayIndex(catstr
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6148 catfp
= BytesIO(catstr
);
6149 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6150 return listcatfiles
;
6152 create_alias_function("", __file_format_name__
, "StringToArrayIndex", ArchiveFileStringToArrayIndex
);
6154 def TarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6156 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
6157 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6158 return listcatfiles
;
6160 def ZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6162 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
6163 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6164 return listcatfiles
;
6166 if(not rarfile_support
):
6167 def RarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6170 if(rarfile_support
):
6171 def RarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6173 catfp
= PackArchiveFileFromRarFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
6174 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6175 return listcatfiles
;
6177 if(not py7zr_support
):
6178 def SevenZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6182 def SevenZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
6184 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
6185 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
6186 return listcatfiles
;
6188 def ListDirToArrayIndex(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6189 outarray
= BytesIO();
6190 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
, True);
6191 listcatfiles
= ArchiveFileToArrayIndex(outarray
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
)
6192 return listcatfiles
;
6194 def RePackArchiveFile(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6195 if(isinstance(infile
, dict)):
6196 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6197 listcatfiles
= prelistcatfiles
['list'];
6199 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6200 infile
= RemoveWindowsPath(infile
);
6202 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6203 listcatfiles
= prelistcatfiles
['list'];
6205 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6206 if(outfile
!="-" and not hasattr(infile
, "read") and not hasattr(outfile
, "write")):
6207 outfile
= RemoveWindowsPath(outfile
);
6208 checksumtype
= checksumtype
.lower();
6209 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
6210 checksumtype
="crc32";
6211 if(checksumtype
=="none"):
6213 if(not compression
or compression
=="catfile" or compression
==formatspecs
['format_lower']):
6214 compression
= "auto";
6215 if(compression
not in compressionlist
and compression
is None):
6216 compression
= "auto";
6218 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6219 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
6220 if(os
.path
.exists(outfile
)):
6223 except OSError as e
:
6225 if(not listcatfiles
):
6230 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6232 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6235 fbasename
= os
.path
.splitext(outfile
)[0];
6236 fextname
= os
.path
.splitext(outfile
)[1];
6237 catfp
= CompressOpenFile(outfile
, compresswholefile
, compressionlevel
);
6238 catver
= formatspecs
['format_ver'];
6239 fileheaderver
= str(int(catver
.replace(".", "")));
6240 lenlist
= len(listcatfiles
['ffilelist']);
6241 fnumfiles
= int(listcatfiles
['fnumfiles']);
6242 if(lenlist
>fnumfiles
or lenlist
<fnumfiles
):
6243 fnumfiles
= lenlist
;
6244 AppendFileHeader(catfp
, fnumfiles
, checksumtype
, formatspecs
);
6245 lenlist
= len(listcatfiles
['ffilelist']);
6246 fnumfiles
= int(listcatfiles
['fnumfiles']);
6248 lcfx
= int(listcatfiles
['fnumfiles']);
6249 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6250 lcfx
= int(lenlist
);
6252 lcfx
= int(listcatfiles
['fnumfiles']);
6260 if(re
.findall("^[.|/]", listcatfiles
['ffilelist'][reallcfi
]['fname'])):
6261 fname
= listcatfiles
['ffilelist'][reallcfi
]['fname'];
6263 fname
= "./"+listcatfiles
['ffilelist'][reallcfi
]['fname'];
6265 VerbosePrintOut(fname
);
6266 fheadersize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fheadersize']), 'x').lower();
6267 fsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fsize']), 'x').lower();
6268 flinkname
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6269 fatime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fatime']), 'x').lower();
6270 fmtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmtime']), 'x').lower();
6271 fctime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fctime']), 'x').lower();
6272 fbtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fbtime']), 'x').lower();
6273 fmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmode']), 'x').lower();
6274 fchmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fchmode']), 'x').lower();
6275 fuid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fuid']), 'x').lower();
6276 funame
= listcatfiles
['ffilelist'][reallcfi
]['funame'];
6277 fgid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fgid']), 'x').lower();
6278 fgname
= listcatfiles
['ffilelist'][reallcfi
]['fgname'];
6279 finode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['finode']), 'x').lower();
6280 flinkcount
= format(int(listcatfiles
['ffilelist'][reallcfi
]['flinkcount']), 'x').lower();
6281 fwinattributes
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fwinattributes']), 'x').lower();
6282 fcompression
= listcatfiles
['ffilelist'][reallcfi
]['fcompression'];
6283 fcsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fcsize']), 'x').lower();
6284 fdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fminor']), 'x').lower();
6285 fdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmajor']), 'x').lower();
6286 frdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frminor']), 'x').lower();
6287 frdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frmajor']), 'x').lower();
6288 if(len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] and len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>0):
6289 listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] = len(listcatfiles
['ffilelist'][reallcfi
]['fextralist']);
6290 if(not followlink
and len(extradata
)<0):
6291 extradata
= listcatfiles
['ffilelist'][reallcfi
]['fextralist'];
6292 fcontents
= listcatfiles
['ffilelist'][reallcfi
]['fcontents'];
6294 fcsize
= format(int(0), 'x').lower();
6295 if(not compresswholefile
):
6296 fcontents
.seek(0, 2);
6297 ucfsize
= fcontents
.tell();
6298 fcontents
.seek(0, 0);
6299 if(compression
=="auto"):
6300 ilsize
= len(compressionlistalt
);
6303 while(ilmin
< ilsize
):
6304 cfcontents
= BytesIO();
6305 shutil
.copyfileobj(fcontents
, cfcontents
);
6306 fcontents
.seek(0, 0);
6307 cfcontents
.seek(0, 0);
6308 cfcontents
= CompressArchiveFile(cfcontents
, compressionlistalt
[ilmin
], compressionlevel
, formatspecs
);
6310 cfcontents
.seek(0, 2);
6311 ilcsize
.append(cfcontents
.tell());
6315 ilcsize
.append(sys
.maxint
);
6316 except AttributeError:
6317 ilcsize
.append(sys
.maxsize
);
6319 ilcmin
= ilcsize
.index(min(ilcsize
));
6320 compression
= compressionlistalt
[ilcmin
];
6321 fcontents
.seek(0, 0);
6322 cfcontents
= BytesIO();
6323 shutil
.copyfileobj(fcontents
, cfcontents
);
6324 cfcontents
.seek(0, 0);
6325 cfcontents
= CompressArchiveFile(cfcontents
, compression
, compressionlevel
, formatspecs
);
6326 cfcontents
.seek(0, 2);
6327 cfsize
= cfcontents
.tell();
6328 if(ucfsize
> cfsize
):
6329 fcsize
= format(int(cfsize
), 'x').lower();
6330 fcompression
= compression
;
6332 fcontents
= cfcontents
;
6334 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']==1 or listcatfiles
['ffilelist'][reallcfi
]['ftype']==2):
6335 getflinkpath
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6336 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6337 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6338 fheadersize
= format(int(flinkinfo
['fheadersize']), 'x').lower();
6339 fsize
= format(int(flinkinfo
['fsize']), 'x').lower();
6340 flinkname
= flinkinfo
['flinkname'];
6341 fatime
= format(int(flinkinfo
['fatime']), 'x').lower();
6342 fmtime
= format(int(flinkinfo
['fmtime']), 'x').lower();
6343 fctime
= format(int(flinkinfo
['fctime']), 'x').lower();
6344 fbtime
= format(int(flinkinfo
['fbtime']), 'x').lower();
6345 fmode
= format(int(flinkinfo
['fmode']), 'x').lower();
6346 fchmode
= format(int(flinkinfo
['fchmode']), 'x').lower();
6347 fuid
= format(int(flinkinfo
['fuid']), 'x').lower();
6348 funame
= flinkinfo
['funame'];
6349 fgid
= format(int(flinkinfo
['fgid']), 'x').lower();
6350 fgname
= flinkinfo
['fgname'];
6351 finode
= format(int(flinkinfo
['finode']), 'x').lower();
6352 flinkcount
= format(int(flinkinfo
['flinkcount']), 'x').lower();
6353 fwinattributes
= format(int(flinkinfo
['fwinattributes']), 'x').lower();
6354 fcompression
= flinkinfo
['fcompression'];
6355 fcsize
= format(int(flinkinfo
['fcsize']), 'x').lower();
6356 fdev_minor
= format(int(flinkinfo
['fminor']), 'x').lower();
6357 fdev_major
= format(int(flinkinfo
['fmajor']), 'x').lower();
6358 frdev_minor
= format(int(flinkinfo
['frminor']), 'x').lower();
6359 frdev_major
= format(int(flinkinfo
['frmajor']), 'x').lower();
6360 if(len(flinkinfo
['fextralist'])>flinkinfo
['fextrafields'] and len(flinkinfo
['fextralist'])>0):
6361 flinkinfo
['fextrafields'] = len(flinkinfo
['fextralist']);
6362 if(len(extradata
)<0):
6363 extradata
= flinkinfo
['fextralist'];
6364 fcontents
= flinkinfo
['fcontents'];
6365 ftypehex
= format(flinkinfo
['ftype'], 'x').lower();
6367 ftypehex
= format(listcatfiles
['ffilelist'][reallcfi
]['ftype'], 'x').lower();
6368 fcurfid
= format(curfid
, 'x').lower();
6369 if(not followlink
and finode
!=0):
6370 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']!=1):
6371 fcurinode
= format(int(curinode
), 'x').lower();
6372 inodetofile
.update({curinode
: fname
});
6373 filetoinode
.update({fname
: curinode
});
6374 curinode
= curinode
+ 1;
6376 fcurinode
= format(int(filetoinode
[flinkname
]), 'x').lower();
6378 fcurinode
= format(int(curinode
), 'x').lower();
6379 curinode
= curinode
+ 1;
6380 curfid
= curfid
+ 1;
6381 if(fcompression
=="none"):
6383 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fcompression
, fcsize
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
6384 catfp
= AppendFileHeaderWithContent(catfp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
6387 reallcfi
= reallcfi
+ 1;
6389 catfp
.write(AppendNullBytes([0, 0], formatspecs
['format_delimiter']).encode("UTF-8"));
6390 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6391 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6394 os
.fsync(catfp
.fileno());
6395 except io
.UnsupportedOperation
:
6397 except AttributeError:
6399 except OSError as e
:
6403 if(hasattr(sys
.stdout
, "buffer")):
6404 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
6406 shutil
.copyfileobj(catfp
, sys
.stdout
);
6407 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6408 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
6410 upload_file_to_internet_file(catfp
, outfile
);
6418 create_alias_function("RePack", __file_format_name__
, "", RePackArchiveFile
);
6420 def RePackArchiveFileFromString(catstr
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6421 catfp
= BytesIO(catstr
);
6422 listcatfiles
= RePackArchiveFile(catfp
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6423 return listcatfiles
;
6425 create_alias_function("RePack", __file_format_name__
, "FromString", RePackArchiveFileFromString
);
6427 def PackArchiveFileFromListDir(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6428 outarray
= BytesIO();
6429 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
6430 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6431 return listcatfiles
;
6433 create_alias_function("Pack", __file_format_name__
, "FromListDir", PackArchiveFileFromListDir
);
6435 def UnPackArchiveFile(infile
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, preservepermissions
=True, preservetime
=True, verbose
=False, returnfp
=False):
6436 if(outdir
is not None):
6437 outdir
= RemoveWindowsPath(outdir
);
6439 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6440 if(isinstance(infile
, dict)):
6441 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6442 listcatfiles
= prelistcatfiles
['list'];
6444 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6445 infile
= RemoveWindowsPath(infile
);
6447 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6448 listcatfiles
= prelistcatfiles
['list'];
6450 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6451 if(not listcatfiles
):
6453 lenlist
= len(listcatfiles
['ffilelist']);
6454 fnumfiles
= int(listcatfiles
['fnumfiles']);
6456 lcfx
= int(listcatfiles
['fnumfiles']);
6457 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6458 lcfx
= int(lenlist
);
6460 lcfx
= int(listcatfiles
['fnumfiles']);
6466 userinfo
= pwd
.getpwuid(listcatfiles
['ffilelist'][lcfi
]['fuid']);
6467 funame
= userinfo
.pw_name
;
6476 groupinfo
= grp
.getgrgid(listcatfiles
['ffilelist'][lcfi
]['fgid']);
6477 fgname
= groupinfo
.gr_name
;
6483 VerbosePrintOut(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6484 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6485 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6486 listcatfiles
['ffilelist'][lcfi
]['fcontents'].seek(0, 0);
6487 shutil
.copyfileobj(listcatfiles
['ffilelist'][lcfi
]['fcontents'], fpc
);
6490 os
.fsync(fpc
.fileno());
6491 except io
.UnsupportedOperation
:
6493 except AttributeError:
6495 except OSError as e
:
6497 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6498 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6499 if(preservepermissions
):
6500 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6502 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6503 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6505 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6506 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6507 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6512 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6513 funame
= userinfo
.pw_name
;
6522 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6523 fgname
= groupinfo
.gr_name
;
6528 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6529 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6530 flinkinfo
['fcontents'].seek(0, 0);
6531 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6534 os
.fsync(fpc
.fileno());
6535 except io
.UnsupportedOperation
:
6537 except AttributeError:
6539 except OSError as e
:
6541 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6542 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6543 if(preservepermissions
):
6544 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6546 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6547 if(flinkinfo
['ftype']==1):
6548 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6549 if(flinkinfo
['ftype']==2):
6550 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6551 if(flinkinfo
['ftype']==5):
6552 if(preservepermissions
):
6553 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6555 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6556 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6557 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6558 if(preservepermissions
):
6559 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6561 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6562 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6563 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6565 os
.link(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6566 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6568 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6569 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6570 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6575 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6576 funame
= userinfo
.pw_name
;
6585 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6586 fgname
= groupinfo
.gr_name
;
6591 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6592 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6593 flinkinfo
['fcontents'].seek(0, 0);
6594 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6597 os
.fsync(fpc
.fileno());
6598 except io
.UnsupportedOperation
:
6600 except AttributeError:
6602 except OSError as e
:
6604 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6605 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6606 if(preservepermissions
):
6607 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6609 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6610 if(flinkinfo
['ftype']==1):
6611 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6612 if(flinkinfo
['ftype']==2):
6613 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6614 if(flinkinfo
['ftype']==5):
6615 if(preservepermissions
):
6616 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6618 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6619 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6620 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6621 if(preservepermissions
):
6622 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6624 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6625 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6626 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6628 os
.symlink(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6629 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6630 if(preservepermissions
):
6631 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6633 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6634 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6635 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6636 if(preservepermissions
):
6637 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6639 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6640 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6 and hasattr(os
, "mkfifo")):
6641 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6644 return listcatfiles
['ffilelist']['catfp'];
6648 create_alias_function("UnPack", __file_format_name__
, "", UnPackArchiveFile
);
6650 if(hasattr(shutil
, "register_unpack_format")):
6651 def UnPackArchiveFileFunc(archive_name
, extract_dir
=None, **kwargs
):
6652 return UnPackArchiveFile(archive_name
, extract_dir
, False, 0, 0, False, __file_format_dict__
['format_delimiter'], False, False);
6653 create_alias_function("UnPack", __file_format_name__
, "Func", UnPackArchiveFileFunc
);
6655 def UnPackArchiveFileString(catstr
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6656 catfp
= BytesIO(catstr
);
6657 listcatfiles
= UnPackArchiveFile(catfp
, outdir
, followlink
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6658 return listcatfiles
;
6660 create_alias_function("UnPack", __file_format_name__
, "String", UnPackArchiveFileString
);
6662 def ArchiveFileListFiles(infile
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6663 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6664 if(isinstance(infile
, dict)):
6665 listcatfiles
= infile
;
6667 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6668 infile
= RemoveWindowsPath(infile
);
6669 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, True, skipchecksum
, formatspecs
, returnfp
);
6670 if(not listcatfiles
):
6672 lenlist
= len(listcatfiles
['ffilelist']);
6673 fnumfiles
= int(listcatfiles
['fnumfiles']);
6675 lcfx
= int(listcatfiles
['fnumfiles']);
6676 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6677 lcfx
= int(lenlist
);
6679 lcfx
= int(listcatfiles
['fnumfiles']);
6682 returnval
.update({lcfi
: listcatfiles
['ffilelist'][lcfi
]['fname']});
6684 VerbosePrintOut(listcatfiles
['ffilelist'][lcfi
]['fname']);
6686 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6687 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'];
6688 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6689 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " link to " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6690 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6691 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " -> " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6692 fuprint
= listcatfiles
['ffilelist'][lcfi
]['funame'];
6693 if(len(fuprint
)<=0):
6694 fuprint
= listcatfiles
['ffilelist'][lcfi
]['fuid'];
6695 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgname'];
6696 if(len(fgprint
)<=0):
6697 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgid'];
6698 VerbosePrintOut(PrintPermissionString(listcatfiles
['ffilelist'][lcfi
]['fmode'], listcatfiles
['ffilelist'][lcfi
]['ftype']) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(listcatfiles
['ffilelist'][lcfi
]['fsize']).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(listcatfiles
['ffilelist'][lcfi
]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6701 return listcatfiles
['catfp'];
6705 create_alias_function("", __file_format_name__
, "ListFiles", ArchiveFileListFiles
);
6707 def ArchiveFileStringListFiles(catstr
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
6708 catfp
= BytesIO(catstr
);
6709 listcatfiles
= ArchiveFileListFiles(catstr
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6710 return listcatfiles
;
6712 create_alias_function("", __file_format_name__
, "StringListFiles", ArchiveFileStringListFiles
);
6714 def TarFileListFiles(infile
, verbose
=False, returnfp
=False):
6715 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6718 if(hasattr(sys
.stdin
, "buffer")):
6719 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6721 shutil
.copyfileobj(sys
.stdin
, infile
);
6726 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6727 infile
= download_file_from_internet_file(infile
);
6732 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6734 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
6736 if(not tarfile
.is_tarfile(infile
)):
6738 except AttributeError:
6739 if(not is_tarfile(infile
)):
6744 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
6745 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
6747 tarfp
= tarfile
.open(infile
, "r");
6748 except FileNotFoundError
:
6752 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
6753 returnval
.update({lcfi
: member
.name
});
6754 fpremode
= member
.mode
;
6755 ffullmode
= member
.mode
;
6759 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6761 elif(member
.isdev()):
6762 ffullmode
= member
.mode
;
6764 elif(member
.islnk()):
6765 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6767 elif(member
.issym()):
6768 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
6770 elif(member
.ischr()):
6771 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
6773 elif(member
.isblk()):
6774 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
6776 elif(member
.isdir()):
6777 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
6779 elif(member
.isfifo()):
6780 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
6782 elif(member
.issparse()):
6783 ffullmode
= member
.mode
;
6786 VerbosePrintOut(member
.name
);
6788 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6789 printfname
= member
.name
;
6791 printfname
= member
.name
+ " link to " + member
.linkname
;
6792 elif(member
.issym()):
6793 printfname
= member
.name
+ " -> " + member
.linkname
;
6794 fuprint
= member
.uname
;
6795 if(len(fuprint
)<=0):
6796 fuprint
= member
.uid
;
6797 fgprint
= member
.gname
;
6798 if(len(fgprint
)<=0):
6799 fgprint
= member
.gid
;
6800 VerbosePrintOut(PrintPermissionString(ffullmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(member
.mtime
).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6803 return listcatfiles
['catfp'];
6807 def ZipFileListFiles(infile
, verbose
=False, returnfp
=False):
6808 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6811 if(hasattr(sys
.stdin
, "buffer")):
6812 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6814 shutil
.copyfileobj(sys
.stdin
, infile
);
6819 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6820 infile
= download_file_from_internet_file(infile
);
6825 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6829 if(not zipfile
.is_zipfile(infile
)):
6832 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
6833 except FileNotFoundError
:
6837 ziptest
= zipfp
.testzip();
6839 VerbosePrintOut("Bad file found!");
6840 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
6841 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
6842 fwinattributes
= int(zipinfo
.external_attr
);
6843 if(not member
.is_dir()):
6844 fmode
= int(stat
.S_IFREG
+ 438);
6845 fchmode
= int(stat
.S_IMODE(fmode
));
6846 ftypemod
= int(stat
.S_IFMT(fmode
));
6847 elif(member
.is_dir()):
6848 fmode
= int(stat
.S_IFDIR
+ 511);
6849 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6850 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6851 elif(zipinfo
.create_system
==3):
6852 fwinattributes
=int(0);
6853 fmode
= int(zipinfo
.external_attr
);
6854 fchmode
= int(stat
.S_IMODE(fmode
));
6855 ftypemod
= int(stat
.S_IFMT(fmode
));
6857 fwinattributes
= int(0);
6858 if(not member
.is_dir()):
6859 fmode
= int(stat
.S_IFREG
+ 438);
6860 fchmode
= int(stat
.S_IMODE(fmode
));
6861 ftypemod
= int(stat
.S_IFMT(fmode
));
6862 elif(member
.is_dir()):
6863 fmode
= int(stat
.S_IFDIR
+ 511);
6864 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6865 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6866 returnval
.update({lcfi
: member
.filename
});
6868 VerbosePrintOut(member
.filename
);
6870 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6872 for fmodval
in str(oct(fmode
))[-3:]:
6873 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
6874 if(not member
.is_dir()):
6876 permissionstr
= "-" + permissionstr
;
6877 elif(member
.is_dir()):
6879 permissionstr
= "d" + permissionstr
;
6880 printfname
= member
.filename
;
6882 fuid
= int(os
.getuid());
6883 except AttributeError:
6888 fgid
= int(os
.getgid());
6889 except AttributeError:
6896 userinfo
= pwd
.getpwuid(os
.getuid());
6897 funame
= userinfo
.pw_name
;
6900 except AttributeError:
6908 groupinfo
= grp
.getgrgid(os
.getgid());
6909 fgname
= groupinfo
.gr_name
;
6912 except AttributeError:
6917 if(len(fuprint
)<=0):
6918 fuprint
= str(fuid
);
6920 if(len(fgprint
)<=0):
6921 fgprint
= str(fgid
);
6922 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(int(time
.mktime(member
.date_time
+ (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6925 return listcatfiles
['catfp'];
6929 if(not rarfile_support
):
6930 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
6931 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6932 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6935 if(rarfile_support
):
6936 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
6937 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6938 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6940 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
6944 rarfp
= rarfile
.RarFile(infile
, "r");
6945 rartest
= rarfp
.testrar();
6947 VerbosePrintOut("Bad file found!");
6948 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
6951 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
6954 member
.external_attr
6956 except AttributeError:
6958 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
6961 member
.external_attr
6963 except AttributeError:
6968 if(is_unix
and member
.external_attr
!=0):
6969 fpremode
= int(member
.external_attr
);
6970 elif(member
.is_file()):
6971 fpremode
= int(stat
.S_IFREG
+ 438);
6972 elif(member
.is_symlink()):
6973 fpremode
= int(stat
.S_IFLNK
+ 438);
6974 elif(member
.is_dir()):
6975 fpremode
= int(stat
.S_IFDIR
+ 511);
6976 if(is_windows
and member
.external_attr
!=0):
6977 fwinattributes
= int(member
.external_attr
);
6979 fwinattributes
= int(0);
6980 if(is_unix
and member
.external_attr
!=0):
6981 fmode
= int(member
.external_attr
);
6982 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
6983 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
6984 elif(member
.is_file()):
6985 fmode
= int(stat
.S_IFREG
+ 438);
6986 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
6987 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
6988 elif(member
.is_symlink()):
6989 fmode
= int(stat
.S_IFLNK
+ 438);
6990 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
6991 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
6992 elif(member
.is_dir()):
6993 fmode
= int(stat
.S_IFDIR
+ 511);
6994 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6995 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6996 returnval
.update({lcfi
: member
.filename
});
6998 VerbosePrintOut(member
.filename
);
7000 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7002 for fmodval
in str(oct(fmode
))[-3:]:
7003 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7004 if(member
.is_file()):
7006 permissionstr
= "-" + permissionstr
;
7007 printfname
= member
.filename
;
7008 elif(member
.is_symlink()):
7010 permissionstr
= "l" + permissionstr
;
7011 printfname
= member
.name
+ " -> " + member
.read().decode("UTF-8");
7012 elif(member
.is_dir()):
7014 permissionstr
= "d" + permissionstr
;
7015 printfname
= member
.filename
;
7017 fuid
= int(os
.getuid());
7018 except AttributeError:
7023 fgid
= int(os
.getgid());
7024 except AttributeError:
7031 userinfo
= pwd
.getpwuid(os
.getuid());
7032 funame
= userinfo
.pw_name
;
7035 except AttributeError:
7043 groupinfo
= grp
.getgrgid(os
.getgid());
7044 fgname
= groupinfo
.gr_name
;
7047 except AttributeError:
7052 if(len(fuprint
)<=0):
7053 fuprint
= str(fuid
);
7055 if(len(fgprint
)<=0):
7056 fgprint
= str(fgid
);
7057 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + member
.mtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7060 return listcatfiles
['catfp'];
7064 if(not py7zr_support
):
7065 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7066 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7067 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7071 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
7072 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7073 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
7077 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
7078 file_content
= szpfp
.readall();
7079 #sztest = szpfp.testzip();
7080 sztestalt
= szpfp
.test();
7082 VerbosePrintOut("Bad file found!");
7083 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
7084 if(re
.findall("^[.|/]", member
.filename
)):
7085 fname
= member
.filename
;
7087 fname
= "./"+member
.filename
;
7088 if(not member
.is_directory
):
7089 fpremode
= int(stat
.S_IFREG
+ 438);
7090 elif(member
.is_directory
):
7091 fpremode
= int(stat
.S_IFDIR
+ 511);
7092 fwinattributes
= int(0);
7093 if(member
.is_directory
):
7094 fmode
= int(stat
.S_IFDIR
+ 511);
7095 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
7096 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
7098 fmode
= int(stat
.S_IFLNK
+ 438);
7099 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
7100 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
7101 returnval
.update({lcfi
: member
.filename
});
7103 VerbosePrintOut(member
.filename
);
7105 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7107 for fmodval
in str(oct(fmode
))[-3:]:
7108 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
7110 if(not member
.is_directory
):
7112 permissionstr
= "-" + permissionstr
;
7113 printfname
= member
.filename
;
7114 elif(member
.is_directory
):
7116 permissionstr
= "d" + permissionstr
;
7117 printfname
= member
.filename
;
7119 fsize
= len(file_content
[member
.filename
].read());
7120 file_content
[member
.filename
].close();
7122 fuid
= int(os
.getuid());
7123 except AttributeError:
7128 fgid
= int(os
.getgid());
7129 except AttributeError:
7136 userinfo
= pwd
.getpwuid(os
.getuid());
7137 funame
= userinfo
.pw_name
;
7140 except AttributeError:
7148 groupinfo
= grp
.getgrgid(os
.getgid());
7149 fgname
= groupinfo
.gr_name
;
7152 except AttributeError:
7157 if(len(fuprint
)<=0):
7158 fuprint
= str(fuid
);
7160 if(len(fgprint
)<=0):
7161 fgprint
= str(fgid
);
7162 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(fsize
).rjust(15) + " " + member
.creationtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
7165 return listcatfiles
['catfp'];
7169 def InFileListFiles(infile
, verbose
=False, formatspecs
=__file_format_dict__
, returnfp
=False):
7170 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
7171 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
7172 if(checkcompressfile
=="tarfile"):
7173 return TarFileListFiles(infile
, verbose
, returnfp
);
7174 elif(checkcompressfile
=="zipfile"):
7175 return ZipFileListFiles(infile
, verbose
, returnfp
);
7176 elif(checkcompressfile
=="catfile"):
7177 return ArchiveFileListFiles(infile
, 0, 0, False, formatspecs
, verbose
, returnfp
);
7178 elif(rarfile_support
and checkcompressfile
=="rarfile"):
7179 return RarFileListFiles(infile
, verbose
, returnfp
);
7180 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
7181 return SevenZipFileListFiles(infile
, verbose
, returnfp
);
7186 def ListDirListFiles(infiles
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7187 outarray
= BytesIO();
7188 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, formatspecs
, False, True);
7189 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7190 return listcatfiles
;
7192 def ListDirListFilesAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7193 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, formatspecs
, verbose
);
7194 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
7195 return listcatfiles
;
7197 def PackArchiveFileFromListDirAlt(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compresswholefile
=True, compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7198 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, False, checksumtype
, extradata
, formatspecs
, False);
7199 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, followlink
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
7200 return listcatfiles
;
7202 create_alias_function("Pack", __file_format_name__
, "FromListDirAlt", PackArchiveFileFromListDirAlt
);
7204 def PackArchiveFileFromTarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7205 outarray
= TarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7206 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7207 return listcatfiles
;
7209 create_alias_function("Pack", __file_format_name__
, "FromTarFileAlt", PackArchiveFileFromTarFileAlt
);
7211 def PackArchiveFileFromZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7212 outarray
= ZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7213 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7214 return listcatfiles
;
7216 create_alias_function("Pack", __file_format_name__
, "FromZipFileAlt", PackArchiveFileFromZipFileAlt
);
7218 if(not rarfile_support
):
7219 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7222 if(rarfile_support
):
7223 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7224 outarray
= RarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7225 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7226 return listcatfiles
;
7228 create_alias_function("Pack", __file_format_name__
, "FromRarFileAlt", PackArchiveFileFromRarFileAlt
);
7230 if(not py7zr_support
):
7231 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7235 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compresswholefile
=True, compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_dict__
, verbose
=False, returnfp
=False):
7236 outarray
= SevenZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
7237 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compresswholefile
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
7238 return listcatfiles
;
7240 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt
);
7242 def download_file_from_ftp_file(url
):
7243 urlparts
= urlparse(url
);
7244 file_name
= os
.path
.basename(urlparts
.path
);
7245 file_dir
= os
.path
.dirname(urlparts
.path
);
7246 if(urlparts
.username
is not None):
7247 ftp_username
= urlparts
.username
;
7249 ftp_username
= "anonymous";
7250 if(urlparts
.password
is not None):
7251 ftp_password
= urlparts
.password
;
7252 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7253 ftp_password
= "anonymous";
7256 if(urlparts
.scheme
=="ftp"):
7258 elif(urlparts
.scheme
=="ftps"):
7262 if(urlparts
.scheme
=="sftp"):
7264 return download_file_from_pysftp_file(url
);
7266 return download_file_from_sftp_file(url
);
7267 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7268 return download_file_from_http_file(url
);
7269 ftp_port
= urlparts
.port
;
7270 if(urlparts
.port
is None):
7273 ftp
.connect(urlparts
.hostname
, ftp_port
);
7274 except socket
.gaierror
:
7275 log
.info("Error With URL "+url
);
7277 except socket
.timeout
:
7278 log
.info("Error With URL "+url
);
7280 ftp
.login(urlparts
.username
, urlparts
.password
);
7281 if(urlparts
.scheme
=="ftps"):
7283 ftpfile
= BytesIO();
7284 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
7285 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7290 def download_file_from_ftp_string(url
):
7291 ftpfile
= download_file_from_ftp_file(url
);
7292 return ftpfile
.read();
7294 def upload_file_to_ftp_file(ftpfile
, url
):
7295 urlparts
= urlparse(url
);
7296 file_name
= os
.path
.basename(urlparts
.path
);
7297 file_dir
= os
.path
.dirname(urlparts
.path
);
7298 if(urlparts
.username
is not None):
7299 ftp_username
= urlparts
.username
;
7301 ftp_username
= "anonymous";
7302 if(urlparts
.password
is not None):
7303 ftp_password
= urlparts
.password
;
7304 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7305 ftp_password
= "anonymous";
7308 if(urlparts
.scheme
=="ftp"):
7310 elif(urlparts
.scheme
=="ftps"):
7314 if(urlparts
.scheme
=="sftp"):
7316 return upload_file_to_pysftp_file(url
);
7318 return upload_file_to_sftp_file(url
);
7319 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7321 ftp_port
= urlparts
.port
;
7322 if(urlparts
.port
is None):
7325 ftp
.connect(urlparts
.hostname
, ftp_port
);
7326 except socket
.gaierror
:
7327 log
.info("Error With URL "+url
);
7329 except socket
.timeout
:
7330 log
.info("Error With URL "+url
);
7332 ftp
.login(urlparts
.username
, urlparts
.password
);
7333 if(urlparts
.scheme
=="ftps"):
7335 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
7340 def upload_file_to_ftp_string(ftpstring
, url
):
7341 ftpfileo
= BytesIO(ftpstring
);
7342 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
7346 def download_file_from_http_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7347 # Parse the URL to extract username and password if present
7348 urlparts
= urlparse(url
);
7349 username
= urlparts
.username
;
7350 password
= urlparts
.password
;
7351 # Rebuild the URL without the username and password
7352 netloc
= urlparts
.hostname
;
7353 if(urlparts
.scheme
=="sftp"):
7355 return download_file_from_pysftp_file(url
);
7357 return download_file_from_sftp_file(url
);
7358 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7359 return download_file_from_ftp_file(url
);
7361 netloc
+= ':' + str(urlparts
.port
);
7362 rebuilt_url
= urlunparse((urlparts
.scheme
, netloc
, urlparts
.path
, urlparts
.params
, urlparts
.query
, urlparts
.fragment
));
7363 # Create a temporary file object
7364 httpfile
= BytesIO();
7366 # Use the requests library if available
7367 if username
and password
:
7368 response
= requests
.get(rebuilt_url
, headers
=headers
, auth
=(username
, password
), stream
=True);
7370 response
= requests
.get(rebuilt_url
, headers
=headers
, stream
=True);
7371 response
.raw
.decode_content
= True
7372 shutil
.copyfileobj(response
.raw
, httpfile
);
7374 # Build a Request object for urllib
7375 request
= Request(rebuilt_url
, headers
=headers
);
7376 # Create an opener object for handling URLs
7377 if username
and password
:
7378 # Create a password manager
7379 password_mgr
= HTTPPasswordMgrWithDefaultRealm();
7380 # Add the username and password
7381 password_mgr
.add_password(None, rebuilt_url
, username
, password
);
7382 # Create an authentication handler using the password manager
7383 auth_handler
= HTTPBasicAuthHandler(password_mgr
);
7384 # Build the opener with the authentication handler
7385 opener
= build_opener(auth_handler
);
7387 opener
= build_opener();
7388 with opener
.open(request
) as response
:
7389 shutil
.copyfileobj(response
, httpfile
);
7390 # Reset file pointer to the start
7391 httpfile
.seek(0, 0);
7392 # Return the temporary file object
7395 def download_file_from_http_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7396 httpfile
= download_file_from_http_file(url
, headers
);
7397 return ftpfile
.read();
7400 def download_file_from_sftp_file(url
):
7401 urlparts
= urlparse(url
);
7402 file_name
= os
.path
.basename(urlparts
.path
);
7403 file_dir
= os
.path
.dirname(urlparts
.path
);
7404 sftp_port
= urlparts
.port
;
7405 if(urlparts
.port
is None):
7408 sftp_port
= urlparts
.port
;
7409 if(urlparts
.username
is not None):
7410 sftp_username
= urlparts
.username
;
7412 sftp_username
= "anonymous";
7413 if(urlparts
.password
is not None):
7414 sftp_password
= urlparts
.password
;
7415 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7416 sftp_password
= "anonymous";
7419 if(urlparts
.scheme
=="ftp"):
7420 return download_file_from_ftp_file(url
);
7421 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7422 return download_file_from_http_file(url
);
7423 if(urlparts
.scheme
!="sftp"):
7425 ssh
= paramiko
.SSHClient();
7426 ssh
.load_system_host_keys();
7427 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7429 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7430 except paramiko
.ssh_exception
.SSHException
:
7432 except socket
.gaierror
:
7433 log
.info("Error With URL "+url
);
7435 except socket
.timeout
:
7436 log
.info("Error With URL "+url
);
7438 sftp
= ssh
.open_sftp();
7439 sftpfile
= BytesIO();
7440 sftp
.getfo(urlparts
.path
, sftpfile
);
7443 sftpfile
.seek(0, 0);
7446 def download_file_from_sftp_file(url
):
7450 def download_file_from_sftp_string(url
):
7451 sftpfile
= download_file_from_sftp_file(url
);
7452 return sftpfile
.read();
7454 def download_file_from_ftp_string(url
):
7458 def upload_file_to_sftp_file(sftpfile
, url
):
7459 urlparts
= urlparse(url
);
7460 file_name
= os
.path
.basename(urlparts
.path
);
7461 file_dir
= os
.path
.dirname(urlparts
.path
);
7462 sftp_port
= urlparts
.port
;
7463 if(urlparts
.port
is None):
7466 sftp_port
= urlparts
.port
;
7467 if(urlparts
.username
is not None):
7468 sftp_username
= urlparts
.username
;
7470 sftp_username
= "anonymous";
7471 if(urlparts
.password
is not None):
7472 sftp_password
= urlparts
.password
;
7473 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7474 sftp_password
= "anonymous";
7477 if(urlparts
.scheme
=="ftp"):
7478 return upload_file_to_ftp_file(url
);
7479 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7481 if(urlparts
.scheme
!="sftp"):
7483 ssh
= paramiko
.SSHClient();
7484 ssh
.load_system_host_keys();
7485 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7487 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7488 except paramiko
.ssh_exception
.SSHException
:
7490 except socket
.gaierror
:
7491 log
.info("Error With URL "+url
);
7493 except socket
.timeout
:
7494 log
.info("Error With URL "+url
);
7496 sftp
= ssh
.open_sftp();
7497 sftp
.putfo(sftpfile
, urlparts
.path
);
7500 sftpfile
.seek(0, 0);
7503 def upload_file_to_sftp_file(sftpfile
, url
):
7507 def upload_file_to_sftp_string(sftpstring
, url
):
7508 sftpfileo
= BytesIO(sftpstring
);
7509 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
7513 def upload_file_to_sftp_string(url
):
7517 def download_file_from_pysftp_file(url
):
7518 urlparts
= urlparse(url
);
7519 file_name
= os
.path
.basename(urlparts
.path
);
7520 file_dir
= os
.path
.dirname(urlparts
.path
);
7521 sftp_port
= urlparts
.port
;
7522 if(urlparts
.port
is None):
7525 sftp_port
= urlparts
.port
;
7526 if(urlparts
.username
is not None):
7527 sftp_username
= urlparts
.username
;
7529 sftp_username
= "anonymous";
7530 if(urlparts
.password
is not None):
7531 sftp_password
= urlparts
.password
;
7532 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7533 sftp_password
= "anonymous";
7536 if(urlparts
.scheme
=="ftp"):
7537 return download_file_from_ftp_file(url
);
7538 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7539 return download_file_from_http_file(url
);
7540 if(urlparts
.scheme
!="sftp"):
7543 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7544 except paramiko
.ssh_exception
.SSHException
:
7546 except socket
.gaierror
:
7547 log
.info("Error With URL "+url
);
7549 except socket
.timeout
:
7550 log
.info("Error With URL "+url
);
7552 sftp
= ssh
.open_sftp();
7553 sftpfile
= BytesIO();
7554 sftp
.getfo(urlparts
.path
, sftpfile
);
7557 sftpfile
.seek(0, 0);
7560 def download_file_from_pysftp_file(url
):
7564 def download_file_from_pysftp_string(url
):
7565 sftpfile
= download_file_from_pysftp_file(url
);
7566 return sftpfile
.read();
7568 def download_file_from_ftp_string(url
):
7572 def upload_file_to_pysftp_file(sftpfile
, url
):
7573 urlparts
= urlparse(url
);
7574 file_name
= os
.path
.basename(urlparts
.path
);
7575 file_dir
= os
.path
.dirname(urlparts
.path
);
7576 sftp_port
= urlparts
.port
;
7577 if(urlparts
.port
is None):
7580 sftp_port
= urlparts
.port
;
7581 if(urlparts
.username
is not None):
7582 sftp_username
= urlparts
.username
;
7584 sftp_username
= "anonymous";
7585 if(urlparts
.password
is not None):
7586 sftp_password
= urlparts
.password
;
7587 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7588 sftp_password
= "anonymous";
7591 if(urlparts
.scheme
=="ftp"):
7592 return upload_file_to_ftp_file(url
);
7593 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7595 if(urlparts
.scheme
!="sftp"):
7598 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7599 except paramiko
.ssh_exception
.SSHException
:
7601 except socket
.gaierror
:
7602 log
.info("Error With URL "+url
);
7604 except socket
.timeout
:
7605 log
.info("Error With URL "+url
);
7607 sftp
= ssh
.open_sftp();
7608 sftp
.putfo(sftpfile
, urlparts
.path
);
7611 sftpfile
.seek(0, 0);
7614 def upload_file_to_pysftp_file(sftpfile
, url
):
7618 def upload_file_to_pysftp_string(sftpstring
, url
):
7619 sftpfileo
= BytesIO(sftpstring
);
7620 sftpfile
= upload_file_to_pysftp_files(ftpfileo
, url
);
7624 def upload_file_to_pysftp_string(url
):
7627 def download_file_from_internet_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7628 urlparts
= urlparse(url
);
7629 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7630 return download_file_from_http_file(url
, headers
);
7631 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7632 return download_file_from_ftp_file(url
);
7633 elif(urlparts
.scheme
=="sftp"):
7634 if(__use_pysftp__
and havepysftp
):
7635 return download_file_from_pysftp_file(url
);
7637 return download_file_from_sftp_file(url
);
7642 def download_file_from_internet_uncompress_file(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7643 fp
= download_file_from_internet_file(url
);
7644 fp
= UncompressArchiveFile(fp
, formatspecs
);
7650 def download_file_from_internet_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7651 urlparts
= urlparse(url
);
7652 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7653 return download_file_from_http_string(url
, headers
);
7654 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7655 return download_file_from_ftp_string(url
);
7656 elif(urlparts
.scheme
=="sftp"):
7657 if(__use_pysftp__
and havepysftp
):
7658 return download_file_from_pysftp_string(url
);
7660 return download_file_from_sftp_string(url
);
7665 def download_file_from_internet_uncompress_string(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_dict__
):
7666 fp
= download_file_from_internet_string(url
);
7667 fp
= UncompressArchiveFile(fp
, formatspecs
);
7673 def upload_file_to_internet_file(ifp
, url
):
7674 urlparts
= urlparse(url
);
7675 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7677 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7678 return upload_file_to_ftp_file(ifp
, url
);
7679 elif(urlparts
.scheme
=="sftp"):
7680 if(__use_pysftp__
and havepysftp
):
7681 return upload_file_to_pysftp_file(ifp
, url
);
7683 return upload_file_to_sftp_file(ifp
, url
);
7688 def upload_file_to_internet_compress_file(ifp
, url
, formatspecs
=__file_format_dict__
):
7689 catfp
= CompressArchiveFile(catfp
, compression
, compressionlevel
, formatspecs
);
7693 upload_file_to_internet_file(catfp
, outfile
);
7696 def upload_file_to_internet_string(ifp
, url
):
7697 urlparts
= urlparse(url
);
7698 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7700 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7701 return upload_file_to_ftp_string(ifp
, url
);
7702 elif(urlparts
.scheme
=="sftp"):
7703 if(__use_pysftp__
and havepysftp
):
7704 return upload_file_to_pysftp_string(ifp
, url
);
7706 return upload_file_to_sftp_string(ifp
, url
);
7711 def upload_file_to_internet_compress_string(ifp
, url
, formatspecs
=__file_format_dict__
):
7712 catfp
= CompressArchiveFile(BytesIO(ifp
), compression
, compressionlevel
, formatspecs
);
7716 upload_file_to_internet_file(catfp
, outfile
);
7720 if(hasattr(shutil
, "register_archive_format")):
7721 # Register the packing format
7722 shutil
.register_archive_format(__file_format_name__
, PackArchiveFileFunc
, description
='Pack concatenated files');
7723 except shutil
.RegistryError
:
7727 if(hasattr(shutil
, "register_unpack_format")):
7728 # Register the unpacking format
7729 shutil
.register_unpack_format(__file_format_name__
, archivefile_extensions
, UnPackArchiveFileFunc
, description
='UnPack concatenated files');
7730 except shutil
.RegistryError
: