2 # -*- coding: utf-8 -*-
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/2/2024 Ver. 0.9.8 RC 1 - Author: cooldude2k $
20 from __future__
import absolute_import
, division
, print_function
, unicode_literals
;
21 import io
, os
, re
, sys
, time
, stat
, zlib
, base64
, shutil
, socket
, hashlib
, datetime
, logging
, binascii
, tempfile
, zipfile
, platform
;
22 from ftplib
import FTP
, FTP_TLS
;
23 if(sys
.version
[0]=="2"):
24 from urlparse
import urlparse
, urlunparse
;
25 elif(sys
.version
[0]>="3"):
26 from urllib
.parse
import urlunparse
;
27 from urllib
.parse
import urlparse
;
29 if os
.name
== 'nt': # Only modify if on Windows
30 if sys
.version
[0] == "2":
32 sys
.stdout
= codecs
.getwriter('utf-8')(sys
.stdout
);
33 sys
.stderr
= codecs
.getwriter('utf-8')(sys
.stderr
);
36 sys
.stdout
= io
.TextIOWrapper(sys
.stdout
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
37 sys
.stderr
= io
.TextIOWrapper(sys
.stderr
.buffer, encoding
='utf-8', errors
='replace', line_buffering
=True);
39 hashlib_guaranteed
= False;
40 os
.environ
["PYTHONIOENCODING"] = "UTF-8";
41 os
.environ
["LC_CTYPE"] = "UTF-8";
45 sys
.setdefaultencoding('UTF-8');
48 except AttributeError:
52 except AttributeError:
56 from zlib
import crc32
;
58 from binascii
import crc32
;
60 if(sys
.version_info
[0]==2):
61 FileNotFoundError
= IOError;
63 rarfile_support
= False;
66 rarfile_support
= True;
68 rarfile_support
= False;
70 py7zr_support
= False;
75 py7zr_support
= False;
78 from xtarfile
import is_tarfile
;
81 from safetar
import is_tarfile
;
83 from tarfile
import is_tarfile
;
86 import xtarfile
as tarfile
;
89 import safetar
as tarfile
;
107 haverequests
= False;
112 haverequests
= False;
116 logging
.getLogger("urllib3").setLevel(logging
.WARNING
);
120 from urllib
.request
import Request
, build_opener
, HTTPBasicAuthHandler
;
121 from urllib
.parse
import urlparse
;
124 from urllib2
import Request
, build_opener
, HTTPBasicAuthHandler
;
125 from urlparse
import urlparse
;
127 if(sys
.version
[0]=="2"):
129 from io
import StringIO
, BytesIO
;
132 from cStringIO
import StringIO
;
133 from cStringIO
import StringIO
as BytesIO
;
135 from StringIO
import StringIO
;
136 from StringIO
import StringIO
as BytesIO
;
137 elif(sys
.version
[0]>="3"):
138 from io
import StringIO
, BytesIO
;
143 from cStringIO
import StringIO
as BytesIO
;
149 from StringIO
import StringIO
as BytesIO
;
155 from io
import BytesIO
;
160 __use_pysftp__
= False;
162 __use_pysftp__
= False;
163 __file_format_name__
= "CatFile";
164 __program_name__
= "Py"+__file_format_name__
;
165 __file_format_lower__
= __file_format_name__
.lower();
166 __file_format_magic__
= __file_format_name__
;
167 __file_format_len__
= len(__file_format_magic__
);
168 __file_format_hex__
= binascii
.hexlify(__file_format_magic__
.encode("UTF-8")).decode("UTF-8");
169 __file_format_delimiter__
= "\x00";
170 __file_format_ver__
= "001";
171 __use_new_style__
= True;
172 __use_advanced_list__
= True;
173 __use_alt_inode__
= False;
174 __file_format_list__
= [__file_format_name__
, __file_format_magic__
, __file_format_lower__
, __file_format_len__
, __file_format_hex__
, __file_format_delimiter__
, __file_format_ver__
, __use_new_style__
, __use_advanced_list__
, __use_alt_inode__
];
175 __project__
= __program_name__
;
176 __project_url__
= "https://github.com/GameMaker2k/PyCatFile";
177 __version_info__
= (0, 9, 8, "RC 1", 1);
178 __version_date_info__
= (2024, 5, 2, "RC 1", 1);
179 __version_date__
= str(__version_date_info__
[0]) + "." + str(__version_date_info__
[1]).zfill(2) + "." + str(__version_date_info__
[2]).zfill(2);
180 __revision__
= __version_info__
[3];
181 __revision_id__
= "$Id$";
182 if(__version_info__
[4] is not None):
183 __version_date_plusrc__
= __version_date__
+ "-" + str(__version_date_info__
[4]);
184 if(__version_info__
[4] is None):
185 __version_date_plusrc__
= __version_date__
;
186 if(__version_info__
[3] is not None):
187 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]) + " " + str(__version_info__
[3]);
188 if(__version_info__
[3] is None):
189 __version__
= str(__version_info__
[0]) + "." + str(__version_info__
[1]) + "." + str(__version_info__
[2]);
191 PyBitness
= platform
.architecture();
192 if(PyBitness
=="32bit" or PyBitness
=="32"):
194 elif(PyBitness
=="64bit" or PyBitness
=="64"):
199 geturls_ua_pycatfile_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
200 if(platform
.python_implementation()!=""):
201 py_implementation
= platform
.python_implementation();
202 if(platform
.python_implementation()==""):
203 py_implementation
= "Python";
204 geturls_ua_pycatfile_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
205 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
206 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
207 geturls_headers_pycatfile_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
208 geturls_headers_pycatfile_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
209 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
210 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
212 compressionlist
= ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
213 outextlist
= ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
214 outextlistwd
= ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
216 tarfile_mimetype
= "application/tar";
217 tarfile_tar_mimetype
= tarfile_mimetype
;
218 zipfile_mimetype
= "application/zip";
219 zipfile_zip_mimetype
= zipfile_mimetype
;
220 rarfile_mimetype
= "application/rar";
221 rarfile_rar_mimetype
= rarfile_mimetype
;
222 archivefile_mimetype
= "application/x-"+__file_format_list__
[1]+"";
223 archivefile_cat_mimetype
= archivefile_mimetype
;
224 archivefile_gzip_mimetype
= "application/x-"+__file_format_list__
[1]+"+gzip";
225 archivefile_gz_mimetype
= archivefile_gzip_mimetype
;
226 archivefile_bzip2_mimetype
= "application/x-"+__file_format_list__
[1]+"+bzip2";
227 archivefile_bz2_mimetype
= archivefile_bzip2_mimetype
;
228 archivefile_lz4_mimetype
= "application/x-"+__file_format_list__
[1]+"+lz4";
229 archivefile_lzop_mimetype
= "application/x-"+__file_format_list__
[1]+"+lzop";
230 archivefile_lzo_mimetype
= archivefile_lzop_mimetype
;
231 archivefile_zstandard_mimetype
= "application/x-"+__file_format_list__
[1]+"+zstandard";
232 archivefile_zstd_mimetype
= archivefile_zstandard_mimetype
;
233 archivefile_lzma_mimetype
= "application/x-"+__file_format_list__
[1]+"+lzma";
234 archivefile_xz_mimetype
= "application/x-"+__file_format_list__
[1]+"+xz";
235 archivefile_extensions
= ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
237 if __name__
== "__main__":
239 curscrpath
= os
.path
.dirname(sys
.argv
[0]);
243 curscrpath
= curscrpath
.replace(os
.sep
, "/");
244 curscrpath
= curscrpath
+ "/";
245 scrfile
= curscrpath
+ "catfile.py";
246 if(os
.path
.exists(scrfile
) and os
.path
.isfile(scrfile
)):
247 scrcmd
= subprocess
.Popen([sys
.executable
, scrfile
] + sys
.argv
[1:]);
250 def VerbosePrintOut(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
256 "warning": logging
.warning
,
257 "error": logging
.error
,
258 "critical": logging
.critical
,
259 "exception": logging
.exception
,
260 "logalt": lambda x
: logging
.log(dgblevel
, x
),
261 "debug": logging
.debug
263 log_function
= log_functions
.get(outtype
);
265 log_function(dbgtxt
);
269 def VerbosePrintOutReturn(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
270 VerbosePrintOut(dbgtxt
, outtype
, dbgenable
, dgblevel
);
273 def RemoveWindowsPath(dpath
):
277 dpath
= dpath
.replace(os
.path
.sep
, "/");
278 dpath
= dpath
.rstrip("/");
279 if(dpath
=="." or dpath
==".."):
283 def NormalizeRelativePath(inpath
):
284 inpath
= RemoveWindowsPath(inpath
);
285 if(os
.path
.isabs(inpath
)):
288 if(inpath
.startswith("./") or inpath
.startswith("../")):
291 outpath
= "./" + inpath
;
294 def PrependPath(base_dir
, child_path
):
295 # Check if base_dir is None or empty, if so, return child_path as is
298 # Ensure base_dir ends with exactly one slash
299 if not base_dir
.endswith('/'):
301 # Check if child_path starts with ./ or ../ (indicating a relative path)
302 if child_path
.startswith('./') or child_path
.startswith('../'):
303 # For relative paths, we don't alter the child_path
304 return base_dir
+ child_path
;
306 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
307 return base_dir
+ child_path
.lstrip('/');
309 def ListDir(dirpath
, followlink
=False, duplicates
=False):
310 if(isinstance(dirpath
, (list, tuple, ))):
311 dirpath
= list(filter(None, dirpath
));
312 elif(isinstance(dirpath
, (str, ))):
313 dirpath
= list(filter(None, [dirpath
]));
315 for mydirfile
in dirpath
:
316 if(not os
.path
.exists(mydirfile
)):
318 mydirfile
= NormalizeRelativePath(mydirfile
);
319 if(os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
)):
320 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
));
321 if(os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
)):
322 for root
, dirs
, filenames
in os
.walk(mydirfile
):
324 dpath
= RemoveWindowsPath(dpath
);
325 if(dpath
not in retlist
and not duplicates
):
326 retlist
.append(dpath
);
328 retlist
.append(dpath
);
329 for file in filenames
:
330 fpath
= os
.path
.join(root
, file);
331 fpath
= RemoveWindowsPath(fpath
);
332 if(fpath
not in retlist
and not duplicates
):
333 retlist
.append(fpath
);
335 retlist
.append(fpath
);
337 retlist
.append(RemoveWindowsPath(mydirfile
));
340 def ListDirAdvanced(dirpath
, followlink
=False, duplicates
=False):
341 if isinstance(dirpath
, (list, tuple)):
342 dirpath
= list(filter(None, dirpath
));
343 elif isinstance(dirpath
, str):
344 dirpath
= list(filter(None, [dirpath
]));
346 for mydirfile
in dirpath
:
347 if not os
.path
.exists(mydirfile
):
349 mydirfile
= NormalizeRelativePath(mydirfile
);
350 if os
.path
.exists(mydirfile
) and os
.path
.islink(mydirfile
) and followlink
:
351 mydirfile
= RemoveWindowsPath(os
.path
.realpath(mydirfile
))
352 if os
.path
.exists(mydirfile
) and os
.path
.isdir(mydirfile
):
353 for root
, dirs
, filenames
in os
.walk(mydirfile
):
354 # Sort dirs and filenames alphabetically in place
355 dirs
.sort(key
=lambda x
: x
.lower());
356 filenames
.sort(key
=lambda x
: x
.lower());
357 dpath
= RemoveWindowsPath(root
);
358 if not duplicates
and dpath
not in retlist
:
359 retlist
.append(dpath
);
361 retlist
.append(dpath
);
362 for file in filenames
:
363 fpath
= os
.path
.join(root
, file);
364 fpath
= RemoveWindowsPath(fpath
);
365 if not duplicates
and fpath
not in retlist
:
366 retlist
.append(fpath
);
368 retlist
.append(fpath
);
370 retlist
.append(RemoveWindowsPath(mydirfile
));
373 def create_alias_function(prefix
, base_name
, suffix
, target_function
):
374 # Define a new function that wraps the target function
375 def alias_function(*args
, **kwargs
):
376 return target_function(*args
, **kwargs
);
378 # Create the function name by combining the prefix, base name, and the suffix
379 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
);
381 # Add the new function to the global namespace
382 globals()[function_name
] = alias_function
;
384 def create_alias_function_alt(prefix
, base_name
, suffix
, target_function
):
385 # Create the function name by combining the prefix, base name, and the suffix
386 # Use the format method for string formatting, compatible with Python 2 and 3
387 function_name
= "{}{}{}".format(prefix
, base_name
, suffix
)
388 # Add the new function (alias of the target_function) to the global namespace
389 # This line is compatible as-is with both Python 2 and 3
390 globals()[function_name
] = target_function
392 # initial_value can be 0xFFFF or 0x0000
393 def crc16_ansi(msg
, initial_value
=0xFFFF):
394 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
395 poly
= 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
396 crc
= initial_value
; # Initial value
398 crc ^
= b
<< 8; # XOR byte into CRC top byte
399 for _
in range(8): # Process each bit
400 if crc
& 0x8000: # If the top bit is set
401 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
403 crc
= crc
<< 1; # Just shift left
404 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
407 # initial_value can be 0xFFFF or 0x0000
408 def crc16_ibm(msg
, initial_value
=0xFFFF):
409 return crc16_ansi(msg
, initial_value
);
411 # initial_value is 0xFFFF
413 return crc16_ansi(msg
, 0xFFFF);
415 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
416 def crc16_ccitt(msg
, initial_value
=0xFFFF):
417 # CRC-16-CCITT polynomial
418 poly
= 0x1021; # Polynomial for CRC-16-CCITT
419 # Use the specified initial value
422 crc ^
= b
<< 8; # XOR byte into CRC top byte
423 for _
in range(8): # Process each bit
424 if crc
& 0x8000: # If the top bit is set
425 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with the polynomial
427 crc
= crc
<< 1; # Just shift left
428 crc
&= 0xFFFF; # Ensure CRC remains 16-bit
431 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
432 def crc64_ecma(msg
, initial_value
=0x0000000000000000):
433 # CRC-64-ECMA polynomial and initial value
434 poly
= 0x42F0E1EBA9EA3693;
435 crc
= initial_value
; # Initial value for CRC-64-ECMA
437 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
438 for _
in range(8): # Process each bit
439 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
440 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
442 crc
<<= 1; # Just shift left if the MSB is 0
443 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
446 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
447 def crc64_iso(msg
, initial_value
=0xFFFFFFFFFFFFFFFF):
448 # CRC-64-ISO polynomial and initial value
449 poly
= 0x000000000000001B;
450 crc
= initial_value
; # Common initial value for CRC-64-ISO
452 crc ^
= b
<< 56; # XOR byte into the most significant byte of the CRC
453 for _
in range(8): # Process each bit
454 if crc
& (1 << 63): # Check if the leftmost (most significant) bit is set
455 crc
= (crc
<< 1) ^ poly
; # Shift left and XOR with poly if the MSB is 1
457 crc
<<= 1; # Just shift left if the MSB is 0
458 crc
&= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
461 def GetDataFromArray(data
, path
, default
=None):
465 element
= element
[key
];
467 except (KeyError, TypeError, IndexError):
470 def GetDataFromArrayAlt(structure
, path
, default
=None):
473 if isinstance(element
, dict) and key
in element
:
474 element
= element
[key
];
475 elif isinstance(element
, list) and isinstance(key
, int) and -len(element
) <= key
< len(element
):
476 element
= element
[key
];
481 def GetHeaderChecksum(inlist
=[], checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_list__
):
482 if isinstance(inlist
, list):
483 fileheader
= AppendNullBytes(inlist
, formatspecs
[5]);
485 fileheader
= AppendNullByte(inlist
, formatspecs
[5]);
487 fileheader
= fileheader
.encode('UTF-8');
488 if(checksumtype
=="none" or checksumtype
==""):
489 catfileheadercshex
= format(0, 'x').lower();
490 elif(checksumtype
=="crc16" or checksumtype
=="crc16_ansi" or checksumtype
=="crc16_ibm"):
491 catfileheadercshex
= format(crc16(fileheader
) & 0xffff, '04x').lower();
492 elif(checksumtype
=="crc16_ccitt"):
493 catfileheadercshex
= format(crc16_ccitt(fileheader
) & 0xffff, '04x').lower();
494 elif(checksumtype
=="adler32"):
495 catfileheadercshex
= format(zlib
.adler32(fileheader
) & 0xffffffff, '08x').lower();
496 elif(checksumtype
=="crc32"):
497 catfileheadercshex
= format(crc32(fileheader
) & 0xffffffff, '08x').lower();
498 elif(checksumtype
=="crc64_ecma"):
499 catfileheadercshex
= format(crc64_ecma(fileheader
) & 0xffffffffffffffff, '016x').lower();
500 elif(checksumtype
=="crc64" or checksumtype
=="crc64_iso"):
501 catfileheadercshex
= format(crc64_iso(fileheader
) & 0xffffffffffffffff, '016x').lower();
502 elif(CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
)):
503 checksumoutstr
= hashlib
.new(checksumtype
);
504 checksumoutstr
.update(fileheader
);
505 catfileheadercshex
= checksumoutstr
.hexdigest().lower();
507 catfileheadercshex
= format(0, 'x').lower();
508 return catfileheadercshex
;
510 def GetFileChecksum(instr
, checksumtype
="crc32", encodedata
=True, formatspecs
=__file_format_list__
):
512 instr
= instr
.encode('UTF-8');
513 if(checksumtype
=="none" or checksumtype
==""):
514 catinstrcshex
= format(0, 'x').lower();
515 elif(checksumtype
=="crc16" or checksumtype
=="crc16_ansi" or checksumtype
=="crc16_ibm"):
516 catinstrcshex
= format(crc16(instr
) & 0xffff, '04x').lower();
517 elif(checksumtype
=="crc16_ccitt"):
518 catinstrcshex
= format(crc16_ccitt(instr
) & 0xffff, '04x').lower();
519 elif(checksumtype
=="adler32"):
520 catinstrcshex
= format(zlib
.adler32(instr
) & 0xffffffff, '08x').lower();
521 elif(checksumtype
=="crc32"):
522 catinstrcshex
= format(crc32(instr
) & 0xffffffff, '08x').lower();
523 elif(checksumtype
=="crc64_ecma"):
524 catinstrcshex
= format(crc64_ecma(instr
) & 0xffffffffffffffff, '016x').lower();
525 elif(checksumtype
=="crc64" or checksumtype
=="crc64_iso"):
526 catinstrcshex
= format(crc64_iso(instr
) & 0xffffffffffffffff, '016x').lower();
527 elif(CheckSumSupportAlt(checksumtype
, hashlib_guaranteed
)):
528 checksumoutstr
= hashlib
.new(checksumtype
);
529 checksumoutstr
.update(instr
);
530 catinstrcshex
= checksumoutstr
.hexdigest().lower();
532 catinstrcshex
= format(0, 'x').lower();
533 return catinstrcshex
;
535 def ValidateHeaderChecksum(inlist
=[], checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_list__
):
536 catfileheadercshex
= GetHeaderChecksum(inlist
, checksumtype
, True, formatspecs
);
537 inchecksum
= inchecksum
.lower();
538 catfileheadercshex
= catfileheadercshex
.lower();
539 if(inchecksum
==catfileheadercshex
):
544 def ValidateFileChecksum(infile
, checksumtype
="crc32", inchecksum
="0", formatspecs
=__file_format_list__
):
545 catinfilecshex
= GetFileChecksum(infile
, checksumtype
, True, formatspecs
);
546 inchecksum
= inchecksum
.lower();
547 catinfilecshex
= catinfilecshex
.lower();
548 if(inchecksum
==catinfilecshex
):
553 def ReadTillNullByte(fp
, delimiter
=__file_format_delimiter__
):
556 nullbyte
= delimiter
.encode("UTF-8");
558 curbyte
= fp
.read(1);
559 if(curbyte
==nullbyte
or not curbyte
):
561 curfullbyte
= curfullbyte
+ curbyte
;
562 return curfullbyte
.decode('UTF-8');
564 def ReadUntilNullByte(fp
, delimiter
=__file_format_delimiter__
):
565 return ReadTillNullByte(fp
, delimiter
);
567 def SeekToEndOfFile(fp
):
571 if(lasttell
==fp
.tell()):
573 lasttell
= fp
.tell();
576 def ReadFileHeaderData(fp
, rounds
=0, delimiter
=__file_format_delimiter__
):
580 while(rocount
<roend
):
581 HeaderOut
.append(ReadTillNullByte(fp
, delimiter
));
582 rocount
= rocount
+ 1;
585 def ReadFileHeaderDataBySize(fp
, delimiter
=__file_format_delimiter__
):
586 headerpresize
= ReadTillNullByte(fp
, delimiter
);
587 headersize
= int(headerpresize
, 16);
590 headercontent
= str(fp
.read(headersize
).decode('UTF-8')).split(delimiter
);
593 roend
= int(len(headercontent
));
594 HeaderOut
= [headerpresize
];
595 while(rocount
<roend
):
596 HeaderOut
.append(headercontent
[rocount
]);
597 rocount
= rocount
+ 1;
600 def ReadFileHeaderDataWoSize(fp
, delimiter
=__file_format_delimiter__
):
601 preheaderdata
= ReadFileHeaderData(fp
, 2, delimiter
);
602 headersize
= int(preheaderdata
[0], 16);
603 headernumfields
= int(preheaderdata
[1], 16);
604 if(headersize
<=0 or headernumfields
<=0):
606 headerdata
= ReadFileHeaderData(fp
, headernumfields
, delimiter
);
607 HeaderOut
= preheaderdata
+ headerdata
;
610 def ReadFileHeaderDataBySizeWithContent(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
611 delimiter
= formatspecs
[5];
612 fheaderstart
= fp
.tell();
613 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
614 if(len(HeaderOut
)==0):
616 if(re
.findall("^[.|/]", HeaderOut
[3])):
617 fname
= HeaderOut
[3];
619 fname
= "./"+HeaderOut
[3];
620 fchecksumtype
= HeaderOut
[-3].lower();
621 fcs
= HeaderOut
[-2].lower();
622 fccs
= HeaderOut
[-1].lower();
623 fsize
= int(HeaderOut
[5], 16);
624 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
625 if(fcs
!=newfcs
and not skipchecksum
):
626 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
628 fhend
= fp
.tell() - 1;
629 fcontentstart
= fp
.tell();
630 fcontents
= BytesIO();
631 if(fsize
>0 and not listonly
):
632 fcontents
.write(fp
.read(fsize
));
633 elif(fsize
>0 and listonly
):
635 fcontents
.seek(0, 0);
636 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
637 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
638 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
641 fcontentend
= fp
.tell() - 1;
642 HeaderOut
.append(fcontents
);
645 def ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
646 delimiter
= formatspecs
[5];
647 fheaderstart
= fp
.tell();
649 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
651 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
652 if(len(HeaderOut
)==0):
654 fheadsize
= int(HeaderOut
[0], 16);
655 fnumfields
= int(HeaderOut
[1], 16);
656 ftype
= int(HeaderOut
[2], 16);
657 if(re
.findall("^[.|/]", HeaderOut
[3])):
658 fname
= HeaderOut
[3];
660 fname
= "./"+HeaderOut
[3];
661 fbasedir
= os
.path
.dirname(fname
);
662 flinkname
= HeaderOut
[4];
663 fsize
= int(HeaderOut
[5], 16);
664 fatime
= int(HeaderOut
[6], 16);
665 fmtime
= int(HeaderOut
[7], 16);
666 fctime
= int(HeaderOut
[8], 16);
667 fbtime
= int(HeaderOut
[9], 16);
668 fmode
= int(HeaderOut
[10], 16);
669 fchmode
= stat
.S_IMODE(fmode
);
670 ftypemod
= stat
.S_IFMT(fmode
);
671 fwinattributes
= int(HeaderOut
[11], 16);
672 fuid
= int(HeaderOut
[12], 16);
673 funame
= HeaderOut
[13];
674 fgid
= int(HeaderOut
[14], 16);
675 fgname
= HeaderOut
[15];
676 fid
= int(HeaderOut
[16], 16);
677 finode
= int(HeaderOut
[17], 16);
678 flinkcount
= int(HeaderOut
[18], 16);
679 fdev_minor
= int(HeaderOut
[19], 16);
680 fdev_major
= int(HeaderOut
[20], 16);
681 frdev_minor
= int(HeaderOut
[21], 16);
682 frdev_major
= int(HeaderOut
[22], 16);
683 fextrasize
= int(HeaderOut
[23], 16);
684 fextrafields
= int(HeaderOut
[24], 16);
685 extrafieldslist
= [];
687 extraend
= extrastart
+ fextrafields
;
688 extrafieldslist
= [];
689 if(extrastart
<extraend
):
690 extrafieldslist
.append(HeaderOut
[extrastart
]);
691 extrastart
= extrastart
+ 1;
692 fchecksumtype
= HeaderOut
[extrastart
].lower();
693 fcs
= HeaderOut
[extrastart
+ 1].lower();
694 fccs
= HeaderOut
[extrastart
+ 2].lower();
695 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
696 if(fcs
!=newfcs
and not skipchecksum
):
697 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
699 fhend
= fp
.tell() - 1;
700 fcontentstart
= fp
.tell();
701 fcontents
= BytesIO();
702 pyhascontents
= False;
703 if(fsize
>0 and not listonly
):
704 fcontents
.write(fp
.read(catfsize
));
705 pyhascontents
= True;
706 elif(fsize
>0 and listonly
):
708 pyhascontents
= False;
709 fcontents
.seek(0, 0);
710 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
711 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
712 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
715 fcontentend
= fp
.tell() - 1;
716 fcontents
.seek(0, 0);
717 catlist
= {'fheadersize': fheadsize
, 'fhstart': fheaderstart
, 'fhend': fhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': fchecksumtype
, 'fnumfields': fnumfields
+ 2, 'frawheader': HeaderOut
, 'fextrafields': fextrafields
, 'fextrafieldsize': fextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': fcs
, 'fcontentchecksum': fccs
, 'fhascontents': pyhascontents
, 'fcontentstart': fcontentstart
, 'fcontentend': fcontentend
, 'fcontents': fcontents
};
720 def ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
721 delimiter
= formatspecs
[5];
722 fheaderstart
= fp
.tell();
724 HeaderOut
= ReadFileHeaderDataBySize(fp
, delimiter
);
726 HeaderOut
= ReadFileHeaderDataWoSize(fp
, delimiter
);
727 if(len(HeaderOut
)==0):
729 fheadsize
= int(HeaderOut
[0], 16);
730 fnumfields
= int(HeaderOut
[1], 16);
731 ftype
= int(HeaderOut
[2], 16);
732 if(re
.findall("^[.|/]", HeaderOut
[3])):
733 fname
= HeaderOut
[3];
735 fname
= "./"+HeaderOut
[3];
736 fbasedir
= os
.path
.dirname(fname
);
737 flinkname
= HeaderOut
[4];
738 fsize
= int(HeaderOut
[5], 16);
739 fatime
= int(HeaderOut
[6], 16);
740 fmtime
= int(HeaderOut
[7], 16);
741 fctime
= int(HeaderOut
[8], 16);
742 fbtime
= int(HeaderOut
[9], 16);
743 fmode
= int(HeaderOut
[10], 16);
744 fchmode
= stat
.S_IMODE(fmode
);
745 ftypemod
= stat
.S_IFMT(fmode
);
746 fwinattributes
= int(HeaderOut
[11], 16);
747 fuid
= int(HeaderOut
[12], 16);
748 funame
= HeaderOut
[13];
749 fgid
= int(HeaderOut
[14], 16);
750 fgname
= HeaderOut
[15];
751 fid
= int(HeaderOut
[16], 16);
752 finode
= int(HeaderOut
[17], 16);
753 flinkcount
= int(HeaderOut
[18], 16);
754 fdev_minor
= int(HeaderOut
[19], 16);
755 fdev_major
= int(HeaderOut
[20], 16);
756 frdev_minor
= int(HeaderOut
[21], 16);
757 frdev_major
= int(HeaderOut
[22], 16);
758 fextrasize
= int(HeaderOut
[23], 16);
759 fextrafields
= int(HeaderOut
[24], 16);
760 extrafieldslist
= [];
762 extraend
= extrastart
+ fextrafields
;
763 extrafieldslist
= [];
764 if(extrastart
<extraend
):
765 extrafieldslist
.append(HeaderOut
[extrastart
]);
766 extrastart
= extrastart
+ 1;
767 fchecksumtype
= HeaderOut
[extrastart
].lower();
768 fcs
= HeaderOut
[extrastart
+ 1].lower();
769 fccs
= HeaderOut
[extrastart
+ 2].lower();
770 newfcs
= GetHeaderChecksum(HeaderOut
[:-2], HeaderOut
[-3].lower(), True, formatspecs
);
771 if(fcs
!=newfcs
and not skipchecksum
):
772 VerbosePrintOut("File Header Checksum Error with file " + fname
+ " at offset " + str(fheaderstart
));
774 fhend
= fp
.tell() - 1;
775 fcontentstart
= fp
.tell();
776 fcontents
= BytesIO();
777 pyhascontents
= False;
778 if(fsize
>0 and not listonly
):
779 fcontents
.write(fp
.read(fsize
));
780 pyhascontents
= True;
781 elif(fsize
>0 and listonly
):
783 pyhascontents
= False;
784 fcontents
.seek(0, 0);
785 newfccs
= GetFileChecksum(fcontents
.read(), HeaderOut
[-3].lower(), False, formatspecs
);
786 if(fccs
!=newfccs
and not skipchecksum
and not listonly
):
787 VerbosePrintOut("File Content Checksum Error with file " + fname
+ " at offset " + str(fcontentstart
));
790 fcontentend
= fp
.tell() - 1;
791 catlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrafieldslist
, fchecksumtype
, fcontents
];
794 def ReadFileDataBySizeWithContent(fp
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
795 delimiter
= formatspecs
[5];
799 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
802 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
803 if(not headercheck
and not skipchecksum
):
804 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
806 fnumfiles
= int(catheader
[1], 16);
809 while(countnum
< fnumfiles
):
810 HeaderOut
= ReadFileHeaderDataBySizeWithContent(fp
, listonly
, skipchecksum
, formatspecs
)
811 if(len(HeaderOut
)==0):
813 flist
.append(HeaderOut
);
814 countnum
= countnum
+ 1;
817 def ReadFileDataBySizeWithContentToArray(fp
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
818 delimiter
= formatspecs
[5];
822 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
825 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
826 if(not headercheck
and not skipchecksum
):
827 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
829 catstring
= catheader
[0];
830 catversion
= re
.findall(r
"([\d]+)$", catstring
);
831 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
832 fprenumfiles
= catheader
[1];
833 fnumfiles
= int(fprenumfiles
, 16);
834 fprechecksumtype
= catheader
[2];
835 fprechecksum
= catheader
[3];
836 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
837 if(seekstart
<0 and seekstart
>fnumfiles
):
839 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
841 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
842 seekend
= fnumfiles
- abs(seekend
);
845 while(il
< seekstart
):
846 prefhstart
= fp
.tell();
847 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
[5]);
848 if(len(preheaderdata
)==0):
850 prefsize
= int(preheaderdata
[5], 16);
851 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-3].lower(), True, formatspecs
);
852 prefcs
= preheaderdata
[-2];
853 if(prefcs
!=prenewfcs
and not skipchecksum
):
854 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
856 valid_archive
= False;
857 invalid_archive
= True;
858 prefhend
= fp
.tell() - 1;
859 prefcontentstart
= fp
.tell();
860 prefcontents
= BytesIO();
861 pyhascontents
= False;
863 prefcontents
.write(fp
.read(prefsize
));
864 prefcontents
.seek(0, 0);
865 prenewfccs
= GetFileChecksum(prefcontents
.read(), preheaderdata
[-3].lower(), False, formatspecs
);
866 prefccs
= preheaderdata
[-1];
867 pyhascontents
= True;
868 if(prefccs
!=prenewfccs
and not skipchecksum
):
869 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
874 countnum
= seekstart
;
875 while(countnum
< seekend
):
876 catlist
['ffilelist'].update({realidnum
: {'fid': realidnum
, 'fidalt': realidnum
}});
877 HeaderOut
= ReadFileHeaderDataBySizeWithContentToArray(fp
, listonly
, skipchecksum
, formatspecs
);
880 catlist
['ffilelist'][realidnum
].update(HeaderOut
);
881 countnum
= countnum
+ 1;
882 realidnum
= realidnum
+ 1;
885 def ReadFileDataBySizeWithContentToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
886 if(isinstance(infile
, dict)):
887 listcatfiles
= infile
;
889 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
891 listcatfiles
= ReadFileDataBySizeWithContentToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
);
892 if(not listcatfiles
):
894 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
895 lenlist
= len(listcatfiles
['ffilelist']);
897 lcfx
= int(listcatfiles
['fnumfiles']);
898 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
901 lcfx
= int(listcatfiles
['fnumfiles']);
903 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
904 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
905 catarray
['filetoid'].update(filetoidarray
);
906 catarray
['idtofile'].update(idtofilearray
);
907 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
908 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
909 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
910 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
911 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
912 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
913 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
914 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
915 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
916 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
917 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
918 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
919 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
920 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
921 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
922 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
923 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
924 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
925 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
926 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
927 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
928 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
929 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
930 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
931 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
932 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
933 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
934 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
935 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
936 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
937 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
941 def ReadFileDataBySizeWithContentToList(fp
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
942 delimiter
= formatspecs
[5];
946 catheader
= ReadFileHeaderData(fp
, 4, delimiter
);
949 headercheck
= ValidateHeaderChecksum(catheader
[:-1], catheader
[2], catheader
[3], formatspecs
);
950 if(not headercheck
and not skipchecksum
):
951 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
953 catstring
= catheader
[0];
954 catversion
= re
.findall(r
"([\d]+)$", catstring
);
955 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
956 fprenumfiles
= catheader
[1];
957 fnumfiles
= int(fprenumfiles
, 16);
958 fprechecksumtype
= catheader
[2];
959 fprechecksum
= catheader
[3];
961 if(seekstart
<0 and seekstart
>fnumfiles
):
963 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
965 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
966 seekend
= fnumfiles
- abs(seekend
);
969 while(il
< seekstart
):
970 prefhstart
= fp
.tell();
971 preheaderdata
= ReadFileHeaderDataBySize(fp
, formatspecs
[5]);
972 if(len(preheaderdata
)==0):
974 prefsize
= int(preheaderdata
[5], 16);
975 prenewfcs
= GetHeaderChecksum(preheaderdata
[:-2], preheaderdata
[-3].lower(), True, formatspecs
);
976 prefcs
= preheaderdata
[-2];
977 if(prefcs
!=prenewfcs
and not skipchecksum
):
978 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
980 valid_archive
= False;
981 invalid_archive
= True;
982 prefhend
= fp
.tell() - 1;
983 prefcontentstart
= fp
.tell();
985 pyhascontents
= False;
987 prefcontents
= fp
.read(prefsize
);
988 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
989 prefccs
= preheaderdata
[-1];
990 pyhascontents
= True;
991 if(prefccs
!=prenewfccs
and not skipchecksum
):
992 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
997 countnum
= seekstart
;
998 while(countnum
< seekend
):
999 HeaderOut
= ReadFileHeaderDataBySizeWithContentToList(fp
, listonly
, skipchecksum
, formatspecs
);
1002 catlist
.append(HeaderOut
);
1003 countnum
= countnum
+ 1;
1004 realidnum
= realidnum
+ 1;
1007 def ReadInFileBySizeWithContentToArray(infile
, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
1008 delimiter
= formatspecs
[5];
1009 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1012 fp
= UncompressArchiveFile(fp
, formatspecs
);
1013 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1014 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
1021 if(hasattr(sys
.stdin
, "buffer")):
1022 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1024 shutil
.copyfileobj(sys
.stdin
, fp
);
1026 fp
= UncompressArchiveFile(fp
, formatspecs
);
1030 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1031 fp
= download_file_from_internet_file(infile
);
1032 fp
= UncompressArchiveFile(fp
, formatspecs
);
1038 infile
= RemoveWindowsPath(infile
);
1039 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1040 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
1042 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1043 if(not compresscheck
):
1044 fextname
= os
.path
.splitext(infile
)[1];
1045 if(fextname
==".gz"):
1046 compresscheck
= "gzip";
1047 elif(fextname
==".bz2"):
1048 compresscheck
= "bzip2";
1049 elif(fextname
==".zst"):
1050 compresscheck
= "zstd";
1051 elif(fextname
==".lz4" or fextname
==".clz4"):
1052 compresscheck
= "lz4";
1053 elif(fextname
==".lzo" or fextname
==".lzop"):
1054 compresscheck
= "lzo";
1055 elif(fextname
==".lzma" or fextname
==".xz"):
1056 compresscheck
= "lzma";
1059 if(not compresscheck
):
1061 fp
= UncompressFile(infile
, formatspecs
, "rb");
1062 return ReadFileDataBySizeWithContentToArray(fp
, listonly
, skipchecksum
, formatspecs
);
1064 def ReadInFileBySizeWithContentToList(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
1065 delimiter
= formatspecs
[5];
1066 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1069 fp
= UncompressArchiveFile(fp
, formatspecs
);
1070 checkcompressfile
= CheckCompressionSubType(fp
, formatspecs
, True);
1071 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
1078 if(hasattr(sys
.stdin
, "buffer")):
1079 shutil
.copyfileobj(sys
.stdin
.buffer, fp
);
1081 shutil
.copyfileobj(sys
.stdin
, fp
);
1083 fp
= UncompressArchiveFile(fp
, formatspecs
);
1087 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
1088 fp
= download_file_from_internet_file(infile
);
1089 fp
= UncompressArchiveFile(fp
, formatspecs
);
1095 infile
= RemoveWindowsPath(infile
);
1096 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
1097 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
1099 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
1100 if(not compresscheck
):
1101 fextname
= os
.path
.splitext(infile
)[1];
1102 if(fextname
==".gz"):
1103 compresscheck
= "gzip";
1104 elif(fextname
==".bz2"):
1105 compresscheck
= "bzip2";
1106 elif(fextname
==".zst"):
1107 compresscheck
= "zstd";
1108 elif(fextname
==".lz4" or fextname
==".clz4"):
1109 compresscheck
= "lz4";
1110 elif(fextname
==".lzo" or fextname
==".lzop"):
1111 compresscheck
= "lzo";
1112 elif(fextname
==".lzma" or fextname
==".xz"):
1113 compresscheck
= "lzma";
1116 if(not compresscheck
):
1118 fp
= UncompressFile(infile
, formatspecs
, "rb");
1119 return ReadFileDataBySizeWithContentToList(fp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
);
1121 def ReadInFileBySizeWithContentToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
):
1122 if(isinstance(infile
, dict)):
1123 listcatfiles
= infile
;
1125 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
1126 infile
= RemoveWindowsPath(infile
);
1127 listcatfiles
= ReadInFileBySizeWithContentToArray(infile
, listonly
, skipchecksum
, formatspecs
);
1128 if(not listcatfiles
):
1130 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1131 lenlist
= len(listcatfiles
['ffilelist']);
1133 lcfx
= int(listcatfiles
['fnumfiles']);
1134 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
1135 lcfx
= int(lenlist
);
1137 lcfx
= int(listcatfiles
['fnumfiles']);
1139 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
1140 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
1141 catarray
['filetoid'].update(filetoidarray
);
1142 catarray
['idtofile'].update(idtofilearray
);
1143 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
1144 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
1145 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
1146 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
1147 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
1148 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
1149 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1150 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1151 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
1152 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1153 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1154 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
1155 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
1156 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
1157 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
1158 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
1159 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1160 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1161 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
1162 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
1163 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
1164 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1165 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1166 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
1167 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
1168 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
1169 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
1170 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
1171 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
1172 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
1173 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
1177 def AppendNullByte(indata
, delimiter
=__file_format_delimiter__
):
1178 outdata
= str(indata
) + delimiter
;
1181 def AppendNullBytes(indata
=[], delimiter
=__file_format_delimiter__
):
1186 outdata
= outdata
+ AppendNullByte(indata
[inum
], delimiter
);
1190 def AppendFileHeader(fp
, numfiles
, checksumtype
="crc32", formatspecs
=__file_format_list__
):
1191 delimiter
= formatspecs
[5];
1192 catver
= formatspecs
[6];
1193 fileheaderver
= str(int(catver
.replace(".", "")));
1194 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
1195 fp
.write(fileheader
.encode('UTF-8'));
1196 fnumfiles
= format(int(numfiles
), 'x').lower();
1197 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
[5]);
1198 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
1199 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
1200 fp
.write(fnumfilesa
.encode('UTF-8'));
1203 os
.fsync(fp
.fileno());
1204 except io
.UnsupportedOperation
:
1206 except AttributeError:
1210 def MakeEmptyFilePointer(fp
, checksumtype
="crc32", formatspecs
=__file_format_list__
):
1211 AppendFileHeader(fp
, 0, checksumtype
, formatspecs
);
1214 def MakeEmptyFile(outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", formatspecs
=__file_format_list__
, returnfp
=False):
1215 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1216 if(os
.path
.exists(outfile
)):
1220 catfpfp
= BytesIO();
1221 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1223 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1226 fbasename
= os
.path
.splitext(outfile
)[0];
1227 fextname
= os
.path
.splitext(outfile
)[1];
1228 catfp
= CompressOpenFile(outfile
, compressionlevel
);
1229 catfp
= AppendFileHeader(catfp
, 0, checksumtype
, formatspecs
);
1230 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1231 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
1234 os
.fsync(catfp
.fileno());
1235 except io
.UnsupportedOperation
:
1237 except AttributeError:
1241 if(hasattr(sys
.stdout
, "buffer")):
1242 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1244 shutil
.copyfileobj(catfp
, sys
.stdout
);
1245 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1246 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
1248 upload_file_to_internet_file(catfp
, outfile
);
1256 def AppendFileHeaderWithContent(fp
, filevalues
=[], extradata
=[], filecontent
="", checksumtype
="crc32", formatspecs
=__file_format_list__
):
1257 extrafields
= format(len(extradata
), 'x').lower();
1258 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
1259 if(len(extradata
)>0):
1260 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
1261 extrasizelen
= format(len(extrasizestr
), 'x').lower();
1262 catoutlen
= len(filevalues
) + len(extradata
) + 5;
1263 catoutlenhex
= format(catoutlen
, 'x').lower();
1264 catoutlist
= filevalues
;
1265 catoutlist
.insert(0, catoutlenhex
);
1266 catoutlist
.append(extrasizelen
);
1267 catoutlist
.append(extrafields
);
1268 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
1269 if(len(extradata
)>0):
1270 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
1271 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
1272 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1273 catfilecontentcshex
= GetFileChecksum(filecontent
, checksumtype
, False, formatspecs
);
1274 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
1275 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
1276 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
1277 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
1278 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
1279 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
1280 nullstrecd
= formatspecs
[5].encode('UTF-8');
1281 catfileout
= catfileoutstrecd
+ filecontent
+ nullstrecd
;
1282 fp
.write(catfileout
);
1285 os
.fsync(fp
.fileno());
1286 except io
.UnsupportedOperation
:
1288 except AttributeError:
1292 def AppendFilesWithContent(infiles
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_list__
, verbose
=False):
1293 advancedlist
= formatspecs
[8];
1294 altinode
= formatspecs
[9];
1296 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1298 for line
in sys
.stdin
:
1299 infilelist
.append(line
.strip());
1300 infilelist
= list(filter(None, infilelist
));
1301 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
1302 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
1304 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
1305 for line
in finfile
:
1306 infilelist
.append(line
.strip());
1307 infilelist
= list(filter(None, infilelist
));
1309 if(isinstance(infiles
, (list, tuple, ))):
1310 infilelist
= list(filter(None, infiles
));
1311 elif(isinstance(infiles
, (str, ))):
1312 infilelist
= list(filter(None, [infiles
]));
1314 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
1316 GetDirList
= ListDir(infilelist
, followlink
, False);
1324 inodetocatinode
= {};
1325 numfiles
= int(len(GetDirList
));
1326 fnumfiles
= format(numfiles
, 'x').lower();
1327 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1328 for curfname
in GetDirList
:
1329 catfhstart
= fp
.tell();
1330 if(re
.findall("^[.|/]", curfname
)):
1333 fname
= "./"+curfname
;
1335 VerbosePrintOut(fname
);
1336 if(not followlink
or followlink
is None):
1337 fstatinfo
= os
.lstat(fname
);
1339 fstatinfo
= os
.stat(fname
);
1340 fpremode
= fstatinfo
.st_mode
;
1341 finode
= fstatinfo
.st_ino
;
1342 flinkcount
= fstatinfo
.st_nlink
;
1344 if(stat
.S_ISREG(fpremode
)):
1346 elif(stat
.S_ISLNK(fpremode
)):
1348 elif(stat
.S_ISCHR(fpremode
)):
1350 elif(stat
.S_ISBLK(fpremode
)):
1352 elif(stat
.S_ISDIR(fpremode
)):
1354 elif(stat
.S_ISFIFO(fpremode
)):
1356 elif(stat
.S_ISSOCK(fpremode
)):
1358 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
1360 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
1362 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
1367 fcurfid
= format(int(curfid
), 'x').lower();
1368 if(not followlink
and finode
!=0):
1370 if(finode
in inodelist
):
1372 flinkname
= inodetofile
[finode
];
1374 fcurinode
= format(int(finode
), 'x').lower();
1376 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
1377 if(finode
not in inodelist
):
1378 inodelist
.append(finode
);
1379 inodetofile
.update({finode
: fname
});
1380 inodetocatinode
.update({finode
: curinode
});
1382 fcurinode
= format(int(finode
), 'x').lower();
1384 fcurinode
= format(int(curinode
), 'x').lower();
1385 curinode
= curinode
+ 1;
1387 fcurinode
= format(int(curinode
), 'x').lower();
1388 curinode
= curinode
+ 1;
1389 curfid
= curfid
+ 1;
1391 flinkname
= os
.readlink(fname
);
1392 fdev
= fstatinfo
.st_dev
;
1393 getfdev
= GetDevMajorMinor(fdev
);
1394 fdev_minor
= getfdev
[0];
1395 fdev_major
= getfdev
[1];
1396 frdev
= fstatinfo
.st_dev
;
1397 if(hasattr(fstatinfo
, "st_rdev")):
1398 frdev
= fstatinfo
.st_rdev
;
1400 frdev
= fstatinfo
.st_dev
;
1401 getfrdev
= GetDevMajorMinor(frdev
);
1402 frdev_minor
= getfrdev
[0];
1403 frdev_major
= getfrdev
[1];
1404 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
1405 fsize
= format(int("0"), 'x').lower();
1406 elif(ftype
==0 or ftype
==7):
1407 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
1409 fsize
= format(int(fstatinfo
.st_size
)).lower();
1410 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
1411 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
1412 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1413 if(hasattr(fstatinfo
, "st_birthtime")):
1414 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
1416 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
1417 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
1418 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
1419 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
1420 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
1421 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
1426 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
1427 funame
= userinfo
.pw_name
;
1436 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
1437 fgname
= groupinfo
.gr_name
;
1442 fdev_minor
= format(int(fdev_minor
), 'x').lower();
1443 fdev_major
= format(int(fdev_major
), 'x').lower();
1444 frdev_minor
= format(int(frdev_minor
), 'x').lower();
1445 frdev_major
= format(int(frdev_major
), 'x').lower();
1446 finode
= format(int(finode
), 'x').lower();
1447 flinkcount
= format(int(flinkcount
), 'x').lower();
1448 if(hasattr(fstatinfo
, "st_file_attributes")):
1449 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
1451 fwinattributes
= format(int(0), 'x').lower();
1452 fcontents
= BytesIO();
1454 if(ftype
==0 or ftype
==7):
1455 with
open(fname
, "rb") as fpc
:
1456 shutil
.copyfileobj(fpc
, fcontents
);
1457 if(followlink
and (ftype
==1 or ftype
==2)):
1458 flstatinfo
= os
.stat(flinkname
);
1459 with
open(flinkname
, "rb") as fpc
:
1460 shutil
.copyfileobj(fpc
, fcontents
);
1461 fcontents
.seek(0, 0);
1462 ftypehex
= format(ftype
, 'x').lower();
1463 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
1464 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
.read(), checksumtype
, formatspecs
);
1466 catfp
.write(AppendNullBytes([0, 0], formatspecs
[5]).encode("UTF-8"));
1470 def AppendListsWithContent(inlist
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_list__
, verbose
=False):
1472 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
1473 GetDirList
= inlist
;
1481 inodetocatinode
= {};
1482 numfiles
= int(len(GetDirList
));
1483 fnumfiles
= format(numfiles
, 'x').lower();
1484 AppendFileHeader(fp
, fnumfiles
, checksumtype
, formatspecs
);
1485 for curfname
in GetDirList
:
1486 ftype
= format(curfname
[0], 'x').lower();
1487 if(re
.findall("^[.|/]", curfname
[1])):
1488 fname
= curfname
[1];
1490 fname
= "./"+curfname
[1];
1491 fbasedir
= os
.path
.dirname(fname
);
1492 flinkname
= curfname
[2];
1493 fsize
= format(curfname
[3], 'x').lower();
1494 fatime
= format(curfname
[4], 'x').lower();
1495 fmtime
= format(curfname
[5], 'x').lower();
1496 fctime
= format(curfname
[6], 'x').lower();
1497 fbtime
= format(curfname
[7], 'x').lower();
1498 fmode
= format(curfname
[8], 'x').lower();
1499 fwinattributes
= format(curfname
[9], 'x').lower();
1500 fuid
= format(curfname
[10], 'x').lower();
1501 funame
= curfname
[11];
1502 fgid
= format(curfname
[12], 'x').lower();
1503 fgname
= curfname
[13];
1504 fid
= format(curfname
[14], 'x').lower();
1505 finode
= format(curfname
[15], 'x').lower();
1506 flinkcount
= format(curfname
[16], 'x').lower();
1507 fdev_minor
= format(curfname
[17], 'x').lower();
1508 fdev_major
= format(curfname
[18], 'x').lower();
1509 frdev_minor
= format(curfname
[19], 'x').lower();
1510 frdev_major
= format(curfname
[20], 'x').lower();
1511 extradata
= curfname
[21];
1512 fchecksumtype
= curfname
[22];
1513 fcontents
= curfname
[23];
1514 catoutlist
= [ftype
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fuid
, funame
, fgid
, fgname
, fid
, finode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
];
1515 fp
= AppendFileHeaderWithContent(fp
, catoutlist
, extradata
, fcontents
, checksumtype
, formatspecs
);
1517 catfp
.write(AppendNullBytes([0, 0], formatspecs
[5]).encode("UTF-8"));
1521 def AppendInFileWithContent(infile
, fp
, dirlistfromtxt
=False, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_list__
, verbose
=False):
1522 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, False, formatspecs
);
1523 return AppendListsWithContent(inlist
, fp
, dirlistfromtxt
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
);
1525 def AppendFilesWithContentToOutFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
1526 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1527 if(os
.path
.exists(outfile
)):
1531 catfpfp
= BytesIO();
1532 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1534 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1537 fbasename
= os
.path
.splitext(outfile
)[0];
1538 fextname
= os
.path
.splitext(outfile
)[1];
1539 catfp
= CompressOpenFile(outfile
, compressionlevel
);
1540 catfp
= AppendFilesWithContent(infiles
, catfp
, dirlistfromtxt
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
);
1541 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1542 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
1545 os
.fsync(catfp
.fileno());
1546 except io
.UnsupportedOperation
:
1548 except AttributeError:
1552 if(hasattr(sys
.stdout
, "buffer")):
1553 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1555 shutil
.copyfileobj(catfp
, sys
.stdout
);
1556 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1557 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
1559 upload_file_to_internet_file(catfp
, outfile
);
1567 def AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
1568 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
1569 if(os
.path
.exists(outfile
)):
1573 catfpfp
= BytesIO();
1574 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1576 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1579 fbasename
= os
.path
.splitext(outfile
)[0];
1580 fextname
= os
.path
.splitext(outfile
)[1];
1581 catfp
= CompressOpenFile(outfile
, compressionlevel
);
1582 catfp
= AppendListsWithContent(inlist
, catfp
, dirlistfromtxt
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
);
1583 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
1584 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
1587 os
.fsync(catfp
.fileno());
1588 except io
.UnsupportedOperation
:
1590 except AttributeError:
1594 if(hasattr(sys
.stdout
, "buffer")):
1595 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
1597 shutil
.copyfileobj(catfp
, sys
.stdout
);
1598 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
1599 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
1601 upload_file_to_internet_file(catfp
, outfile
);
1609 def AppendInFileWithContentToOutFile(infile
, outfile
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, filevalues
=[], extradata
=[], followlink
=False, checksumtype
="crc32", formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
1610 inlist
= ReadInFileBySizeWithContentToList(infile
, 0, 0, False, False, formatspecs
);
1611 return AppendListsWithContentToOutFile(inlist
, outfile
, dirlistfromtxt
, compression
, compressionlevel
, filevalues
, extradata
, followlink
, checksumtype
, formatspecs
, verbose
, returnfp
);
1613 def PrintPermissionString(fchmode
, ftype
):
1614 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1616 for fmodval
in str(oct(fchmode
))[-3:]:
1617 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
1618 if(ftype
==0 or ftype
==7):
1619 permissionstr
= "-" + permissionstr
;
1621 permissionstr
= "h" + permissionstr
;
1623 permissionstr
= "l" + permissionstr
;
1625 permissionstr
= "c" + permissionstr
;
1627 permissionstr
= "b" + permissionstr
;
1629 permissionstr
= "d" + permissionstr
;
1631 permissionstr
= "f" + permissionstr
;
1633 permissionstr
= "D" + permissionstr
;
1635 permissionstr
= "p" + permissionstr
;
1637 permissionstr
= "w" + permissionstr
;
1639 permissionoutstr
= stat
.filemode(fchmode
);
1640 except AttributeError:
1641 permissionoutstr
= permissionstr
;
1643 permissionoutstr
= permissionstr
;
1644 return permissionoutstr
;
1646 def PrintPermissionStringAlt(fchmode
, ftype
):
1648 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1649 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1651 # Translate file mode into permission string
1652 permissionstr
= ''.join([permissions
[i
] for i
in str(oct(fchmode
))[-3:]]);
1653 # Append file type indicator
1655 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1656 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1658 file_type
= type_indicators
.get(ftype
, '-');
1659 permissionstr
= file_type
+ permissionstr
;
1661 permissionoutstr
= stat
.filemode(fchmode
);
1662 except AttributeError:
1663 permissionoutstr
= permissionstr
;
1664 return permissionoutstr
;
1666 def CompressionSupport():
1667 compression_list
= [];
1670 compression_list
.append("gz");
1671 compression_list
.append("gzip");
1676 compression_list
.append("bz2");
1677 compression_list
.append("bzip2");
1682 compression_list
.append("lz4");
1687 compression_list
.append("lzo");
1688 compression_list
.append("lzop");
1693 compression_list
.append("zstd");
1694 compression_list
.append("zstandard");
1699 compression_list
.append("lzma");
1700 compression_list
.append("xz");
1703 return compression_list
;
1705 def CheckCompressionType(infile
, formatspecs
=__file_format_list__
, closefp
=True):
1706 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
1710 catfp
= open(infile
, "rb");
1711 except FileNotFoundError
:
1714 prefp
= catfp
.read(2);
1716 if(prefp
==binascii
.unhexlify("1f8b")):
1719 prefp
= catfp
.read(3);
1720 if(prefp
==binascii
.unhexlify("425a68")):
1722 if(prefp
==binascii
.unhexlify("5d0000")):
1725 prefp
= catfp
.read(4);
1726 if(prefp
==binascii
.unhexlify("28b52ffd")):
1728 if(prefp
==binascii
.unhexlify("04224d18")):
1730 if(prefp
==binascii
.unhexlify("504B0304")):
1731 filetype
= "zipfile";
1733 prefp
= catfp
.read(5);
1734 if(prefp
==binascii
.unhexlify("7573746172")):
1735 filetype
= "tarfile";
1737 prefp
= catfp
.read(6);
1738 if(prefp
==binascii
.unhexlify("fd377a585a00")):
1740 if(prefp
==binascii
.unhexlify("377abcaf271c")):
1741 filetype
= "7zipfile";
1743 prefp
= catfp
.read(7);
1744 if(prefp
==binascii
.unhexlify("526172211a0700")):
1745 filetype
= "rarfile";
1746 if(prefp
==binascii
.unhexlify("43617446696c65")):
1747 filetype
= "catfile";
1749 prefp
= catfp
.read(8);
1750 if(prefp
==binascii
.unhexlify("526172211a070100")):
1751 filetype
= "rarfile";
1753 prefp
= catfp
.read(formatspecs
[3]);
1754 if(prefp
==binascii
.unhexlify(formatspecs
[4])):
1755 filetype
= formatspecs
[2];
1757 prefp
= catfp
.read(9);
1758 if(prefp
==binascii
.unhexlify("894c5a4f000d0a1a0a")):
1761 prefp
= catfp
.read(10);
1762 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
1763 filetype
= "tarfile";
1765 if(filetype
=="gzip" or filetype
=="bzip2" or filetype
=="lzma" or filetype
=="zstd" or filetype
=="lz4"):
1766 if(is_tarfile(catfp
)):
1767 filetype
= "tarfile";
1769 if(is_tarfile(catfp
)):
1770 filetype
= "tarfile";
1771 elif(zipfile
.is_zipfile(catfp
)):
1772 filetype
= "zipfile";
1773 elif(rarfile
.is_rarfile(catfp
) or rarfile
.is_rarfile_sfx(catfp
)):
1774 filetype
= "rarile";
1782 def CheckCompressionTypeFromString(instring
, formatspecs
=__file_format_list__
, closefp
=True):
1784 instringsfile
= BytesIO(instring
);
1786 instringsfile
= BytesIO(instring
.encode("UTF-8"));
1787 return CheckCompressionType(instringsfile
, formatspecs
, closefp
);
1789 def GetCompressionMimeType(infile
, formatspecs
=__file_format_list__
):
1790 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
1791 if(compresscheck
=="gzip" or compresscheck
=="gz"):
1792 return archivefile_gzip_mimetype
;
1793 if(compresscheck
=="bzip2" or compresscheck
=="bz2"):
1794 return archivefile_bzip2_mimetype
;
1795 if(compresscheck
=="zstd" or compresscheck
=="zstandard"):
1796 return archivefile_zstandard_mimetype
;
1797 if(compresscheck
=="lz4"):
1798 return archivefile_lz4_mimetype
;
1799 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
1800 return archivefile_lzop_mimetype
;
1801 if(compresscheck
=="lzma"):
1802 return archivefile_lzma_mimetype
;
1803 if(compresscheck
=="xz"):
1804 return archivefile_xz_mimetype
;
1805 if(compresscheck
=="catfile" or compresscheck
=="cat" or compresscheck
==formatspecs
[2]):
1806 return archivefile_cat_mimetype
;
1807 if(not compresscheck
):
1811 def UncompressArchiveFile(fp
, formatspecs
=__file_format_list__
):
1812 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
1814 compresscheck
= CheckCompressionType(fp
, formatspecs
, False);
1815 if(compresscheck
=="gzip"):
1820 catfp
= gzip
.GzipFile(fileobj
=fp
, mode
="rb");
1821 if(compresscheck
=="bzip2"):
1827 catfp
.write(bz2
.decompress(fp
.read()));
1828 if(compresscheck
=="zstd"):
1834 catfp
.write(zstandard
.decompress(fp
.read()));
1835 if(compresscheck
=="lz4"):
1841 catfp
.write(lz4
.frame
.decompress(fp
.read()));
1842 if(compresscheck
=="lzo" or compresscheck
=="lzop"):
1848 catfp
.write(lzo
.decompress(fp
.read()));
1849 if(compresscheck
=="lzma" or compresscheck
=="xz"):
1855 catfp
.write(lzma
.decompress(fp
.read()));
1856 if(compresscheck
=="catfile" or compresscheck
==formatspecs
[2]):
1858 if(not compresscheck
):
1864 with fp
as fpcontent
:
1866 catfp
.write(lzma
.decompress(fp
.read()));
1867 except lzma
.LZMAError
:
1871 create_alias_function("Uncompress", __file_format_name__
, "", UncompressArchiveFile
);
1873 def UncompressFile(infile
, formatspecs
=__file_format_list__
, mode
="rb"):
1874 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
1875 if(sys
.version_info
[0]==2 and compresscheck
):
1881 if(compresscheck
=="gzip"):
1887 filefp
= gzip
.open(infile
, mode
, encoding
="UTF-8");
1888 except (ValueError, TypeError) as e
:
1889 filefp
= gzip
.open(infile
, mode
);
1890 if(compresscheck
=="bzip2"):
1896 filefp
= bz2
.open(infile
, mode
, encoding
="UTF-8");
1897 except (ValueError, TypeError) as e
:
1898 filefp
= bz2
.open(infile
, mode
);
1899 if(compresscheck
=="zstd"):
1905 filefp
= zstandard
.open(infile
, mode
, encoding
="UTF-8");
1906 except (ValueError, TypeError) as e
:
1907 filefp
= zstandard
.open(infile
, mode
);
1908 if(compresscheck
=="lz4"):
1914 filefp
= lz4
.frame
.open(infile
, mode
, encoding
="UTF-8");
1915 except (ValueError, TypeError) as e
:
1916 filefp
= lz4
.frame
.open(infile
, mode
);
1917 if(compresscheck
=="lzo"):
1923 filefp
= lzo
.open(infile
, mode
, encoding
="UTF-8");
1924 except (ValueError, TypeError) as e
:
1925 filefp
= lzo
.open(infile
, mode
);
1926 if(compresscheck
=="lzma"):
1932 filefp
= lzma
.open(infile
, mode
, encoding
="UTF-8");
1933 except (ValueError, TypeError) as e
:
1934 filefp
= lzma
.open(infile
, mode
);
1935 if(compresscheck
=="catfile" or compresscheck
==formatspecs
[2]):
1937 filefp
= open(infile
, mode
, encoding
="UTF-8");
1938 except (ValueError, TypeError) as e
:
1939 filefp
= open(infile
, mode
);
1940 if(not compresscheck
):
1942 filefp
= open(infile
, mode
, encoding
="UTF-8");
1943 except (ValueError, TypeError) as e
:
1944 filefp
= open(infile
, mode
);
1945 except FileNotFoundError
:
1949 def UncompressString(infile
):
1950 compresscheck
= CheckCompressionTypeFromString(infile
, formatspecs
, False);
1951 if(compresscheck
=="gzip"):
1956 fileuz
= gzip
.decompress(infile
);
1957 if(compresscheck
=="bzip2"):
1962 fileuz
= bz2
.decompress(infile
);
1963 if(compresscheck
=="zstd"):
1968 fileuz
= zstandard
.decompress(infile
);
1969 if(compresscheck
=="lz4"):
1974 fileuz
= lz4
.frame
.decompress(infile
);
1975 if(compresscheck
=="lzo"):
1980 fileuz
= lzo
.decompress(infile
);
1981 if(compresscheck
=="lzma"):
1986 fileuz
= lzma
.decompress(infile
);
1987 if(not compresscheck
):
1989 if(hasattr(fileuz
, 'decode')):
1990 fileuz
= fileuz
.decode("UTF-8");
1993 def UncompressStringAlt(infile
):
1994 filefp
= StringIO();
1995 outstring
= UncompressString(infile
);
1996 filefp
.write(outstring
);
2000 def CheckCompressionSubType(infile
, formatspecs
=__file_format_list__
, closefp
=True):
2001 compresscheck
= CheckCompressionType(infile
, formatspecs
, False);
2002 if(not compresscheck
):
2003 fextname
= os
.path
.splitext(infile
)[1];
2004 if(fextname
==".gz"):
2005 compresscheck
= "gzip";
2006 elif(fextname
==".bz2"):
2007 compresscheck
= "bzip2";
2008 elif(fextname
==".zst"):
2009 compresscheck
= "zstd";
2010 elif(fextname
==".lz4"):
2011 compresscheck
= "lz4";
2012 elif(fextname
==".lzo" or fextname
==".lzop"):
2013 compresscheck
= "lzo";
2014 elif(fextname
==".lzma" or fextname
==".xz"):
2015 compresscheck
= "lzma";
2018 if(compresscheck
=="gzip" or compresscheck
=="bzip2" or compresscheck
=="lzma" or compresscheck
=="zstd" or compresscheck
=="lz4"):
2019 if(is_tarfile(infile
)):
2020 filetype
= "tarfile";
2021 if(not compresscheck
):
2022 if(is_tarfile(infile
)):
2024 elif(zipfile
.is_zipfile(infile
)):
2026 elif(rarfile
.is_rarfile(infile
) or rarfile
.is_rarfile_sfx(infile
)):
2031 if(compresscheck
=="catfile"):
2033 if(compresscheck
==formatspecs
[2]):
2034 return formatspecs
[2];
2035 if(compresscheck
=="tarfile"):
2037 if(compresscheck
=="zipfile"):
2039 if(rarfile_support
and compresscheck
=="rarfile"):
2041 if(py7zr_support
and compresscheck
=="7zipfile"):
2043 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2044 catfp
= UncompressArchiveFile(infile
, formatspecs
[2]);
2047 if(compresscheck
=="gzip"):
2052 catfp
= gzip
.GzipFile(infile
, "rb");
2053 if(compresscheck
=="bzip2"):
2058 catfp
= bz2
.BZ2File(infile
, "rb");
2059 if(compresscheck
=="lz4"):
2064 catfp
= lz4
.frame
.open(infile
, "rb");
2065 if(compresscheck
=="zstd"):
2070 catfp
= zstandard
.open(infile
, "rb");
2071 if(compresscheck
=="lzma" or compresscheck
=="xz"):
2076 catfp
= lzma
.open(infile
, "rb");
2077 except FileNotFoundError
:
2080 prefp
= catfp
.read(5);
2081 if(prefp
==binascii
.unhexlify("7573746172")):
2082 filetype
= "tarfile";
2084 prefp
= catfp
.read(7);
2085 if(prefp
==binascii
.unhexlify("43617446696c65")):
2086 filetype
= "catfile";
2088 prefp
= catfp
.read(formatspecs
[3]);
2089 if(prefp
==binascii
.unhexlify(formatspecs
[4])):
2090 filetype
= formatspecs
[2];
2092 prefp
= catfp
.read(10);
2093 if(prefp
==binascii
.unhexlify("7061785f676c6f62616c")):
2094 filetype
= "tarfile";
2100 def GZipCompress(data
, compresslevel
=9):
2105 tmpfp
= tempfile
.NamedTemporaryFile("wb", delete
=False);
2107 tmpfp
= gzip
.GzipFile(tmpfp
.name
, mode
="wb", compresslevel
=compresslevel
);
2111 catfp
= open(tmpfp
.name
, "rb");
2112 except FileNotFoundError
:
2114 catdata
= catfp
.read();
2118 def CompressArchiveFile(fp
, compression
="auto", compressionlevel
=None, formatspecs
=__file_format_list__
):
2119 if(not hasattr(fp
, "read") and not hasattr(fp
, "write")):
2122 if(not compression
or compression
or compression
=="catfile" or compression
==formatspecs
[2]):
2124 if(compression
not in compressionlist
and compression
is None):
2125 compression
= "auto";
2126 if(compression
=="gzip"):
2132 if(compressionlevel
is None):
2133 compressionlevel
= 9;
2135 compressionlevel
= int(compressionlevel
);
2136 catfp
.write(gzip
.compress(fp
.read(), compresslevel
=compressionlevel
));
2137 if(compression
=="bzip2"):
2143 if(compressionlevel
is None):
2144 compressionlevel
= 9;
2146 compressionlevel
= int(compressionlevel
);
2147 catfp
.write(bz2
.compress(fp
.read(), compresslevel
=compressionlevel
));
2148 if(compression
=="lz4"):
2154 if(compressionlevel
is None):
2155 compressionlevel
= 9;
2157 compressionlevel
= int(compressionlevel
);
2158 catfp
.write(lz4
.frame
.compress(fp
.read(), compression_level
=compressionlevel
));
2159 if(compression
=="lzo" or compression
=="lzop"):
2165 if(compressionlevel
is None):
2166 compressionlevel
= 9;
2168 compressionlevel
= int(compressionlevel
);
2169 catfp
.write(lzo
.compress(fp
.read(), compresslevel
=compressionlevel
));
2170 if(compression
=="zstd"):
2176 if(compressionlevel
is None):
2177 compressionlevel
= 10;
2179 compressionlevel
= int(compressionlevel
);
2180 catfp
.write(zstandard
.compress(fp
.read(), level
=compressionlevel
));
2181 if(compression
=="lzma"):
2187 if(compressionlevel
is None):
2188 compressionlevel
= 9;
2190 compressionlevel
= int(compressionlevel
);
2191 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]));
2192 if(compression
=="xz"):
2198 if(compressionlevel
is None):
2199 compressionlevel
= 9;
2201 compressionlevel
= int(compressionlevel
);
2202 catfp
.write(lzma
.compress(fp
.read(), format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]));
2203 if(compression
=="auto" or compression
is None):
2208 create_alias_function("Compress", __file_format_name__
, "", CompressArchiveFile
);
2210 def CompressOpenFile(outfile
, compressionlevel
=None):
2211 if(outfile
is None):
2213 fbasename
= os
.path
.splitext(outfile
)[0];
2214 fextname
= os
.path
.splitext(outfile
)[1];
2215 if(compressionlevel
is None and fextname
!=".zst"):
2216 compressionlevel
= 9;
2217 elif(compressionlevel
is None and fextname
==".zst"):
2218 compressionlevel
= 10;
2220 compressionlevel
= int(compressionlevel
);
2221 if(sys
.version_info
[0]==2):
2226 if(fextname
not in outextlistwd
):
2228 outfp
= open(outfile
, "wb", encoding
="UTF-8");
2229 except (ValueError, TypeError) as e
:
2230 outfp
= open(outfile
, "wb");
2231 elif(fextname
==".gz"):
2237 outfp
= gzip
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2238 except (ValueError, TypeError) as e
:
2239 outfp
= gzip
.open(outfile
, mode
, compressionlevel
);
2240 elif(fextname
==".bz2"):
2246 outfp
= bz2
.open(outfile
, mode
, compressionlevel
, encoding
="UTF-8");
2247 except (ValueError, TypeError) as e
:
2248 outfp
= bz2
.open(outfile
, mode
, compressionlevel
);
2249 elif(fextname
==".zst"):
2255 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
), encoding
="UTF-8");
2256 except (ValueError, TypeError) as e
:
2257 outfp
= zstandard
.open(outfile
, mode
, zstandard
.ZstdCompressor(level
=compressionlevel
));
2258 elif(fextname
==".xz"):
2264 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}], encoding
="UTF-8");
2265 except (ValueError, TypeError) as e
:
2266 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_XZ
, filters
=[{"id": lzma
.FILTER_LZMA2
, "preset": compressionlevel
}]);
2267 elif(fextname
==".lz4"):
2273 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
, encoding
="UTF-8");
2274 except (ValueError, TypeError) as e
:
2275 outfp
= lz4
.frame
.open(outfile
, mode
, compression_level
=compressionlevel
);
2276 elif(fextname
==".lzo"):
2282 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
, encoding
="UTF-8");
2283 except (ValueError, TypeError) as e
:
2284 outfp
= lzo
.open(outfile
, mode
, compresslevel
=compressionlevel
);
2285 elif(fextname
==".lzma"):
2291 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}], encoding
="UTF-8");
2292 except (ValueError, TypeError) as e
:
2293 outfp
= lzma
.open(outfile
, mode
, format
=lzma
.FORMAT_ALONE
, filters
=[{"id": lzma
.FILTER_LZMA1
, "preset": compressionlevel
}]);
2294 except FileNotFoundError
:
2298 def GetDevMajorMinor(fdev
):
2300 if(hasattr(os
, "minor")):
2301 retdev
.append(os
.minor(fdev
));
2304 if(hasattr(os
, "major")):
2305 retdev
.append(os
.major(fdev
));
2310 def CheckSumSupport(checkfor
, guaranteed
=True):
2312 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2314 hash_list
= sorted(list(hashlib
.algorithms_available
));
2315 checklistout
= sorted(hash_list
+ ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2316 if(checkfor
in checklistout
):
2321 def CheckSumSupportAlt(checkfor
, guaranteed
=True):
2323 hash_list
= sorted(list(hashlib
.algorithms_guaranteed
));
2325 hash_list
= sorted(list(hashlib
.algorithms_available
));
2326 checklistout
= hash_list
;
2327 if(checkfor
in checklistout
):
2332 def PackArchiveFile(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
2333 advancedlist
= formatspecs
[8];
2334 altinode
= formatspecs
[9];
2335 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2336 outfile
= RemoveWindowsPath(outfile
);
2337 checksumtype
= checksumtype
.lower();
2338 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2339 checksumtype
="crc32";
2340 if(checksumtype
=="none"):
2342 if(not compression
or compression
or compression
=="catfile" or compression
==formatspecs
[2]):
2344 if(compression
not in compressionlist
and compression
is None):
2345 compression
= "auto";
2347 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2348 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2349 if(os
.path
.exists(outfile
)):
2354 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2356 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2359 fbasename
= os
.path
.splitext(outfile
)[0];
2360 fextname
= os
.path
.splitext(outfile
)[1];
2361 catfp
= CompressOpenFile(outfile
, compressionlevel
);
2362 catver
= formatspecs
[6];
2363 fileheaderver
= str(int(catver
.replace(".", "")));
2364 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
2365 catfp
.write(fileheader
.encode('UTF-8'));
2368 for line
in sys
.stdin
:
2369 infilelist
.append(line
.strip());
2370 infilelist
= list(filter(None, infilelist
));
2371 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
2372 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
2374 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
2375 for line
in finfile
:
2376 infilelist
.append(line
.strip());
2377 infilelist
= list(filter(None, infilelist
));
2379 if(isinstance(infiles
, (list, tuple, ))):
2380 infilelist
= list(filter(None, infiles
));
2381 elif(isinstance(infiles
, (str, ))):
2382 infilelist
= list(filter(None, [infiles
]));
2384 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
2386 GetDirList
= ListDir(infilelist
, followlink
, False);
2394 inodetocatinode
= {};
2395 numfiles
= int(len(GetDirList
));
2396 fnumfiles
= format(int(len(GetDirList
)), 'x').lower();
2397 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
[5]);
2398 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
2399 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
2400 catfp
.write(fnumfilesa
.encode('UTF-8'));
2403 os
.fsync(catfp
.fileno());
2404 except io
.UnsupportedOperation
:
2406 except AttributeError:
2408 for curfname
in GetDirList
:
2409 catfhstart
= catfp
.tell();
2410 if(re
.findall("^[.|/]", curfname
)):
2413 fname
= "./"+curfname
;
2415 VerbosePrintOut(fname
);
2416 if(not followlink
or followlink
is None):
2417 fstatinfo
= os
.lstat(fname
);
2419 fstatinfo
= os
.stat(fname
);
2420 fpremode
= fstatinfo
.st_mode
;
2421 finode
= fstatinfo
.st_ino
;
2422 flinkcount
= fstatinfo
.st_nlink
;
2424 if(stat
.S_ISREG(fpremode
)):
2426 elif(stat
.S_ISLNK(fpremode
)):
2428 elif(stat
.S_ISCHR(fpremode
)):
2430 elif(stat
.S_ISBLK(fpremode
)):
2432 elif(stat
.S_ISDIR(fpremode
)):
2434 elif(stat
.S_ISFIFO(fpremode
)):
2436 elif(stat
.S_ISSOCK(fpremode
)):
2438 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
2440 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
2442 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
2447 fcurfid
= format(int(curfid
), 'x').lower();
2448 if(not followlink
and finode
!=0):
2450 if(finode
in inodelist
):
2452 flinkname
= inodetofile
[finode
];
2454 fcurinode
= format(int(finode
), 'x').lower();
2456 fcurinode
= format(int(inodetocatinode
[finode
]), 'x').lower();
2457 if(finode
not in inodelist
):
2458 inodelist
.append(finode
);
2459 inodetofile
.update({finode
: fname
});
2460 inodetocatinode
.update({finode
: curinode
});
2462 fcurinode
= format(int(finode
), 'x').lower();
2464 fcurinode
= format(int(curinode
), 'x').lower();
2465 curinode
= curinode
+ 1;
2467 fcurinode
= format(int(curinode
), 'x').lower();
2468 curinode
= curinode
+ 1;
2469 curfid
= curfid
+ 1;
2471 flinkname
= os
.readlink(fname
);
2472 fdev
= fstatinfo
.st_dev
;
2473 getfdev
= GetDevMajorMinor(fdev
);
2474 fdev_minor
= getfdev
[0];
2475 fdev_major
= getfdev
[1];
2476 frdev
= fstatinfo
.st_dev
;
2477 if(hasattr(fstatinfo
, "st_rdev")):
2478 frdev
= fstatinfo
.st_rdev
;
2480 frdev
= fstatinfo
.st_dev
;
2481 getfrdev
= GetDevMajorMinor(frdev
);
2482 frdev_minor
= getfrdev
[0];
2483 frdev_major
= getfrdev
[1];
2484 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
2485 fsize
= format(int("0"), 'x').lower();
2486 elif(ftype
==0 or ftype
==7):
2487 fsize
= format(int(fstatinfo
.st_size
), 'x').lower();
2489 fsize
= format(int(fstatinfo
.st_size
)).lower();
2490 fatime
= format(int(fstatinfo
.st_atime
), 'x').lower();
2491 fmtime
= format(int(fstatinfo
.st_mtime
), 'x').lower();
2492 fctime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2493 if(hasattr(fstatinfo
, "st_birthtime")):
2494 fbtime
= format(int(fstatinfo
.st_birthtime
), 'x').lower();
2496 fbtime
= format(int(fstatinfo
.st_ctime
), 'x').lower();
2497 fmode
= format(int(fstatinfo
.st_mode
), 'x').lower();
2498 fchmode
= format(int(stat
.S_IMODE(fstatinfo
.st_mode
)), 'x').lower();
2499 ftypemod
= format(int(stat
.S_IFMT(fstatinfo
.st_mode
)), 'x').lower();
2500 fuid
= format(int(fstatinfo
.st_uid
), 'x').lower();
2501 fgid
= format(int(fstatinfo
.st_gid
), 'x').lower();
2506 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
2507 funame
= userinfo
.pw_name
;
2516 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
2517 fgname
= groupinfo
.gr_name
;
2522 fdev_minor
= format(int(fdev_minor
), 'x').lower();
2523 fdev_major
= format(int(fdev_major
), 'x').lower();
2524 frdev_minor
= format(int(frdev_minor
), 'x').lower();
2525 frdev_major
= format(int(frdev_major
), 'x').lower();
2526 finode
= format(int(finode
), 'x').lower();
2527 flinkcount
= format(int(flinkcount
), 'x').lower();
2528 if(hasattr(fstatinfo
, "st_file_attributes")):
2529 fwinattributes
= format(int(fstatinfo
.st_file_attributes
), 'x').lower();
2531 fwinattributes
= format(int(0), 'x').lower();
2532 fcontents
= BytesIO();
2533 if(ftype
==0 or ftype
==7):
2534 with
open(fname
, "rb") as fpc
:
2535 shutil
.copyfileobj(fpc
, fcontents
);
2536 if(followlink
and (ftype
==1 or ftype
==2)):
2537 flstatinfo
= os
.stat(flinkname
);
2538 with
open(flinkname
, "rb") as fpc
:
2539 shutil
.copyfileobj(fpc
, fcontents
);
2540 fcontents
.seek(0, 0);
2541 ftypehex
= format(ftype
, 'x').lower();
2542 extrafields
= format(len(extradata
), 'x').lower();
2543 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
2544 if(len(extradata
)>0):
2545 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
2546 extrasizelen
= format(len(extrasizestr
), 'x').lower();
2547 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrasizelen
, extrafields
];
2548 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
2549 catoutlenhex
= format(catoutlen
, 'x').lower();
2550 catoutlist
.insert(0, catoutlenhex
);
2551 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
2552 if(len(extradata
)>0):
2553 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
2554 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
2555 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
2556 fcontents
.seek(0, 0);
2557 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
2558 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
2559 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
2560 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
2561 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
2562 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
2563 catfhend
= (catfp
.tell() - 1) + len(catfileoutstr
);
2564 catfcontentstart
= catfp
.tell() + len(catfileoutstr
);
2565 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
2566 nullstrecd
= formatspecs
[5].encode('UTF-8');
2567 fcontents
.seek(0, 0);
2568 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
2569 catfcontentend
= (catfp
.tell() - 1) + len(catfileout
);
2570 catfp
.write(catfileout
);
2573 os
.fsync(catfp
.fileno());
2574 except io
.UnsupportedOperation
:
2576 except AttributeError:
2579 catfp
.write(AppendNullBytes([0, 0], formatspecs
[5]).encode("UTF-8"));
2580 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2581 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
2584 os
.fsync(catfp
.fileno());
2585 except io
.UnsupportedOperation
:
2587 except AttributeError:
2591 if(hasattr(sys
.stdout
, "buffer")):
2592 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
2594 shutil
.copyfileobj(catfp
, sys
.stdout
);
2595 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2596 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
2598 upload_file_to_internet_file(catfp
, outfile
);
2606 create_alias_function("Pack", __file_format_name__
, "", PackArchiveFile
);
2608 if(hasattr(shutil
, "register_archive_format")):
2609 def PackArchiveFileFunc(archive_name
, source_dir
, **kwargs
):
2610 return PackArchiveFile(source_dir
, archive_name
, False, "auto", None, False, "crc32", [], __file_format_delimiter__
, False, False);
2611 create_alias_function("Pack", __file_format_name__
, "Func", PackArchiveFileFunc
);
2613 def PackArchiveFileFromDirList(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, followlink
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
2614 return PackArchiveFile(infiles
, outfile
, dirlistfromtxt
, compression
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
2616 create_alias_function("Pack", __file_format_name__
, "FromDirList", PackArchiveFileFromDirList
);
2618 def PackArchiveFileFromTarFile(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
2619 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2620 outfile
= RemoveWindowsPath(outfile
);
2621 checksumtype
= checksumtype
.lower();
2622 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2623 checksumtype
="crc32";
2624 if(checksumtype
=="none"):
2626 if(not compression
or compression
or compression
=="catfile" or compression
==formatspecs
[2]):
2628 if(compression
not in compressionlist
and compression
is None):
2629 compression
= "auto";
2631 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2632 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2633 if(os
.path
.exists(outfile
)):
2638 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2640 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2643 fbasename
= os
.path
.splitext(outfile
)[0];
2644 fextname
= os
.path
.splitext(outfile
)[1];
2645 catfp
= CompressOpenFile(outfile
, compressionlevel
);
2646 catver
= formatspecs
[6];
2647 fileheaderver
= str(int(catver
.replace(".", "")));
2648 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
2649 catfp
.write(fileheader
.encode('UTF-8'));
2655 inodetocatinode
= {};
2658 if(hasattr(sys
.stdin
, "buffer")):
2659 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
2661 shutil
.copyfileobj(sys
.stdin
, infile
);
2666 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
2667 infile
= download_file_from_internet_file(infile
);
2672 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
2674 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
2676 if(not tarfile
.is_tarfile(infile
)):
2678 except AttributeError:
2679 if(not is_tarfile(infile
)):
2684 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
2685 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
2687 tarfp
= tarfile
.open(infile
, "r");
2688 except FileNotFoundError
:
2690 numfiles
= int(len(tarfp
.getmembers()));
2691 fnumfiles
= format(int(len(tarfp
.getmembers())), 'x').lower();
2692 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
[5]);
2693 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
2694 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
2695 catfp
.write(fnumfilesa
.encode('UTF-8'));
2698 os
.fsync(catfp
.fileno());
2699 except io
.UnsupportedOperation
:
2701 except AttributeError:
2703 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
2704 catfhstart
= catfp
.tell();
2705 if(re
.findall("^[.|/]", member
.name
)):
2706 fname
= member
.name
;
2708 fname
= "./"+member
.name
;
2710 VerbosePrintOut(fname
);
2711 fpremode
= member
.mode
;
2712 ffullmode
= member
.mode
;
2716 ffullmode
= member
.mode
+ stat
.S_IFREG
;
2718 elif(member
.isdev()):
2719 ffullmode
= member
.mode
;
2721 elif(member
.islnk()):
2722 ffullmode
= member
.mode
+ stat
.S_IFREG
;
2724 elif(member
.issym()):
2725 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
2727 elif(member
.ischr()):
2728 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
2730 elif(member
.isblk()):
2731 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
2733 elif(member
.isdir()):
2734 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
2736 elif(member
.isfifo()):
2737 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
2739 elif(member
.issparse()):
2740 ffullmode
= member
.mode
;
2743 ffullmode
= member
.mode
;
2746 fcurfid
= format(int(curfid
), 'x').lower();
2747 fcurinode
= format(int(curfid
), 'x').lower();
2748 curfid
= curfid
+ 1;
2750 flinkname
= member
.linkname
;
2751 fdev_minor
= format(int(member
.devminor
), 'x').lower();
2752 fdev_major
= format(int(member
.devmajor
), 'x').lower();
2753 frdev_minor
= format(int(member
.devminor
), 'x').lower();
2754 frdev_major
= format(int(member
.devmajor
), 'x').lower();
2755 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
2756 fsize
= format(int("0"), 'x').lower();
2757 elif(ftype
==0 or ftype
==7):
2758 fsize
= format(int(member
.size
), 'x').lower();
2760 fsize
= format(int(member
.size
), 'x').lower();
2761 fatime
= format(int(member
.mtime
), 'x').lower();
2762 fmtime
= format(int(member
.mtime
), 'x').lower();
2763 fctime
= format(int(member
.mtime
), 'x').lower();
2764 fbtime
= format(int(member
.mtime
), 'x').lower();
2765 fmode
= format(int(ffullmode
), 'x').lower();
2766 fchmode
= format(int(stat
.S_IMODE(ffullmode
)), 'x').lower();
2767 ftypemod
= format(int(stat
.S_IFMT(ffullmode
)), 'x').lower();
2768 fuid
= format(int(member
.uid
), 'x').lower();
2769 fgid
= format(int(member
.gid
), 'x').lower();
2770 funame
= member
.uname
;
2771 fgname
= member
.gname
;
2772 flinkcount
= format(int(flinkcount
), 'x').lower();
2773 fwinattributes
= format(int(0), 'x').lower();
2774 fcontents
= BytesIO();
2775 if(ftype
==0 or ftype
==7):
2776 with tarfp
.extractfile(member
) as fpc
:
2777 shutil
.copyfileobj(fpc
, fcontents
);
2778 fcontents
.seek(0, 0);
2779 ftypehex
= format(ftype
, 'x').lower();
2780 extrafields
= format(len(extradata
), 'x').lower();
2781 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
2782 if(len(extradata
)>0):
2783 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
2784 extrasizelen
= format(len(extrasizestr
), 'x').lower();
2785 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrasizelen
, extrafields
];
2786 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
2787 catoutlenhex
= format(catoutlen
, 'x').lower();
2788 catoutlist
.insert(0, catoutlenhex
);
2789 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
2790 if(len(extradata
)>0):
2791 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
2792 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
2793 catfhend
= (catfp
.tell() - 1) + len(catfileoutstr
);
2794 catfcontentstart
= catfp
.tell() + len(catfileoutstr
);
2795 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
2796 fcontents
.seek(0, 0);
2797 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
2798 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
2799 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
2800 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
2801 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
2802 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
2803 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
2804 nullstrecd
= formatspecs
[5].encode('UTF-8');
2805 fcontents
.seek(0, 0);
2806 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
2807 catfcontentend
= (catfp
.tell() - 1) + len(catfileout
);
2808 catfp
.write(catfileout
);
2811 os
.fsync(catfp
.fileno());
2812 except io
.UnsupportedOperation
:
2814 except AttributeError:
2817 catfp
.write(AppendNullBytes([0, 0], formatspecs
[5]).encode("UTF-8"));
2818 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2819 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
2822 os
.fsync(catfp
.fileno());
2823 except io
.UnsupportedOperation
:
2825 except AttributeError:
2829 if(hasattr(sys
.stdout
, "buffer")):
2830 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
2832 shutil
.copyfileobj(catfp
, sys
.stdout
);
2833 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2834 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
2836 upload_file_to_internet_file(catfp
, outfile
);
2844 create_alias_function("Pack", __file_format_name__
, "FromTarFile", PackArchiveFileFromTarFile
);
2846 def PackArchiveFileFromZipFile(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
2847 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2848 outfile
= RemoveWindowsPath(outfile
);
2849 checksumtype
= checksumtype
.lower();
2850 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
2851 checksumtype
="crc32";
2852 if(checksumtype
=="none"):
2854 if(not compression
or compression
or compression
=="catfile" or compression
==formatspecs
[2]):
2856 if(compression
not in compressionlist
and compression
is None):
2857 compression
= "auto";
2859 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
2860 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
2861 if(os
.path
.exists(outfile
)):
2866 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
2868 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
2871 fbasename
= os
.path
.splitext(outfile
)[0];
2872 fextname
= os
.path
.splitext(outfile
)[1];
2873 catfp
= CompressOpenFile(outfile
, compressionlevel
);
2874 catver
= formatspecs
[6];
2875 fileheaderver
= str(int(catver
.replace(".", "")));
2876 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
2877 catfp
.write(fileheader
.encode('UTF-8'));
2883 inodetocatinode
= {};
2886 if(hasattr(sys
.stdin
, "buffer")):
2887 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
2889 shutil
.copyfileobj(sys
.stdin
, infile
);
2894 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
2895 infile
= download_file_from_internet_file(infile
);
2900 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
2904 if(not zipfile
.is_zipfile(infile
)):
2907 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
2908 except FileNotFoundError
:
2910 ziptest
= zipfp
.testzip();
2912 VerbosePrintOut("Bad file found!");
2913 numfiles
= int(len(zipfp
.infolist()));
2914 fnumfiles
= format(int(len(zipfp
.infolist())), 'x').lower();
2915 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
[5]);
2916 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
2917 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
2918 catfp
.write(fnumfilesa
.encode('UTF-8'));
2921 os
.fsync(catfp
.fileno());
2922 except io
.UnsupportedOperation
:
2924 except AttributeError:
2926 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
2927 catfhstart
= catfp
.tell();
2928 if(re
.findall("^[.|/]", member
.filename
)):
2929 fname
= member
.filename
;
2931 fname
= "./"+member
.filename
;
2932 zipinfo
= zipfp
.getinfo(member
.filename
);
2934 VerbosePrintOut(fname
);
2935 if(not member
.is_dir()):
2936 fpremode
= int(stat
.S_IFREG
+ 438);
2937 elif(member
.is_dir()):
2938 fpremode
= int(stat
.S_IFDIR
+ 511);
2941 if(not member
.is_dir()):
2943 elif(member
.is_dir()):
2946 fcurfid
= format(int(curfid
), 'x').lower();
2947 fcurinode
= format(int(curfid
), 'x').lower();
2948 curfid
= curfid
+ 1;
2949 fdev_minor
= format(int(0), 'x').lower();
2950 fdev_major
= format(int(0), 'x').lower();
2951 frdev_minor
= format(int(0), 'x').lower();
2952 frdev_major
= format(int(0), 'x').lower();
2954 fsize
= format(int("0"), 'x').lower();
2956 fsize
= format(int(member
.file_size
), 'x').lower();
2958 fsize
= format(int(member
.file_size
), 'x').lower();
2959 fatime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
2960 fmtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
2961 fctime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
2962 fbtime
= format(int(time
.mktime(member
.date_time
+ (0, 0, -1))), 'x').lower();
2963 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
2964 fwinattributes
= format(int(zipinfo
.external_attr
), 'x').lower();
2965 if(not member
.is_dir()):
2966 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
2967 fchmode
= stat
.S_IMODE(fmode
);
2968 ftypemod
= stat
.S_IFMT(fmode
);
2969 elif(member
.is_dir()):
2970 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
2971 fchmode
= stat
.S_IMODE(fmode
);
2972 ftypemod
= stat
.S_IFMT(fmode
);
2973 elif(zipinfo
.create_system
==3):
2974 fwinattributes
= format(int(0), 'x').lower();
2975 fmode
= format(int(zipinfo
.external_attr
), 'x').lower();
2976 fchmode
= stat
.S_IMODE(fmode
);
2977 ftypemod
= stat
.S_IFMT(fmode
);
2979 fwinattributes
= format(int(0), 'x').lower();
2980 if(not member
.is_dir()):
2981 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
2982 fchmode
= stat
.S_IMODE(fmode
);
2983 ftypemod
= stat
.S_IFMT(fmode
);
2984 elif(member
.is_dir()):
2985 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
2986 fchmode
= stat
.S_IMODE(fmode
);
2987 ftypemod
= stat
.S_IFMT(fmode
);
2989 fuid
= format(int(os
.getuid()), 'x').lower();
2990 except AttributeError:
2991 fuid
= format(int(0), 'x').lower();
2993 fuid
= format(int(0), 'x').lower();
2995 fgid
= format(int(os
.getgid()), 'x').lower();
2996 except AttributeError:
2997 fgid
= format(int(0), 'x').lower();
2999 fgid
= format(int(0), 'x').lower();
3003 userinfo
= pwd
.getpwuid(os
.getuid());
3004 funame
= userinfo
.pw_name
;
3007 except AttributeError:
3015 groupinfo
= grp
.getgrgid(os
.getgid());
3016 fgname
= groupinfo
.gr_name
;
3019 except AttributeError:
3023 fcontents
= BytesIO();
3025 fcontents
.write(zipfp
.read(member
.filename
));
3026 fcontents
.seek(0, 0);
3027 ftypehex
= format(ftype
, 'x').lower();
3028 extrafields
= format(len(extradata
), 'x').lower();
3029 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
3030 if(len(extradata
)>0):
3031 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
3032 extrasizelen
= format(len(extrasizestr
), 'x').lower();
3033 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrasizelen
, extrafields
];
3034 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
3035 catoutlenhex
= format(catoutlen
, 'x').lower();
3036 catoutlist
.insert(0, catoutlenhex
);
3037 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
3038 if(len(extradata
)>0):
3039 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
3040 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
3041 catfhend
= (catfp
.tell() - 1) + len(catfileoutstr
);
3042 catfcontentstart
= catfp
.tell() + len(catfileoutstr
);
3043 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
3044 fcontents
.seek(0, 0);
3045 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
3046 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
3047 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
3048 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
3049 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
3050 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
3051 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
3052 nullstrecd
= formatspecs
[5].encode('UTF-8');
3053 fcontents
.seek(0, 0);
3054 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
3055 catfcontentend
= (catfp
.tell() - 1) + len(catfileout
);
3056 catfp
.write(catfileout
);
3059 os
.fsync(catfp
.fileno());
3060 except io
.UnsupportedOperation
:
3062 except AttributeError:
3065 catfp
.write(AppendNullBytes([0, 0], formatspecs
[5]).encode("UTF-8"));
3066 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3067 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
3070 os
.fsync(catfp
.fileno());
3071 except io
.UnsupportedOperation
:
3073 except AttributeError:
3077 if(hasattr(sys
.stdout
, "buffer")):
3078 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3080 shutil
.copyfileobj(catfp
, sys
.stdout
);
3081 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3082 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
3084 upload_file_to_internet_file(catfp
, outfile
);
3092 create_alias_function("Pack", __file_format_name__
, "FromZipFile", PackArchiveFileFromZipFile
);
3094 if(not rarfile_support
):
3095 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
3098 if(rarfile_support
):
3099 def PackArchiveFileFromRarFile(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
3100 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3101 outfile
= RemoveWindowsPath(outfile
);
3102 checksumtype
= checksumtype
.lower();
3103 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3104 checksumtype
="crc32";
3105 if(checksumtype
=="none"):
3107 if(not compression
or compression
or compression
=="catfile" or compression
==formatspecs
[2]):
3109 if(compression
not in compressionlist
and compression
is None):
3110 compression
= "auto";
3112 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3113 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3114 if(os
.path
.exists(outfile
)):
3119 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3121 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3124 fbasename
= os
.path
.splitext(outfile
)[0];
3125 fextname
= os
.path
.splitext(outfile
)[1];
3126 catfp
= CompressOpenFile(outfile
, compressionlevel
);
3127 catver
= formatspecs
[6];
3128 fileheaderver
= str(int(catver
.replace(".", "")));
3129 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
3130 catfp
.write(fileheader
.encode('UTF-8'));
3136 inodetocatinode
= {};
3137 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3139 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
3141 rarfp
= rarfile
.RarFile(infile
, "r");
3142 rartest
= rarfp
.testrar();
3144 VerbosePrintOut("Bad file found!");
3145 numfiles
= int(len(rarfp
.infolist()));
3146 fnumfiles
= format(int(len(rarfp
.infolist())), 'x').lower();
3147 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
[5]);
3148 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
3149 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
3150 catfp
.write(fnumfilesa
.encode('UTF-8'));
3153 os
.fsync(catfp
.fileno());
3154 except io
.UnsupportedOperation
:
3156 except AttributeError:
3158 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
3161 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
3164 member
.external_attr
3166 except AttributeError:
3168 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
3171 member
.external_attr
3173 except AttributeError:
3178 catfhstart
= catfp
.tell();
3179 if(re
.findall("^[.|/]", member
.filename
)):
3180 fname
= member
.filename
;
3182 fname
= "./"+member
.filename
;
3183 rarinfo
= rarfp
.getinfo(member
.filename
);
3185 VerbosePrintOut(fname
);
3186 if(is_unix
and member
.external_attr
!=0):
3187 fpremode
= int(member
.external_attr
);
3188 elif(member
.is_file()):
3189 fpremode
= int(stat
.S_IFREG
+ 438);
3190 elif(member
.is_symlink()):
3191 fpremode
= int(stat
.S_IFLNK
+ 438);
3192 elif(member
.is_dir()):
3193 fpremode
= int(stat
.S_IFDIR
+ 511);
3194 if(is_windows
and member
.external_attr
!=0):
3195 fwinattributes
= format(int(member
.external_attr
), 'x').lower();
3197 fwinattributes
= format(int(0), 'x').lower();
3200 if(member
.is_file()):
3202 elif(member
.is_symlink()):
3204 elif(member
.is_dir()):
3208 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
3209 fcurfid
= format(int(curfid
), 'x').lower();
3210 fcurinode
= format(int(curfid
), 'x').lower();
3211 curfid
= curfid
+ 1;
3212 fdev_minor
= format(int(0), 'x').lower();
3213 fdev_major
= format(int(0), 'x').lower();
3214 frdev_minor
= format(int(0), 'x').lower();
3215 frdev_major
= format(int(0), 'x').lower();
3217 fsize
= format(int("0"), 'x').lower();
3219 fsize
= format(int(member
.file_size
), 'x').lower();
3221 fsize
= format(int(member
.file_size
), 'x').lower();
3224 fatime
= format(int(member
.atime
.timestamp()), 'x').lower();
3226 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3227 except AttributeError:
3228 fatime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3229 fmtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3232 fctime
= format(int(member
.ctime
.timestamp()), 'x').lower();
3234 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3235 except AttributeError:
3236 fctime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3237 fbtime
= format(int(member
.mtime
.timestamp()), 'x').lower();
3238 if(is_unix
and member
.external_attr
!=0):
3239 fmode
= format(int(member
.external_attr
), 'x').lower();
3240 fchmode
= format(int(stat
.S_IMODE(member
.external_attr
)), 'x').lower();
3241 ftypemod
= format(int(stat
.S_IFMT(member
.external_attr
)), 'x').lower();
3242 elif(member
.is_file()):
3243 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3244 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3245 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3246 elif(member
.is_symlink()):
3247 fmode
= format(int(stat
.S_IFLNK
+ 438), 'x').lower();
3248 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3249 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3250 elif(member
.is_dir()):
3251 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3252 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3253 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3255 fuid
= format(int(os
.getuid()), 'x').lower();
3256 except AttributeError:
3257 fuid
= format(int(0), 'x').lower();
3259 fuid
= format(int(0), 'x').lower();
3261 fgid
= format(int(os
.getgid()), 'x').lower();
3262 except AttributeError:
3263 fgid
= format(int(0), 'x').lower();
3265 fgid
= format(int(0), 'x').lower();
3269 userinfo
= pwd
.getpwuid(os
.getuid());
3270 funame
= userinfo
.pw_name
;
3273 except AttributeError:
3281 groupinfo
= grp
.getgrgid(os
.getgid());
3282 fgname
= groupinfo
.gr_name
;
3285 except AttributeError:
3289 fcontents
= BytesIO();
3291 fcontents
.write(rarfp
.read(member
.filename
));
3292 fcontents
.seek(0, 0);
3293 ftypehex
= format(ftype
, 'x').lower();
3294 extrafields
= format(len(extradata
), 'x').lower();
3295 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
3296 if(len(extradata
)>0):
3297 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
3298 extrasizelen
= format(len(extrasizestr
), 'x').lower();
3299 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrasizelen
, extrafields
];
3300 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
3301 catoutlenhex
= format(catoutlen
, 'x').lower();
3302 catoutlist
.insert(0, catoutlenhex
);
3303 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
3304 if(len(extradata
)>0):
3305 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
3306 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
3307 catfhend
= (catfp
.tell() - 1) + len(catfileoutstr
);
3308 catfcontentstart
= catfp
.tell() + len(catfileoutstr
);
3309 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
3310 fcontents
.seek(0, 0);
3311 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
3312 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
3313 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
3314 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
3315 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
3316 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5])
3317 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
3318 nullstrecd
= formatspecs
[5].encode('UTF-8');
3319 fcontents
.seek(0, 0);
3320 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
3321 catfcontentend
= (catfp
.tell() - 1) + len(catfileout
);
3322 catfp
.write(catfileout
);
3325 os
.fsync(catfp
.fileno());
3326 except io
.UnsupportedOperation
:
3328 except AttributeError:
3331 catfp
.write(AppendNullBytes([0, 0], formatspecs
[5]).encode("UTF-8"));
3332 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3333 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
)
3336 os
.fsync(catfp
.fileno());
3337 except io
.UnsupportedOperation
:
3339 except AttributeError:
3343 if(hasattr(sys
.stdout
, "buffer")):
3344 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3346 shutil
.copyfileobj(catfp
, sys
.stdout
);
3347 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3348 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
3350 upload_file_to_internet_file(catfp
, outfile
);
3358 create_alias_function("Pack", __file_format_name__
, "FromRarFile", PackArchiveFileFromRarFile
);
3360 if(not py7zr_support
):
3361 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
3365 def PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
3366 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3367 outfile
= RemoveWindowsPath(outfile
);
3368 checksumtype
= checksumtype
.lower();
3369 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
3370 checksumtype
="crc32";
3371 if(checksumtype
=="none"):
3373 if(not compression
or compression
or compression
=="catfile" or compression
==formatspecs
[2]):
3375 if(compression
not in compressionlist
and compression
is None):
3376 compression
= "auto";
3378 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3379 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
3380 if(os
.path
.exists(outfile
)):
3385 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3387 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3390 fbasename
= os
.path
.splitext(outfile
)[0];
3391 fextname
= os
.path
.splitext(outfile
)[1];
3392 catfp
= CompressOpenFile(outfile
, compressionlevel
);
3393 catver
= formatspecs
[6];
3394 fileheaderver
= str(int(catver
.replace(".", "")));
3395 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
3396 catfp
.write(fileheader
.encode('UTF-8'));
3402 inodetocatinode
= {};
3403 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
3405 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
3406 file_content
= szpfp
.readall();
3407 #sztest = szpfp.testzip();
3408 sztestalt
= szpfp
.test();
3410 VerbosePrintOut("Bad file found!");
3411 numfiles
= int(len(szpfp
.list()));
3412 fnumfiles
= format(int(len(szpfp
.list())), 'x').lower();
3413 fnumfilesa
= AppendNullBytes([fnumfiles
, checksumtype
], formatspecs
[5]);
3414 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
3415 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
3416 catfp
.write(fnumfilesa
.encode('UTF-8'));
3419 os
.fsync(catfp
.fileno());
3420 except io
.UnsupportedOperation
:
3422 except AttributeError:
3424 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
3425 catfhstart
= catfp
.tell();
3426 if(re
.findall("^[.|/]", member
.filename
)):
3427 fname
= member
.filename
;
3429 fname
= "./"+member
.filename
;
3431 VerbosePrintOut(fname
);
3432 if(not member
.is_directory
):
3433 fpremode
= int(stat
.S_IFREG
+ 438);
3434 elif(member
.is_directory
):
3435 fpremode
= int(stat
.S_IFDIR
+ 511);
3436 fwinattributes
= format(int(0), 'x').lower();
3439 if(member
.is_directory
):
3444 fcurfid
= format(int(curfid
), 'x').lower();
3445 fcurinode
= format(int(curfid
), 'x').lower();
3446 curfid
= curfid
+ 1;
3447 fdev_minor
= format(int(0), 'x').lower();
3448 fdev_major
= format(int(0), 'x').lower();
3449 frdev_minor
= format(int(0), 'x').lower();
3450 frdev_major
= format(int(0), 'x').lower();
3452 fsize
= format(int("0"), 'x').lower();
3453 fatime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3454 fmtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3455 fctime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3456 fbtime
= format(int(member
.creationtime
.timestamp()), 'x').lower();
3457 if(member
.is_directory
):
3458 fmode
= format(int(stat
.S_IFDIR
+ 511), 'x').lower();
3459 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3460 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511))), 'x').lower();
3462 fmode
= format(int(stat
.S_IFREG
+ 438), 'x').lower();
3463 fchmode
= format(int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438))), 'x').lower();
3464 ftypemod
= format(int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438))), 'x').lower();
3466 fuid
= format(int(os
.getuid()), 'x').lower();
3467 except AttributeError:
3468 fuid
= format(int(0), 'x').lower();
3470 fuid
= format(int(0), 'x').lower();
3472 fgid
= format(int(os
.getgid()), 'x').lower();
3473 except AttributeError:
3474 fgid
= format(int(0), 'x').lower();
3476 fgid
= format(int(0), 'x').lower();
3480 userinfo
= pwd
.getpwuid(os
.getuid());
3481 funame
= userinfo
.pw_name
;
3484 except AttributeError:
3492 groupinfo
= grp
.getgrgid(os
.getgid());
3493 fgname
= groupinfo
.gr_name
;
3496 except AttributeError:
3500 fcontents
= BytesIO();
3502 fcontents
.write(file_content
[member
.filename
].read());
3503 fsize
= format(fcontents
.tell(), 'x').lower();
3504 file_content
[member
.filename
].close();
3505 fcontents
.seek(0, 0);
3506 ftypehex
= format(ftype
, 'x').lower();
3507 extrafields
= format(len(extradata
), 'x').lower();
3508 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
3509 if(len(extradata
)>0):
3510 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
3511 extrasizelen
= format(len(extrasizestr
), 'x').lower();
3512 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrasizelen
, extrafields
];
3513 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
3514 catoutlenhex
= format(catoutlen
, 'x').lower();
3515 catoutlist
.insert(0, catoutlenhex
);
3516 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
3517 if(len(extradata
)>0):
3518 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
3519 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
3520 catfhend
= (catfp
.tell() - 1) + len(catfileoutstr
);
3521 catfcontentstart
= catfp
.tell() + len(catfileoutstr
);
3522 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
3523 fcontents
.seek(0, 0);
3524 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
3525 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
3526 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
3527 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
3528 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
3529 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5])
3530 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
3531 nullstrecd
= formatspecs
[5].encode('UTF-8');
3532 fcontents
.seek(0, 0);
3533 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
3534 catfcontentend
= (catfp
.tell() - 1) + len(catfileout
);
3535 catfp
.write(catfileout
);
3538 os
.fsync(catfp
.fileno());
3539 except io
.UnsupportedOperation
:
3541 except AttributeError:
3544 catfp
.write(AppendNullBytes([0, 0], formatspecs
[5]).encode("UTF-8"));
3545 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
3546 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
)
3549 os
.fsync(catfp
.fileno());
3550 except io
.UnsupportedOperation
:
3552 except AttributeError:
3556 if(hasattr(sys
.stdout
, "buffer")):
3557 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
3559 shutil
.copyfileobj(catfp
, sys
.stdout
);
3560 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
3561 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
3563 upload_file_to_internet_file(catfp
, outfile
);
3571 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFile", PackArchiveFileFromSevenZipFile
);
3573 def PackArchiveFileFromInFile(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
3574 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
3576 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
3577 if(checkcompressfile
=="tarfile"):
3578 return PackArchiveFileFromTarFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3579 elif(checkcompressfile
=="zipfile"):
3580 return PackArchiveFileFromZipFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3581 elif(checkcompressfile
=="catfile"):
3582 return RePackArchiveFile(infile
, outfile
, compression
, compressionlevel
, False, 0, 0, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
3583 elif(rarfile_support
and checkcompressfile
=="rarfile"):
3584 return PackArchiveFileFromRarFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3585 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
3586 return PackArchiveFileFromSevenZipFile(infile
, outfile
, compression
, compressionlevel
, checksumtype
, extradata
, formatspecs
, verbose
, returnfp
);
3591 create_alias_function("Pack", __file_format_name__
, "FromInFile", PackArchiveFileFromInFile
);
3593 def ArchiveFileSeekToFileNum(infile
, seekto
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
3594 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
3597 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3598 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
3599 if(checkcompressfile
=="tarfile"):
3600 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3601 if(checkcompressfile
=="zipfile"):
3602 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3603 if(rarfile_support
and checkcompressfile
=="rarfile"):
3604 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3605 if(py7zr_support
and checkcompressfile
=="7zipfile"):
3606 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3607 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
3614 if(hasattr(sys
.stdin
, "buffer")):
3615 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
3617 shutil
.copyfileobj(sys
.stdin
, catfp
);
3619 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3623 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3624 catfp
= download_file_from_internet_file(infile
);
3626 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3631 infile
= RemoveWindowsPath(infile
);
3632 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
3633 if(checkcompressfile
=="tarfile"):
3634 return TarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3635 if(checkcompressfile
=="zipfile"):
3636 return ZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3637 if(rarfile_support
and checkcompressfile
=="rarfile"):
3638 return RarFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3639 if(py7zr_support
and checkcompressfile
=="7zipfile"):
3640 return SevenZipFileToArray(infile
, seekto
, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3641 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
3643 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
3644 if(not compresscheck
):
3645 fextname
= os
.path
.splitext(infile
)[1];
3646 if(fextname
==".gz"):
3647 compresscheck
= "gzip";
3648 elif(fextname
==".bz2"):
3649 compresscheck
= "bzip2";
3650 elif(fextname
==".zst"):
3651 compresscheck
= "zstd";
3652 elif(fextname
==".lz4" or fextname
==".clz4"):
3653 compresscheck
= "lz4";
3654 elif(fextname
==".lzo" or fextname
==".lzop"):
3655 compresscheck
= "lzo";
3656 elif(fextname
==".lzma" or fextname
==".xz"):
3657 compresscheck
= "lzma";
3660 if(not compresscheck
):
3662 catfp
= UncompressFile(infile
, formatspecs
, "rb");
3667 SeekToEndOfFile(catfp);
3669 SeekToEndOfFile(catfp);
3670 CatSize = catfp.tell();
3671 CatSizeEnd = CatSize;
3679 curloc
= catfp
.tell();
3682 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
[5]);
3684 catfp
.seek(curloc
, 0);
3685 catstring
= catheader
[0];
3686 catversion
= re
.findall(r
"([\d]+)$", catstring
);
3687 fprenumfiles
= catheader
[1];
3688 fnumfiles
= int(fprenumfiles
, 16);
3689 fprechecksumtype
= catheader
[2];
3690 fprechecksum
= catheader
[3];
3691 fileheader
= AppendNullByte(catstring
, formatspecs
[5]);
3692 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
3693 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
[5]);
3694 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
3695 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
3696 fheadtell
= len(fileheader
);
3697 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
3698 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
3700 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
3701 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
3702 if(seekto
>=fnumfiles
):
3703 seekto
= fnumfiles
- 1;
3709 prefhstart
= catfp
.tell();
3711 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
[5]);
3713 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
[5]);
3714 if(len(preheaderdata
)==0):
3716 prefheadsize
= int(preheaderdata
[0], 16);
3717 prefnumfields
= int(preheaderdata
[1], 16);
3718 preftype
= int(preheaderdata
[2], 16);
3719 if(re
.findall("^[.|/]", preheaderdata
[3])):
3720 prefname
= preheaderdata
[3];
3722 prefname
= "./"+preheaderdata
[3];
3723 prefbasedir
= os
.path
.dirname(prefname
);
3724 preflinkname
= preheaderdata
[4];
3725 prefsize
= int(preheaderdata
[5], 16);
3726 prefatime
= int(preheaderdata
[6], 16);
3727 prefmtime
= int(preheaderdata
[7], 16);
3728 prefctime
= int(preheaderdata
[8], 16);
3729 prefbtime
= int(preheaderdata
[9], 16);
3730 prefmode
= int(preheaderdata
[10], 16);
3731 prefchmode
= stat
.S_IMODE(prefmode
);
3732 preftypemod
= stat
.S_IFMT(prefmode
);
3733 prefwinattributes
= int(preheaderdata
[11], 16);
3734 prefuid
= int(preheaderdata
[12], 16);
3735 prefuname
= preheaderdata
[13];
3736 prefgid
= int(preheaderdata
[14], 16);
3737 prefgname
= preheaderdata
[15];
3738 fid
= int(preheaderdata
[16], 16);
3739 finode
= int(preheaderdata
[17], 16);
3740 flinkcount
= int(preheaderdata
[18], 16);
3741 prefdev_minor
= int(preheaderdata
[19], 16);
3742 prefdev_major
= int(preheaderdata
[20], 16);
3743 prefrdev_minor
= int(preheaderdata
[22], 16);
3744 prefrdev_major
= int(preheaderdata
[23], 16);
3745 prefextrasize
= int(preheaderdata
[24], 16);
3746 prefextrafields
= int(preheaderdata
[25], 16);
3747 extrafieldslist
= [];
3749 extraend
= extrastart
+ prefextrafields
;
3750 extrafieldslist
= [];
3751 if(extrastart
<extraend
):
3752 extrafieldslist
.append(preheaderdata
[extrastart
]);
3753 extrastart
= extrastart
+ 1;
3754 prefchecksumtype
= preheaderdata
[extrastart
].lower();
3755 prefcs
= preheaderdata
[extrastart
+ 1].lower();
3756 prefccs
= preheaderdata
[extrastart
+ 2].lower();
3758 hcmax
= len(preheaderdata
) - 2;
3761 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
[5]);
3763 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
3764 if(prefcs
!=prenewfcs
and not skipchecksum
):
3765 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
3767 valid_archive
= False;
3768 invalid_archive
= True;
3769 prefhend
= catfp
.tell() - 1;
3770 prefcontentstart
= catfp
.tell();
3772 pyhascontents
= False;
3774 prefcontents
= catfp
.read(prefsize
);
3775 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
3776 pyhascontents
= True;
3777 if(prefccs
!=prenewfccs
and not skipchecksum
):
3778 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
3782 catfp
.seek(seekstart
, 0);
3784 catfheadsize
= int(preheaderdata
[0], 16);
3785 catfnumfields
= int(preheaderdata
[1], 16);
3786 catftype
= int(preheaderdata
[2], 16);
3787 if(re
.findall("^[.|/]", preheaderdata
[3])):
3788 catfname
= preheaderdata
[3];
3790 catfname
= "./"+preheaderdata
[3];
3791 catflinkname
= preheaderdata
[4];
3792 catfsize
= int(preheaderdata
[5], 16);
3793 catfbasedir
= os
.path
.dirname(catfname
);
3794 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
3796 catlist
.update({'catfp': catfp
});
3801 create_alias_function("", __file_format_name__
, "SeekToFileNum", ArchiveFileSeekToFileNum
);
3803 def ArchiveFileSeekToFileName(infile
, seekfile
=None, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
3804 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
3807 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3808 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
3809 if(checkcompressfile
=="tarfile"):
3810 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3811 if(checkcompressfile
=="zipfile"):
3812 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3813 if(rarfile_support
and checkcompressfile
=="rarfile"):
3814 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3815 if(py7zr_support
and checkcompressfile
=="7zipfile"):
3816 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3817 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
3824 if(hasattr(sys
.stdin
, "buffer")):
3825 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
3827 shutil
.copyfileobj(sys
.stdin
, catfp
);
3829 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3833 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
3834 catfp
= download_file_from_internet_file(infile
);
3835 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
3841 infile
= RemoveWindowsPath(infile
);
3842 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
3843 if(checkcompressfile
=="tarfile"):
3844 return TarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3845 if(checkcompressfile
=="zipfile"):
3846 return ZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3847 if(rarfile_support
and checkcompressfile
=="rarfile"):
3848 return RarFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3849 if(py7zr_support
and checkcompressfile
=="7zipfile"):
3850 return SevenZipFileToArray(infile
, 0, 0, listonly
, skipchecksum
, formatspecs
, returnfp
);
3851 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
3853 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
3854 if(not compresscheck
):
3855 fextname
= os
.path
.splitext(infile
)[1];
3856 if(fextname
==".gz"):
3857 compresscheck
= "gzip";
3858 elif(fextname
==".bz2"):
3859 compresscheck
= "bzip2";
3860 elif(fextname
==".zst"):
3861 compresscheck
= "zstd";
3862 elif(fextname
==".lz4" or fextname
==".clz4"):
3863 compresscheck
= "lz4";
3864 elif(fextname
==".lzo" or fextname
==".lzop"):
3865 compresscheck
= "lzo";
3866 elif(fextname
==".lzma" or fextname
==".xz"):
3867 compresscheck
= "lzma";
3870 if(not compresscheck
):
3872 catfp
= UncompressFile(infile
, formatspecs
, "rb");
3877 SeekToEndOfFile(catfp);
3879 SeekToEndOfFile(catfp);
3880 CatSize = catfp.tell();
3881 CatSizeEnd = CatSize;
3889 curloc
= catfp
.tell();
3892 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
[5]);
3894 catfp
.seek(curloc
, 0);
3895 catstring
= catheader
[0];
3896 catversion
= re
.findall(r
"([\d]+)$", catstring
);
3897 fprenumfiles
= catheader
[1];
3898 fnumfiles
= int(fprenumfiles
, 16);
3899 fprechecksumtype
= catheader
[2];
3900 fprechecksum
= catheader
[3];
3901 fileheader
= AppendNullByte(catstring
, formatspecs
[5]);
3902 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
3903 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
[5]);
3904 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
3905 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
3906 fheadtell
= len(fileheader
);
3907 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
3908 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
3910 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
3911 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
3912 seekto
= fnumfiles
- 1
3917 prefhstart
= catfp
.tell();
3919 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
[5]);
3921 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
[5]);
3922 if(len(preheaderdata
)==0):
3924 prefheadsize
= int(preheaderdata
[0], 16);
3925 prefnumfields
= int(preheaderdata
[1], 16);
3926 preftype
= int(preheaderdata
[2], 16);
3927 if(re
.findall("^[.|/]", preheaderdata
[3])):
3928 prefname
= preheaderdata
[3];
3930 prefname
= "./"+preheaderdata
[3];
3931 prefbasedir
= os
.path
.dirname(prefname
);
3932 preflinkname
= preheaderdata
[4];
3933 prefsize
= int(preheaderdata
[5], 16);
3934 prefatime
= int(preheaderdata
[6], 16);
3935 prefmtime
= int(preheaderdata
[7], 16);
3936 prefctime
= int(preheaderdata
[8], 16);
3937 prefbtime
= int(preheaderdata
[9], 16);
3938 prefmode
= int(preheaderdata
[10], 16);
3939 prefchmode
= stat
.S_IMODE(prefmode
);
3940 preftypemod
= stat
.S_IFMT(prefmode
);
3941 prefwinattributes
= int(preheaderdata
[11], 16);
3942 prefuid
= int(preheaderdata
[12], 16);
3943 prefuname
= preheaderdata
[13];
3944 prefgid
= int(preheaderdata
[14], 16);
3945 prefgname
= preheaderdata
[15];
3946 fid
= int(preheaderdata
[16], 16);
3947 finode
= int(preheaderdata
[17], 16);
3948 flinkcount
= int(preheaderdata
[18], 16);
3949 prefdev_minor
= int(preheaderdata
[19], 16);
3950 prefdev_major
= int(preheaderdata
[20], 16);
3951 prefrdev_minor
= int(preheaderdata
[22], 16);
3952 prefrdev_major
= int(preheaderdata
[23], 16);
3953 prefextrasize
= int(preheaderdata
[24], 16);
3954 prefextrafields
= int(preheaderdata
[25], 16);
3955 extrafieldslist
= [];
3957 extraend
= extrastart
+ prefextrafields
;
3958 extrafieldslist
= [];
3959 if(extrastart
<extraend
):
3960 extrafieldslist
.append(preheaderdata
[extrastart
]);
3961 extrastart
= extrastart
+ 1;
3962 prefchecksumtype
= preheaderdata
[extrastart
].lower();
3963 prefcs
= preheaderdata
[extrastart
+ 1].lower();
3964 prefccs
= preheaderdata
[extrastart
+ 2].lower();
3966 hcmax
= len(preheaderdata
) - 2;
3969 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
[5]);
3971 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
3972 if(prefcs
!=prenewfcs
and not skipchecksum
):
3973 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
3975 valid_archive
= False;
3976 invalid_archive
= True;
3977 prefhend
= catfp
.tell() - 1;
3978 prefcontentstart
= catfp
.tell();
3980 pyhascontents
= False;
3982 prefcontents
= catfp
.read(prefsize
);
3983 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
3984 pyhascontents
= True;
3985 if(prefccs
!=prenewfccs
and not skipchecksum
):
3986 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
3991 prefname
= preheaderdata
[2];
3992 if(re
.findall("^[.|/]", preheaderdata
[2])):
3993 prefname
= preheaderdata
[2];
3995 prefname
= "./"+preheaderdata
[2];
3996 if(prefname
==seekfile
):
3999 catfp
.seek(seekstart
, 0);
4001 catfheadsize
= int(preheaderdata
[0], 16);
4002 catfnumfields
= int(preheaderdata
[1], 16);
4003 catftype
= int(preheaderdata
[2], 16);
4004 if(re
.findall("^[.|/]", preheaderdata
[3])):
4005 catfname
= preheaderdata
[3];
4007 catfname
= "./"+preheaderdata
[3];
4008 catflinkname
= preheaderdata
[4];
4009 catfsize
= int(preheaderdata
[5], 16);
4010 catfbasedir
= os
.path
.dirname(catfname
);
4012 catlist
= {'fid': fileidnum
, 'foffset': catfp
.tell(), 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
};
4016 catlist
.update({'catfp': catfp
});
4021 create_alias_function("", __file_format_name__
, "SeekToFileName", ArchiveFileSeekToFileName
);
4023 def ArchiveFileValidate(infile
, formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
4025 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
4026 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4029 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4030 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4031 if(checkcompressfile
=="tarfile"):
4032 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4033 if(checkcompressfile
=="zipfile"):
4034 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4035 if(rarfile_support
and checkcompressfile
=="rarfile"):
4036 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4037 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4038 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4039 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
4046 if(hasattr(sys
.stdin
, "buffer")):
4047 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4049 shutil
.copyfileobj(sys
.stdin
, catfp
);
4051 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4055 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4056 catfp
= download_file_from_internet_file(infile
);
4057 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4063 infile
= RemoveWindowsPath(infile
);
4064 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4065 if(checkcompressfile
=="tarfile"):
4066 return TarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4067 if(checkcompressfile
=="zipfile"):
4068 return ZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4069 if(rarfile_support
and checkcompressfile
=="rarfile"):
4070 return RarFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4071 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4072 return SevenZipFileToArray(infile
, 0, 0, False, False, formatspecs
, returnfp
);
4073 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
4075 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4076 if(not compresscheck
):
4077 fextname
= os
.path
.splitext(infile
)[1];
4078 if(fextname
==".gz"):
4079 compresscheck
= "gzip";
4080 elif(fextname
==".bz2"):
4081 compresscheck
= "bzip2";
4082 elif(fextname
==".zst"):
4083 compresscheck
= "zstd";
4084 elif(fextname
==".lz4" or fextname
==".clz4"):
4085 compresscheck
= "lz4";
4086 elif(fextname
==".lzo" or fextname
==".lzop"):
4087 compresscheck
= "lzo";
4088 elif(fextname
==".lzma" or fextname
==".xz"):
4089 compresscheck
= "lzma";
4092 if(not compresscheck
):
4094 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4099 SeekToEndOfFile(catfp);
4101 SeekToEndOfFile(catfp);
4102 CatSize = catfp.tell();
4103 CatSizeEnd = CatSize;
4111 curloc
= catfp
.tell();
4114 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
[5]);
4116 catfp
.seek(curloc
, 0);
4117 catstring
= catheader
[0];
4118 catversion
= re
.findall(r
"([\d]+)$", catstring
);
4119 fprenumfiles
= catheader
[1];
4120 fnumfiles
= int(fprenumfiles
, 16);
4121 fprechecksumtype
= catheader
[2];
4122 fprechecksum
= catheader
[3];
4124 fileheader
= AppendNullByte(catstring
, formatspecs
[5]);
4125 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4126 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
[5]);
4127 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4128 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
4129 valid_archive
= True;
4130 invalid_archive
= False;
4132 VerbosePrintOut(infile
);
4133 VerbosePrintOut("Number of Records " + str(fnumfiles
));
4134 if(fprechecksum
==catfileheadercshex
):
4136 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4139 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4140 valid_archive
= False;
4141 invalid_archive
= True;
4143 VerbosePrintOut("");
4144 while(il
<fnumfiles
):
4145 catfhstart
= catfp
.tell();
4147 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
[5]);
4149 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
[5]);
4150 if(len(catheaderdata
)==0):
4152 catfheadsize
= int(catheaderdata
[0], 16);
4153 catfnumfields
= int(catheaderdata
[1], 16);
4154 catftype
= int(catheaderdata
[2], 16);
4155 if(re
.findall("^[.|/]", catheaderdata
[3])):
4156 catfname
= catheaderdata
[3];
4158 catfname
= "./"+catheaderdata
[3];
4159 catfbasedir
= os
.path
.dirname(catfname
);
4160 catflinkname
= catheaderdata
[4];
4161 catfsize
= int(catheaderdata
[5], 16);
4162 catfatime
= int(catheaderdata
[6], 16);
4163 catfmtime
= int(catheaderdata
[7], 16);
4164 catfctime
= int(catheaderdata
[8], 16);
4165 catfbtime
= int(catheaderdata
[9], 16);
4166 catfmode
= int(catheaderdata
[10], 16);
4167 catfchmode
= stat
.S_IMODE(catfmode
);
4168 catftypemod
= stat
.S_IFMT(catfmode
);
4169 prefwinattributes
= int(catheaderdata
[11], 16);
4170 catfuid
= int(catheaderdata
[12], 16);
4171 catfuname
= catheaderdata
[13];
4172 catfgid
= int(catheaderdata
[14], 16);
4173 catfgname
= catheaderdata
[15];
4174 fid
= int(catheaderdata
[16], 16);
4175 finode
= int(catheaderdata
[17], 16);
4176 flinkcount
= int(catheaderdata
[18], 16);
4177 catfdev_minor
= int(catheaderdata
[19], 16);
4178 catfdev_major
= int(catheaderdata
[20], 16);
4179 catfrdev_minor
= int(catheaderdata
[21], 16);
4180 catfrdev_major
= int(catheaderdata
[22], 16);
4181 catfextrasize
= int(catheaderdata
[23], 16);
4182 catfextrafields
= int(catheaderdata
[24], 16);
4183 extrafieldslist
= [];
4185 extraend
= extrastart
+ catfextrafields
;
4186 extrafieldslist
= [];
4187 if(extrastart
<extraend
):
4188 extrafieldslist
.append(catheaderdata
[extrastart
]);
4189 extrastart
= extrastart
+ 1;
4190 catfchecksumtype
= catheaderdata
[extrastart
].lower();
4191 catfcs
= catheaderdata
[extrastart
+ 1].lower();
4192 catfccs
= catheaderdata
[extrastart
+ 2].lower();
4194 hcmax
= len(catheaderdata
) - 2;
4197 hout
= hout
+ AppendNullByte(catheaderdata
[hc
], formatspecs
[5]);
4199 catnewfcs
= GetFileChecksum(hout
, catheaderdata
[-3].lower(), True, formatspecs
);
4201 VerbosePrintOut(catfname
);
4202 VerbosePrintOut("Record Number " + str(il
) + "; File ID " + str(fid
) + "; iNode Number " + str(finode
));
4203 if(catfcs
==catnewfcs
):
4205 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart
));
4208 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart
));
4209 valid_archive
= False;
4210 invalid_archive
= True;
4211 catfhend
= catfp
.tell() - 1;
4212 catfcontentstart
= catfp
.tell();
4214 pyhascontents
= False;
4216 catfcontents
= catfp
.read(catfsize
);
4217 catnewfccs
= GetFileChecksum(catfcontents
, catheaderdata
[-3].lower(), False, formatspecs
);
4218 pyhascontents
= True;
4219 if(catfccs
==catnewfccs
):
4221 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart
));
4224 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart
));
4225 valid_archive
= False;
4226 invalid_archive
= True;
4228 VerbosePrintOut("");
4241 create_alias_function("", __file_format_name__
, "Validate", ArchiveFileValidate
);
4243 def ArchiveFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
4244 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4247 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4248 checkcompressfile
= CheckCompressionSubType(catfp
, formatspecs
, True);
4249 if(checkcompressfile
=="tarfile"):
4250 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4251 if(checkcompressfile
=="zipfile"):
4252 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4253 if(rarfile_support
and checkcompressfile
=="rarfile"):
4254 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4255 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4256 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4257 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
4264 if(hasattr(sys
.stdin
, "buffer")):
4265 shutil
.copyfileobj(sys
.stdin
.buffer, catfp
);
4267 shutil
.copyfileobj(sys
.stdin
, catfp
);
4269 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4273 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4274 catfp
= download_file_from_internet_file(infile
);
4275 catfp
= UncompressArchiveFile(catfp
, formatspecs
);
4281 infile
= RemoveWindowsPath(infile
);
4282 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4283 if(checkcompressfile
=="tarfile"):
4284 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4285 if(checkcompressfile
=="zipfile"):
4286 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4287 if(rarfile_support
and checkcompressfile
=="rarfile"):
4288 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4289 if(py7zr_support
and checkcompressfile
=="7zipfile"):
4290 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4291 if(checkcompressfile
!="catfile" and checkcompressfile
!=formatspecs
[2]):
4293 compresscheck
= CheckCompressionType(infile
, formatspecs
, True);
4294 if(not compresscheck
):
4295 fextname
= os
.path
.splitext(infile
)[1];
4296 if(fextname
==".gz"):
4297 compresscheck
= "gzip";
4298 elif(fextname
==".bz2"):
4299 compresscheck
= "bzip2";
4300 elif(fextname
==".zst"):
4301 compresscheck
= "zstd";
4302 elif(fextname
==".lz4" or fextname
==".clz4"):
4303 compresscheck
= "lz4";
4304 elif(fextname
==".lzo" or fextname
==".lzop"):
4305 compresscheck
= "lzo";
4306 elif(fextname
==".lzma" or fextname
==".xz"):
4307 compresscheck
= "lzma";
4310 if(not compresscheck
):
4312 catfp
= UncompressFile(infile
, formatspecs
, "rb");
4317 SeekToEndOfFile(catfp);
4319 SeekToEndOfFile(catfp);
4320 CatSize = catfp.tell();
4321 CatSizeEnd = CatSize;
4329 curloc
= catfp
.tell();
4332 catheader
= ReadFileHeaderData(catfp
, 4, formatspecs
[5]);
4334 catfp
.seek(curloc
, 0);
4335 catstring
= catheader
[0];
4336 catversion
= re
.findall(r
"([\d]+)$", catstring
);
4337 fprenumfiles
= catheader
[1];
4338 fnumfiles
= int(fprenumfiles
, 16);
4339 fprechecksumtype
= catheader
[2];
4340 fprechecksum
= catheader
[3];
4341 fileheader
= AppendNullByte(catstring
, formatspecs
[5]);
4342 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4343 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, fprechecksumtype
], formatspecs
[5]);
4344 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4345 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
4346 fheadtell
= len(fileheader
);
4347 if(fprechecksum
!=catfileheadercshex
and not skipchecksum
):
4348 VerbosePrintOut("File Header Checksum Error with file " + infile
+ " at offset " + str(0));
4350 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4351 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': fprechecksumtype
, 'fheaderchecksum': fprechecksum
, 'ffilelist': {}};
4352 if(seekstart
<0 and seekstart
>fnumfiles
):
4354 if(seekend
==0 or seekend
>fnumfiles
and seekend
<seekstart
):
4355 seekend
= fnumfiles
;
4356 elif(seekend
<0 and abs(seekend
)<=fnumfiles
and abs(seekend
)>=seekstart
):
4357 seekend
= fnumfiles
- abs(seekend
);
4360 while(il
< seekstart
):
4361 prefhstart
= catfp
.tell();
4363 preheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
[5]);
4365 preheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
[5]);
4366 if(len(preheaderdata
)==0):
4368 prefheadsize
= int(preheaderdata
[0], 16);
4369 prefnumfields
= int(preheaderdata
[1], 16);
4370 if(re
.findall("^[.|/]", preheaderdata
[3])):
4371 prefname
= preheaderdata
[3];
4373 prefname
= "./"+preheaderdata
[3];
4374 prefsize
= int(preheaderdata
[5], 16);
4375 prefextrasize
= int(preheaderdata
[23], 16);
4376 prefextrafields
= int(preheaderdata
[24], 16);
4377 extrafieldslist
= [];
4379 extraend
= extrastart
+ prefextrafields
;
4380 extrafieldslist
= [];
4381 if(extrastart
<extraend
):
4382 extrafieldslist
.append(preheaderdata
[extrastart
]);
4383 extrastart
= extrastart
+ 1;
4384 prefchecksumtype
= preheaderdata
[extrastart
].lower();
4385 prefcs
= preheaderdata
[extrastart
+ 1].lower();
4386 prefccs
= preheaderdata
[extrastart
+ 2].lower();
4388 hcmax
= len(preheaderdata
) - 2;
4391 hout
= hout
+ AppendNullByte(preheaderdata
[hc
], formatspecs
[5]);
4393 prenewfcs
= GetFileChecksum(hout
, preheaderdata
[-3].lower(), True, formatspecs
);
4394 if(prefcs
!=prenewfcs
and not skipchecksum
):
4395 VerbosePrintOut("File Header Checksum Error with file " + prefname
+ " at offset " + str(prefhstart
));
4397 valid_archive
= False;
4398 invalid_archive
= True;
4399 prefhend
= catfp
.tell() - 1;
4400 prefcontentstart
= catfp
.tell();
4402 pyhascontents
= False;
4404 prefcontents
= catfp
.read(prefsize
);
4405 prenewfccs
= GetFileChecksum(prefcontents
, preheaderdata
[-3].lower(), False, formatspecs
);
4406 pyhascontents
= True;
4407 if(prefccs
!=prenewfccs
and not skipchecksum
):
4408 VerbosePrintOut("File Content Checksum Error with file " + prefname
+ " at offset " + str(prefcontentstart
));
4412 fileidnum
= seekstart
;
4414 while(fileidnum
<seekend
):
4415 catfhstart
= catfp
.tell();
4417 catheaderdata
= ReadFileHeaderDataBySize(catfp
, formatspecs
[5]);
4419 catheaderdata
= ReadFileHeaderDataWoSize(catfp
, formatspecs
[5]);
4420 if(len(catheaderdata
)==0):
4422 catfheadsize
= int(catheaderdata
[0], 16);
4423 catfnumfields
= int(catheaderdata
[1], 16);
4424 catftype
= int(catheaderdata
[2], 16);
4425 if(re
.findall("^[.|/]", catheaderdata
[3])):
4426 catfname
= catheaderdata
[3];
4428 catfname
= "./"+catheaderdata
[3];
4429 catfbasedir
= os
.path
.dirname(catfname
);
4430 catflinkname
= catheaderdata
[4];
4431 catfsize
= int(catheaderdata
[5], 16);
4432 catfatime
= int(catheaderdata
[6], 16);
4433 catfmtime
= int(catheaderdata
[7], 16);
4434 catfctime
= int(catheaderdata
[8], 16);
4435 catfbtime
= int(catheaderdata
[9], 16);
4436 catfmode
= int(catheaderdata
[10], 16);
4437 catfchmode
= stat
.S_IMODE(catfmode
);
4438 catftypemod
= stat
.S_IFMT(catfmode
);
4439 catfwinattributes
= int(catheaderdata
[11], 16);
4440 catfuid
= int(catheaderdata
[12], 16);
4441 catfuname
= catheaderdata
[13];
4442 catfgid
= int(catheaderdata
[14], 16);
4443 catfgname
= catheaderdata
[15];
4444 fid
= int(catheaderdata
[16], 16);
4445 finode
= int(catheaderdata
[17], 16);
4446 flinkcount
= int(catheaderdata
[18], 16);
4447 catfdev_minor
= int(catheaderdata
[19], 16);
4448 catfdev_major
= int(catheaderdata
[20], 16);
4449 catfrdev_minor
= int(catheaderdata
[21], 16);
4450 catfrdev_major
= int(catheaderdata
[22], 16);
4451 catfextrasize
= int(catheaderdata
[23], 16);
4452 catfextrafields
= int(catheaderdata
[24], 16);
4453 extrafieldslist
= [];
4455 extraend
= extrastart
+ catfextrafields
;
4456 extrafieldslist
= [];
4457 if(extrastart
<extraend
):
4458 extrafieldslist
.append(catheaderdata
[extrastart
]);
4459 extrastart
= extrastart
+ 1;
4460 catfchecksumtype
= catheaderdata
[extrastart
].lower();
4461 catfcs
= catheaderdata
[extrastart
+ 1].lower();
4462 catfccs
= catheaderdata
[extrastart
+ 2].lower();
4464 hcmax
= len(catheaderdata
) - 2;
4467 hout
= hout
+ AppendNullByte(catheaderdata
[hc
], formatspecs
[5]);
4469 catnewfcs
= GetFileChecksum(hout
, catheaderdata
[-3].lower(), True, formatspecs
);
4470 if(catfcs
!=catnewfcs
and not skipchecksum
):
4471 VerbosePrintOut("File Header Checksum Error with file " + catfname
+ " at offset " + str(catfhstart
));
4473 catfhend
= catfp
.tell() - 1;
4474 catfcontentstart
= catfp
.tell();
4475 catfcontents
= BytesIO();
4476 pyhascontents
= False;
4477 catfcontents
.seek(0, 0);
4478 if(catfsize
>0 and not listonly
):
4479 catfcontents
.write(catfp
.read(catfsize
));
4480 catnewfccs
= GetFileChecksum(catfcontents
.read(), catheaderdata
[-3].lower(), False, formatspecs
);
4481 pyhascontents
= True;
4482 if(catfccs
!=catnewfccs
and skipchecksum
):
4483 VerbosePrintOut("File Content Checksum Error with file " + catfname
+ " at offset " + str(catfcontentstart
));
4485 if(catfsize
>0 and listonly
):
4486 catfp
.seek(catfsize
, 1);
4487 pyhascontents
= False;
4489 catfcontentend
= catfp
.tell() - 1;
4490 catfcontents
.seek(0, 0);
4491 catlist
['ffilelist'].update({realidnum
: {'fid': realidnum
, 'fidalt': fileidnum
, 'fheadersize': catfheadsize
, 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': catftype
, 'fname': catfname
, 'fbasedir': catfbasedir
, 'flinkname': catflinkname
, 'fsize': catfsize
, 'fatime': catfatime
, 'fmtime': catfmtime
, 'fctime': catfctime
, 'fbtime': catfbtime
, 'fmode': catfmode
, 'fchmode': catfchmode
, 'ftypemod': catftypemod
, 'fwinattributes': catfwinattributes
, 'fuid': catfuid
, 'funame': catfuname
, 'fgid': catfgid
, 'fgname': catfgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': catfdev_minor
, 'fmajor': catfdev_major
, 'frminor': catfrdev_minor
, 'frmajor': catfrdev_major
, 'fchecksumtype': catfchecksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': catfextrasize
, 'fextralist': extrafieldslist
, 'fheaderchecksum': catfcs
, 'fcontentchecksum': catfccs
, 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': catfcontents
} });
4492 fileidnum
= fileidnum
+ 1;
4493 realidnum
= realidnum
+ 1;
4495 catlist
.update({'catfp': catfp
});
4500 create_alias_function("", __file_format_name__
, "ToArray", ArchiveFileToArray
);
4502 def ArchiveFileStringToArray(catstr
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
4503 catfp
= BytesIO(catstr
);
4504 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4505 return listcatfiles
;
4507 create_alias_function("", __file_format_name__
, "StringToArray", ArchiveFileStringToArray
);
4509 def TarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
4511 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4512 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4513 return listcatfiles
;
4515 def ZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
4517 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4518 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4519 return listcatfiles
;
4521 if(not rarfile_support
):
4522 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
4525 if(rarfile_support
):
4526 def RarFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
4528 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4529 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4530 return listcatfiles
;
4532 if(not py7zr_support
):
4533 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
4537 def SevenZipFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
4539 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
4540 listcatfiles
= ArchiveFileToArray(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4541 return listcatfiles
;
4543 def InFileToArray(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
4544 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
4545 if(checkcompressfile
=="tarfile"):
4546 return TarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4547 elif(checkcompressfile
=="zipfile"):
4548 return ZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4549 elif(checkcompressfile
=="catfile"):
4550 return ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4551 elif(rarfile_support
and checkcompressfile
=="rarfile"):
4552 return RarFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4553 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
4554 return SevenZipFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
4559 def ListDirToArrayAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
4560 catver
= formatspecs
[6];
4561 fileheaderver
= str(int(catver
.replace(".", "")));
4562 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
4563 advancedlist
= formatspecs
[8];
4564 altinode
= formatspecs
[9];
4567 for line
in sys
.stdin
:
4568 infilelist
.append(line
.strip());
4569 infilelist
= list(filter(None, infilelist
));
4570 elif(infiles
!="-" and dirlistfromtxt
and os
.path
.exists(infiles
) and (os
.path
.isfile(infiles
) or infiles
=="/dev/null" or infiles
=="NUL")):
4571 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
4573 with
UncompressFile(infiles
, formatspecs
, "r") as finfile
:
4574 for line
in finfile
:
4575 infilelist
.append(line
.strip());
4576 infilelist
= list(filter(None, infilelist
));
4578 if(isinstance(infiles
, (list, tuple, ))):
4579 infilelist
= list(filter(None, infiles
));
4580 elif(isinstance(infiles
, (str, ))):
4581 infilelist
= list(filter(None, [infiles
]));
4583 GetDirList
= ListDirAdvanced(infilelist
, followlink
, False);
4585 GetDirList
= ListDir(infilelist
, followlink
, False);
4593 inodetocatinode
= {};
4595 fnumfiles
= int(len(GetDirList
));
4596 catver
= formatspecs
[6];
4597 fileheaderver
= str(int(catver
.replace(".", "")));
4598 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
4599 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4600 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
[5]);
4601 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
4602 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4603 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4604 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
4605 fheadtell
= len(fileheader
);
4606 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
4607 for curfname
in GetDirList
:
4608 if(re
.findall("^[.|/]", curfname
)):
4611 fname
= "./"+curfname
;
4613 VerbosePrintOut(fname
);
4614 if(not followlink
or followlink
is None):
4615 fstatinfo
= os
.lstat(fname
);
4617 fstatinfo
= os
.stat(fname
);
4618 fpremode
= fstatinfo
.st_mode
;
4619 finode
= fstatinfo
.st_ino
;
4620 flinkcount
= fstatinfo
.st_nlink
;
4622 if(stat
.S_ISREG(fpremode
)):
4624 elif(stat
.S_ISLNK(fpremode
)):
4626 elif(stat
.S_ISCHR(fpremode
)):
4628 elif(stat
.S_ISBLK(fpremode
)):
4630 elif(stat
.S_ISDIR(fpremode
)):
4632 elif(stat
.S_ISFIFO(fpremode
)):
4634 elif(stat
.S_ISSOCK(fpremode
)):
4636 elif(hasattr(stat
, "S_ISDOOR") and stat
.S_ISDOOR(fpremode
)):
4638 elif(hasattr(stat
, "S_ISPORT") and stat
.S_ISPORT(fpremode
)):
4640 elif(hasattr(stat
, "S_ISWHT") and stat
.S_ISWHT(fpremode
)):
4645 fbasedir
= os
.path
.dirname(fname
);
4647 if(not followlink
and finode
!=0):
4649 if(finode
in inodelist
):
4651 flinkname
= inodetofile
[finode
];
4655 fcurinode
= inodetocatinode
[finode
];
4656 if(finode
not in inodelist
):
4657 inodelist
.append(finode
);
4658 inodetofile
.update({finode
: fname
});
4659 inodetocatinode
.update({finode
: curinode
});
4663 fcurinode
= curinode
;
4664 curinode
= curinode
+ 1;
4666 fcurinode
= curinode
;
4667 curinode
= curinode
+ 1;
4668 curfid
= curfid
+ 1;
4670 flinkname
= os
.readlink(fname
);
4671 fdev
= fstatinfo
.st_dev
;
4672 getfdev
= GetDevMajorMinor(fdev
);
4673 fdev_minor
= getfdev
[0];
4674 fdev_major
= getfdev
[1];
4675 frdev
= fstatinfo
.st_dev
;
4676 if(hasattr(fstatinfo
, "st_rdev")):
4677 frdev
= fstatinfo
.st_rdev
;
4679 frdev
= fstatinfo
.st_dev
;
4680 getfrdev
= GetDevMajorMinor(frdev
);
4681 frdev_minor
= getfrdev
[0];
4682 frdev_major
= getfrdev
[1];
4683 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
4685 if(ftype
==0 or ftype
==7):
4686 fsize
= fstatinfo
.st_size
;
4687 fatime
= fstatinfo
.st_atime
;
4688 fmtime
= fstatinfo
.st_mtime
;
4689 fctime
= fstatinfo
.st_ctime
;
4690 if(hasattr(fstatinfo
, "st_birthtime")):
4691 fbtime
= fstatinfo
.st_birthtime
;
4693 fbtime
= fstatinfo
.st_ctime
;
4694 fmode
= fstatinfo
.st_mode
;
4695 fchmode
= stat
.S_IMODE(fstatinfo
.st_mode
);
4696 ftypemod
= stat
.S_IFMT(fstatinfo
.st_mode
);
4697 fuid
= fstatinfo
.st_uid
;
4698 fgid
= fstatinfo
.st_gid
;
4703 userinfo
= pwd
.getpwuid(fstatinfo
.st_uid
);
4704 funame
= userinfo
.pw_name
;
4713 groupinfo
= grp
.getgrgid(fstatinfo
.st_gid
);
4714 fgname
= groupinfo
.gr_name
;
4719 fdev_minor
= fdev_minor
;
4720 fdev_major
= fdev_major
;
4721 frdev_minor
= frdev_minor
;
4722 frdev_major
= frdev_major
;
4724 flinkcount
= flinkcount
;
4725 if(hasattr(fstatinfo
, "st_file_attributes")):
4726 fwinattributes
= fstatinfo
.st_file_attributes
;
4729 fcontents
= BytesIO();
4730 if(ftype
==0 or ftype
==7):
4731 with
open(fname
, "rb") as fpc
:
4732 shutil
.copyfileobj(fpc
, fcontents
);
4733 if(followlink
and (ftype
==1 or ftype
==2)):
4734 flstatinfo
= os
.stat(flinkname
);
4735 with
open(flinkname
, "rb") as fpc
:
4736 shutil
.copyfileobj(fpc
, fcontents
);
4737 fcontents
.seek(0, 0);
4738 ftypehex
= format(ftype
, 'x').lower();
4739 extrafields
= len(extradata
);
4740 extrafieldslist
= extradata
;
4741 catfextrafields
= extrafields
;
4742 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
4743 if(len(extradata
)>0):
4744 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
4745 extrasizelen
= len(extrasizestr
);
4746 extrasizelenhex
= format(extrasizelen
, 'x').lower();
4747 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
4748 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
4749 catoutlenhex
= format(catoutlen
, 'x').lower();
4750 catoutlist
.insert(0, catoutlenhex
);
4751 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
4752 if(len(extradata
)>0):
4753 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
4754 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
4755 catfnumfields
= catoutlen
;
4756 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
4757 fcontents
.seek(0, 0);
4758 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
4759 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
4760 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
4761 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
4762 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
4763 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
4764 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
4765 nullstrecd
= formatspecs
[5].encode('UTF-8');
4766 fheadtell
+= len(catfileoutstr
) + 1;
4767 catfcontentend
= fheadtell
- 1;
4768 fcontents
.seek(0, 0);
4769 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
4770 pyhascontents
= False;
4771 if(int(fsize
)>0 and not listonly
):
4772 pyhascontents
= True;
4773 if(int(fsize
)>0 and listonly
):
4774 fcontents
= BytesIO();
4775 pyhascontents
= False;
4776 fcontents
.seek(0, 0);
4777 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
4778 fileidnum
= fileidnum
+ 1;
4781 def TarFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
4787 inodetocatinode
= {};
4791 if(hasattr(sys
.stdin
, "buffer")):
4792 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
4794 shutil
.copyfileobj(sys
.stdin
, infile
);
4799 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4800 infile
= download_file_from_internet_file(infile
);
4805 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
4807 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
4809 if(not tarfile
.is_tarfile(infile
)):
4811 except AttributeError:
4812 if(not is_tarfile(infile
)):
4817 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
4818 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
4820 tarfp
= tarfile
.open(infile
, "r");
4821 except FileNotFoundError
:
4823 fnumfiles
= int(len(tarfp
.getmembers()));
4824 catver
= formatspecs
[6];
4825 fileheaderver
= str(int(catver
.replace(".", "")));
4826 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
4827 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4828 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
[5]);
4829 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
4830 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4831 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
4832 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
4833 fheadtell
= len(fileheader
);
4834 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
4835 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
4836 if(re
.findall("^[.|/]", member
.name
)):
4837 fname
= member
.name
;
4839 fname
= "./"+member
.name
;
4841 VerbosePrintOut(fname
);
4842 fpremode
= member
.mode
;
4843 ffullmode
= member
.mode
;
4847 ffullmode
= member
.mode
+ stat
.S_IFREG
;
4849 elif(member
.isdev()):
4850 ffullmode
= member
.mode
;
4852 elif(member
.islnk()):
4853 ffullmode
= member
.mode
+ stat
.S_IFREG
;
4855 elif(member
.issym()):
4856 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
4858 elif(member
.ischr()):
4859 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
4861 elif(member
.isblk()):
4862 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
4864 elif(member
.isdir()):
4865 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
4867 elif(member
.isfifo()):
4868 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
4870 elif(member
.issparse()):
4871 ffullmode
= member
.mode
;
4874 ffullmode
= member
.mode
;
4877 fbasedir
= os
.path
.dirname(fname
);
4881 curfid
= curfid
+ 1;
4883 flinkname
= member
.linkname
;
4884 fdev_minor
= member
.devminor
;
4885 fdev_major
= member
.devmajor
;
4886 frdev_minor
= member
.devminor
;
4887 frdev_major
= member
.devmajor
;
4888 if(ftype
==1 or ftype
==2 or ftype
==3 or ftype
==4 or ftype
==5 or ftype
==6):
4890 elif(ftype
==0 or ftype
==7):
4891 fsize
= member
.size
;
4893 fsize
= member
.size
;
4894 fatime
= member
.mtime
;
4895 fmtime
= member
.mtime
;
4896 fctime
= member
.mtime
;
4897 fbtime
= member
.mtime
;
4899 fchmode
= stat
.S_IMODE(ffullmode
);
4900 ftypemod
= stat
.S_IFMT(ffullmode
);
4903 funame
= member
.uname
;
4904 fgname
= member
.gname
;
4905 flinkcount
= flinkcount
;
4906 fwinattributes
= int(0);
4907 fcontents
= BytesIO();
4908 if(ftype
==0 or ftype
==7):
4909 with tarfp
.extractfile(member
) as fpc
:
4910 shutil
.copyfileobj(fpc
, fcontents
);
4911 fcontents
.seek(0, 0);
4912 ftypehex
= format(ftype
, 'x').lower();
4913 extrafields
= len(extradata
);
4914 extrafieldslist
= extradata
;
4915 catfextrafields
= extrafields
;
4916 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
4917 if(len(extradata
)>0):
4918 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
4919 extrasizelen
= len(extrasizestr
);
4920 extrasizelenhex
= format(extrasizelen
, 'x').lower();
4921 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
4922 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
4923 catoutlenhex
= format(catoutlen
, 'x').lower();
4924 catoutlist
.insert(0, catoutlenhex
);
4925 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
4926 if(len(extradata
)>0):
4927 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
4928 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
4929 catfnumfields
= catoutlen
;
4930 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
4931 fcontents
.seek(0, 0);
4932 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
4933 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
4934 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
4935 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
4936 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
4937 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
4938 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
4939 nullstrecd
= formatspecs
[5].encode('UTF-8');
4940 fheadtell
+= len(catfileoutstr
) + 1;
4941 catfcontentend
= fheadtell
- 1;
4942 fcontents
.seek(0, 0);
4943 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
4944 pyhascontents
= False;
4945 if(int(fsize
)>0 and not listonly
):
4946 pyhascontents
= True;
4947 if(int(fsize
)>0 and listonly
):
4949 pyhascontents
= False;
4950 fcontents
.seek(0, 0);
4951 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
4952 fileidnum
= fileidnum
+ 1;
4955 def ZipFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
4961 inodetocatinode
= {};
4965 if(hasattr(sys
.stdin
, "buffer")):
4966 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
4968 shutil
.copyfileobj(sys
.stdin
, infile
);
4973 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
4974 infile
= download_file_from_internet_file(infile
);
4979 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
4983 if(not zipfile
.is_zipfile(infile
)):
4986 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
4987 except FileNotFoundError
:
4989 ziptest
= zipfp
.testzip();
4991 VerbosePrintOut("Bad file found!");
4992 fnumfiles
= int(len(zipfp
.infolist()));
4993 catver
= formatspecs
[6];
4994 fileheaderver
= str(int(catver
.replace(".", "")));
4995 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
4996 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
4997 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
4998 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
4999 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
[5]);
5000 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5001 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
5002 fheadtell
= len(fileheader
);
5003 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5004 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
5005 if(re
.findall("^[.|/]", member
.filename
)):
5006 fname
= member
.filename
;
5008 fname
= "./"+member
.filename
;
5009 zipinfo
= zipfp
.getinfo(member
.filename
);
5011 VerbosePrintOut(fname
);
5012 if(not member
.is_dir()):
5013 fpremode
= stat
.S_IFREG
+ 438;
5014 elif(member
.is_dir()):
5015 fpremode
= stat
.S_IFDIR
+ 511;
5018 if(not member
.is_dir()):
5020 elif(member
.is_dir()):
5023 fbasedir
= os
.path
.dirname(fname
);
5027 curfid
= curfid
+ 1;
5035 fsize
= member
.file_size
;
5037 fsize
= member
.file_size
;
5038 fatime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5039 fmtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5040 fctime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5041 fbtime
= time
.mktime(member
.date_time
+ (0, 0, -1));
5042 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
5043 fwinattributes
= int(zipinfo
.external_attr
);
5044 if(not member
.is_dir()):
5045 fmode
= int(stat
.S_IFREG
+ 438);
5046 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5047 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5048 elif(member
.is_dir()):
5049 fmode
= int(stat
.S_IFDIR
+ 511);
5050 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5051 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5052 elif(zipinfo
.create_system
==3):
5053 fwinattributes
= int(0);
5054 fmode
= int(zipinfo
.external_attr
);
5056 fwinattributes
= int(0);
5057 if(not member
.is_dir()):
5058 fmode
= int(stat
.S_IFREG
+ 438);
5059 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
5060 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
5061 elif(member
.is_dir()):
5062 fmode
= int(stat
.S_IFDIR
+ 511);
5063 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
5064 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
5067 except AttributeError:
5073 except AttributeError:
5080 userinfo
= pwd
.getpwuid(os
.getuid());
5081 funame
= userinfo
.pw_name
;
5084 except AttributeError:
5092 groupinfo
= grp
.getgrgid(os
.getgid());
5093 fgname
= groupinfo
.gr_name
;
5096 except AttributeError:
5100 fcontents
= BytesIO();
5102 fcontents
.write(zipfp
.read(member
.filename
));
5103 fcontents
.seek(0, 0);
5104 ftypehex
= format(ftype
, 'x').lower();
5105 extrafields
= len(extradata
);
5106 extrafieldslist
= extradata
;
5107 catfextrafields
= extrafields
;
5108 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
5109 if(len(extradata
)>0):
5110 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
5111 extrasizelen
= len(extrasizestr
);
5112 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5113 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5114 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5115 catoutlenhex
= format(catoutlen
, 'x').lower();
5116 catoutlist
.insert(0, catoutlenhex
);
5117 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
5118 if(len(extradata
)>0):
5119 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
5120 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
5121 catfnumfields
= catoutlen
;
5122 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5123 fcontents
.seek(0, 0);
5124 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5125 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
5126 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5127 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
5128 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5129 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
5130 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5131 nullstrecd
= formatspecs
[5].encode('UTF-8');
5132 fheadtell
+= len(catfileoutstr
) + 1;
5133 catfcontentend
= fheadtell
- 1;
5134 fcontents
.seek(0, 0);
5135 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5136 pyhascontents
= False;
5137 if(int(fsize
)>0 and not listonly
):
5138 pyhascontents
= True;
5139 if(int(fsize
)>0 and listonly
):
5140 fcontents
= BytesIO();
5141 pyhascontents
= False;
5142 fcontents
.seek(0, 0);
5143 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5144 fileidnum
= fileidnum
+ 1;
5147 if(not rarfile_support
):
5148 def RarFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5151 if(rarfile_support
):
5152 def RarFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5158 inodetocatinode
= {};
5160 if(not os
.path
.exists(infiles
) or not os
.path
.isfile(infiles
)):
5162 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
5164 rarfp
= rarfile
.RarFile(infile
, "r");
5165 rartest
= rarfp
.testrar();
5167 VerbosePrintOut("Bad file found!");
5168 fnumfiles
= int(len(rarfp
.infolist()));
5169 catver
= formatspecs
[6];
5170 fileheaderver
= str(int(catver
.replace(".", "")));
5171 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
5172 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5173 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5174 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5175 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
[5]);
5176 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5177 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
5178 fheadtell
= len(fileheader
);
5179 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5180 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
5183 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
5186 member
.external_attr
5188 except AttributeError:
5190 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
5193 member
.external_attr
5195 except AttributeError:
5200 if(re
.findall("^[.|/]", member
.filename
)):
5201 fname
= member
.filename
;
5203 fname
= "./"+member
.filename
;
5204 rarinfo
= rarfp
.getinfo(member
.filename
);
5206 VerbosePrintOut(fname
);
5207 if(is_unix
and member
.external_attr
!=0):
5208 fpremode
= int(member
.external_attr
);
5209 elif(member
.is_file()):
5210 fpremode
= stat
.S_IFREG
+ 438;
5211 elif(member
.is_symlink()):
5212 fpremode
= stat
.S_IFLNK
+ 438;
5213 elif(member
.is_dir()):
5214 fpremode
= stat
.S_IFDIR
+ 511;
5215 if(is_windows
and member
.external_attr
!=0):
5216 fwinattributes
= int(member
.external_attr
);
5218 fwinattributes
= int(0);
5221 if(member
.is_file()):
5223 elif(member
.is_symlink()):
5225 elif(member
.is_dir()):
5229 flinkname
= rarfp
.read(member
.filename
).decode("UTF-8");
5230 fbasedir
= os
.path
.dirname(fname
);
5234 curfid
= curfid
+ 1;
5242 fsize
= member
.file_size
;
5245 fatime
= int(member
.atime
.timestamp());
5247 fatime
= int(member
.mtime
.timestamp());
5248 except AttributeError:
5249 fatime
= int(member
.mtime
.timestamp());
5250 fmtime
= int(member
.mtime
.timestamp());
5253 fctime
= int(member
.ctime
.timestamp());
5255 fctime
= int(member
.mtime
.timestamp());
5256 except AttributeError:
5257 fctime
= int(member
.mtime
.timestamp());
5258 fbtime
= int(member
.mtime
.timestamp());
5259 if(is_unix
and member
.external_attr
!=0):
5260 fmode
= int(member
.external_attr
);
5261 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
5262 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
5263 elif(member
.is_file()):
5264 fmode
= int(stat
.S_IFREG
+ 438)
5265 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5266 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5267 elif(member
.is_symlink()):
5268 fmode
= int(stat
.S_IFLNK
+ 438)
5269 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5270 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5271 elif(member
.is_dir()):
5272 fmode
= int(stat
.S_IFDIR
+ 511)
5273 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5274 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5277 except AttributeError:
5283 except AttributeError:
5290 userinfo
= pwd
.getpwuid(os
.getuid());
5291 funame
= userinfo
.pw_name
;
5294 except AttributeError:
5302 groupinfo
= grp
.getgrgid(os
.getgid());
5303 fgname
= groupinfo
.gr_name
;
5306 except AttributeError:
5310 fcontents
= BytesIO();
5312 fcontents
.write(rarfp
.read(member
.filename
));
5313 fcontents
.seek(0, 0);
5314 ftypehex
= format(ftype
, 'x').lower();
5315 extrafields
= len(extradata
);
5316 extrafieldslist
= extradata
;
5317 catfextrafields
= extrafields
;
5318 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
5319 if(len(extradata
)>0):
5320 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
5321 extrasizelen
= len(extrasizestr
);
5322 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5323 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5324 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5325 catoutlenhex
= format(catoutlen
, 'x').lower();
5326 catoutlist
.insert(0, catoutlenhex
);
5327 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
5328 if(len(extradata
)>0):
5329 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
5330 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
5331 catfnumfields
= 24 + catfextrafields
;
5332 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5333 fcontents
.seek(0, 0);
5334 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5335 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
5336 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5337 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
5338 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5339 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
5340 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5341 nullstrecd
= formatspecs
[5].encode('UTF-8');
5342 fheadtell
+= len(catfileoutstr
) + 1;
5343 catfcontentend
= fheadtell
- 1;
5344 fcontents
.seek(0, 0);
5345 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5346 pyhascontents
= False;
5347 if(int(fsize
)>0 and not listonly
):
5348 pyhascontents
= True;
5349 if(int(fsize
)>0 and listonly
):
5350 fcontents
= BytesIO();
5351 pyhascontents
= False;
5352 fcontents
.seek(0, 0);
5353 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5354 fileidnum
= fileidnum
+ 1;
5357 if(not py7zr_support
):
5358 def SevenZipFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5362 def SevenZipFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5368 inodetocatinode
= {};
5370 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
5371 file_content
= szpfp
.readall();
5372 #sztest = szpfp.testzip();
5373 sztestalt
= szpfp
.test();
5375 VerbosePrintOut("Bad file found!");
5376 numfiles
= int(len(szpfp
.list()));
5377 catver
= formatspecs
[6];
5378 fileheaderver
= str(int(catver
.replace(".", "")));
5379 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
5380 catversion
= re
.findall(r
"([\d]+)$", fileheader
);
5381 catversions
= re
.search(r
'(.*?)(\d+)$', catstring
).groups();
5382 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5383 fileheader
= fileheader
+ AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
[5]);
5384 catfileheadercshex
= GetFileChecksum(fileheader
, fprechecksumtype
, True, formatspecs
);
5385 fileheader
= fileheader
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
5386 fheadtell
= len(fileheader
);
5387 catlist
= {'fnumfiles': fnumfiles
, 'fformat': catversions
[0], 'fversion': catversions
[1], 'fformatspecs': formatspecs
, 'fchecksumtype': checksumtype
, 'fheaderchecksum': catfileheadercshex
, 'ffilelist': {}};
5388 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
5389 if(re
.findall("^[.|/]", member
.filename
)):
5390 fname
= member
.filename
;
5392 fname
= "./"+member
.filename
;
5393 if(not member
.is_directory
):
5394 fpremode
= int(stat
.S_IFREG
+ 438);
5395 elif(member
.is_directory
):
5396 fpremode
= int(stat
.S_IFDIR
+ 511);
5397 fwinattributes
= int(0);
5400 if(member
.is_directory
):
5405 fbasedir
= os
.path
.dirname(fname
);
5409 curfid
= curfid
+ 1;
5416 fatime
= int(member
.creationtime
.timestamp());
5417 fmtime
= int(member
.creationtime
.timestamp());
5418 fctime
= int(member
.creationtime
.timestamp());
5419 fbtime
= int(member
.creationtime
.timestamp());
5420 if(member
.is_directory
):
5421 fmode
= int(stat
.S_IFDIR
+ 511)
5422 fchmode
= int(stat
.S_IMODE(stat
.S_IFDIR
+ 511));
5423 ftypemod
= int(stat
.S_IFMT(stat
.S_IFDIR
+ 511));
5425 fmode
= int(stat
.S_IFLNK
+ 438)
5426 fchmode
= int(stat
.S_IMODE(stat
.S_IFREG
+ 438));
5427 ftypemod
= int(stat
.S_IFMT(stat
.S_IFREG
+ 438));
5430 except AttributeError:
5436 except AttributeError:
5443 userinfo
= pwd
.getpwuid(os
.getuid());
5444 funame
= userinfo
.pw_name
;
5447 except AttributeError:
5455 groupinfo
= grp
.getgrgid(os
.getgid());
5456 fgname
= groupinfo
.gr_name
;
5459 except AttributeError:
5463 fcontents
= BytesIO();
5465 fcontents
.write(file_content
[member
.filename
].read());
5466 fsize
= format(fcontents
.tell(), 'x').lower();
5468 fcontents
.seek(0, 0);
5469 ftypehex
= format(ftype
, 'x').lower();
5470 extrafields
= len(extradata
);
5471 extrafieldslist
= extradata
;
5472 catfextrafields
= extrafields
;
5473 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
5474 if(len(extradata
)>0):
5475 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
5476 extrasizelen
= len(extrasizestr
);
5477 extrasizelenhex
= format(extrasizelen
, 'x').lower();
5478 catoutlist
= [ftypehex
, fname
, flinkname
, format(int(fsize
), 'x').lower(), format(int(fatime
), 'x').lower(), format(int(fmtime
), 'x').lower(), format(int(fctime
), 'x').lower(), format(int(fbtime
), 'x').lower(), format(int(fmode
), 'x').lower(), format(int(fwinattributes
), 'x').lower(), format(int(fuid
), 'x').lower(), funame
, format(int(fgid
), 'x').lower(), fgname
, format(int(fcurfid
), 'x').lower(), format(int(fcurinode
), 'x').lower(), format(int(flinkcount
), 'x').lower(), format(int(fdev_minor
), 'x').lower(), format(int(fdev_major
), 'x').lower(), format(int(frdev_minor
), 'x').lower(), format(int(frdev_major
), 'x').lower(), extrasizelenhex
, format(catfextrafields
, 'x').lower()];
5479 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
5480 catoutlenhex
= format(catoutlen
, 'x').lower();
5481 catoutlist
.insert(0, catoutlenhex
);
5482 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
5483 if(len(extradata
)>0):
5484 catfileoutstr
= catfileoutstr
+ AppendNullBytes(extradata
, formatspecs
[5]);
5485 catfileoutstr
= catfileoutstr
+ AppendNullByte(checksumtype
, formatspecs
[5]);
5486 catfnumfields
= 24 + catfextrafields
;
5487 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5488 fcontents
.seek(0, 0);
5489 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
5490 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
5491 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
5492 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
5493 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
5494 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
5495 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
5496 nullstrecd
= formatspecs
[5].encode('UTF-8');
5497 fheadtell
+= len(catfileoutstr
) + 1;
5498 catfcontentend
= fheadtell
- 1;
5499 fcontents
.seek(0, 0);
5500 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
5501 pyhascontents
= False;
5502 if(int(fsize
)>0 and not listonly
):
5503 pyhascontents
= True;
5504 if(int(fsize
)>0 and listonly
):
5506 pyhascontents
= False;
5507 fcontents
.seek(0, 0);
5508 catlist
['ffilelist'].update({fileidnum
: {'fid': fileidnum
, 'fidalt': fileidnum
, 'fheadersize': int(catheaersize
, 16), 'fhstart': catfhstart
, 'fhend': catfhend
, 'ftype': ftype
, 'fname': fname
, 'fbasedir': fbasedir
, 'flinkname': flinkname
, 'fsize': fsize
, 'fatime': fatime
, 'fmtime': fmtime
, 'fctime': fctime
, 'fbtime': fbtime
, 'fmode': fmode
, 'fchmode': fchmode
, 'ftypemod': ftypemod
, 'fwinattributes': fwinattributes
, 'fuid': fuid
, 'funame': funame
, 'fgid': fgid
, 'fgname': fgname
, 'finode': finode
, 'flinkcount': flinkcount
, 'fminor': fdev_minor
, 'fmajor': fdev_major
, 'frminor': frdev_minor
, 'frmajor': frdev_major
, 'fchecksumtype': checksumtype
, 'fnumfields': catfnumfields
+ 2, 'frawheader': catheaderdata
, 'fextrafields': catfextrafields
, 'fextrafieldsize': extrasizelen
, 'fextralist': extrafieldslist
, 'fheaderchecksum': int(catfileheadercshex
, 16), 'fcontentchecksum': int(catfilecontentcshex
, 16), 'fhascontents': pyhascontents
, 'fcontentstart': catfcontentstart
, 'fcontentend': catfcontentend
, 'fcontents': fcontents
} });
5509 fileidnum
= fileidnum
+ 1;
5512 def InFileToArrayAlt(infiles
, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5513 checkcompressfile
= CheckCompressionSubType(infiles
, formatspecs
, True);
5514 if(checkcompressfile
=="tarfile"):
5515 return TarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5516 elif(checkcompressfile
=="zipfile"):
5517 return ZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5518 elif(checkcompressfile
=="catfile"):
5519 return ArchiveFileToArray(infile
, 0, 0, listonly
, False, formatspecs
, False);
5520 elif(rarfile_support
and checkcompressfile
=="rarfile"):
5521 return RarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5522 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
5523 return SevenZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5528 def ListDirToArray(infiles
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
5529 outarray
= BytesIO();
5530 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
5531 listcatfiles
= ArchiveFileToArray(outarray
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5532 return listcatfiles
;
5534 def ArchiveFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
5535 if(isinstance(infile
, dict)):
5536 listcatfiles
= infile
;
5538 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
5539 infile
= RemoveWindowsPath(infile
);
5540 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5541 if(not listcatfiles
):
5543 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5545 catarray
.update({'catfp': listcatfiles
['catfp']});
5546 lenlist
= len(listcatfiles
['ffilelist']);
5548 lcfx
= int(listcatfiles
['fnumfiles']);
5549 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5550 lcfx
= int(lenlist
);
5552 lcfx
= int(listcatfiles
['fnumfiles']);
5554 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5555 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5556 catarray
['filetoid'].update(filetoidarray
);
5557 catarray
['idtofile'].update(idtofilearray
);
5558 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5559 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5560 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5561 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5562 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5563 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5564 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5565 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5566 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5567 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5568 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5569 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5570 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5571 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5572 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5573 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5574 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5575 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5576 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5577 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5578 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5579 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5580 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5581 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5582 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5583 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5584 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5585 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5586 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5587 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5588 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5592 create_alias_function("", __file_format_name__
, "ToArrayIndex", ArchiveFileToArrayIndex
);
5594 def ListDirToArrayIndexAlt(infiles
, dirlistfromtxt
=False, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5595 listcatfiles
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5596 if(not listcatfiles
):
5598 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5599 lenlist
= len(listcatfiles
['ffilelist']);
5601 lcfx
= int(listcatfiles
['fnumfiles']);
5602 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5603 lcfx
= int(lenlist
);
5605 lcfx
= int(listcatfiles
['fnumfiles']);
5607 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5608 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5609 catarray
['filetoid'].update(filetoidarray
);
5610 catarray
['idtofile'].update(idtofilearray
);
5611 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5612 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5613 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5614 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5615 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5616 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5617 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5618 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5619 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5620 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5621 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5622 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5623 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5624 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5625 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5626 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5627 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5628 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5629 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5630 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5631 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5632 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5633 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5634 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5635 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5636 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5637 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5638 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5639 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5640 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5641 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5645 def TarFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5646 listcatfiles
= TarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5647 if(not listcatfiles
):
5649 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5650 lenlist
= len(listcatfiles
['ffilelist']);
5652 lcfx
= int(listcatfiles
['fnumfiles']);
5653 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5654 lcfx
= int(lenlist
);
5656 lcfx
= int(listcatfiles
['fnumfiles']);
5658 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5659 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5660 catarray
['filetoid'].update(filetoidarray
);
5661 catarray
['idtofile'].update(idtofilearray
);
5662 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5663 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5664 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5665 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5666 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5667 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5668 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5669 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5670 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5671 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5672 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5673 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5674 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5675 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5676 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5677 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5678 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5679 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5680 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5681 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5682 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5683 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5684 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5685 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5686 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5687 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5688 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5689 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5690 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5691 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5692 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5696 def ZipFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5697 listcatfiles
= ZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5698 if(not listcatfiles
):
5700 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5701 lenlist
= len(listcatfiles
['ffilelist']);
5703 lcfx
= int(listcatfiles
['fnumfiles']);
5704 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5705 lcfx
= int(lenlist
);
5707 lcfx
= int(listcatfiles
['fnumfiles']);
5709 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5710 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5711 catarray
['filetoid'].update(filetoidarray
);
5712 catarray
['idtofile'].update(idtofilearray
);
5713 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5714 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5715 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5716 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5717 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5718 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5719 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5720 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5721 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5722 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5723 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5724 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5725 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5726 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5727 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5728 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5729 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5730 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5731 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5732 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5733 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5734 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5735 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5736 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5737 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5738 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5739 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5740 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5741 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5742 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5743 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5747 if(not rarfile_support
):
5748 def RarFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5751 if(rarfile_support
):
5752 def RarFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5753 listcatfiles
= RarFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5754 if(not listcatfiles
):
5756 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5757 lenlist
= len(listcatfiles
['ffilelist']);
5759 lcfx
= int(listcatfiles
['fnumfiles']);
5760 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5761 lcfx
= int(lenlist
);
5763 lcfx
= int(listcatfiles
['fnumfiles']);
5765 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5766 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5767 catarray
['filetoid'].update(filetoidarray
);
5768 catarray
['idtofile'].update(idtofilearray
);
5769 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5770 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5771 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5772 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5773 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5774 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5775 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5776 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5777 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5778 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5779 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5780 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5781 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5782 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5783 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5784 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5785 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5786 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5787 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5788 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5789 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5790 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5791 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5792 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5793 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5794 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5795 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5796 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5797 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5798 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5799 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5803 if(not py7zr_support
):
5804 def SevenZipFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5808 def SevenZipFileToArrayIndexAlt(infiles
, seekstart
=0, seekend
=0, listonly
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False):
5809 listcatfiles
= SevenZipFileToArrayAlt(infiles
, listonly
, checksumtype
, extradata
, formatspecs
, verbose
);
5810 if(not listcatfiles
):
5812 catarray
= {'list': listcatfiles
, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5813 lenlist
= len(listcatfiles
['ffilelist']);
5815 lcfx
= int(listcatfiles
['fnumfiles']);
5816 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5817 lcfx
= int(lenlist
);
5819 lcfx
= int(listcatfiles
['fnumfiles']);
5821 filetoidarray
= {listcatfiles
['ffilelist'][lcfi
]['fname']: listcatfiles
['ffilelist'][lcfi
]['fid']};
5822 idtofilearray
= {listcatfiles
['ffilelist'][lcfi
]['fid']: listcatfiles
['ffilelist'][lcfi
]['fname']};
5823 catarray
['filetoid'].update(filetoidarray
);
5824 catarray
['idtofile'].update(idtofilearray
);
5825 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
5826 catarray
['filetypes']['files']['filetoid'].update(filetoidarray
);
5827 catarray
['filetypes']['files']['idtofile'].update(idtofilearray
);
5828 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
5829 catarray
['filetypes']['hardlinks']['filetoid'].update(filetoidarray
);
5830 catarray
['filetypes']['hardlinks']['idtofile'].update(idtofilearray
);
5831 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5832 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5833 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
5834 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5835 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5836 catarray
['filetypes']['links']['filetoid'].update(filetoidarray
);
5837 catarray
['filetypes']['links']['idtofile'].update(idtofilearray
);
5838 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==3):
5839 catarray
['filetypes']['character']['filetoid'].update(filetoidarray
);
5840 catarray
['filetypes']['character']['idtofile'].update(idtofilearray
);
5841 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5842 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5843 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==4):
5844 catarray
['filetypes']['block']['filetoid'].update(filetoidarray
);
5845 catarray
['filetypes']['block']['idtofile'].update(idtofilearray
);
5846 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5847 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5848 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
5849 catarray
['filetypes']['directories']['filetoid'].update(filetoidarray
);
5850 catarray
['filetypes']['directories']['idtofile'].update(idtofilearray
);
5851 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6):
5852 catarray
['filetypes']['symlinks']['filetoid'].update(filetoidarray
);
5853 catarray
['filetypes']['symlinks']['idtofile'].update(idtofilearray
);
5854 catarray
['filetypes']['devices']['filetoid'].update(filetoidarray
);
5855 catarray
['filetypes']['devices']['idtofile'].update(idtofilearray
);
5859 def ArchiveFileStringToArrayIndex(catstr
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
5860 catfp
= BytesIO(catstr
);
5861 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5862 return listcatfiles
;
5864 create_alias_function("", __file_format_name__
, "StringToArrayIndex", ArchiveFileStringToArrayIndex
);
5866 def TarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
5868 catfp
= PackArchiveFileFromTarFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
5869 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5870 return listcatfiles
;
5872 def ZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
5874 catfp
= PackArchiveFileFromZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
5875 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5876 return listcatfiles
;
5878 if(not rarfile_support
):
5879 def RarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
5882 if(rarfile_support
):
5883 def RarFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
5885 catfp
= PackArchiveFileFromRarFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
5886 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5887 return listcatfiles
;
5889 if(not py7zr_support
):
5890 def SevenZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
5894 def SevenZipFileToArrayIndex(infile
, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, formatspecs
=__file_format_list__
, returnfp
=False):
5896 catfp
= PackArchiveFileFromSevenZipFile(infile
, catfp
, "auto", None, "crc32", [], formatspecs
, False, True);
5897 listcatfiles
= ArchiveFileToArrayIndex(catfp
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
);
5898 return listcatfiles
;
5900 def ListDirToArrayIndex(infiles
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, listonly
=False, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
5901 outarray
= BytesIO();
5902 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compressionlevel
, followlink
, checksumtype
, formatspecs
, verbose
, True);
5903 listcatfiles
= ArchiveFileToArrayIndex(outarray
, seekstart
, seekend
, listonly
, skipchecksum
, formatspecs
, returnfp
)
5904 return listcatfiles
;
5906 def RePackArchiveFile(infile
, outfile
, compression
="auto", compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
5907 if(isinstance(infile
, dict)):
5908 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
5909 listcatfiles
= prelistcatfiles
['list'];
5911 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
5912 infile
= RemoveWindowsPath(infile
);
5914 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
5915 listcatfiles
= prelistcatfiles
['list'];
5917 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
5918 if(outfile
!="-" and not hasattr(infile
, "read") and not hasattr(outfile
, "write")):
5919 outfile
= RemoveWindowsPath(outfile
);
5920 checksumtype
= checksumtype
.lower();
5921 if(not CheckSumSupport(checksumtype
, hashlib_guaranteed
)):
5922 checksumtype
="crc32";
5923 if(checksumtype
=="none"):
5925 if(not compression
or compression
or compression
=="catfile" or compression
==formatspecs
[2]):
5927 if(compression
not in compressionlist
and compression
is None):
5928 compression
= "auto";
5930 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
5931 if(outfile
!="-" and not hasattr(outfile
, "read") and not hasattr(outfile
, "write")):
5932 if(os
.path
.exists(outfile
)):
5934 if(not listcatfiles
):
5939 elif(hasattr(outfile
, "read") or hasattr(outfile
, "write")):
5941 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
5944 fbasename
= os
.path
.splitext(outfile
)[0];
5945 fextname
= os
.path
.splitext(outfile
)[1];
5946 catfp
= CompressOpenFile(outfile
, compressionlevel
);
5947 catver
= formatspecs
[6];
5948 fileheaderver
= str(int(catver
.replace(".", "")));
5949 fileheader
= AppendNullByte(formatspecs
[1] + fileheaderver
, formatspecs
[5]);
5950 catfp
.write(fileheader
.encode('UTF-8'));
5951 lenlist
= len(listcatfiles
['ffilelist']);
5952 fnumfiles
= int(listcatfiles
['fnumfiles']);
5953 if(lenlist
>fnumfiles
or lenlist
<fnumfiles
):
5954 fnumfiles
= lenlist
;
5955 fnumfileshex
= format(int(fnumfiles
), 'x').lower();
5956 fnumfilesa
= AppendNullBytes([fnumfileshex
, checksumtype
], formatspecs
[5]);
5957 catfileheadercshex
= GetFileChecksum(fileheader
+ fnumfilesa
, checksumtype
, True, formatspecs
);
5958 fnumfilesa
= fnumfilesa
+ AppendNullByte(catfileheadercshex
, formatspecs
[5]);
5959 catfp
.write(fnumfilesa
.encode('UTF-8'));
5962 os
.fsync(catfp
.fileno());
5963 except io
.UnsupportedOperation
:
5965 except AttributeError:
5967 lenlist
= len(listcatfiles
['ffilelist']);
5968 fnumfiles
= int(listcatfiles
['fnumfiles']);
5970 lcfx
= int(listcatfiles
['fnumfiles']);
5971 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
5972 lcfx
= int(lenlist
);
5974 lcfx
= int(listcatfiles
['fnumfiles']);
5982 catfhstart
= catfp
.tell();
5983 if(re
.findall("^[.|/]", listcatfiles
['ffilelist'][reallcfi
]['fname'])):
5984 fname
= listcatfiles
['ffilelist'][reallcfi
]['fname'];
5986 fname
= "./"+listcatfiles
['ffilelist'][reallcfi
]['fname'];
5988 VerbosePrintOut(fname
);
5989 fheadersize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fheadersize']), 'x').lower();
5990 fsize
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fsize']), 'x').lower();
5991 flinkname
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
5992 fatime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fatime']), 'x').lower();
5993 fmtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmtime']), 'x').lower();
5994 fctime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fctime']), 'x').lower();
5995 fbtime
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fbtime']), 'x').lower();
5996 fmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmode']), 'x').lower();
5997 fchmode
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fchmode']), 'x').lower();
5998 fuid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fuid']), 'x').lower();
5999 funame
= listcatfiles
['ffilelist'][reallcfi
]['funame'];
6000 fgid
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fgid']), 'x').lower();
6001 fgname
= listcatfiles
['ffilelist'][reallcfi
]['fgname'];
6002 finode
= listcatfiles
['ffilelist'][reallcfi
]['finode'];
6003 flinkcount
= listcatfiles
['ffilelist'][reallcfi
]['flinkcount'];
6004 fwinattributes
= flinkinfo
['ffilelist'][reallcfi
]['fwinattributes'];
6005 fdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fminor']), 'x').lower();
6006 fdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fmajor']), 'x').lower();
6007 frdev_minor
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frminor']), 'x').lower();
6008 frdev_major
= format(int(listcatfiles
['ffilelist'][reallcfi
]['frmajor']), 'x').lower();
6009 if(len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] and len(listcatfiles
['ffilelist'][reallcfi
]['fextralist'])>0):
6010 listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] = len(listcatfiles
['ffilelist'][reallcfi
]['fextralist']);
6011 if(len(extradata
)>0):
6012 listcatfiles
['ffilelist'][reallcfi
]['fextrafields'] = len(extradata
);
6013 listcatfiles
['ffilelist'][reallcfi
]['fextralist'] = extradata
;
6014 extrafields
= format(int(listcatfiles
['ffilelist'][reallcfi
]['fextrafields']), 'x').lower();
6015 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
6016 if(len(extradata
)>0):
6017 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
6018 extrasizelen
= format(len(extrasizestr
), 'x').lower();
6019 fcontents
= listcatfiles
['ffilelist'][reallcfi
]['fcontents'];
6021 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']==1 or listcatfiles
['ffilelist'][reallcfi
]['ftype']==2):
6022 getflinkpath
= listcatfiles
['ffilelist'][reallcfi
]['flinkname'];
6023 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6024 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6025 fheadersize
= format(int(flinkinfo
['fheadersize']), 'x').lower();
6026 fsize
= format(int(flinkinfo
['fsize']), 'x').lower();
6027 flinkname
= flinkinfo
['flinkname'];
6028 fatime
= format(int(flinkinfo
['fatime']), 'x').lower();
6029 fmtime
= format(int(flinkinfo
['fmtime']), 'x').lower();
6030 fctime
= format(int(flinkinfo
['fctime']), 'x').lower();
6031 fbtime
= format(int(flinkinfo
['fbtime']), 'x').lower();
6032 fmode
= format(int(flinkinfo
['fmode']), 'x').lower();
6033 fchmode
= format(int(flinkinfo
['fchmode']), 'x').lower();
6034 fuid
= format(int(flinkinfo
['fuid']), 'x').lower();
6035 funame
= flinkinfo
['funame'];
6036 fgid
= format(int(flinkinfo
['fgid']), 'x').lower();
6037 fgname
= flinkinfo
['fgname'];
6038 finode
= flinkinfo
['finode'];
6039 flinkcount
= flinkinfo
['flinkcount'];
6040 fwinattributes
= flinkinfo
['fwinattributes'];
6041 fdev_minor
= format(int(flinkinfo
['fminor']), 'x').lower();
6042 fdev_major
= format(int(flinkinfo
['fmajor']), 'x').lower();
6043 frdev_minor
= format(int(flinkinfo
['frminor']), 'x').lower();
6044 frdev_major
= format(int(flinkinfo
['frmajor']), 'x').lower();
6045 if(len(flinkinfo
['fextralist'])>flinkinfo
['fextrafields'] and len(flinkinfo
['fextralist'])>0):
6046 flinkinfo
['fextrafields'] = len(flinkinfo
['fextralist']);
6047 if(len(extradata
)>0):
6048 flinkinfo
['fextrafields'] = len(extradata
);
6049 flinkinfo
['fextralist'] = extradata
;
6050 extrafields
= format(int(flinkinfo
['fextrafields']), 'x').lower();
6051 extrasizestr
= AppendNullByte(extrafields
, formatspecs
[5]);
6052 if(len(extradata
)>0):
6053 extrasizestr
= extrasizestr
+ AppendNullBytes(extradata
, formatspecs
[5]);
6054 extrasizelen
= format(len(extrasizestr
), 'x').lower();
6055 fcontents
= flinkinfo
['fcontents'];
6056 ftypehex
= format(flinkinfo
['ftype'], 'x').lower();
6058 ftypehex
= format(listcatfiles
['ffilelist'][reallcfi
]['ftype'], 'x').lower();
6059 fcurfid
= format(curfid
, 'x').lower();
6060 if(not followlink
and finode
!=0):
6061 if(listcatfiles
['ffilelist'][reallcfi
]['ftype']!=1):
6062 fcurinode
= format(int(curinode
), 'x').lower();
6063 inodetofile
.update({curinode
: fname
});
6064 filetoinode
.update({fname
: curinode
});
6065 curinode
= curinode
+ 1;
6067 fcurinode
= format(int(filetoinode
[flinkname
]), 'x').lower();
6069 fcurinode
= format(int(curinode
), 'x').lower();
6070 curinode
= curinode
+ 1;
6071 curfid
= curfid
+ 1;
6072 catoutlist
= [ftypehex
, fname
, flinkname
, fsize
, fatime
, fmtime
, fctime
, fbtime
, fmode
, fwinattributes
, fuid
, funame
, fgid
, fgname
, fcurfid
, fcurinode
, flinkcount
, fdev_minor
, fdev_major
, frdev_minor
, frdev_major
, extrasizelen
, extrafields
];
6073 catoutlen
= len(catoutlist
) + len(extradata
) + 3;
6074 catoutlenhex
= format(catoutlen
, 'x').lower();
6075 catoutlist
.insert(0, catoutlenhex
);
6076 catfileoutstr
= AppendNullBytes(catoutlist
, formatspecs
[5]);
6077 if(listcatfiles
['ffilelist'][reallcfi
]['fextrafields']>0):
6078 extrafieldslist
= [];
6080 exil
= listcatfiles
['ffilelist'][reallcfi
]['fextrafields'];
6082 extrafieldslist
.append(listcatfiles
['ffilelist'][reallcfi
]['fextralist']);
6084 catfileoutstr
+= AppendNullBytes([extrafieldslist
], formatspecs
[5]);
6085 catfileoutstr
+= AppendNullBytes([checksumtype
], formatspecs
[5]);
6086 catfhend
= (catfp
.tell() - 1) + len(catfileoutstr
);
6087 catfcontentstart
= catfp
.tell() + len(catfileoutstr
);
6088 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
6089 fcontents
.seek(0, 0);
6090 catfilecontentcshex
= GetFileChecksum(fcontents
.read(), checksumtype
, False, formatspecs
);
6091 tmpfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
6092 catheaersize
= format(int(len(tmpfileoutstr
) - 1), 'x').lower();
6093 catfileoutstr
= AppendNullByte(catheaersize
, formatspecs
[5]) + catfileoutstr
;
6094 catfileheadercshex
= GetFileChecksum(catfileoutstr
, checksumtype
, True, formatspecs
);
6095 catfileoutstr
= catfileoutstr
+ AppendNullBytes([catfileheadercshex
, catfilecontentcshex
], formatspecs
[5]);
6096 catfileoutstrecd
= catfileoutstr
.encode('UTF-8');
6097 nullstrecd
= formatspecs
[5].encode('UTF-8');
6098 fcontents
.seek(0, 0);
6099 catfileout
= catfileoutstrecd
+ fcontents
.read() + nullstrecd
;
6100 catfcontentend
= (catfp
.tell() - 1) + len(catfileout
);
6101 catfp
.write(catfileout
);
6104 os
.fsync(catfp
.fileno());
6105 except io
.UnsupportedOperation
:
6107 except AttributeError:
6110 reallcfi
= reallcfi
+ 1;
6112 catfp
.write(AppendNullBytes([0, 0], formatspecs
[5]).encode("UTF-8"));
6113 if(outfile
=="-" or hasattr(outfile
, "read") or hasattr(outfile
, "write")):
6114 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
6117 os
.fsync(catfp
.fileno());
6118 except io
.UnsupportedOperation
:
6120 except AttributeError:
6124 if(hasattr(sys
.stdout
, "buffer")):
6125 shutil
.copyfileobj(catfp
, sys
.stdout
.buffer);
6127 shutil
.copyfileobj(catfp
, sys
.stdout
);
6128 elif(re
.findall(r
"^(ftp|ftps|sftp)\:\/\/", str(outfile
))):
6129 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
6131 upload_file_to_internet_file(catfp
, outfile
);
6139 create_alias_function("RePack", __file_format_name__
, "", RePackArchiveFile
);
6141 def RePackArchiveFileFromString(catstr
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", skipchecksum
=False, extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6142 catfp
= BytesIO(catstr
);
6143 listcatfiles
= RePackArchiveFile(catfp
, compression
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6144 return listcatfiles
;
6146 create_alias_function("RePack", __file_format_name__
, "FromString", RePackArchiveFileFromString
);
6148 def PackArchiveFileFromListDir(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6149 outarray
= BytesIO();
6150 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compressionlevel
, followlink
, checksumtype
, extradata
, formatspecs
, verbose
, True);
6151 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compressionlevel
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6152 return listcatfiles
;
6154 create_alias_function("Pack", __file_format_name__
, "FromListDir", PackArchiveFileFromListDir
);
6156 def UnPackArchiveFile(infile
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_list__
, preservepermissions
=True, preservetime
=True, verbose
=False, returnfp
=False):
6157 if(outdir
is not None):
6158 outdir
= RemoveWindowsPath(outdir
);
6160 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6161 if(isinstance(infile
, dict)):
6162 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6163 listcatfiles
= prelistcatfiles
['list'];
6165 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6166 infile
= RemoveWindowsPath(infile
);
6168 prelistcatfiles
= ArchiveFileToArrayIndex(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6169 listcatfiles
= prelistcatfiles
['list'];
6171 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, False, skipchecksum
, formatspecs
, returnfp
);
6172 if(not listcatfiles
):
6174 lenlist
= len(listcatfiles
['ffilelist']);
6175 fnumfiles
= int(listcatfiles
['fnumfiles']);
6177 lcfx
= int(listcatfiles
['fnumfiles']);
6178 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6179 lcfx
= int(lenlist
);
6181 lcfx
= int(listcatfiles
['fnumfiles']);
6187 userinfo
= pwd
.getpwuid(listcatfiles
['ffilelist'][lcfi
]['fuid']);
6188 funame
= userinfo
.pw_name
;
6197 groupinfo
= grp
.getgrgid(listcatfiles
['ffilelist'][lcfi
]['fgid']);
6198 fgname
= groupinfo
.gr_name
;
6204 VerbosePrintOut(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6205 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==0 or listcatfiles
['ffilelist'][lcfi
]['ftype']==7):
6206 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6207 listcatfiles
['ffilelist'][lcfi
]['fcontents'].seek(0, 0);
6208 shutil
.copyfileobj(listcatfiles
['ffilelist'][lcfi
]['fcontents'], fpc
);
6211 os
.fsync(fpc
.fileno());
6212 except io
.UnsupportedOperation
:
6214 except AttributeError:
6216 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6217 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6218 if(preservepermissions
):
6219 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6221 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6222 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6224 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6225 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6226 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6231 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6232 funame
= userinfo
.pw_name
;
6241 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6242 fgname
= groupinfo
.gr_name
;
6247 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6248 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6249 flinkinfo
['fcontents'].seek(0, 0);
6250 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6253 os
.fsync(fpc
.fileno());
6254 except io
.UnsupportedOperation
:
6256 except AttributeError:
6258 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6259 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6260 if(preservepermissions
):
6261 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6263 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6264 if(flinkinfo
['ftype']==1):
6265 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6266 if(flinkinfo
['ftype']==2):
6267 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6268 if(flinkinfo
['ftype']==5):
6269 if(preservepermissions
):
6270 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6272 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6273 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6274 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6275 if(preservepermissions
):
6276 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6278 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6279 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6280 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6282 os
.link(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6283 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6285 getflinkpath
= listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6286 flinkid
= prelistcatfiles
['filetoid'][getflinkpath
];
6287 flinkinfo
= listcatfiles
['ffilelist'][flinkid
];
6292 userinfo
= pwd
.getpwuid(flinkinfo
['fuid']);
6293 funame
= userinfo
.pw_name
;
6302 groupinfo
= grp
.getgrgid(flinkinfo
['fgid']);
6303 fgname
= groupinfo
.gr_name
;
6308 if(flinkinfo
['ftype']==0 or flinkinfo
['ftype']==7):
6309 with
open(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), "wb") as fpc
:
6310 flinkinfo
['fcontents'].seek(0, 0);
6311 shutil
.copyfileobj(flinkinfo
['fcontents'], fpc
);
6314 os
.fsync(fpc
.fileno());
6315 except io
.UnsupportedOperation
:
6317 except AttributeError:
6319 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6320 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6321 if(preservepermissions
):
6322 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6324 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6325 if(flinkinfo
['ftype']==1):
6326 os
.link(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6327 if(flinkinfo
['ftype']==2):
6328 os
.symlink(flinkinfo
['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6329 if(flinkinfo
['ftype']==5):
6330 if(preservepermissions
):
6331 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6333 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6334 if(hasattr(os
, "chown") and funame
==flinkinfo
['funame'] and fgname
==flinkinfo
['fgname'] and preservepermissions
):
6335 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fuid'], flinkinfo
['fgid']);
6336 if(preservepermissions
):
6337 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6339 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (flinkinfo
['fatime'], flinkinfo
['fmtime']));
6340 if(flinkinfo
['ftype']==6 and hasattr(os
, "mkfifo")):
6341 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), flinkinfo
['fchmode']);
6343 os
.symlink(listcatfiles
['ffilelist'][lcfi
]['flinkname'], PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6344 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==5):
6345 if(preservepermissions
):
6346 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6348 os
.mkdir(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']));
6349 if(hasattr(os
, "chown") and funame
==listcatfiles
['ffilelist'][lcfi
]['funame'] and fgname
==listcatfiles
['ffilelist'][lcfi
]['fgname'] and preservepermissions
):
6350 os
.chown(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fuid'], listcatfiles
['ffilelist'][lcfi
]['fgid']);
6351 if(preservepermissions
):
6352 os
.chmod(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6354 os
.utime(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), (listcatfiles
['ffilelist'][lcfi
]['fatime'], listcatfiles
['ffilelist'][lcfi
]['fmtime']));
6355 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==6 and hasattr(os
, "mkfifo")):
6356 os
.mkfifo(PrependPath(outdir
, listcatfiles
['ffilelist'][lcfi
]['fname']), listcatfiles
['ffilelist'][lcfi
]['fchmode']);
6359 return listcatfiles
['ffilelist']['catfp'];
6363 create_alias_function("UnPack", __file_format_name__
, "", UnPackArchiveFile
);
6365 if(hasattr(shutil
, "register_unpack_format")):
6366 def UnPackArchiveFileFunc(archive_name
, extract_dir
=None, **kwargs
):
6367 return UnPackArchiveFile(archive_name
, extract_dir
, False, 0, 0, False, __file_format_delimiter__
, False, False);
6368 create_alias_function("UnPack", __file_format_name__
, "Func", UnPackArchiveFileFunc
);
6370 def UnPackArchiveFileString(catstr
, outdir
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6371 catfp
= BytesIO(catstr
);
6372 listcatfiles
= UnPackArchiveFile(catfp
, outdir
, followlink
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6373 return listcatfiles
;
6375 create_alias_function("UnPack", __file_format_name__
, "String", UnPackArchiveFileString
);
6377 def ArchiveFileListFiles(infile
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6378 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6379 if(isinstance(infile
, dict)):
6380 listcatfiles
= infile
;
6382 if(infile
!="-" and not hasattr(infile
, "read") and not hasattr(infile
, "write")):
6383 infile
= RemoveWindowsPath(infile
);
6384 listcatfiles
= ArchiveFileToArray(infile
, seekstart
, seekend
, True, skipchecksum
, formatspecs
, returnfp
);
6385 if(not listcatfiles
):
6387 lenlist
= len(listcatfiles
['ffilelist']);
6388 fnumfiles
= int(listcatfiles
['fnumfiles']);
6390 lcfx
= int(listcatfiles
['fnumfiles']);
6391 if(lenlist
>listcatfiles
['fnumfiles'] or lenlist
<listcatfiles
['fnumfiles']):
6392 lcfx
= int(lenlist
);
6394 lcfx
= int(listcatfiles
['fnumfiles']);
6397 returnval
.update({lcfi
: listcatfiles
['ffilelist'][lcfi
]['fname']});
6399 VerbosePrintOut(listcatfiles
['ffilelist'][lcfi
]['fname']);
6401 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6402 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'];
6403 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==1):
6404 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " link to " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6405 if(listcatfiles
['ffilelist'][lcfi
]['ftype']==2):
6406 printfname
= listcatfiles
['ffilelist'][lcfi
]['fname'] + " -> " + listcatfiles
['ffilelist'][lcfi
]['flinkname'];
6407 fuprint
= listcatfiles
['ffilelist'][lcfi
]['funame'];
6408 if(len(fuprint
)<=0):
6409 fuprint
= listcatfiles
['ffilelist'][lcfi
]['fuid'];
6410 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgname'];
6411 if(len(fgprint
)<=0):
6412 fgprint
= listcatfiles
['ffilelist'][lcfi
]['fgid'];
6413 VerbosePrintOut(PrintPermissionString(listcatfiles
['ffilelist'][lcfi
]['fmode'], listcatfiles
['ffilelist'][lcfi
]['ftype']) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(listcatfiles
['ffilelist'][lcfi
]['fsize']).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(listcatfiles
['ffilelist'][lcfi
]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6416 return listcatfiles
['catfp'];
6420 create_alias_function("", __file_format_name__
, "ListFiles", ArchiveFileListFiles
);
6422 def ArchiveFileStringListFiles(catstr
, seekstart
=0, seekend
=0, skipchecksum
=False, formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6423 catfp
= BytesIO(catstr
);
6424 listcatfiles
= ArchiveFileListFiles(catstr
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6425 return listcatfiles
;
6427 create_alias_function("", __file_format_name__
, "StringListFiles", ArchiveFileStringListFiles
);
6429 def TarFileListFiles(infile
, verbose
=False, returnfp
=False):
6430 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6433 if(hasattr(sys
.stdin
, "buffer")):
6434 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6436 shutil
.copyfileobj(sys
.stdin
, infile
);
6441 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6442 infile
= download_file_from_internet_file(infile
);
6447 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6449 elif(os
.path
.exists(infile
) and os
.path
.isfile(infile
)):
6451 if(not tarfile
.is_tarfile(infile
)):
6453 except AttributeError:
6454 if(not is_tarfile(infile
)):
6459 if(hasattr(infile
, "read") or hasattr(infile
, "write")):
6460 tarfp
= tarfile
.open(fileobj
=infile
, mode
="r");
6462 tarfp
= tarfile
.open(infile
, "r");
6463 except FileNotFoundError
:
6467 for member
in sorted(tarfp
.getmembers(), key
=lambda x
: x
.name
):
6468 returnval
.update({lcfi
: member
.name
});
6469 fpremode
= member
.mode
;
6470 ffullmode
= member
.mode
;
6474 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6476 elif(member
.isdev()):
6477 ffullmode
= member
.mode
;
6479 elif(member
.islnk()):
6480 ffullmode
= member
.mode
+ stat
.S_IFREG
;
6482 elif(member
.issym()):
6483 ffullmode
= member
.mode
+ stat
.S_IFLNK
;
6485 elif(member
.ischr()):
6486 ffullmode
= member
.mode
+ stat
.S_IFCHR
;
6488 elif(member
.isblk()):
6489 ffullmode
= member
.mode
+ stat
.S_IFBLK
;
6491 elif(member
.isdir()):
6492 ffullmode
= member
.mode
+ stat
.S_IFDIR
;
6494 elif(member
.isfifo()):
6495 ffullmode
= member
.mode
+ stat
.S_IFIFO
;
6497 elif(member
.issparse()):
6498 ffullmode
= member
.mode
;
6501 VerbosePrintOut(member
.name
);
6503 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6504 printfname
= member
.name
;
6506 printfname
= member
.name
+ " link to " + member
.linkname
;
6507 elif(member
.issym()):
6508 printfname
= member
.name
+ " -> " + member
.linkname
;
6509 fuprint
= member
.uname
;
6510 if(len(fuprint
)<=0):
6511 fuprint
= member
.uid
;
6512 fgprint
= member
.gname
;
6513 if(len(fgprint
)<=0):
6514 fgprint
= member
.gid
;
6515 VerbosePrintOut(PrintPermissionString(ffullmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(member
.mtime
).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6518 return listcatfiles
['catfp'];
6522 def ZipFileListFiles(infile
, verbose
=False, returnfp
=False):
6523 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6526 if(hasattr(sys
.stdin
, "buffer")):
6527 shutil
.copyfileobj(sys
.stdin
.buffer, infile
);
6529 shutil
.copyfileobj(sys
.stdin
, infile
);
6534 elif(re
.findall(r
"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile
))):
6535 infile
= download_file_from_internet_file(infile
);
6540 elif(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6544 if(not zipfile
.is_zipfile(infile
)):
6547 zipfp
= zipfile
.ZipFile(infile
, "r", allowZip64
=True);
6548 except FileNotFoundError
:
6552 ziptest
= zipfp
.testzip();
6554 VerbosePrintOut("Bad file found!");
6555 for member
in sorted(zipfp
.infolist(), key
=lambda x
: x
.filename
):
6556 if(zipinfo
.create_system
==0 or zipinfo
.create_system
==10):
6557 fwinattributes
= int(zipinfo
.external_attr
);
6558 if(not member
.is_dir()):
6559 fmode
= int(stat
.S_IFREG
+ 438);
6560 fchmode
= int(stat
.S_IMODE(fmode
));
6561 ftypemod
= int(stat
.S_IFMT(fmode
));
6562 elif(member
.is_dir()):
6563 fmode
= int(stat
.S_IFDIR
+ 511);
6564 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6565 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6566 elif(zipinfo
.create_system
==3):
6567 fwinattributes
=int(0);
6568 fmode
= int(zipinfo
.external_attr
);
6569 fchmode
= int(stat
.S_IMODE(fmode
));
6570 ftypemod
= int(stat
.S_IFMT(fmode
));
6572 fwinattributes
= int(0);
6573 if(not member
.is_dir()):
6574 fmode
= int(stat
.S_IFREG
+ 438);
6575 fchmode
= int(stat
.S_IMODE(fmode
));
6576 ftypemod
= int(stat
.S_IFMT(fmode
));
6577 elif(member
.is_dir()):
6578 fmode
= int(stat
.S_IFDIR
+ 511);
6579 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6580 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6581 returnval
.update({lcfi
: member
.filename
});
6583 VerbosePrintOut(member
.filename
);
6585 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6587 for fmodval
in str(oct(fmode
))[-3:]:
6588 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
6589 if(not member
.is_dir()):
6591 permissionstr
= "-" + permissionstr
;
6592 elif(member
.is_dir()):
6594 permissionstr
= "d" + permissionstr
;
6595 printfname
= member
.filename
;
6597 fuid
= int(os
.getuid());
6598 except AttributeError:
6603 fgid
= int(os
.getgid());
6604 except AttributeError:
6611 userinfo
= pwd
.getpwuid(os
.getuid());
6612 funame
= userinfo
.pw_name
;
6615 except AttributeError:
6623 groupinfo
= grp
.getgrgid(os
.getgid());
6624 fgname
= groupinfo
.gr_name
;
6627 except AttributeError:
6632 if(len(fuprint
)<=0):
6633 fuprint
= str(fuid
);
6635 if(len(fgprint
)<=0):
6636 fgprint
= str(fgid
);
6637 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + datetime
.datetime
.utcfromtimestamp(int(time
.mktime(member
.date_time
+ (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6640 return listcatfiles
['catfp'];
6644 if(not rarfile_support
):
6645 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
6646 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6647 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6650 if(rarfile_support
):
6651 def RarFileListFiles(infile
, verbose
=False, returnfp
=False):
6652 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6653 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6655 if(not rarfile
.is_rarfile(infile
) and not rarfile
.is_rarfile_sfx(infile
)):
6659 rarfp
= rarfile
.RarFile(infile
, "r");
6660 rartest
= rarfp
.testrar();
6662 VerbosePrintOut("Bad file found!");
6663 for member
in sorted(rarfp
.infolist(), key
=lambda x
: x
.filename
):
6666 if(member
.host_os
==rarfile
.RAR_OS_UNIX
):
6669 member
.external_attr
6671 except AttributeError:
6673 elif(member
.host_os
==rarfile
.RAR_OS_WIN32
):
6676 member
.external_attr
6678 except AttributeError:
6683 if(is_unix
and member
.external_attr
!=0):
6684 fpremode
= int(member
.external_attr
);
6685 elif(member
.is_file()):
6686 fpremode
= int(stat
.S_IFREG
+ 438);
6687 elif(member
.is_symlink()):
6688 fpremode
= int(stat
.S_IFLNK
+ 438);
6689 elif(member
.is_dir()):
6690 fpremode
= int(stat
.S_IFDIR
+ 511);
6691 if(is_windows
and member
.external_attr
!=0):
6692 fwinattributes
= int(member
.external_attr
);
6694 fwinattributes
= int(0);
6695 if(is_unix
and member
.external_attr
!=0):
6696 fmode
= int(member
.external_attr
);
6697 fchmode
= int(stat
.S_IMODE(member
.external_attr
));
6698 ftypemod
= int(stat
.S_IFMT(member
.external_attr
));
6699 elif(member
.is_file()):
6700 fmode
= int(stat
.S_IFREG
+ 438);
6701 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFREG
+ 438)));
6702 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFREG
+ 438)));
6703 elif(member
.is_symlink()):
6704 fmode
= int(stat
.S_IFLNK
+ 438);
6705 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
6706 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
6707 elif(member
.is_dir()):
6708 fmode
= int(stat
.S_IFDIR
+ 511);
6709 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6710 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6711 returnval
.update({lcfi
: member
.filename
});
6713 VerbosePrintOut(member
.filename
);
6715 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6717 for fmodval
in str(oct(fmode
))[-3:]:
6718 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
6719 if(member
.is_file()):
6721 permissionstr
= "-" + permissionstr
;
6722 printfname
= member
.filename
;
6723 elif(member
.is_symlink()):
6725 permissionstr
= "l" + permissionstr
;
6726 printfname
= member
.name
+ " -> " + member
.read().decode("UTF-8");
6727 elif(member
.is_dir()):
6729 permissionstr
= "d" + permissionstr
;
6730 printfname
= member
.filename
;
6732 fuid
= int(os
.getuid());
6733 except AttributeError:
6738 fgid
= int(os
.getgid());
6739 except AttributeError:
6746 userinfo
= pwd
.getpwuid(os
.getuid());
6747 funame
= userinfo
.pw_name
;
6750 except AttributeError:
6758 groupinfo
= grp
.getgrgid(os
.getgid());
6759 fgname
= groupinfo
.gr_name
;
6762 except AttributeError:
6767 if(len(fuprint
)<=0):
6768 fuprint
= str(fuid
);
6770 if(len(fgprint
)<=0):
6771 fgprint
= str(fgid
);
6772 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(member
.file_size
).rjust(15) + " " + member
.mtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6775 return listcatfiles
['catfp'];
6779 if(not py7zr_support
):
6780 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
6781 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6782 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6786 def SevenZipFileListFiles(infile
, verbose
=False, returnfp
=False):
6787 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6788 if(not os
.path
.exists(infile
) or not os
.path
.isfile(infile
)):
6792 szpfp
= py7zr
.SevenZipFile(infile
, mode
="r");
6793 file_content
= szpfp
.readall();
6794 #sztest = szpfp.testzip();
6795 sztestalt
= szpfp
.test();
6797 VerbosePrintOut("Bad file found!");
6798 for member
in sorted(szpfp
.list(), key
=lambda x
: x
.filename
):
6799 if(re
.findall("^[.|/]", member
.filename
)):
6800 fname
= member
.filename
;
6802 fname
= "./"+member
.filename
;
6803 if(not member
.is_directory
):
6804 fpremode
= int(stat
.S_IFREG
+ 438);
6805 elif(member
.is_directory
):
6806 fpremode
= int(stat
.S_IFDIR
+ 511);
6807 fwinattributes
= int(0);
6808 if(member
.is_directory
):
6809 fmode
= int(stat
.S_IFDIR
+ 511);
6810 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFDIR
+ 511)));
6811 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFDIR
+ 511)));
6813 fmode
= int(stat
.S_IFLNK
+ 438);
6814 fchmode
= int(stat
.S_IMODE(int(stat
.S_IFLNK
+ 438)));
6815 ftypemod
= int(stat
.S_IFMT(int(stat
.S_IFLNK
+ 438)));
6816 returnval
.update({lcfi
: member
.filename
});
6818 VerbosePrintOut(member
.filename
);
6820 permissions
= { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6822 for fmodval
in str(oct(fmode
))[-3:]:
6823 permissionstr
= permissionstr
+ permissions
['access'].get(fmodval
, '---');
6825 if(not member
.is_directory
):
6827 permissionstr
= "-" + permissionstr
;
6828 printfname
= member
.filename
;
6829 elif(member
.is_directory
):
6831 permissionstr
= "d" + permissionstr
;
6832 printfname
= member
.filename
;
6834 fsize
= len(file_content
[member
.filename
].read());
6835 file_content
[member
.filename
].close();
6837 fuid
= int(os
.getuid());
6838 except AttributeError:
6843 fgid
= int(os
.getgid());
6844 except AttributeError:
6851 userinfo
= pwd
.getpwuid(os
.getuid());
6852 funame
= userinfo
.pw_name
;
6855 except AttributeError:
6863 groupinfo
= grp
.getgrgid(os
.getgid());
6864 fgname
= groupinfo
.gr_name
;
6867 except AttributeError:
6872 if(len(fuprint
)<=0):
6873 fuprint
= str(fuid
);
6875 if(len(fgprint
)<=0):
6876 fgprint
= str(fgid
);
6877 VerbosePrintOut(PrintPermissionString(fmode
, ftype
) + " " + str(str(fuprint
) + "/" + str(fgprint
) + " " + str(fsize
).rjust(15) + " " + member
.creationtime
.strftime('%Y-%m-%d %H:%M') + " " + printfname
));
6880 return listcatfiles
['catfp'];
6884 def InFileListFiles(infile
, verbose
=False, formatspecs
=__file_format_list__
, returnfp
=False):
6885 logging
.basicConfig(format
="%(message)s", stream
=sys
.stdout
, level
=logging
.DEBUG
);
6886 checkcompressfile
= CheckCompressionSubType(infile
, formatspecs
, True);
6887 if(checkcompressfile
=="tarfile"):
6888 return TarFileListFiles(infile
, verbose
, returnfp
);
6889 elif(checkcompressfile
=="zipfile"):
6890 return ZipFileListFiles(infile
, verbose
, returnfp
);
6891 elif(checkcompressfile
=="catfile"):
6892 return ArchiveFileListFiles(infile
, 0, 0, False, formatspecs
, verbose
, returnfp
);
6893 elif(rarfile_support
and checkcompressfile
=="rarfile"):
6894 return RarFileListFiles(infile
, verbose
, returnfp
);
6895 elif(py7zr_support
and checkcompressfile
=="7zipfile"):
6896 return SevenZipFileListFiles(infile
, verbose
, returnfp
);
6901 def ListDirListFiles(infiles
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, followlink
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6902 outarray
= BytesIO();
6903 packcat
= PackArchiveFile(infiles
, outarray
, dirlistfromtxt
, compression
, compressionlevel
, followlink
, checksumtype
, formatspecs
, False, True);
6904 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6905 return listcatfiles
;
6907 def ListDirListFilesAlt(infiles
, dirlistfromtxt
=False, followlink
=False, listonly
=False, seekstart
=0, seekend
=0, skipchecksum
=False, checksumtype
="crc32", formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6908 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, listonly
, checksumtype
, formatspecs
, verbose
);
6909 listcatfiles
= ArchiveFileListFiles(outarray
, seekstart
, seekend
, skipchecksum
, formatspecs
, verbose
, returnfp
);
6910 return listcatfiles
;
6912 def PackArchiveFileFromListDirAlt(infiles
, outfile
, dirlistfromtxt
=False, compression
="auto", compressionlevel
=None, followlink
=False, skipchecksum
=False, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6913 outarray
= ListDirToArrayAlt(infiles
, dirlistfromtxt
, followlink
, False, checksumtype
, extradata
, formatspecs
, False);
6914 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compressionlevel
, followlink
, checksumtype
, skipchecksum
, extradata
, formatspecs
, verbose
, returnfp
);
6915 return listcatfiles
;
6917 create_alias_function("Pack", __file_format_name__
, "FromListDirAlt", PackArchiveFileFromListDirAlt
);
6919 def PackArchiveFileFromTarFileAlt(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6920 outarray
= TarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
6921 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
6922 return listcatfiles
;
6924 create_alias_function("Pack", __file_format_name__
, "FromTarFileAlt", PackArchiveFileFromTarFileAlt
);
6926 def PackArchiveFileFromZipFileAlt(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6927 outarray
= ZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
6928 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
6929 return listcatfiles
;
6931 create_alias_function("Pack", __file_format_name__
, "FromZipFileAlt", PackArchiveFileFromZipFileAlt
);
6933 if(not rarfile_support
):
6934 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6937 if(rarfile_support
):
6938 def PackArchiveFileFromRarFileAlt(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6939 outarray
= RarFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
6940 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
6941 return listcatfiles
;
6943 create_alias_function("Pack", __file_format_name__
, "FromRarFileAlt", PackArchiveFileFromRarFileAlt
);
6945 if(not py7zr_support
):
6946 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6950 def PackArchiveFileFromSevenZipFileAlt(infile
, outfile
, compression
="auto", compressionlevel
=None, checksumtype
="crc32", extradata
=[], formatspecs
=__file_format_list__
, verbose
=False, returnfp
=False):
6951 outarray
= SevenZipFileToArrayAlt(infile
, False, checksumtype
, extradata
, formatspecs
, False);
6952 listcatfiles
= RePackArchiveFile(outarray
, outfile
, compression
, compressionlevel
, False, checksumtype
, False, extradata
, formatspecs
, verbose
, returnfp
);
6953 return listcatfiles
;
6955 create_alias_function("Pack", __file_format_name__
, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt
);
6957 def download_file_from_ftp_file(url
):
6958 urlparts
= urlparse(url
);
6959 file_name
= os
.path
.basename(urlparts
.path
);
6960 file_dir
= os
.path
.dirname(urlparts
.path
);
6961 if(urlparts
.username
is not None):
6962 ftp_username
= urlparts
.username
;
6964 ftp_username
= "anonymous";
6965 if(urlparts
.password
is not None):
6966 ftp_password
= urlparts
.password
;
6967 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
6968 ftp_password
= "anonymous";
6971 if(urlparts
.scheme
=="ftp"):
6973 elif(urlparts
.scheme
=="ftps"):
6977 if(urlparts
.scheme
=="sftp"):
6979 return download_file_from_pysftp_file(url
);
6981 return download_file_from_sftp_file(url
);
6982 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
6983 return download_file_from_http_file(url
);
6984 ftp_port
= urlparts
.port
;
6985 if(urlparts
.port
is None):
6988 ftp
.connect(urlparts
.hostname
, ftp_port
);
6989 except socket
.gaierror
:
6990 log
.info("Error With URL "+url
);
6992 except socket
.timeout
:
6993 log
.info("Error With URL "+url
);
6995 ftp
.login(urlparts
.username
, urlparts
.password
);
6996 if(urlparts
.scheme
=="ftps"):
6998 ftpfile
= BytesIO();
6999 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
7000 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7005 def download_file_from_ftp_string(url
):
7006 ftpfile
= download_file_from_ftp_file(url
);
7007 return ftpfile
.read();
7009 def upload_file_to_ftp_file(ftpfile
, url
):
7010 urlparts
= urlparse(url
);
7011 file_name
= os
.path
.basename(urlparts
.path
);
7012 file_dir
= os
.path
.dirname(urlparts
.path
);
7013 if(urlparts
.username
is not None):
7014 ftp_username
= urlparts
.username
;
7016 ftp_username
= "anonymous";
7017 if(urlparts
.password
is not None):
7018 ftp_password
= urlparts
.password
;
7019 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7020 ftp_password
= "anonymous";
7023 if(urlparts
.scheme
=="ftp"):
7025 elif(urlparts
.scheme
=="ftps"):
7029 if(urlparts
.scheme
=="sftp"):
7031 return upload_file_to_pysftp_file(url
);
7033 return upload_file_to_sftp_file(url
);
7034 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7036 ftp_port
= urlparts
.port
;
7037 if(urlparts
.port
is None):
7040 ftp
.connect(urlparts
.hostname
, ftp_port
);
7041 except socket
.gaierror
:
7042 log
.info("Error With URL "+url
);
7044 except socket
.timeout
:
7045 log
.info("Error With URL "+url
);
7047 ftp
.login(urlparts
.username
, urlparts
.password
);
7048 if(urlparts
.scheme
=="ftps"):
7050 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
7055 def upload_file_to_ftp_string(ftpstring
, url
):
7056 ftpfileo
= BytesIO(ftpstring
);
7057 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
7061 def download_file_from_http_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7062 # Parse the URL to extract username and password if present
7063 urlparts
= urlparse(url
);
7064 username
= urlparts
.username
;
7065 password
= urlparts
.password
;
7066 # Rebuild the URL without the username and password
7067 netloc
= urlparts
.hostname
;
7068 if(urlparts
.scheme
=="sftp"):
7070 return download_file_from_pysftp_file(url
);
7072 return download_file_from_sftp_file(url
);
7073 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7074 return download_file_from_ftp_file(url
);
7076 netloc
+= ':' + str(urlparts
.port
);
7077 rebuilt_url
= urlunparse((urlparts
.scheme
, netloc
, urlparts
.path
, urlparts
.params
, urlparts
.query
, urlparts
.fragment
));
7078 # Create a temporary file object
7079 httpfile
= BytesIO();
7081 # Use the requests library if available
7082 if username
and password
:
7083 response
= requests
.get(rebuilt_url
, headers
=headers
, auth
=(username
, password
), stream
=True);
7085 response
= requests
.get(rebuilt_url
, headers
=headers
, stream
=True);
7086 response
.raw
.decode_content
= True
7087 shutil
.copyfileobj(response
.raw
, httpfile
);
7089 # Build a Request object for urllib
7090 request
= Request(rebuilt_url
, headers
=headers
);
7091 # Create an opener object for handling URLs
7092 if username
and password
:
7093 # Create a password manager
7094 password_mgr
= HTTPPasswordMgrWithDefaultRealm();
7095 # Add the username and password
7096 password_mgr
.add_password(None, rebuilt_url
, username
, password
);
7097 # Create an authentication handler using the password manager
7098 auth_handler
= HTTPBasicAuthHandler(password_mgr
);
7099 # Build the opener with the authentication handler
7100 opener
= build_opener(auth_handler
);
7102 opener
= build_opener();
7103 with opener
.open(request
) as response
:
7104 shutil
.copyfileobj(response
, httpfile
);
7105 # Reset file pointer to the start
7106 httpfile
.seek(0, 0);
7107 # Return the temporary file object
7110 def download_file_from_http_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7111 httpfile
= download_file_from_http_file(url
, headers
);
7112 return ftpfile
.read();
7115 def download_file_from_sftp_file(url
):
7116 urlparts
= urlparse(url
);
7117 file_name
= os
.path
.basename(urlparts
.path
);
7118 file_dir
= os
.path
.dirname(urlparts
.path
);
7119 sftp_port
= urlparts
.port
;
7120 if(urlparts
.port
is None):
7123 sftp_port
= urlparts
.port
;
7124 if(urlparts
.username
is not None):
7125 sftp_username
= urlparts
.username
;
7127 sftp_username
= "anonymous";
7128 if(urlparts
.password
is not None):
7129 sftp_password
= urlparts
.password
;
7130 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7131 sftp_password
= "anonymous";
7134 if(urlparts
.scheme
=="ftp"):
7135 return download_file_from_ftp_file(url
);
7136 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7137 return download_file_from_http_file(url
);
7138 if(urlparts
.scheme
!="sftp"):
7140 ssh
= paramiko
.SSHClient();
7141 ssh
.load_system_host_keys();
7142 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7144 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7145 except paramiko
.ssh_exception
.SSHException
:
7147 except socket
.gaierror
:
7148 log
.info("Error With URL "+url
);
7150 except socket
.timeout
:
7151 log
.info("Error With URL "+url
);
7153 sftp
= ssh
.open_sftp();
7154 sftpfile
= BytesIO();
7155 sftp
.getfo(urlparts
.path
, sftpfile
);
7158 sftpfile
.seek(0, 0);
7161 def download_file_from_sftp_file(url
):
7165 def download_file_from_sftp_string(url
):
7166 sftpfile
= download_file_from_sftp_file(url
);
7167 return sftpfile
.read();
7169 def download_file_from_ftp_string(url
):
7173 def upload_file_to_sftp_file(sftpfile
, url
):
7174 urlparts
= urlparse(url
);
7175 file_name
= os
.path
.basename(urlparts
.path
);
7176 file_dir
= os
.path
.dirname(urlparts
.path
);
7177 sftp_port
= urlparts
.port
;
7178 if(urlparts
.port
is None):
7181 sftp_port
= urlparts
.port
;
7182 if(urlparts
.username
is not None):
7183 sftp_username
= urlparts
.username
;
7185 sftp_username
= "anonymous";
7186 if(urlparts
.password
is not None):
7187 sftp_password
= urlparts
.password
;
7188 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7189 sftp_password
= "anonymous";
7192 if(urlparts
.scheme
=="ftp"):
7193 return upload_file_to_ftp_file(url
);
7194 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7196 if(urlparts
.scheme
!="sftp"):
7198 ssh
= paramiko
.SSHClient();
7199 ssh
.load_system_host_keys();
7200 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
7202 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7203 except paramiko
.ssh_exception
.SSHException
:
7205 except socket
.gaierror
:
7206 log
.info("Error With URL "+url
);
7208 except socket
.timeout
:
7209 log
.info("Error With URL "+url
);
7211 sftp
= ssh
.open_sftp();
7212 sftp
.putfo(sftpfile
, urlparts
.path
);
7215 sftpfile
.seek(0, 0);
7218 def upload_file_to_sftp_file(sftpfile
, url
):
7222 def upload_file_to_sftp_string(sftpstring
, url
):
7223 sftpfileo
= BytesIO(sftpstring
);
7224 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
7228 def upload_file_to_sftp_string(url
):
7232 def download_file_from_pysftp_file(url
):
7233 urlparts
= urlparse(url
);
7234 file_name
= os
.path
.basename(urlparts
.path
);
7235 file_dir
= os
.path
.dirname(urlparts
.path
);
7236 sftp_port
= urlparts
.port
;
7237 if(urlparts
.port
is None):
7240 sftp_port
= urlparts
.port
;
7241 if(urlparts
.username
is not None):
7242 sftp_username
= urlparts
.username
;
7244 sftp_username
= "anonymous";
7245 if(urlparts
.password
is not None):
7246 sftp_password
= urlparts
.password
;
7247 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7248 sftp_password
= "anonymous";
7251 if(urlparts
.scheme
=="ftp"):
7252 return download_file_from_ftp_file(url
);
7253 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7254 return download_file_from_http_file(url
);
7255 if(urlparts
.scheme
!="sftp"):
7258 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7259 except paramiko
.ssh_exception
.SSHException
:
7261 except socket
.gaierror
:
7262 log
.info("Error With URL "+url
);
7264 except socket
.timeout
:
7265 log
.info("Error With URL "+url
);
7267 sftp
= ssh
.open_sftp();
7268 sftpfile
= BytesIO();
7269 sftp
.getfo(urlparts
.path
, sftpfile
);
7272 sftpfile
.seek(0, 0);
7275 def download_file_from_pysftp_file(url
):
7279 def download_file_from_pysftp_string(url
):
7280 sftpfile
= download_file_from_pysftp_file(url
);
7281 return sftpfile
.read();
7283 def download_file_from_ftp_string(url
):
7287 def upload_file_to_pysftp_file(sftpfile
, url
):
7288 urlparts
= urlparse(url
);
7289 file_name
= os
.path
.basename(urlparts
.path
);
7290 file_dir
= os
.path
.dirname(urlparts
.path
);
7291 sftp_port
= urlparts
.port
;
7292 if(urlparts
.port
is None):
7295 sftp_port
= urlparts
.port
;
7296 if(urlparts
.username
is not None):
7297 sftp_username
= urlparts
.username
;
7299 sftp_username
= "anonymous";
7300 if(urlparts
.password
is not None):
7301 sftp_password
= urlparts
.password
;
7302 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
7303 sftp_password
= "anonymous";
7306 if(urlparts
.scheme
=="ftp"):
7307 return upload_file_to_ftp_file(url
);
7308 elif(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7310 if(urlparts
.scheme
!="sftp"):
7313 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
7314 except paramiko
.ssh_exception
.SSHException
:
7316 except socket
.gaierror
:
7317 log
.info("Error With URL "+url
);
7319 except socket
.timeout
:
7320 log
.info("Error With URL "+url
);
7322 sftp
= ssh
.open_sftp();
7323 sftp
.putfo(sftpfile
, urlparts
.path
);
7326 sftpfile
.seek(0, 0);
7329 def upload_file_to_pysftp_file(sftpfile
, url
):
7333 def upload_file_to_pysftp_string(sftpstring
, url
):
7334 sftpfileo
= BytesIO(sftpstring
);
7335 sftpfile
= upload_file_to_pysftp_files(ftpfileo
, url
);
7339 def upload_file_to_pysftp_string(url
):
7342 def download_file_from_internet_file(url
, headers
=geturls_headers_pycatfile_python_alt
):
7343 urlparts
= urlparse(url
);
7344 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7345 return download_file_from_http_file(url
, headers
);
7346 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7347 return download_file_from_ftp_file(url
);
7348 elif(urlparts
.scheme
=="sftp"):
7349 if(__use_pysftp__
and havepysftp
):
7350 return download_file_from_pysftp_file(url
);
7352 return download_file_from_sftp_file(url
);
7357 def download_file_from_internet_uncompress_file(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_list__
):
7358 fp
= download_file_from_internet_file(url
);
7359 fp
= UncompressArchiveFile(fp
, formatspecs
);
7365 def download_file_from_internet_string(url
, headers
=geturls_headers_pycatfile_python_alt
):
7366 urlparts
= urlparse(url
);
7367 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7368 return download_file_from_http_string(url
, headers
);
7369 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7370 return download_file_from_ftp_string(url
);
7371 elif(urlparts
.scheme
=="sftp"):
7372 if(__use_pysftp__
and havepysftp
):
7373 return download_file_from_pysftp_string(url
);
7375 return download_file_from_sftp_string(url
);
7380 def download_file_from_internet_uncompress_string(url
, headers
=geturls_headers_pycatfile_python_alt
, formatspecs
=__file_format_list__
):
7381 fp
= download_file_from_internet_string(url
);
7382 fp
= UncompressArchiveFile(fp
, formatspecs
);
7388 def upload_file_to_internet_file(ifp
, url
):
7389 urlparts
= urlparse(url
);
7390 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7392 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7393 return upload_file_to_ftp_file(ifp
, url
);
7394 elif(urlparts
.scheme
=="sftp"):
7395 if(__use_pysftp__
and havepysftp
):
7396 return upload_file_to_pysftp_file(ifp
, url
);
7398 return upload_file_to_sftp_file(ifp
, url
);
7403 def upload_file_to_internet_compress_file(ifp
, url
, formatspecs
=__file_format_list__
):
7404 catfp
= CompressArchiveFile(catfp
, compression
, formatspecs
);
7408 upload_file_to_internet_file(catfp
, outfile
);
7411 def upload_file_to_internet_string(ifp
, url
):
7412 urlparts
= urlparse(url
);
7413 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
7415 elif(urlparts
.scheme
=="ftp" or urlparts
.scheme
=="ftps"):
7416 return upload_file_to_ftp_string(ifp
, url
);
7417 elif(urlparts
.scheme
=="sftp"):
7418 if(__use_pysftp__
and havepysftp
):
7419 return upload_file_to_pysftp_string(ifp
, url
);
7421 return upload_file_to_sftp_string(ifp
, url
);
7426 def upload_file_to_internet_compress_string(ifp
, url
, formatspecs
=__file_format_list__
):
7427 catfp
= CompressArchiveFile(BytesIO(ifp
), compression
, formatspecs
);
7431 upload_file_to_internet_file(catfp
, outfile
);
7435 if(hasattr(shutil
, "register_archive_format")):
7436 # Register the packing format
7437 shutil
.register_archive_format(__file_format_name__
, PackArchiveFileFunc
, description
='Pack concatenated files');
7438 except shutil
.RegistryError
:
7442 if(hasattr(shutil
, "register_unpack_format")):
7443 # Register the unpacking format
7444 shutil
.register_unpack_format(__file_format_name__
, archivefile_extensions
, UnPackArchiveFileFunc
, description
='UnPack concatenated files');
7445 except shutil
.RegistryError
: