4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the Revised BSD License.
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 Revised BSD License for more details.
12 Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
13 Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
15 $FileInfo: pywwwgetold.py - Last Update: 10/5/2023 Ver. 2.0.2 RC 1 - Author: cooldude2k $
18 from __future__
import division
, absolute_import
, print_function
;
19 import re
, os
, sys
, hashlib
, shutil
, platform
, tempfile
, urllib
, zlib
, time
, argparse
, subprocess
, socket
, email
.utils
, datetime
, time
;
20 import logging
as log
;
21 from ftplib
import FTP
, FTP_TLS
;
22 from base64
import b64encode
;
24 from cgi
import parse_qsl
;
26 from urlparse
import parse_qsl
;
33 havemechanize
= False;
38 havemechanize
= False;
66 from httplib2
import HTTPConnectionWithTimeout
, HTTPSConnectionWithTimeout
;
106 if(sys
.version
[0]=="2"):
108 from io
import StringIO
, BytesIO
;
111 from cStringIO
import StringIO
;
112 from cStringIO
import StringIO
as BytesIO
;
114 from StringIO
import StringIO
;
115 from StringIO
import StringIO
as BytesIO
;
116 # From http://python-future.org/compatible_idioms.html
117 from urlparse
import urlparse
, urlunparse
, urlsplit
, urlunsplit
, urljoin
;
118 from urllib
import urlencode
;
119 from urllib
import urlopen
as urlopenalt
;
120 from urllib2
import urlopen
, Request
, install_opener
, HTTPError
, URLError
, build_opener
, HTTPCookieProcessor
;
121 import urlparse
, cookielib
;
122 from httplib
import HTTPConnection
, HTTPSConnection
;
123 if(sys
.version
[0]>="3"):
124 from io
import StringIO
, BytesIO
;
125 # From http://python-future.org/compatible_idioms.html
126 from urllib
.parse
import urlparse
, urlunparse
, urlsplit
, urlunsplit
, urljoin
, urlencode
;
127 from urllib
.request
import urlopen
, Request
, install_opener
, build_opener
, HTTPCookieProcessor
;
128 from urllib
.error
import HTTPError
, URLError
;
129 import urllib
.parse
as urlparse
;
130 import http
.cookiejar
as cookielib
;
131 from http
.client
import HTTPConnection
, HTTPSConnection
;
133 __program_name__
= "PyWWW-Get";
134 __program_alt_name__
= "PyWWWGet";
135 __program_small_name__
= "wwwget";
136 __project__
= __program_name__
;
137 __project_url__
= "https://github.com/GameMaker2k/PyWWW-Get";
138 __version_info__
= (2, 0, 2, "RC 1", 1);
139 __version_date_info__
= (2023, 10, 5, "RC 1", 1);
140 __version_date__
= str(__version_date_info__
[0])+"."+str(__version_date_info__
[1]).zfill(2)+"."+str(__version_date_info__
[2]).zfill(2);
141 __revision__
= __version_info__
[3];
142 __revision_id__
= "$Id$";
143 if(__version_info__
[4] is not None):
144 __version_date_plusrc__
= __version_date__
+"-"+str(__version_date_info__
[4]);
145 if(__version_info__
[4] is None):
146 __version_date_plusrc__
= __version_date__
;
147 if(__version_info__
[3] is not None):
148 __version__
= str(__version_info__
[0])+"."+str(__version_info__
[1])+"."+str(__version_info__
[2])+" "+str(__version_info__
[3]);
149 if(__version_info__
[3] is None):
150 __version__
= str(__version_info__
[0])+"."+str(__version_info__
[1])+"."+str(__version_info__
[2]);
152 tmpfileprefix
= "py"+str(sys
.version_info
[0])+__program_small_name__
+str(__version_info__
[0])+"-";
154 pytempdir
= tempfile
.gettempdir();
156 PyBitness
= platform
.architecture();
157 if(PyBitness
=="32bit" or PyBitness
=="32"):
159 elif(PyBitness
=="64bit" or PyBitness
=="64"):
164 compression_supported_list
= ['identity', 'gzip', 'deflate', 'bzip2'];
166 compression_supported_list
.append('br');
168 compression_supported_list
.append('zstd');
170 compression_supported_list
.append('lzma');
171 compression_supported_list
.append('xz');
172 compression_supported
= ', '.join(compression_supported_list
);
174 geturls_cj
= cookielib
.CookieJar();
175 windowsNT4_ua_string
= "Windows NT 4.0";
176 windowsNT4_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
177 windows2k_ua_string
= "Windows NT 5.0";
178 windows2k_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
179 windowsXP_ua_string
= "Windows NT 5.1";
180 windowsXP_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
181 windowsXP64_ua_string
= "Windows NT 5.2; Win64; x64";
182 windowsXP64_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
183 windows7_ua_string
= "Windows NT 6.1; Win64; x64";
184 windows7_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
185 windows8_ua_string
= "Windows NT 6.2; Win64; x64";
186 windows8_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
187 windows81_ua_string
= "Windows NT 6.3; Win64; x64";
188 windows81_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
189 windows10_ua_string
= "Windows NT 10.0; Win64; x64";
190 windows10_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
191 windows11_ua_string
= "Windows NT 11.0; Win64; x64";
192 windows11_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
193 geturls_ua_firefox_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:109.0) Gecko/20100101 Firefox/117.0";
194 geturls_ua_seamonkey_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
195 geturls_ua_chrome_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
196 geturls_ua_chromium_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
197 geturls_ua_palemoon_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
198 geturls_ua_opera_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
199 geturls_ua_vivaldi_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
200 geturls_ua_internet_explorer_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; Trident/7.0; rv:11.0) like Gecko";
201 geturls_ua_microsoft_edge_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
202 geturls_ua_pywwwget_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
203 if(platform
.python_implementation()!=""):
204 py_implementation
= platform
.python_implementation();
205 if(platform
.python_implementation()==""):
206 py_implementation
= "Python";
207 geturls_ua_pywwwget_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
208 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
209 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
210 geturls_ua
= geturls_ua_firefox_windows7
;
211 geturls_headers_firefox_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
212 geturls_headers_seamonkey_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 geturls_headers_chrome_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
214 geturls_headers_chrome_windows7
.update(windows7_ua_addon
);
215 geturls_headers_chromium_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
216 geturls_headers_chromium_windows7
.update(windows7_ua_addon
);
217 geturls_headers_palemoon_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
218 geturls_headers_opera_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
219 geturls_headers_opera_windows7
.update(windows7_ua_addon
);
220 geturls_headers_vivaldi_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
221 geturls_headers_vivaldi_windows7
.update(windows7_ua_addon
);
222 geturls_headers_internet_explorer_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
223 geturls_headers_microsoft_edge_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
224 geturls_headers_microsoft_edge_windows7
.update(windows7_ua_addon
);
225 geturls_headers_pywwwget_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
226 geturls_headers_pywwwget_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
227 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
228 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
229 geturls_headers
= geturls_headers_firefox_windows7
;
230 geturls_download_sleep
= 0;
232 def verbose_printout(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
233 if(outtype
=="print" and dbgenable
):
236 elif(outtype
=="log" and dbgenable
):
237 logging
.info(dbgtxt
);
239 elif(outtype
=="warning" and dbgenable
):
240 logging
.warning(dbgtxt
);
242 elif(outtype
=="error" and dbgenable
):
243 logging
.error(dbgtxt
);
245 elif(outtype
=="critical" and dbgenable
):
246 logging
.critical(dbgtxt
);
248 elif(outtype
=="exception" and dbgenable
):
249 logging
.exception(dbgtxt
);
251 elif(outtype
=="logalt" and dbgenable
):
252 logging
.log(dgblevel
, dbgtxt
);
254 elif(outtype
=="debug" and dbgenable
):
255 logging
.debug(dbgtxt
);
263 def verbose_printout_return(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
264 dbgout
= verbose_printout(dbgtxt
, outtype
, dbgenable
, dgblevel
);
269 def add_url_param(url
, **params
):
271 parts
= list(urlparse
.urlsplit(url
));
272 d
= dict(parse_qsl(parts
[n
])); # use cgi.parse_qs for list values
274 parts
[n
]=urlencode(d
);
275 return urlparse
.urlunsplit(parts
);
277 os
.environ
["PATH"] = os
.environ
["PATH"] + os
.pathsep
+ os
.path
.dirname(os
.path
.realpath(__file__
)) + os
.pathsep
+ os
.getcwd();
278 def which_exec(execfile):
279 for path
in os
.environ
["PATH"].split(":"):
280 if os
.path
.exists(path
+ "/" + execfile):
281 return path
+ "/" + execfile;
283 def listize(varlist
):
291 newlistreg
.update({ilx
: varlist
[il
]});
292 newlistrev
.update({varlist
[il
]: ilx
});
295 newlistfull
= {1: newlistreg
, 2: newlistrev
, 'reg': newlistreg
, 'rev': newlistrev
};
298 def twolistize(varlist
):
308 newlistnamereg
.update({ilx
: varlist
[il
][0].strip()});
309 newlistnamerev
.update({varlist
[il
][0].strip(): ilx
});
310 newlistdescreg
.update({ilx
: varlist
[il
][1].strip()});
311 newlistdescrev
.update({varlist
[il
][1].strip(): ilx
});
314 newlistnametmp
= {1: newlistnamereg
, 2: newlistnamerev
, 'reg': newlistnamereg
, 'rev': newlistnamerev
};
315 newlistdesctmp
= {1: newlistdescreg
, 2: newlistdescrev
, 'reg': newlistdescreg
, 'rev': newlistdescrev
};
316 newlistfull
= {1: newlistnametmp
, 2: newlistdesctmp
, 'name': newlistnametmp
, 'desc': newlistdesctmp
}
319 def arglistize(proexec
, *varlist
):
323 newarglist
= [proexec
];
325 if varlist
[il
][0] is not None:
326 newarglist
.append(varlist
[il
][0]);
327 if varlist
[il
][1] is not None:
328 newarglist
.append(varlist
[il
][1]);
332 def fix_header_names(header_dict
):
333 if(sys
.version
[0]=="2"):
334 header_dict
= {k
.title(): v
for k
, v
in header_dict
.iteritems()};
335 if(sys
.version
[0]>="3"):
336 header_dict
= {k
.title(): v
for k
, v
in header_dict
.items()};
339 # hms_string by ArcGIS Python Recipes
340 # https://arcpy.wordpress.com/2012/04/20/146/
341 def hms_string(sec_elapsed
):
342 h
= int(sec_elapsed
/ (60 * 60));
343 m
= int((sec_elapsed
% (60 * 60)) / 60);
344 s
= sec_elapsed
% 60.0;
345 return "{}:{:>02}:{:>05.2f}".format(h
, m
, s
);
347 # get_readable_size by Lipis
348 # http://stackoverflow.com/posts/14998888/revisions
349 def get_readable_size(bytes
, precision
=1, unit
="IEC"):
351 if(unit
!="IEC" and unit
!="SI"):
354 units
= [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
355 unitswos
= ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
358 units
= [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
359 unitswos
= ["B","kB","MB","GB","TB","PB","EB","ZB"];
364 if abs(bytes
) < unitsize
:
365 strformat
= "%3."+str(precision
)+"f%s";
366 pre_return_val
= (strformat
% (bytes
, unit
));
367 pre_return_val
= re
.sub(r
"([0]+) ([A-Za-z]+)", r
" \2", pre_return_val
);
368 pre_return_val
= re
.sub(r
"\. ([A-Za-z]+)", r
" \1", pre_return_val
);
369 alt_return_val
= pre_return_val
.split();
370 return_val
= {'Bytes': orgbytes
, 'ReadableWithSuffix': pre_return_val
, 'ReadableWithoutSuffix': alt_return_val
[0], 'ReadableSuffix': alt_return_val
[1]}
373 strformat
= "%."+str(precision
)+"f%s";
374 pre_return_val
= (strformat
% (bytes
, "YiB"));
375 pre_return_val
= re
.sub(r
"([0]+) ([A-Za-z]+)", r
" \2", pre_return_val
);
376 pre_return_val
= re
.sub(r
"\. ([A-Za-z]+)", r
" \1", pre_return_val
);
377 alt_return_val
= pre_return_val
.split();
378 return_val
= {'Bytes': orgbytes
, 'ReadableWithSuffix': pre_return_val
, 'ReadableWithoutSuffix': alt_return_val
[0], 'ReadableSuffix': alt_return_val
[1]}
381 def get_readable_size_from_file(infile
, precision
=1, unit
="IEC", usehashes
=False, usehashtypes
="md5,sha1"):
383 usehashtypes
= usehashtypes
.lower();
384 getfilesize
= os
.path
.getsize(infile
);
385 return_val
= get_readable_size(getfilesize
, precision
, unit
);
387 hashtypelist
= usehashtypes
.split(",");
388 openfile
= open(infile
, "rb");
389 filecontents
= openfile
.read();
392 listnumend
= len(hashtypelist
);
393 while(listnumcount
< listnumend
):
394 hashtypelistlow
= hashtypelist
[listnumcount
].strip();
395 hashtypelistup
= hashtypelistlow
.upper();
396 filehash
= hashlib
.new(hashtypelistup
);
397 filehash
.update(filecontents
);
398 filegethash
= filehash
.hexdigest();
399 return_val
.update({hashtypelistup
: filegethash
});
403 def get_readable_size_from_string(instring
, precision
=1, unit
="IEC", usehashes
=False, usehashtypes
="md5,sha1"):
405 usehashtypes
= usehashtypes
.lower();
406 getfilesize
= len(instring
);
407 return_val
= get_readable_size(getfilesize
, precision
, unit
);
409 hashtypelist
= usehashtypes
.split(",");
411 listnumend
= len(hashtypelist
);
412 while(listnumcount
< listnumend
):
413 hashtypelistlow
= hashtypelist
[listnumcount
].strip();
414 hashtypelistup
= hashtypelistlow
.upper();
415 filehash
= hashlib
.new(hashtypelistup
);
416 if(sys
.version
[0]=="2"):
417 filehash
.update(instring
);
418 if(sys
.version
[0]>="3"):
419 filehash
.update(instring
.encode('utf-8'));
420 filegethash
= filehash
.hexdigest();
421 return_val
.update({hashtypelistup
: filegethash
});
425 def http_status_to_reason(code
):
428 101: 'Switching Protocols',
433 203: 'Non-Authoritative Information',
435 205: 'Reset Content',
436 206: 'Partial Content',
438 208: 'Already Reported',
440 300: 'Multiple Choices',
441 301: 'Moved Permanently',
446 307: 'Temporary Redirect',
447 308: 'Permanent Redirect',
450 402: 'Payment Required',
453 405: 'Method Not Allowed',
454 406: 'Not Acceptable',
455 407: 'Proxy Authentication Required',
456 408: 'Request Timeout',
459 411: 'Length Required',
460 412: 'Precondition Failed',
461 413: 'Payload Too Large',
463 415: 'Unsupported Media Type',
464 416: 'Range Not Satisfiable',
465 417: 'Expectation Failed',
466 421: 'Misdirected Request',
467 422: 'Unprocessable Entity',
469 424: 'Failed Dependency',
470 426: 'Upgrade Required',
471 428: 'Precondition Required',
472 429: 'Too Many Requests',
473 431: 'Request Header Fields Too Large',
474 451: 'Unavailable For Legal Reasons',
475 500: 'Internal Server Error',
476 501: 'Not Implemented',
478 503: 'Service Unavailable',
479 504: 'Gateway Timeout',
480 505: 'HTTP Version Not Supported',
481 506: 'Variant Also Negotiates',
482 507: 'Insufficient Storage',
483 508: 'Loop Detected',
485 511: 'Network Authentication Required'
487 return reasons
.get(code
, 'Unknown Status Code');
489 def ftp_status_to_reason(code
):
491 110: 'Restart marker reply',
492 120: 'Service ready in nnn minutes',
493 125: 'Data connection already open; transfer starting',
494 150: 'File status okay; about to open data connection',
496 202: 'Command not implemented, superfluous at this site',
497 211: 'System status, or system help reply',
498 212: 'Directory status',
501 215: 'NAME system type',
502 220: 'Service ready for new user',
503 221: 'Service closing control connection',
504 225: 'Data connection open; no transfer in progress',
505 226: 'Closing data connection',
506 227: 'Entering Passive Mode',
507 230: 'User logged in, proceed',
508 250: 'Requested file action okay, completed',
509 257: '"PATHNAME" created',
510 331: 'User name okay, need password',
511 332: 'Need account for login',
512 350: 'Requested file action pending further information',
513 421: 'Service not available, closing control connection',
514 425: 'Can\'t open data connection',
515 426: 'Connection closed; transfer aborted',
516 450: 'Requested file action not taken',
517 451: 'Requested action aborted. Local error in processing',
518 452: 'Requested action not taken. Insufficient storage space in system',
519 500: 'Syntax error, command unrecognized',
520 501: 'Syntax error in parameters or arguments',
521 502: 'Command not implemented',
522 503: 'Bad sequence of commands',
523 504: 'Command not implemented for that parameter',
524 530: 'Not logged in',
525 532: 'Need account for storing files',
526 550: 'Requested action not taken. File unavailable',
527 551: 'Requested action aborted. Page type unknown',
528 552: 'Requested file action aborted. Exceeded storage allocation',
529 553: 'Requested action not taken. File name not allowed'
531 return reasons
.get(code
, 'Unknown Status Code');
533 def sftp_status_to_reason(code
):
537 2: 'SSH_FX_NO_SUCH_FILE',
538 3: 'SSH_FX_PERMISSION_DENIED',
540 5: 'SSH_FX_BAD_MESSAGE',
541 6: 'SSH_FX_NO_CONNECTION',
542 7: 'SSH_FX_CONNECTION_LOST',
543 8: 'SSH_FX_OP_UNSUPPORTED'
545 return reasons
.get(code
, 'Unknown Status Code');
547 def make_http_headers_from_dict_to_list(headers
={'Referer': "http://google.com/", 'User-Agent': geturls_ua
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
548 if isinstance(headers
, dict):
550 if(sys
.version
[0]=="2"):
551 for headkey
, headvalue
in headers
.iteritems():
552 returnval
.append((headkey
, headvalue
));
553 if(sys
.version
[0]>="3"):
554 for headkey
, headvalue
in headers
.items():
555 returnval
.append((headkey
, headvalue
));
556 elif isinstance(headers
, list):
562 def make_http_headers_from_dict_to_pycurl(headers
={'Referer': "http://google.com/", 'User-Agent': geturls_ua
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
563 if isinstance(headers
, dict):
565 if(sys
.version
[0]=="2"):
566 for headkey
, headvalue
in headers
.iteritems():
567 returnval
.append(headkey
+": "+headvalue
);
568 if(sys
.version
[0]>="3"):
569 for headkey
, headvalue
in headers
.items():
570 returnval
.append(headkey
+": "+headvalue
);
571 elif isinstance(headers
, list):
577 def make_http_headers_from_pycurl_to_dict(headers
):
579 headers
= headers
.strip().split('\r\n');
580 for header
in headers
:
581 parts
= header
.split(': ', 1)
584 header_dict
[key
.title()] = value
;
587 def make_http_headers_from_list_to_dict(headers
=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua
), ("Accept-Encoding", compression_supported
), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
588 if isinstance(headers
, list):
593 returnval
.update({headers
[mli
][0]: headers
[mli
][1]});
595 elif isinstance(headers
, dict):
601 def get_httplib_support(checkvalue
=None):
602 global haverequests
, havemechanize
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
604 returnval
.append("ftp");
605 returnval
.append("httplib");
607 returnval
.append("httplib2");
608 returnval
.append("urllib");
610 returnval
.append("urllib3");
611 returnval
.append("request3");
612 returnval
.append("request");
614 returnval
.append("requests");
616 returnval
.append("aiohttp");
618 returnval
.append("httpx");
619 returnval
.append("httpx2");
621 returnval
.append("mechanize");
623 returnval
.append("pycurl");
624 if(hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
625 returnval
.append("pycurl2");
626 if(hasattr(pycurl
, "CURL_HTTP_VERSION_3_0")):
627 returnval
.append("pycurl3");
629 returnval
.append("sftp");
631 returnval
.append("pysftp");
632 if(not checkvalue
is None):
633 if(checkvalue
=="urllib1" or checkvalue
=="urllib2"):
634 checkvalue
= "urllib";
635 if(checkvalue
=="httplib1"):
636 checkvalue
= "httplib";
637 if(checkvalue
in returnval
):
643 def check_httplib_support(checkvalue
="urllib"):
644 if(checkvalue
=="urllib1" or checkvalue
=="urllib2"):
645 checkvalue
= "urllib";
646 if(checkvalue
=="httplib1"):
647 checkvalue
= "httplib";
648 returnval
= get_httplib_support(checkvalue
);
651 def get_httplib_support_list():
652 returnval
= get_httplib_support(None);
655 def download_from_url(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", buffersize
=524288, sleep
=-1, timeout
=10):
656 global geturls_download_sleep
, havezstd
, havebrotli
, haveaiohttp
, haverequests
, havemechanize
, havepycurl
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
658 sleep
= geturls_download_sleep
;
661 if(httplibuse
=="urllib1" or httplibuse
=="urllib2" or httplibuse
=="request"):
662 httplibuse
= "urllib";
663 if(httplibuse
=="httplib1"):
664 httplibuse
= "httplib";
665 if(not haverequests
and httplibuse
=="requests"):
666 httplibuse
= "urllib";
667 if(not haveaiohttp
and httplibuse
=="aiohttp"):
668 httplibuse
= "urllib";
669 if(not havehttpx
and httplibuse
=="httpx"):
670 httplibuse
= "urllib";
671 if(not havehttpx
and httplibuse
=="httpx2"):
672 httplibuse
= "urllib";
673 if(not havehttpcore
and httplibuse
=="httpcore"):
674 httplibuse
= "urllib";
675 if(not havehttpcore
and httplibuse
=="httpcore2"):
676 httplibuse
= "urllib";
677 if(not havemechanize
and httplibuse
=="mechanize"):
678 httplibuse
= "urllib";
679 if(not havepycurl
and httplibuse
=="pycurl"):
680 httplibuse
= "urllib";
681 if(not havepycurl
and httplibuse
=="pycurl2"):
682 httplibuse
= "urllib";
683 if(havepycurl
and httplibuse
=="pycurl2" and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
684 httplibuse
= "pycurl";
685 if(not havepycurl
and httplibuse
=="pycurl3"):
686 httplibuse
= "urllib";
687 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
688 httplibuse
= "pycurl2";
689 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
690 httplibuse
= "pycurl";
691 if(not havehttplib2
and httplibuse
=="httplib2"):
692 httplibuse
= "httplib";
693 if(not haveparamiko
and httplibuse
=="sftp"):
695 if(not havepysftp
and httplibuse
=="pysftp"):
697 if(httplibuse
=="urllib" or httplibuse
=="request"):
698 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
699 elif(httplibuse
=="request"):
700 returnval
= download_from_url_with_request(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
701 elif(httplibuse
=="request3"):
702 returnval
= download_from_url_with_request3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
703 elif(httplibuse
=="httplib"):
704 returnval
= download_from_url_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
705 elif(httplibuse
=="httplib2"):
706 returnval
= download_from_url_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
707 elif(httplibuse
=="urllib3" or httplibuse
=="request3"):
708 returnval
= download_from_url_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
709 elif(httplibuse
=="requests"):
710 returnval
= download_from_url_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
711 elif(httplibuse
=="aiohttp"):
712 returnval
= download_from_url_with_aiohttp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
713 elif(httplibuse
=="httpx"):
714 returnval
= download_from_url_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
715 elif(httplibuse
=="httpx2"):
716 returnval
= download_from_url_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
717 elif(httplibuse
=="httpcore"):
718 returnval
= download_from_url_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
719 elif(httplibuse
=="httpcore2"):
720 returnval
= download_from_url_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
721 elif(httplibuse
=="mechanize"):
722 returnval
= download_from_url_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
723 elif(httplibuse
=="pycurl"):
724 returnval
= download_from_url_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
725 elif(httplibuse
=="pycurl2"):
726 returnval
= download_from_url_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
727 elif(httplibuse
=="pycurl3"):
728 returnval
= download_from_url_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
729 elif(httplibuse
=="ftp"):
730 returnval
= download_from_url_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
731 elif(httplibuse
=="sftp"):
732 returnval
= download_from_url_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
733 elif(httplibuse
=="pysftp"):
734 returnval
= download_from_url_with_pysftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
739 def download_from_url_from_list(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", buffersize
=524288, sleep
=-1, timeout
=10):
740 if(isinstance(httpurl
, list)):
742 elif(isinstance(httpurl
, tuple)):
744 elif(isinstance(httpurl
, dict)):
745 httpurl
= httpurl
.values();
748 listsize
= len(httpurl
);
751 while(listcount
<listsize
):
752 ouputval
= download_from_url(httpurl
[listcount
], httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, httplibuse
, buffersize
, sleep
, timeout
);
753 returnval
.append(ouputval
);
757 def download_from_url_file(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
758 global geturls_download_sleep
, havezstd
, havebrotli
, haveaiohttp
, haverequests
, havemechanize
, havepycurl
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
760 sleep
= geturls_download_sleep
;
763 if(httplibuse
=="urllib1" or httplibuse
=="urllib2" or httplibuse
=="request"):
764 httplibuse
= "urllib";
765 if(httplibuse
=="httplib1"):
766 httplibuse
= "httplib";
767 if(not haverequests
and httplibuse
=="requests"):
768 httplibuse
= "urllib";
769 if(not haveaiohttp
and httplibuse
=="aiohttp"):
770 httplibuse
= "urllib";
771 if(not havehttpx
and httplibuse
=="httpx"):
772 httplibuse
= "urllib";
773 if(not havehttpx
and httplibuse
=="httpx2"):
774 httplibuse
= "urllib";
775 if(not havehttpcore
and httplibuse
=="httpcore"):
776 httplibuse
= "urllib";
777 if(not havehttpcore
and httplibuse
=="httpcore2"):
778 httplibuse
= "urllib";
779 if(not havemechanize
and httplibuse
=="mechanize"):
780 httplibuse
= "urllib";
781 if(not havepycurl
and httplibuse
=="pycurl"):
782 httplibuse
= "urllib";
783 if(not havepycurl
and httplibuse
=="pycurl2"):
784 httplibuse
= "urllib";
785 if(havepycurl
and httplibuse
=="pycurl2" and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
786 httplibuse
= "pycurl";
787 if(not havepycurl
and httplibuse
=="pycurl3"):
788 httplibuse
= "urllib";
789 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
790 httplibuse
= "pycurl2";
791 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
792 httplibuse
= "pycurl";
793 if(not havehttplib2
and httplibuse
=="httplib2"):
794 httplibuse
= "httplib";
795 if(not haveparamiko
and httplibuse
=="sftp"):
797 if(not haveparamiko
and httplibuse
=="pysftp"):
799 if(httplibuse
=="urllib" or httplibuse
=="request"):
800 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
801 elif(httplibuse
=="request"):
802 returnval
= download_from_url_file_with_request(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
803 elif(httplibuse
=="request3"):
804 returnval
= download_from_url_file_with_request3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
805 elif(httplibuse
=="httplib"):
806 returnval
= download_from_url_file_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
807 elif(httplibuse
=="httplib2"):
808 returnval
= download_from_url_file_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
809 elif(httplibuse
=="urllib3" or httplibuse
=="request3"):
810 returnval
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
811 elif(httplibuse
=="requests"):
812 returnval
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
813 elif(httplibuse
=="aiohttp"):
814 returnval
= download_from_url_file_with_aiohttp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
815 elif(httplibuse
=="httpx"):
816 returnval
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
817 elif(httplibuse
=="httpx2"):
818 returnval
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
819 elif(httplibuse
=="httpcore"):
820 returnval
= download_from_url_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
821 elif(httplibuse
=="httpcore2"):
822 returnval
= download_from_url_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
823 elif(httplibuse
=="mechanize"):
824 returnval
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
825 elif(httplibuse
=="pycurl"):
826 returnval
= download_from_url_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
827 elif(httplibuse
=="pycurl2"):
828 returnval
= download_from_url_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
829 elif(httplibuse
=="pycurl3"):
830 returnval
= download_from_url_file_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
831 elif(httplibuse
=="ftp"):
832 returnval
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
833 elif(httplibuse
=="sftp"):
834 returnval
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
835 elif(httplibuse
=="pysftp"):
836 returnval
= download_from_url_file_with_pysftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
841 def download_from_url_file_with_list(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
842 if(isinstance(httpurl
, list)):
844 elif(isinstance(httpurl
, tuple)):
846 elif(isinstance(httpurl
, dict)):
847 httpurl
= httpurl
.values();
850 listsize
= len(httpurl
);
853 while(listcount
<listsize
):
854 ouputval
= download_from_url_file(httpurl
[listcount
], httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, httplibuse
, ranges
, buffersize
, sleep
, timeout
);
855 returnval
.append(ouputval
);
859 def download_from_url_to_file(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
860 global geturls_download_sleep
, havezstd
, havebrotli
, haveaiohttp
, haverequests
, havemechanize
, havepycurl
, havehttplib2
, haveurllib3
, havehttpx
, havehttpcore
, haveparamiko
, havepysftp
;
862 sleep
= geturls_download_sleep
;
865 if(httplibuse
=="urllib1" or httplibuse
=="urllib2" or httplibuse
=="request"):
866 httplibuse
= "urllib";
867 if(httplibuse
=="httplib1"):
868 httplibuse
= "httplib";
869 if(not haverequests
and httplibuse
=="requests"):
870 httplibuse
= "urllib";
871 if(not haveaiohttp
and httplibuse
=="aiohttp"):
872 httplibuse
= "urllib";
873 if(not havehttpx
and httplibuse
=="httpx"):
874 httplibuse
= "urllib";
875 if(not havehttpx
and httplibuse
=="httpx2"):
876 httplibuse
= "urllib";
877 if(not havehttpcore
and httplibuse
=="httpcore"):
878 httplibuse
= "urllib";
879 if(not havehttpcore
and httplibuse
=="httpcore2"):
880 httplibuse
= "urllib";
881 if(not havemechanize
and httplibuse
=="mechanize"):
882 httplibuse
= "urllib";
883 if(not havepycurl
and httplibuse
=="pycurl"):
884 httplibuse
= "urllib";
885 if(not havepycurl
and httplibuse
=="pycurl2"):
886 httplibuse
= "urllib";
887 if(havepycurl
and httplibuse
=="pycurl2" and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
888 httplibuse
= "pycurl";
889 if(not havepycurl
and httplibuse
=="pycurl3"):
890 httplibuse
= "urllib";
891 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
892 httplibuse
= "pycurl2";
893 if(havepycurl
and httplibuse
=="pycurl3" and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
894 httplibuse
= "pycurl";
895 if(not havehttplib2
and httplibuse
=="httplib2"):
896 httplibuse
= "httplib";
897 if(not haveparamiko
and httplibuse
=="sftp"):
899 if(not havepysftp
and httplibuse
=="pysftp"):
901 if(httplibuse
=="urllib" or httplibuse
=="request"):
902 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
903 elif(httplibuse
=="request"):
904 returnval
= download_from_url_to_file_with_request(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
905 elif(httplibuse
=="request3"):
906 returnval
= download_from_url_to_file_with_request3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
907 elif(httplibuse
=="httplib"):
908 returnval
= download_from_url_to_file_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
909 elif(httplibuse
=="httplib2"):
910 returnval
= download_from_url_to_file_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
911 elif(httplibuse
=="urllib3" or httplibuse
=="request3"):
912 returnval
= download_from_url_to_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
913 elif(httplibuse
=="requests"):
914 returnval
= download_from_url_to_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
915 elif(httplibuse
=="aiohttp"):
916 returnval
= download_from_url_to_file_with_aiohttp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
917 elif(httplibuse
=="httpx"):
918 returnval
= download_from_url_to_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
919 elif(httplibuse
=="httpx2"):
920 returnval
= download_from_url_to_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
921 elif(httplibuse
=="httpcore"):
922 returnval
= download_from_url_to_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
923 elif(httplibuse
=="httpcore2"):
924 returnval
= download_from_url_to_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
);
925 elif(httplibuse
=="mechanize"):
926 returnval
= download_from_url_to_file_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
927 elif(httplibuse
=="pycurl"):
928 returnval
= download_from_url_to_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
929 elif(httplibuse
=="pycurl2"):
930 returnval
= download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
931 elif(httplibuse
=="pycurl3"):
932 returnval
= download_from_url_to_file_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
933 elif(httplibuse
=="ftp"):
934 returnval
= download_from_url_to_file_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
935 elif(httplibuse
=="sftp"):
936 returnval
= download_from_url_to_file_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
937 elif(httplibuse
=="pysftp"):
938 returnval
= download_from_url_to_file_with_pysftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
943 def download_from_url_to_file_with_list(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
944 if(isinstance(httpurl
, list)):
946 elif(isinstance(httpurl
, tuple)):
948 elif(isinstance(httpurl
, dict)):
949 httpurl
= httpurl
.values();
952 listsize
= len(httpurl
);
955 while(listcount
<listsize
):
956 ouputval
= download_from_url_to_file(httpurl
[listcount
], httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, httplibuse
, outfile
, outpath
, ranges
, buffersize
, sleep
, timeout
);
957 returnval
.append(ouputval
);
961 def download_from_url_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
962 global geturls_download_sleep
, havezstd
, havebrotli
;
964 sleep
= geturls_download_sleep
;
967 urlparts
= urlparse
.urlparse(httpurl
);
968 if(isinstance(httpheaders
, list)):
969 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
970 httpheaders
= fix_header_names(httpheaders
);
971 if(httpuseragent
is not None):
972 if('User-Agent' in httpheaders
):
973 httpheaders
['User-Agent'] = httpuseragent
;
975 httpuseragent
.update({'User-Agent': httpuseragent
});
976 if(httpreferer
is not None):
977 if('Referer' in httpheaders
):
978 httpheaders
['Referer'] = httpreferer
;
980 httpuseragent
.update({'Referer': httpreferer
});
981 if(urlparts
.username
is not None or urlparts
.password
is not None):
982 if(sys
.version
[0]=="2"):
983 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
984 if(sys
.version
[0]>="3"):
985 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
986 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
987 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
988 if(isinstance(httpheaders
, dict)):
989 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
990 geturls_opener
.addheaders
= httpheaders
;
992 if(postdata
is not None and not isinstance(postdata
, dict)):
993 postdata
= urlencode(postdata
);
995 geturls_request
= Request(httpurl
);
996 if(httpmethod
=="GET"):
997 geturls_text
= geturls_opener
.open(geturls_request
);
998 elif(httpmethod
=="POST"):
999 geturls_text
= geturls_opener
.open(geturls_request
, data
=postdata
);
1001 geturls_text
= geturls_opener
.open(geturls_request
);
1002 except HTTPError
as geturls_text_error
:
1003 geturls_text
= geturls_text_error
;
1004 log
.info("Error With URL "+httpurl
);
1006 log
.info("Error With URL "+httpurl
);
1008 except socket
.timeout
:
1009 log
.info("Error With URL "+httpurl
);
1011 httpcodeout
= geturls_text
.getcode();
1013 httpcodereason
= geturls_text
.reason
;
1014 except AttributeError:
1015 httpcodereason
= http_status_to_reason(geturls_text
.getcode());
1017 httpversionout
= geturls_text
.version
;
1018 except AttributeError:
1019 httpversionout
= "1.1";
1020 httpmethodout
= geturls_request
.get_method();
1021 httpurlout
= geturls_text
.geturl();
1022 httpheaderout
= geturls_text
.info();
1023 httpheadersentout
= httpheaders
;
1024 if(isinstance(httpheaderout
, list)):
1025 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
1026 httpheaderout
= fix_header_names(httpheaderout
);
1027 if(sys
.version
[0]=="2"):
1029 prehttpheaderout
= httpheaderout
;
1030 httpheaderkeys
= httpheaderout
.keys();
1031 imax
= len(httpheaderkeys
);
1035 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
1037 except AttributeError:
1039 if(isinstance(httpheadersentout
, list)):
1040 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
1041 httpheadersentout
= fix_header_names(httpheadersentout
);
1042 downloadsize
= httpheaderout
.get('Content-Length');
1043 if(downloadsize
is not None):
1044 downloadsize
= int(downloadsize
);
1045 if downloadsize
is None: downloadsize
= 0;
1048 log
.info("Downloading URL "+httpurl
);
1049 with
BytesIO() as strbuf
:
1051 databytes
= geturls_text
.read(buffersize
);
1052 if not databytes
: break;
1053 datasize
= len(databytes
);
1054 fulldatasize
= datasize
+ fulldatasize
;
1057 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1058 downloaddiff
= fulldatasize
- prevdownsize
;
1059 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1060 prevdownsize
= fulldatasize
;
1061 strbuf
.write(databytes
);
1063 returnval_content
= strbuf
.read();
1064 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1066 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1069 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1071 returnval_content
= zlib
.decompress(returnval_content
);
1074 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1076 returnval_content
= brotli
.decompress(returnval_content
);
1077 except brotli
.error
:
1079 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1081 returnval_content
= zstandard
.decompress(returnval_content
);
1082 except zstandard
.error
:
1084 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
1086 returnval_content
= lzma
.decompress(returnval_content
);
1087 except zstandard
.error
:
1089 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
1091 returnval_content
= bz2
.decompress(returnval_content
);
1092 except zstandard
.error
:
1094 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "urllib"};
1095 geturls_text
.close();
1098 def download_from_url_file_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1099 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
1100 exec_time_start
= time
.time();
1101 myhash
= hashlib
.new("sha1");
1102 if(sys
.version
[0]=="2"):
1103 myhash
.update(httpurl
);
1104 myhash
.update(str(buffersize
));
1105 myhash
.update(str(exec_time_start
));
1106 if(sys
.version
[0]>="3"):
1107 myhash
.update(httpurl
.encode('utf-8'));
1108 myhash
.update(str(buffersize
).encode('utf-8'));
1109 myhash
.update(str(exec_time_start
).encode('utf-8'));
1110 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1112 sleep
= geturls_download_sleep
;
1115 pretmpfilename
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
1116 if(not pretmpfilename
):
1118 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1119 tmpfilename
= f
.name
;
1121 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1122 except AttributeError:
1124 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1129 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1130 f
.write(pretmpfilename
.get('Content'));
1132 exec_time_end
= time
.time();
1133 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1134 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1137 def download_from_url_to_file_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1138 global geturls_download_sleep
, havezstd
, havebrotli
;
1140 sleep
= geturls_download_sleep
;
1143 if(not outfile
=="-"):
1144 outpath
= outpath
.rstrip(os
.path
.sep
);
1145 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1146 if(not os
.path
.exists(outpath
)):
1147 os
.makedirs(outpath
);
1148 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1150 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1152 pretmpfilename
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1153 if(not pretmpfilename
):
1155 tmpfilename
= pretmpfilename
.get('Filename');
1156 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1158 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1159 exec_time_start
= time
.time();
1160 shutil
.move(tmpfilename
, filepath
);
1162 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1163 except AttributeError:
1165 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1170 exec_time_end
= time
.time();
1171 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1172 if(os
.path
.exists(tmpfilename
)):
1173 os
.remove(tmpfilename
);
1174 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1176 pretmpfilename
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1177 tmpfilename
= pretmpfilename
.get('Filename');
1178 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1181 exec_time_start
= time
.time();
1182 with
open(tmpfilename
, 'rb') as ft
:
1185 databytes
= ft
.read(buffersize
[1]);
1186 if not databytes
: break;
1187 datasize
= len(databytes
);
1188 fulldatasize
= datasize
+ fulldatasize
;
1191 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1192 downloaddiff
= fulldatasize
- prevdownsize
;
1193 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1194 prevdownsize
= fulldatasize
;
1197 fdata
= f
.getvalue();
1200 os
.remove(tmpfilename
);
1201 exec_time_end
= time
.time();
1202 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1203 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1206 def download_from_url_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1207 global geturls_download_sleep
, havezstd
, havebrotli
, havezstd
, havebrotli
;
1209 sleep
= geturls_download_sleep
;
1212 urlparts
= urlparse
.urlparse(httpurl
);
1213 if(isinstance(httpheaders
, list)):
1214 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1215 httpheaders
= fix_header_names(httpheaders
);
1216 if(httpuseragent
is not None):
1217 if('User-Agent' in httpheaders
):
1218 httpheaders
['User-Agent'] = httpuseragent
;
1220 httpuseragent
.update({'User-Agent': httpuseragent
});
1221 if(httpreferer
is not None):
1222 if('Referer' in httpheaders
):
1223 httpheaders
['Referer'] = httpreferer
;
1225 httpuseragent
.update({'Referer': httpreferer
});
1226 if(urlparts
.username
is not None or urlparts
.password
is not None):
1227 if(sys
.version
[0]=="2"):
1228 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
1229 if(sys
.version
[0]>="3"):
1230 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1231 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1232 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
1233 geturls_opener
.addheaders
= httpheaders
;
1235 if(urlparts
[0]=="http"):
1236 httpconn
= HTTPConnection(urlparts
[1], timeout
=timeout
);
1237 elif(urlparts
[0]=="https"):
1238 httpconn
= HTTPSConnection(urlparts
[1], timeout
=timeout
);
1241 if(postdata
is not None and not isinstance(postdata
, dict)):
1242 postdata
= urlencode(postdata
);
1244 if(httpmethod
=="GET"):
1245 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1246 elif(httpmethod
=="POST"):
1247 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
1249 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1250 except socket
.timeout
:
1251 log
.info("Error With URL "+httpurl
);
1253 except socket
.gaierror
:
1254 log
.info("Error With URL "+httpurl
);
1256 except BlockingIOError
:
1257 log
.info("Error With URL "+httpurl
);
1259 geturls_text
= httpconn
.getresponse();
1260 httpcodeout
= geturls_text
.status
;
1261 httpcodereason
= geturls_text
.reason
;
1262 if(geturls_text
.version
=="10"):
1263 httpversionout
= "1.0";
1265 httpversionout
= "1.1";
1266 httpmethodout
= geturls_text
._method
;
1267 httpurlout
= httpurl
;
1268 httpheaderout
= geturls_text
.getheaders();
1269 httpheadersentout
= httpheaders
;
1270 if(isinstance(httpheaderout
, list)):
1271 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
1272 if(sys
.version
[0]=="2"):
1274 prehttpheaderout
= httpheaderout
;
1275 httpheaderkeys
= httpheaderout
.keys();
1276 imax
= len(httpheaderkeys
);
1280 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
1282 except AttributeError:
1284 httpheaderout
= fix_header_names(httpheaderout
);
1285 if(isinstance(httpheadersentout
, list)):
1286 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
1287 httpheadersentout
= fix_header_names(httpheadersentout
);
1288 downloadsize
= httpheaderout
.get('Content-Length');
1289 if(downloadsize
is not None):
1290 downloadsize
= int(downloadsize
);
1291 if downloadsize
is None: downloadsize
= 0;
1294 log
.info("Downloading URL "+httpurl
);
1295 with
BytesIO() as strbuf
:
1297 databytes
= geturls_text
.read(buffersize
);
1298 if not databytes
: break;
1299 datasize
= len(databytes
);
1300 fulldatasize
= datasize
+ fulldatasize
;
1303 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1304 downloaddiff
= fulldatasize
- prevdownsize
;
1305 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1306 prevdownsize
= fulldatasize
;
1307 strbuf
.write(databytes
);
1309 returnval_content
= strbuf
.read();
1310 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1312 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1315 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1317 returnval_content
= zlib
.decompress(returnval_content
);
1320 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1322 returnval_content
= brotli
.decompress(returnval_content
);
1323 except brotli
.error
:
1325 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1327 returnval_content
= zstandard
.decompress(returnval_content
);
1328 except zstandard
.error
:
1330 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
1332 returnval_content
= lzma
.decompress(returnval_content
);
1333 except zstandard
.error
:
1335 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
1337 returnval_content
= bz2
.decompress(returnval_content
);
1338 except zstandard
.error
:
1340 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "httplib"};
1341 geturls_text
.close();
1344 def download_from_url_file_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1345 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
1346 exec_time_start
= time
.time();
1347 myhash
= hashlib
.new("sha1");
1348 if(sys
.version
[0]=="2"):
1349 myhash
.update(httpurl
);
1350 myhash
.update(str(buffersize
));
1351 myhash
.update(str(exec_time_start
));
1352 if(sys
.version
[0]>="3"):
1353 myhash
.update(httpurl
.encode('utf-8'));
1354 myhash
.update(str(buffersize
).encode('utf-8'));
1355 myhash
.update(str(exec_time_start
).encode('utf-8'));
1356 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1358 sleep
= geturls_download_sleep
;
1361 pretmpfilename
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
1362 if(not pretmpfilename
):
1364 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1365 tmpfilename
= f
.name
;
1367 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1368 except AttributeError:
1370 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1375 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1376 f
.write(pretmpfilename
.get('Content'));
1378 exec_time_end
= time
.time();
1379 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1380 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1383 def download_from_url_to_file_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1384 global geturls_download_sleep
, havezstd
, havebrotli
;
1386 sleep
= geturls_download_sleep
;
1389 if(not outfile
=="-"):
1390 outpath
= outpath
.rstrip(os
.path
.sep
);
1391 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1392 if(not os
.path
.exists(outpath
)):
1393 os
.makedirs(outpath
);
1394 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1396 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1398 pretmpfilename
= download_from_url_file_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1399 if(not pretmpfilename
):
1401 tmpfilename
= pretmpfilename
.get('Filename');
1402 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1404 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1405 exec_time_start
= time
.time();
1406 shutil
.move(tmpfilename
, filepath
);
1408 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1409 except AttributeError:
1411 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1416 exec_time_end
= time
.time();
1417 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1418 if(os
.path
.exists(tmpfilename
)):
1419 os
.remove(tmpfilename
);
1420 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1422 pretmpfilename
= download_from_url_file_with_httplib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1423 tmpfilename
= pretmpfilename
.get('Filename');
1424 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1427 exec_time_start
= time
.time();
1428 with
open(tmpfilename
, 'rb') as ft
:
1431 databytes
= ft
.read(buffersize
[1]);
1432 if not databytes
: break;
1433 datasize
= len(databytes
);
1434 fulldatasize
= datasize
+ fulldatasize
;
1437 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1438 downloaddiff
= fulldatasize
- prevdownsize
;
1439 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1440 prevdownsize
= fulldatasize
;
1443 fdata
= f
.getvalue();
1446 os
.remove(tmpfilename
);
1447 exec_time_end
= time
.time();
1448 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1449 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1453 def download_from_url_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1454 global geturls_download_sleep
, havezstd
, havebrotli
;
1456 sleep
= geturls_download_sleep
;
1459 urlparts
= urlparse
.urlparse(httpurl
);
1460 if(isinstance(httpheaders
, list)):
1461 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1462 httpheaders
= fix_header_names(httpheaders
);
1463 if(httpuseragent
is not None):
1464 if('User-Agent' in httpheaders
):
1465 httpheaders
['User-Agent'] = httpuseragent
;
1467 httpuseragent
.update({'User-Agent': httpuseragent
});
1468 if(httpreferer
is not None):
1469 if('Referer' in httpheaders
):
1470 httpheaders
['Referer'] = httpreferer
;
1472 httpuseragent
.update({'Referer': httpreferer
});
1473 if(urlparts
.username
is not None or urlparts
.password
is not None):
1474 if(sys
.version
[0]=="2"):
1475 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
1476 if(sys
.version
[0]>="3"):
1477 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1478 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1479 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
1480 geturls_opener
.addheaders
= httpheaders
;
1482 if(urlparts
[0]=="http"):
1483 httpconn
= HTTPConnectionWithTimeout(urlparts
[1], timeout
=timeout
);
1484 elif(urlparts
[0]=="https"):
1485 httpconn
= HTTPSConnectionWithTimeout(urlparts
[1], timeout
=timeout
);
1488 if(postdata
is not None and not isinstance(postdata
, dict)):
1489 postdata
= urlencode(postdata
);
1491 if(httpmethod
=="GET"):
1492 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1493 elif(httpmethod
=="POST"):
1494 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
1496 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1497 except socket
.timeout
:
1498 log
.info("Error With URL "+httpurl
);
1500 except socket
.gaierror
:
1501 log
.info("Error With URL "+httpurl
);
1503 except BlockingIOError
:
1504 log
.info("Error With URL "+httpurl
);
1506 geturls_text
= httpconn
.getresponse();
1507 httpcodeout
= geturls_text
.status
;
1508 httpcodereason
= geturls_text
.reason
;
1509 if(geturls_text
.version
=="10"):
1510 httpversionout
= "1.0";
1512 httpversionout
= "1.1";
1513 httpmethodout
= httpmethod
;
1514 httpurlout
= httpurl
;
1515 httpheaderout
= geturls_text
.getheaders();
1516 httpheadersentout
= httpheaders
;
1517 if(isinstance(httpheaderout
, list)):
1518 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
1519 if(sys
.version
[0]=="2"):
1521 prehttpheaderout
= httpheaderout
;
1522 httpheaderkeys
= httpheaderout
.keys();
1523 imax
= len(httpheaderkeys
);
1527 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
1529 except AttributeError:
1531 httpheaderout
= fix_header_names(httpheaderout
);
1532 if(isinstance(httpheadersentout
, list)):
1533 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
1534 httpheadersentout
= fix_header_names(httpheadersentout
);
1535 downloadsize
= httpheaderout
.get('Content-Length');
1536 if(downloadsize
is not None):
1537 downloadsize
= int(downloadsize
);
1538 if downloadsize
is None: downloadsize
= 0;
1541 log
.info("Downloading URL "+httpurl
);
1542 with
BytesIO() as strbuf
:
1544 databytes
= geturls_text
.read(buffersize
);
1545 if not databytes
: break;
1546 datasize
= len(databytes
);
1547 fulldatasize
= datasize
+ fulldatasize
;
1550 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1551 downloaddiff
= fulldatasize
- prevdownsize
;
1552 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1553 prevdownsize
= fulldatasize
;
1554 strbuf
.write(databytes
);
1556 returnval_content
= strbuf
.read();
1557 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1559 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1562 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1564 returnval_content
= zlib
.decompress(returnval_content
);
1567 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1569 returnval_content
= brotli
.decompress(returnval_content
);
1570 except brotli
.error
:
1572 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1574 returnval_content
= zstandard
.decompress(returnval_content
);
1575 except zstandard
.error
:
1577 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
1579 returnval_content
= lzma
.decompress(returnval_content
);
1580 except zstandard
.error
:
1582 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
1584 returnval_content
= bz2
.decompress(returnval_content
);
1585 except zstandard
.error
:
1587 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "httplib2"};
1588 geturls_text
.close();
1591 if(not havehttplib2
):
1592 def download_from_url_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1593 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
1597 def download_from_url_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1598 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
1599 exec_time_start
= time
.time();
1600 myhash
= hashlib
.new("sha1");
1601 if(sys
.version
[0]=="2"):
1602 myhash
.update(httpurl
);
1603 myhash
.update(str(buffersize
));
1604 myhash
.update(str(exec_time_start
));
1605 if(sys
.version
[0]>="3"):
1606 myhash
.update(httpurl
.encode('utf-8'));
1607 myhash
.update(str(buffersize
).encode('utf-8'));
1608 myhash
.update(str(exec_time_start
).encode('utf-8'));
1609 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1611 sleep
= geturls_download_sleep
;
1614 pretmpfilename
= download_from_url_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
1615 if(not pretmpfilename
):
1617 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1618 tmpfilename
= f
.name
;
1620 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1621 except AttributeError:
1623 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1628 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1629 f
.write(pretmpfilename
.get('Content'));
1631 exec_time_end
= time
.time();
1632 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1633 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1636 if(not havehttplib2
):
1637 def download_from_url_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1638 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
1642 def download_from_url_to_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1643 global geturls_download_sleep
, havezstd
, havebrotli
;
1645 sleep
= geturls_download_sleep
;
1648 if(not outfile
=="-"):
1649 outpath
= outpath
.rstrip(os
.path
.sep
);
1650 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1651 if(not os
.path
.exists(outpath
)):
1652 os
.makedirs(outpath
);
1653 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1655 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1657 pretmpfilename
= download_from_url_file_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1658 if(not pretmpfilename
):
1660 tmpfilename
= pretmpfilename
.get('Filename');
1661 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1663 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1664 exec_time_start
= time
.time();
1665 shutil
.move(tmpfilename
, filepath
);
1667 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1668 except AttributeError:
1670 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1675 exec_time_end
= time
.time();
1676 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1677 if(os
.path
.exists(tmpfilename
)):
1678 os
.remove(tmpfilename
);
1679 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1681 pretmpfilename
= download_from_url_file_with_httplib2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1682 tmpfilename
= pretmpfilename
.get('Filename');
1683 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1686 exec_time_start
= time
.time();
1687 with
open(tmpfilename
, 'rb') as ft
:
1690 databytes
= ft
.read(buffersize
[1]);
1691 if not databytes
: break;
1692 datasize
= len(databytes
);
1693 fulldatasize
= datasize
+ fulldatasize
;
1696 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1697 downloaddiff
= fulldatasize
- prevdownsize
;
1698 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1699 prevdownsize
= fulldatasize
;
1702 fdata
= f
.getvalue();
1705 os
.remove(tmpfilename
);
1706 exec_time_end
= time
.time();
1707 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1708 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1711 if(not havehttplib2
):
1712 def download_from_url_to_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1713 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
1716 def download_from_url_with_request(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1717 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
1720 def download_from_url_file_with_request(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1721 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
1724 def download_from_url_to_file_with_request(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1725 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
1729 def download_from_url_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1730 global geturls_download_sleep
, havezstd
, havebrotli
;
1732 sleep
= geturls_download_sleep
;
1735 urlparts
= urlparse
.urlparse(httpurl
);
1736 if(isinstance(httpheaders
, list)):
1737 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1738 httpheaders
= fix_header_names(httpheaders
);
1739 if(httpuseragent
is not None):
1740 if('User-Agent' in httpheaders
):
1741 httpheaders
['User-Agent'] = httpuseragent
;
1743 httpuseragent
.update({'User-Agent': httpuseragent
});
1744 if(httpreferer
is not None):
1745 if('Referer' in httpheaders
):
1746 httpheaders
['Referer'] = httpreferer
;
1748 httpuseragent
.update({'Referer': httpreferer
});
1749 if(urlparts
.username
is not None or urlparts
.password
is not None):
1750 if(sys
.version
[0]=="2"):
1751 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
1752 if(sys
.version
[0]>="3"):
1753 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1754 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1756 if(postdata
is not None and not isinstance(postdata
, dict)):
1757 postdata
= urlencode(postdata
);
1759 reqsession
= requests
.Session();
1760 if(httpmethod
=="GET"):
1761 geturls_text
= reqsession
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
, stream
=True);
1762 elif(httpmethod
=="POST"):
1763 geturls_text
= reqsession
.post(httpurl
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
, stream
=True);
1765 geturls_text
= reqsession
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
, stream
=True);
1766 except requests
.exceptions
.ConnectTimeout
:
1767 log
.info("Error With URL "+httpurl
);
1769 except requests
.exceptions
.ConnectError
:
1770 log
.info("Error With URL "+httpurl
);
1772 except socket
.timeout
:
1773 log
.info("Error With URL "+httpurl
);
1775 httpcodeout
= geturls_text
.status_code
;
1776 httpcodereason
= geturls_text
.reason
;
1777 if(geturls_text
.raw
.version
=="10"):
1778 httpversionout
= "1.0";
1780 httpversionout
= "1.1";
1781 httpmethodout
= httpmethod
;
1782 httpurlout
= geturls_text
.url
;
1783 httpheaderout
= geturls_text
.headers
;
1784 httpheadersentout
= geturls_text
.request
.headers
;
1785 if(isinstance(httpheaderout
, list)):
1786 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
1787 if(sys
.version
[0]=="2"):
1789 prehttpheaderout
= httpheaderout
;
1790 httpheaderkeys
= httpheaderout
.keys();
1791 imax
= len(httpheaderkeys
);
1795 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
1797 except AttributeError:
1799 httpheaderout
= fix_header_names(httpheaderout
);
1800 if(isinstance(httpheadersentout
, list)):
1801 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
1802 httpheadersentout
= fix_header_names(httpheadersentout
);
1803 downloadsize
= httpheaderout
.get('Content-Length');
1804 if(downloadsize
is not None):
1805 downloadsize
= int(downloadsize
);
1806 if downloadsize
is None: downloadsize
= 0;
1809 log
.info("Downloading URL "+httpurl
);
1810 with
BytesIO() as strbuf
:
1812 databytes
= geturls_text
.raw
.read(buffersize
);
1813 if not databytes
: break;
1814 datasize
= len(databytes
);
1815 fulldatasize
= datasize
+ fulldatasize
;
1818 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1819 downloaddiff
= fulldatasize
- prevdownsize
;
1820 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1821 prevdownsize
= fulldatasize
;
1822 strbuf
.write(databytes
);
1824 returnval_content
= strbuf
.read();
1825 if(httpheaderout
.get("Content-Encoding")=="gzip"):
1827 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
1830 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
1832 returnval_content
= zlib
.decompress(returnval_content
);
1835 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
1837 returnval_content
= brotli
.decompress(returnval_content
);
1838 except brotli
.error
:
1840 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
1842 returnval_content
= zstandard
.decompress(returnval_content
);
1843 except zstandard
.error
:
1845 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
1847 returnval_content
= lzma
.decompress(returnval_content
);
1848 except zstandard
.error
:
1850 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
1852 returnval_content
= bz2
.decompress(returnval_content
);
1853 except zstandard
.error
:
1855 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "requests"};
1856 geturls_text
.close();
1859 if(not haverequests
):
1860 def download_from_url_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1861 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
1865 def download_from_url_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1866 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
1867 exec_time_start
= time
.time();
1868 myhash
= hashlib
.new("sha1");
1869 if(sys
.version
[0]=="2"):
1870 myhash
.update(httpurl
);
1871 myhash
.update(str(buffersize
));
1872 myhash
.update(str(exec_time_start
));
1873 if(sys
.version
[0]>="3"):
1874 myhash
.update(httpurl
.encode('utf-8'));
1875 myhash
.update(str(buffersize
).encode('utf-8'));
1876 myhash
.update(str(exec_time_start
).encode('utf-8'));
1877 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1879 sleep
= geturls_download_sleep
;
1882 pretmpfilename
= download_from_url_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
1883 if(not pretmpfilename
):
1885 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1886 tmpfilename
= f
.name
;
1888 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1889 except AttributeError:
1891 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1896 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1897 f
.write(pretmpfilename
.get('Content'));
1899 exec_time_end
= time
.time();
1900 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1901 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1904 if(not haverequests
):
1905 def download_from_url_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
1906 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
1910 def download_from_url_to_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1911 global geturls_download_sleep
, havezstd
, havebrotli
;
1913 sleep
= geturls_download_sleep
;
1916 if(not outfile
=="-"):
1917 outpath
= outpath
.rstrip(os
.path
.sep
);
1918 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1919 if(not os
.path
.exists(outpath
)):
1920 os
.makedirs(outpath
);
1921 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1923 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1925 pretmpfilename
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1926 if(not pretmpfilename
):
1928 tmpfilename
= pretmpfilename
.get('Filename');
1929 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1931 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1932 exec_time_start
= time
.time();
1933 shutil
.move(tmpfilename
, filepath
);
1935 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
1936 except AttributeError:
1938 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1943 exec_time_end
= time
.time();
1944 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1945 if(os
.path
.exists(tmpfilename
)):
1946 os
.remove(tmpfilename
);
1947 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1949 pretmpfilename
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
1950 tmpfilename
= pretmpfilename
.get('Filename');
1951 downloadsize
= int(os
.path
.getsize(tmpfilename
));
1954 exec_time_start
= time
.time();
1955 with
open(tmpfilename
, 'rb') as ft
:
1958 databytes
= ft
.read(buffersize
[1]);
1959 if not databytes
: break;
1960 datasize
= len(databytes
);
1961 fulldatasize
= datasize
+ fulldatasize
;
1964 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1965 downloaddiff
= fulldatasize
- prevdownsize
;
1966 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1967 prevdownsize
= fulldatasize
;
1970 fdata
= f
.getvalue();
1973 os
.remove(tmpfilename
);
1974 exec_time_end
= time
.time();
1975 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1976 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
1979 if(not haverequests
):
1980 def download_from_url_to_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
1981 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
1985 def download_from_url_with_aiohttp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
1986 global geturls_download_sleep
, havezstd
, havebrotli
;
1988 sleep
= geturls_download_sleep
;
1991 urlparts
= urlparse
.urlparse(httpurl
);
1992 if(isinstance(httpheaders
, list)):
1993 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1994 httpheaders
= fix_header_names(httpheaders
);
1995 if(httpuseragent
is not None):
1996 if('User-Agent' in httpheaders
):
1997 httpheaders
['User-Agent'] = httpuseragent
;
1999 httpuseragent
.update({'User-Agent': httpuseragent
});
2000 if(httpreferer
is not None):
2001 if('Referer' in httpheaders
):
2002 httpheaders
['Referer'] = httpreferer
;
2004 httpuseragent
.update({'Referer': httpreferer
});
2005 if(urlparts
.username
is not None or urlparts
.password
is not None):
2006 if(sys
.version
[0]=="2"):
2007 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
2008 if(sys
.version
[0]>="3"):
2009 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2010 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2012 if(postdata
is not None and not isinstance(postdata
, dict)):
2013 postdata
= urlencode(postdata
);
2015 reqsession
= aiohttp
.ClientSession(cookie_jar
=httpcookie
, headers
=httpheaders
, timeout
=timeout
, read_timeout
=timeout
, conn_timeout
=timeout
, read_bufsize
=buffersize
);
2016 if(httpmethod
=="GET"):
2017 geturls_text
= reqsession
.get(httpurl
);
2018 elif(httpmethod
=="POST"):
2019 geturls_text
= reqsession
.post(httpurl
, data
=postdata
);
2021 geturls_text
= reqsession
.get(httpurl
);
2022 except aiohttp
.exceptions
.ConnectTimeout
:
2023 log
.info("Error With URL "+httpurl
);
2025 except aiohttp
.exceptions
.ConnectError
:
2026 log
.info("Error With URL "+httpurl
);
2028 except socket
.timeout
:
2029 log
.info("Error With URL "+httpurl
);
2031 httpcodeout
= geturls_text
.status
;
2032 httpcodereason
= geturls_text
.reason
;
2033 httpversionout
= geturls_text
.version
;
2034 httpmethodout
= geturls_text
.method
;
2035 httpurlout
= geturls_text
.url
;
2036 httpheaderout
= geturls_text
.headers
;
2037 httpheadersentout
= geturls_text
.request_info
.headers
;
2038 if(isinstance(httpheaderout
, list)):
2039 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
2040 if(sys
.version
[0]=="2"):
2042 prehttpheaderout
= httpheaderout
;
2043 httpheaderkeys
= httpheaderout
.keys();
2044 imax
= len(httpheaderkeys
);
2048 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
2050 except AttributeError:
2052 httpheaderout
= fix_header_names(httpheaderout
);
2053 if(isinstance(httpheadersentout
, list)):
2054 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
2055 httpheadersentout
= fix_header_names(httpheadersentout
);
2056 downloadsize
= httpheaderout
.get('Content-Length');
2057 if(downloadsize
is not None):
2058 downloadsize
= int(downloadsize
);
2059 if downloadsize
is None: downloadsize
= 0;
2062 log
.info("Downloading URL "+httpurl
);
2063 with
BytesIO() as strbuf
:
2065 databytes
= geturls_text
.read(buffersize
);
2066 if not databytes
: break;
2067 datasize
= len(databytes
);
2068 fulldatasize
= datasize
+ fulldatasize
;
2071 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2072 downloaddiff
= fulldatasize
- prevdownsize
;
2073 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2074 prevdownsize
= fulldatasize
;
2075 strbuf
.write(databytes
);
2077 returnval_content
= strbuf
.read();
2078 if(httpheaderout
.get("Content-Encoding")=="gzip"):
2080 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
2083 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
2085 returnval_content
= zlib
.decompress(returnval_content
);
2088 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
2090 returnval_content
= brotli
.decompress(returnval_content
);
2091 except brotli
.error
:
2093 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
2095 returnval_content
= zstandard
.decompress(returnval_content
);
2096 except zstandard
.error
:
2098 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
2100 returnval_content
= lzma
.decompress(returnval_content
);
2101 except zstandard
.error
:
2103 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
2105 returnval_content
= bz2
.decompress(returnval_content
);
2106 except zstandard
.error
:
2108 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "aiohttp"};
2109 geturls_text
.close();
2112 if(not haveaiohttp
):
2113 def download_from_url_with_aiohttp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2114 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2118 def download_from_url_file_with_aiohttp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2119 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
2120 exec_time_start
= time
.time();
2121 myhash
= hashlib
.new("sha1");
2122 if(sys
.version
[0]=="2"):
2123 myhash
.update(httpurl
);
2124 myhash
.update(str(buffersize
));
2125 myhash
.update(str(exec_time_start
));
2126 if(sys
.version
[0]>="3"):
2127 myhash
.update(httpurl
.encode('utf-8'));
2128 myhash
.update(str(buffersize
).encode('utf-8'));
2129 myhash
.update(str(exec_time_start
).encode('utf-8'));
2130 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2132 sleep
= geturls_download_sleep
;
2135 pretmpfilename
= download_from_url_with_aiohttp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
2136 if(not pretmpfilename
):
2138 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2139 tmpfilename
= f
.name
;
2141 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2142 except AttributeError:
2144 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2149 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2150 f
.write(pretmpfilename
.get('Content'));
2152 exec_time_end
= time
.time();
2153 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2154 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2157 if(not haveaiohttp
):
2158 def download_from_url_file_with_aiohttp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2159 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2163 def download_from_url_to_file_with_aiohttp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2164 global geturls_download_sleep
, havezstd
, havebrotli
;
2166 sleep
= geturls_download_sleep
;
2169 if(not outfile
=="-"):
2170 outpath
= outpath
.rstrip(os
.path
.sep
);
2171 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2172 if(not os
.path
.exists(outpath
)):
2173 os
.makedirs(outpath
);
2174 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2176 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2178 pretmpfilename
= download_from_url_file_with_aiohttp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2179 if(not pretmpfilename
):
2181 tmpfilename
= pretmpfilename
.get('Filename');
2182 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2184 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2185 exec_time_start
= time
.time();
2186 shutil
.move(tmpfilename
, filepath
);
2188 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2189 except AttributeError:
2191 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2196 exec_time_end
= time
.time();
2197 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2198 if(os
.path
.exists(tmpfilename
)):
2199 os
.remove(tmpfilename
);
2200 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2202 pretmpfilename
= download_from_url_file_with_aiohttp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2203 tmpfilename
= pretmpfilename
.get('Filename');
2204 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2207 exec_time_start
= time
.time();
2208 with
open(tmpfilename
, 'rb') as ft
:
2211 databytes
= ft
.read(buffersize
[1]);
2212 if not databytes
: break;
2213 datasize
= len(databytes
);
2214 fulldatasize
= datasize
+ fulldatasize
;
2217 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2218 downloaddiff
= fulldatasize
- prevdownsize
;
2219 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2220 prevdownsize
= fulldatasize
;
2223 fdata
= f
.getvalue();
2226 os
.remove(tmpfilename
);
2227 exec_time_end
= time
.time();
2228 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2229 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2232 if(not haveaiohttp
):
2233 def download_from_url_to_file_with_aiohttp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2234 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
2238 def download_from_url_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2239 global geturls_download_sleep
, havezstd
, havebrotli
;
2241 sleep
= geturls_download_sleep
;
2244 urlparts
= urlparse
.urlparse(httpurl
);
2245 if(isinstance(httpheaders
, list)):
2246 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2247 httpheaders
= fix_header_names(httpheaders
);
2248 if(httpuseragent
is not None):
2249 if('User-Agent' in httpheaders
):
2250 httpheaders
['User-Agent'] = httpuseragent
;
2252 httpuseragent
.update({'User-Agent': httpuseragent
});
2253 if(httpreferer
is not None):
2254 if('Referer' in httpheaders
):
2255 httpheaders
['Referer'] = httpreferer
;
2257 httpuseragent
.update({'Referer': httpreferer
});
2258 if(urlparts
.username
is not None or urlparts
.password
is not None):
2259 if(sys
.version
[0]=="2"):
2260 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
2261 if(sys
.version
[0]>="3"):
2262 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2263 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2265 if(postdata
is not None and not isinstance(postdata
, dict)):
2266 postdata
= urlencode(postdata
);
2268 if(httpmethod
=="GET"):
2269 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
2270 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
2271 elif(httpmethod
=="POST"):
2272 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
2273 geturls_text
= httpx_pool
.post(httpurl
, timeout
=timeout
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
2275 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
2276 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
2277 except httpx
.ConnectTimeout
:
2278 log
.info("Error With URL "+httpurl
);
2280 except httpx
.ConnectError
:
2281 log
.info("Error With URL "+httpurl
);
2283 except socket
.timeout
:
2284 log
.info("Error With URL "+httpurl
);
2286 httpcodeout
= geturls_text
.status_code
;
2288 httpcodereason
= geturls_text
.reason_phrase
;
2290 httpcodereason
= http_status_to_reason(geturls_text
.status_code
);
2291 httpversionout
= geturls_text
.http_version
;
2292 httpmethodout
= httpmethod
;
2293 httpurlout
= str(geturls_text
.url
);
2294 httpheaderout
= geturls_text
.headers
;
2295 httpheadersentout
= geturls_text
.request
.headers
;
2296 if(isinstance(httpheaderout
, list)):
2297 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
2298 if(sys
.version
[0]=="2"):
2300 prehttpheaderout
= httpheaderout
;
2301 httpheaderkeys
= httpheaderout
.keys();
2302 imax
= len(httpheaderkeys
);
2306 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
2308 except AttributeError:
2310 httpheaderout
= fix_header_names(httpheaderout
);
2311 if(isinstance(httpheadersentout
, list)):
2312 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
2313 httpheadersentout
= fix_header_names(httpheadersentout
);
2314 downloadsize
= httpheaderout
.get('Content-Length');
2315 if(downloadsize
is not None):
2316 downloadsize
= int(downloadsize
);
2317 if downloadsize
is None: downloadsize
= 0;
2320 log
.info("Downloading URL "+httpurl
);
2321 with
BytesIO() as strbuf
:
2323 databytes
= geturls_text
.read();
2324 if not databytes
: break;
2325 datasize
= len(databytes
);
2326 fulldatasize
= datasize
+ fulldatasize
;
2329 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2330 downloaddiff
= fulldatasize
- prevdownsize
;
2331 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2332 prevdownsize
= fulldatasize
;
2333 strbuf
.write(databytes
);
2336 returnval_content
= strbuf
.read();
2337 geturls_text
.close();
2338 if(httpheaderout
.get("Content-Encoding")=="gzip"):
2340 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
2343 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
2345 returnval_content
= zlib
.decompress(returnval_content
);
2348 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
2350 returnval_content
= brotli
.decompress(returnval_content
);
2351 except brotli
.error
:
2353 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
2355 returnval_content
= zstandard
.decompress(returnval_content
);
2356 except zstandard
.error
:
2358 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
2360 returnval_content
= lzma
.decompress(returnval_content
);
2361 except zstandard
.error
:
2363 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
2365 returnval_content
= bz2
.decompress(returnval_content
);
2366 except zstandard
.error
:
2368 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "httpx"};
2369 geturls_text
.close();
2373 def download_from_url_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2374 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2378 def download_from_url_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2379 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
2380 exec_time_start
= time
.time();
2381 myhash
= hashlib
.new("sha1");
2382 if(sys
.version
[0]=="2"):
2383 myhash
.update(httpurl
);
2384 myhash
.update(str(buffersize
));
2385 myhash
.update(str(exec_time_start
));
2386 if(sys
.version
[0]>="3"):
2387 myhash
.update(httpurl
.encode('utf-8'));
2388 myhash
.update(str(buffersize
).encode('utf-8'));
2389 myhash
.update(str(exec_time_start
).encode('utf-8'));
2390 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2392 sleep
= geturls_download_sleep
;
2395 pretmpfilename
= download_from_url_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
2396 if(not pretmpfilename
):
2398 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2399 tmpfilename
= f
.name
;
2401 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2402 except AttributeError:
2404 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2409 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2410 f
.write(pretmpfilename
.get('Content'));
2412 exec_time_end
= time
.time();
2413 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2414 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2418 def download_from_url_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2419 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2423 def download_from_url_to_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2424 global geturls_download_sleep
, havezstd
, havebrotli
;
2426 sleep
= geturls_download_sleep
;
2429 if(not outfile
=="-"):
2430 outpath
= outpath
.rstrip(os
.path
.sep
);
2431 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2432 if(not os
.path
.exists(outpath
)):
2433 os
.makedirs(outpath
);
2434 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2436 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2438 pretmpfilename
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2439 if(not pretmpfilename
):
2441 tmpfilename
= pretmpfilename
.get('Filename');
2442 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2444 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2445 exec_time_start
= time
.time();
2446 shutil
.move(tmpfilename
, filepath
);
2448 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2449 except AttributeError:
2451 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2456 exec_time_end
= time
.time();
2457 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2458 if(os
.path
.exists(tmpfilename
)):
2459 os
.remove(tmpfilename
);
2460 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2462 pretmpfilename
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2463 tmpfilename
= pretmpfilename
.get('Filename');
2464 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2467 exec_time_start
= time
.time();
2468 with
open(tmpfilename
, 'rb') as ft
:
2471 databytes
= ft
.read(buffersize
[1]);
2472 if not databytes
: break;
2473 datasize
= len(databytes
);
2474 fulldatasize
= datasize
+ fulldatasize
;
2477 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2478 downloaddiff
= fulldatasize
- prevdownsize
;
2479 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2480 prevdownsize
= fulldatasize
;
2483 fdata
= f
.getvalue();
2486 os
.remove(tmpfilename
);
2487 exec_time_end
= time
.time();
2488 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2489 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2493 def download_from_url_to_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2494 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
2498 def download_from_url_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2499 global geturls_download_sleep
, havezstd
, havebrotli
;
2501 sleep
= geturls_download_sleep
;
2504 urlparts
= urlparse
.urlparse(httpurl
);
2505 if(isinstance(httpheaders
, list)):
2506 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2507 httpheaders
= fix_header_names(httpheaders
);
2508 if(httpuseragent
is not None):
2509 if('User-Agent' in httpheaders
):
2510 httpheaders
['User-Agent'] = httpuseragent
;
2512 httpuseragent
.update({'User-Agent': httpuseragent
});
2513 if(httpreferer
is not None):
2514 if('Referer' in httpheaders
):
2515 httpheaders
['Referer'] = httpreferer
;
2517 httpuseragent
.update({'Referer': httpreferer
});
2518 if(urlparts
.username
is not None or urlparts
.password
is not None):
2519 if(sys
.version
[0]=="2"):
2520 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
2521 if(sys
.version
[0]>="3"):
2522 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2523 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2525 if(postdata
is not None and not isinstance(postdata
, dict)):
2526 postdata
= urlencode(postdata
);
2528 if(httpmethod
=="GET"):
2529 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
2530 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
2531 elif(httpmethod
=="POST"):
2532 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
2533 geturls_text
= httpx_pool
.post(httpurl
, timeout
=timeout
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
2535 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
2536 geturls_text
= httpx_pool
.get(httpurl
, timeout
=timeout
, headers
=httpheaders
, cookies
=httpcookie
);
2537 except httpx
.ConnectTimeout
:
2538 log
.info("Error With URL "+httpurl
);
2540 except httpx
.ConnectError
:
2541 log
.info("Error With URL "+httpurl
);
2543 except socket
.timeout
:
2544 log
.info("Error With URL "+httpurl
);
2546 httpcodeout
= geturls_text
.status_code
;
2548 httpcodereason
= geturls_text
.reason_phrase
;
2550 httpcodereason
= http_status_to_reason(geturls_text
.status_code
);
2551 httpversionout
= geturls_text
.http_version
;
2552 httpmethodout
= httpmethod
;
2553 httpurlout
= str(geturls_text
.url
);
2554 httpheaderout
= geturls_text
.headers
;
2555 httpheadersentout
= geturls_text
.request
.headers
;
2556 if(isinstance(httpheaderout
, list)):
2557 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
2558 if(sys
.version
[0]=="2"):
2560 prehttpheaderout
= httpheaderout
;
2561 httpheaderkeys
= httpheaderout
.keys();
2562 imax
= len(httpheaderkeys
);
2566 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
2568 except AttributeError:
2570 httpheaderout
= fix_header_names(httpheaderout
);
2571 if(isinstance(httpheadersentout
, list)):
2572 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
2573 httpheadersentout
= fix_header_names(httpheadersentout
);
2574 downloadsize
= httpheaderout
.get('Content-Length');
2575 if(downloadsize
is not None):
2576 downloadsize
= int(downloadsize
);
2577 if downloadsize
is None: downloadsize
= 0;
2580 log
.info("Downloading URL "+httpurl
);
2581 with
BytesIO() as strbuf
:
2583 databytes
= geturls_text
.read();
2584 if not databytes
: break;
2585 datasize
= len(databytes
);
2586 fulldatasize
= datasize
+ fulldatasize
;
2589 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2590 downloaddiff
= fulldatasize
- prevdownsize
;
2591 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2592 prevdownsize
= fulldatasize
;
2593 strbuf
.write(databytes
);
2596 returnval_content
= strbuf
.read();
2597 geturls_text
.close();
2598 if(httpheaderout
.get("Content-Encoding")=="gzip"):
2600 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
2603 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
2605 returnval_content
= zlib
.decompress(returnval_content
);
2608 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
2610 returnval_content
= brotli
.decompress(returnval_content
);
2611 except brotli
.error
:
2613 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
2615 returnval_content
= zstandard
.decompress(returnval_content
);
2616 except zstandard
.error
:
2618 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
2620 returnval_content
= lzma
.decompress(returnval_content
);
2621 except zstandard
.error
:
2623 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
2625 returnval_content
= bz2
.decompress(returnval_content
);
2626 except zstandard
.error
:
2628 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "httpx2"};
2629 geturls_text
.close();
2633 def download_from_url_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2634 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2638 def download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2639 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
2640 exec_time_start
= time
.time();
2641 myhash
= hashlib
.new("sha1");
2642 if(sys
.version
[0]=="2"):
2643 myhash
.update(httpurl
);
2644 myhash
.update(str(buffersize
));
2645 myhash
.update(str(exec_time_start
));
2646 if(sys
.version
[0]>="3"):
2647 myhash
.update(httpurl
.encode('utf-8'));
2648 myhash
.update(str(buffersize
).encode('utf-8'));
2649 myhash
.update(str(exec_time_start
).encode('utf-8'));
2650 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2652 sleep
= geturls_download_sleep
;
2655 pretmpfilename
= download_from_url_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
2656 if(not pretmpfilename
):
2658 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2659 tmpfilename
= f
.name
;
2661 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2662 except AttributeError:
2664 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2669 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2670 f
.write(pretmpfilename
.get('Content'));
2672 exec_time_end
= time
.time();
2673 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2674 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2678 def download_from_url_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2679 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2683 def download_from_url_to_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2684 global geturls_download_sleep
, havezstd
, havebrotli
;
2686 sleep
= geturls_download_sleep
;
2689 if(not outfile
=="-"):
2690 outpath
= outpath
.rstrip(os
.path
.sep
);
2691 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2692 if(not os
.path
.exists(outpath
)):
2693 os
.makedirs(outpath
);
2694 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2696 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2698 pretmpfilename
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2699 if(not pretmpfilename
):
2701 tmpfilename
= pretmpfilename
.get('Filename');
2702 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2704 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2705 exec_time_start
= time
.time();
2706 shutil
.move(tmpfilename
, filepath
);
2708 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2709 except AttributeError:
2711 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2716 exec_time_end
= time
.time();
2717 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2718 if(os
.path
.exists(tmpfilename
)):
2719 os
.remove(tmpfilename
);
2720 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2722 pretmpfilename
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2723 tmpfilename
= pretmpfilename
.get('Filename');
2724 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2727 exec_time_start
= time
.time();
2728 with
open(tmpfilename
, 'rb') as ft
:
2731 databytes
= ft
.read(buffersize
[1]);
2732 if not databytes
: break;
2733 datasize
= len(databytes
);
2734 fulldatasize
= datasize
+ fulldatasize
;
2737 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2738 downloaddiff
= fulldatasize
- prevdownsize
;
2739 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2740 prevdownsize
= fulldatasize
;
2743 fdata
= f
.getvalue();
2746 os
.remove(tmpfilename
);
2747 exec_time_end
= time
.time();
2748 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2749 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2753 def download_from_url_to_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2754 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
2758 def download_from_url_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2759 global geturls_download_sleep
, havezstd
, havebrotli
;
2761 sleep
= geturls_download_sleep
;
2764 urlparts
= urlparse
.urlparse(httpurl
);
2765 if(isinstance(httpheaders
, list)):
2766 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2767 httpheaders
= fix_header_names(httpheaders
);
2768 if(httpuseragent
is not None):
2769 if('User-Agent' in httpheaders
):
2770 httpheaders
['User-Agent'] = httpuseragent
;
2772 httpuseragent
.update({'User-Agent': httpuseragent
});
2773 if(httpreferer
is not None):
2774 if('Referer' in httpheaders
):
2775 httpheaders
['Referer'] = httpreferer
;
2777 httpuseragent
.update({'Referer': httpreferer
});
2778 if(urlparts
.username
is not None or urlparts
.password
is not None):
2779 if(sys
.version
[0]=="2"):
2780 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
2781 if(sys
.version
[0]>="3"):
2782 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2783 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2785 if(postdata
is not None and not isinstance(postdata
, dict)):
2786 postdata
= urlencode(postdata
);
2788 if(httpmethod
=="GET"):
2789 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=False);
2790 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
2791 elif(httpmethod
=="POST"):
2792 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=False);
2793 geturls_text
= httpx_pool
.request("GET", httpurl
, data
=postdata
, headers
=httpheaders
);
2795 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=False);
2796 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
2797 except httpcore
.ConnectTimeout
:
2798 log
.info("Error With URL "+httpurl
);
2800 except httpcore
.ConnectError
:
2801 log
.info("Error With URL "+httpurl
);
2803 except socket
.timeout
:
2804 log
.info("Error With URL "+httpurl
);
2806 httpcodeout
= geturls_text
.status
;
2807 httpcodereason
= http_status_to_reason(geturls_text
.status
);
2808 httpversionout
= "1.1";
2809 httpmethodout
= httpmethod
;
2810 httpurlout
= str(httpurl
);
2811 httpheaderout
= geturls_text
.headers
;
2812 httpheadersentout
= httpheaders
;
2813 if(isinstance(httpheaderout
, list)):
2814 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
2815 if(sys
.version
[0]=="2"):
2817 prehttpheaderout
= httpheaderout
;
2818 httpheaderkeys
= httpheaderout
.keys();
2819 imax
= len(httpheaderkeys
);
2823 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
2825 except AttributeError:
2827 httpheaderout
= fix_header_names(httpheaderout
);
2828 if(isinstance(httpheadersentout
, list)):
2829 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
2830 httpheadersentout
= fix_header_names(httpheadersentout
);
2831 downloadsize
= httpheaderout
.get('Content-Length');
2832 if(downloadsize
is not None):
2833 downloadsize
= int(downloadsize
);
2834 if downloadsize
is None: downloadsize
= 0;
2837 log
.info("Downloading URL "+httpurl
);
2838 with
BytesIO() as strbuf
:
2840 databytes
= geturls_text
.read();
2841 if not databytes
: break;
2842 datasize
= len(databytes
);
2843 fulldatasize
= datasize
+ fulldatasize
;
2846 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2847 downloaddiff
= fulldatasize
- prevdownsize
;
2848 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2849 prevdownsize
= fulldatasize
;
2850 strbuf
.write(databytes
);
2853 returnval_content
= strbuf
.read();
2854 geturls_text
.close();
2855 if(httpheaderout
.get("Content-Encoding")=="gzip"):
2857 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
2860 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
2862 returnval_content
= zlib
.decompress(returnval_content
);
2865 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
2867 returnval_content
= brotli
.decompress(returnval_content
);
2868 except brotli
.error
:
2870 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
2872 returnval_content
= zstandard
.decompress(returnval_content
);
2873 except zstandard
.error
:
2875 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
2877 returnval_content
= lzma
.decompress(returnval_content
);
2878 except zstandard
.error
:
2880 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
2882 returnval_content
= bz2
.decompress(returnval_content
);
2883 except zstandard
.error
:
2885 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "httpcore"};
2886 geturls_text
.close();
2889 if(not havehttpcore
):
2890 def download_from_url_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
2891 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
2895 def download_from_url_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2896 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
2897 exec_time_start
= time
.time();
2898 myhash
= hashlib
.new("sha1");
2899 if(sys
.version
[0]=="2"):
2900 myhash
.update(httpurl
);
2901 myhash
.update(str(buffersize
));
2902 myhash
.update(str(exec_time_start
));
2903 if(sys
.version
[0]>="3"):
2904 myhash
.update(httpurl
.encode('utf-8'));
2905 myhash
.update(str(buffersize
).encode('utf-8'));
2906 myhash
.update(str(exec_time_start
).encode('utf-8'));
2907 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2909 sleep
= geturls_download_sleep
;
2912 pretmpfilename
= download_from_url_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
2913 if(not pretmpfilename
):
2915 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2916 tmpfilename
= f
.name
;
2918 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2919 except AttributeError:
2921 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2926 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2927 f
.write(pretmpfilename
.get('Content'));
2929 exec_time_end
= time
.time();
2930 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2931 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2934 if(not havehttpcore
):
2935 def download_from_url_file_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
2936 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
2940 def download_from_url_to_file_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
2941 global geturls_download_sleep
, havezstd
, havebrotli
;
2943 sleep
= geturls_download_sleep
;
2946 if(not outfile
=="-"):
2947 outpath
= outpath
.rstrip(os
.path
.sep
);
2948 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2949 if(not os
.path
.exists(outpath
)):
2950 os
.makedirs(outpath
);
2951 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2953 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2955 pretmpfilename
= download_from_url_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2956 if(not pretmpfilename
):
2958 tmpfilename
= pretmpfilename
.get('Filename');
2959 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2961 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2962 exec_time_start
= time
.time();
2963 shutil
.move(tmpfilename
, filepath
);
2965 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
2966 except AttributeError:
2968 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2973 exec_time_end
= time
.time();
2974 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2975 if(os
.path
.exists(tmpfilename
)):
2976 os
.remove(tmpfilename
);
2977 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
2979 pretmpfilename
= download_from_url_file_with_httpcore(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
2980 tmpfilename
= pretmpfilename
.get('Filename');
2981 downloadsize
= int(os
.path
.getsize(tmpfilename
));
2984 exec_time_start
= time
.time();
2985 with
open(tmpfilename
, 'rb') as ft
:
2988 databytes
= ft
.read(buffersize
[1]);
2989 if not databytes
: break;
2990 datasize
= len(databytes
);
2991 fulldatasize
= datasize
+ fulldatasize
;
2994 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2995 downloaddiff
= fulldatasize
- prevdownsize
;
2996 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2997 prevdownsize
= fulldatasize
;
3000 fdata
= f
.getvalue();
3003 os
.remove(tmpfilename
);
3004 exec_time_end
= time
.time();
3005 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3006 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3009 if(not havehttpcore
):
3010 def download_from_url_to_file_with_httpcore(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3011 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3015 def download_from_url_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3016 global geturls_download_sleep
, havezstd
, havebrotli
;
3018 sleep
= geturls_download_sleep
;
3021 urlparts
= urlparse
.urlparse(httpurl
);
3022 if(isinstance(httpheaders
, list)):
3023 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3024 httpheaders
= fix_header_names(httpheaders
);
3025 if(httpuseragent
is not None):
3026 if('User-Agent' in httpheaders
):
3027 httpheaders
['User-Agent'] = httpuseragent
;
3029 httpuseragent
.update({'User-Agent': httpuseragent
});
3030 if(httpreferer
is not None):
3031 if('Referer' in httpheaders
):
3032 httpheaders
['Referer'] = httpreferer
;
3034 httpuseragent
.update({'Referer': httpreferer
});
3035 if(urlparts
.username
is not None or urlparts
.password
is not None):
3036 if(sys
.version
[0]=="2"):
3037 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
3038 if(sys
.version
[0]>="3"):
3039 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
3040 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
3042 if(postdata
is not None and not isinstance(postdata
, dict)):
3043 postdata
= urlencode(postdata
);
3045 if(httpmethod
=="GET"):
3046 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=True);
3047 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
3048 elif(httpmethod
=="POST"):
3049 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=True);
3050 geturls_text
= httpx_pool
.request("GET", httpurl
, data
=postdata
, headers
=httpheaders
);
3052 httpx_pool
= httpcore
.ConnectionPool(http1
=True, http2
=True);
3053 geturls_text
= httpx_pool
.request("GET", httpurl
, headers
=httpheaders
);
3054 except httpcore
.ConnectTimeout
:
3055 log
.info("Error With URL "+httpurl
);
3057 except httpcore
.ConnectError
:
3058 log
.info("Error With URL "+httpurl
);
3060 except socket
.timeout
:
3061 log
.info("Error With URL "+httpurl
);
3063 httpcodeout
= geturls_text
.status
;
3064 httpcodereason
= http_status_to_reason(geturls_text
.status
);
3065 httpversionout
= "1.1";
3066 httpmethodout
= httpmethod
;
3067 httpurlout
= str(httpurl
);
3068 httpheaderout
= geturls_text
.headers
;
3069 httpheadersentout
= httpheaders
;
3070 if(isinstance(httpheaderout
, list)):
3071 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
3072 if(sys
.version
[0]=="2"):
3074 prehttpheaderout
= httpheaderout
;
3075 httpheaderkeys
= httpheaderout
.keys();
3076 imax
= len(httpheaderkeys
);
3080 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
3082 except AttributeError:
3084 httpheaderout
= fix_header_names(httpheaderout
);
3085 if(isinstance(httpheadersentout
, list)):
3086 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
3087 httpheadersentout
= fix_header_names(httpheadersentout
);
3088 downloadsize
= httpheaderout
.get('Content-Length');
3089 if(downloadsize
is not None):
3090 downloadsize
= int(downloadsize
);
3091 if downloadsize
is None: downloadsize
= 0;
3094 log
.info("Downloading URL "+httpurl
);
3095 with
BytesIO() as strbuf
:
3097 databytes
= geturls_text
.read();
3098 if not databytes
: break;
3099 datasize
= len(databytes
);
3100 fulldatasize
= datasize
+ fulldatasize
;
3103 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3104 downloaddiff
= fulldatasize
- prevdownsize
;
3105 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3106 prevdownsize
= fulldatasize
;
3107 strbuf
.write(databytes
);
3110 returnval_content
= strbuf
.read();
3111 geturls_text
.close();
3112 if(httpheaderout
.get("Content-Encoding")=="gzip"):
3114 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
3117 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
3119 returnval_content
= zlib
.decompress(returnval_content
);
3122 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
3124 returnval_content
= brotli
.decompress(returnval_content
);
3125 except brotli
.error
:
3127 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
3129 returnval_content
= zstandard
.decompress(returnval_content
);
3130 except zstandard
.error
:
3132 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
3134 returnval_content
= lzma
.decompress(returnval_content
);
3135 except zstandard
.error
:
3137 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
3139 returnval_content
= bz2
.decompress(returnval_content
);
3140 except zstandard
.error
:
3142 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "httpcore2"};
3143 geturls_text
.close();
3146 if(not havehttpcore
):
3147 def download_from_url_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3148 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
3152 def download_from_url_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3153 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
3154 exec_time_start
= time
.time();
3155 myhash
= hashlib
.new("sha1");
3156 if(sys
.version
[0]=="2"):
3157 myhash
.update(httpurl
);
3158 myhash
.update(str(buffersize
));
3159 myhash
.update(str(exec_time_start
));
3160 if(sys
.version
[0]>="3"):
3161 myhash
.update(httpurl
.encode('utf-8'));
3162 myhash
.update(str(buffersize
).encode('utf-8'));
3163 myhash
.update(str(exec_time_start
).encode('utf-8'));
3164 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
3166 sleep
= geturls_download_sleep
;
3169 pretmpfilename
= download_from_url_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
3170 if(not pretmpfilename
):
3172 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
3173 tmpfilename
= f
.name
;
3175 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3176 except AttributeError:
3178 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3183 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3184 f
.write(pretmpfilename
.get('Content'));
3186 exec_time_end
= time
.time();
3187 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
3188 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
3191 if(not havehttpcore
):
3192 def download_from_url_file_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3193 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
3197 def download_from_url_to_file_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3198 global geturls_download_sleep
, havezstd
, havebrotli
;
3200 sleep
= geturls_download_sleep
;
3203 if(not outfile
=="-"):
3204 outpath
= outpath
.rstrip(os
.path
.sep
);
3205 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
3206 if(not os
.path
.exists(outpath
)):
3207 os
.makedirs(outpath
);
3208 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
3210 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
3212 pretmpfilename
= download_from_url_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3213 if(not pretmpfilename
):
3215 tmpfilename
= pretmpfilename
.get('Filename');
3216 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3218 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
3219 exec_time_start
= time
.time();
3220 shutil
.move(tmpfilename
, filepath
);
3222 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3223 except AttributeError:
3225 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3230 exec_time_end
= time
.time();
3231 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
3232 if(os
.path
.exists(tmpfilename
)):
3233 os
.remove(tmpfilename
);
3234 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3236 pretmpfilename
= download_from_url_file_with_httpcore2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3237 tmpfilename
= pretmpfilename
.get('Filename');
3238 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3241 exec_time_start
= time
.time();
3242 with
open(tmpfilename
, 'rb') as ft
:
3245 databytes
= ft
.read(buffersize
[1]);
3246 if not databytes
: break;
3247 datasize
= len(databytes
);
3248 fulldatasize
= datasize
+ fulldatasize
;
3251 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3252 downloaddiff
= fulldatasize
- prevdownsize
;
3253 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3254 prevdownsize
= fulldatasize
;
3257 fdata
= f
.getvalue();
3260 os
.remove(tmpfilename
);
3261 exec_time_end
= time
.time();
3262 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3263 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3267 def download_from_url_to_file_with_httpcore2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3268 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3272 def download_from_url_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3273 returnval
= download_from_url_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
3276 if(not haveurllib3
):
3277 def download_from_url_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3278 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
3282 def download_from_url_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3283 returnval
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
3286 if(not haveurllib3
):
3287 def download_from_url_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3288 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
3292 def download_from_url_to_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3293 returnval
= download_from_url_to_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3296 if(not haveurllib3
):
3297 def download_from_url_to_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3298 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3302 def download_from_url_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3303 global geturls_download_sleep
, havezstd
, havebrotli
;
3305 sleep
= geturls_download_sleep
;
3308 urlparts
= urlparse
.urlparse(httpurl
);
3309 if(isinstance(httpheaders
, list)):
3310 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3311 httpheaders
= fix_header_names(httpheaders
);
3312 if(httpuseragent
is not None):
3313 if('User-Agent' in httpheaders
):
3314 httpheaders
['User-Agent'] = httpuseragent
;
3316 httpuseragent
.update({'User-Agent': httpuseragent
});
3317 if(httpreferer
is not None):
3318 if('Referer' in httpheaders
):
3319 httpheaders
['Referer'] = httpreferer
;
3321 httpuseragent
.update({'Referer': httpreferer
});
3322 if(urlparts
.username
is not None or urlparts
.password
is not None):
3323 if(sys
.version
[0]=="2"):
3324 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
3325 if(sys
.version
[0]>="3"):
3326 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
3327 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
3329 timeout
= urllib3
.util
.Timeout(connect
=timeout
, read
=timeout
);
3330 urllib_pool
= urllib3
.PoolManager(headers
=httpheaders
, timeout
=timeout
);
3331 if(postdata
is not None and not isinstance(postdata
, dict)):
3332 postdata
= urlencode(postdata
);
3334 if(httpmethod
=="GET"):
3335 geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
3336 elif(httpmethod
=="POST"):
3337 geturls_text
= urllib_pool
.request("POST", httpurl
, body
=postdata
, headers
=httpheaders
, preload_content
=False);
3339 geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
3340 except urllib3
.exceptions
.ConnectTimeoutError
:
3341 log
.info("Error With URL "+httpurl
);
3343 except urllib3
.exceptions
.ConnectError
:
3344 log
.info("Error With URL "+httpurl
);
3346 except urllib3
.exceptions
.MaxRetryError
:
3347 log
.info("Error With URL "+httpurl
);
3349 except socket
.timeout
:
3350 log
.info("Error With URL "+httpurl
);
3353 log
.info("Error With URL "+httpurl
);
3355 httpcodeout
= geturls_text
.status
;
3356 httpcodereason
= geturls_text
.reason
;
3357 if(geturls_text
.version
=="10"):
3358 httpversionout
= "1.0";
3360 httpversionout
= "1.1";
3361 httpmethodout
= httpmethod
;
3362 httpurlout
= geturls_text
.geturl();
3363 httpheaderout
= geturls_text
.info();
3364 httpheadersentout
= httpheaders
;
3365 if(isinstance(httpheaderout
, list)):
3366 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
3367 if(sys
.version
[0]=="2"):
3369 prehttpheaderout
= httpheaderout
;
3370 httpheaderkeys
= httpheaderout
.keys();
3371 imax
= len(httpheaderkeys
);
3375 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
3377 except AttributeError:
3379 httpheaderout
= fix_header_names(httpheaderout
);
3380 if(isinstance(httpheadersentout
, list)):
3381 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
3382 httpheadersentout
= fix_header_names(httpheadersentout
);
3383 downloadsize
= httpheaderout
.get('Content-Length');
3384 if(downloadsize
is not None):
3385 downloadsize
= int(downloadsize
);
3386 if downloadsize
is None: downloadsize
= 0;
3389 log
.info("Downloading URL "+httpurl
);
3390 with
BytesIO() as strbuf
:
3392 databytes
= geturls_text
.read(buffersize
);
3393 if not databytes
: break;
3394 datasize
= len(databytes
);
3395 fulldatasize
= datasize
+ fulldatasize
;
3398 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3399 downloaddiff
= fulldatasize
- prevdownsize
;
3400 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3401 prevdownsize
= fulldatasize
;
3402 strbuf
.write(databytes
);
3404 returnval_content
= strbuf
.read();
3405 if(httpheaderout
.get("Content-Encoding")=="gzip"):
3407 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
3410 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
3412 returnval_content
= zlib
.decompress(returnval_content
);
3415 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
3417 returnval_content
= brotli
.decompress(returnval_content
);
3418 except brotli
.error
:
3420 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
3422 returnval_content
= zstandard
.decompress(returnval_content
);
3423 except zstandard
.error
:
3425 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
3427 returnval_content
= lzma
.decompress(returnval_content
);
3428 except zstandard
.error
:
3430 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
3432 returnval_content
= bz2
.decompress(returnval_content
);
3433 except zstandard
.error
:
3435 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "urllib3"};
3436 geturls_text
.close();
3439 if(not haveurllib3
):
3440 def download_from_url_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3441 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
3445 def download_from_url_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3446 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
3447 exec_time_start
= time
.time();
3448 myhash
= hashlib
.new("sha1");
3449 if(sys
.version
[0]=="2"):
3450 myhash
.update(httpurl
);
3451 myhash
.update(str(buffersize
));
3452 myhash
.update(str(exec_time_start
));
3453 if(sys
.version
[0]>="3"):
3454 myhash
.update(httpurl
.encode('utf-8'));
3455 myhash
.update(str(buffersize
).encode('utf-8'));
3456 myhash
.update(str(exec_time_start
).encode('utf-8'));
3457 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
3459 sleep
= geturls_download_sleep
;
3462 pretmpfilename
= download_from_url_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
3463 if(not pretmpfilename
):
3465 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
3466 tmpfilename
= f
.name
;
3468 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3469 except AttributeError:
3471 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3476 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3477 f
.write(pretmpfilename
.get('Content'));
3479 exec_time_end
= time
.time();
3480 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
3481 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
3484 if(not haveurllib3
):
3485 def download_from_url_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3486 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
3490 def download_from_url_to_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3491 global geturls_download_sleep
, havezstd
, havebrotli
;
3493 sleep
= geturls_download_sleep
;
3496 if(not outfile
=="-"):
3497 outpath
= outpath
.rstrip(os
.path
.sep
);
3498 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
3499 if(not os
.path
.exists(outpath
)):
3500 os
.makedirs(outpath
);
3501 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
3503 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
3505 pretmpfilename
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3506 if(not pretmpfilename
):
3508 tmpfilename
= pretmpfilename
.get('Filename');
3509 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3511 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
3512 exec_time_start
= time
.time();
3513 shutil
.move(tmpfilename
, filepath
);
3515 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3516 except AttributeError:
3518 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3523 exec_time_end
= time
.time();
3524 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
3525 if(os
.path
.exists(tmpfilename
)):
3526 os
.remove(tmpfilename
);
3527 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3529 pretmpfilename
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3530 tmpfilename
= pretmpfilename
.get('Filename');
3531 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3534 exec_time_start
= time
.time();
3535 with
open(tmpfilename
, 'rb') as ft
:
3538 databytes
= ft
.read(buffersize
[1]);
3539 if not databytes
: break;
3540 datasize
= len(databytes
);
3541 fulldatasize
= datasize
+ fulldatasize
;
3544 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3545 downloaddiff
= fulldatasize
- prevdownsize
;
3546 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3547 prevdownsize
= fulldatasize
;
3550 fdata
= f
.getvalue();
3553 os
.remove(tmpfilename
);
3554 exec_time_end
= time
.time();
3555 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3556 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3559 if(not haveurllib3
):
3560 def download_from_url_to_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3561 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3565 def download_from_url_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3566 global geturls_download_sleep
, havezstd
, havebrotli
;
3568 sleep
= geturls_download_sleep
;
3571 urlparts
= urlparse
.urlparse(httpurl
);
3572 if(isinstance(httpheaders
, list)):
3573 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3574 httpheaders
= fix_header_names(httpheaders
);
3575 if(httpuseragent
is not None):
3576 if('User-Agent' in httpheaders
):
3577 httpheaders
['User-Agent'] = httpuseragent
;
3579 httpuseragent
.update({'User-Agent': httpuseragent
});
3580 if(httpreferer
is not None):
3581 if('Referer' in httpheaders
):
3582 httpheaders
['Referer'] = httpreferer
;
3584 httpuseragent
.update({'Referer': httpreferer
});
3585 if(urlparts
.username
is not None or urlparts
.password
is not None):
3586 if(sys
.version
[0]=="2"):
3587 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
3588 if(sys
.version
[0]>="3"):
3589 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
3590 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
3591 geturls_opener
= mechanize
.Browser();
3592 if(isinstance(httpheaders
, dict)):
3593 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
3595 geturls_opener
.addheaders
= httpheaders
;
3596 geturls_opener
.set_cookiejar(httpcookie
);
3597 geturls_opener
.set_handle_robots(False);
3598 if(postdata
is not None and not isinstance(postdata
, dict)):
3599 postdata
= urlencode(postdata
);
3601 if(httpmethod
=="GET"):
3602 geturls_text
= geturls_opener
.open(httpurl
);
3603 elif(httpmethod
=="POST"):
3604 geturls_text
= geturls_opener
.open(httpurl
, data
=postdata
);
3606 geturls_text
= geturls_opener
.open(httpurl
);
3607 except mechanize
.HTTPError
as geturls_text_error
:
3608 geturls_text
= geturls_text_error
;
3609 log
.info("Error With URL "+httpurl
);
3611 log
.info("Error With URL "+httpurl
);
3613 except socket
.timeout
:
3614 log
.info("Error With URL "+httpurl
);
3616 httpcodeout
= geturls_text
.code
;
3617 httpcodereason
= geturls_text
.msg
;
3618 httpversionout
= "1.1";
3619 httpmethodout
= httpmethod
;
3620 httpurlout
= geturls_text
.geturl();
3621 httpheaderout
= geturls_text
.info();
3622 reqhead
= geturls_opener
.request
;
3623 httpheadersentout
= reqhead
.header_items();
3624 if(isinstance(httpheaderout
, list)):
3625 httpheaderout
= dict(make_http_headers_from_list_to_dict(httpheaderout
));
3626 if(sys
.version
[0]=="2"):
3628 prehttpheaderout
= httpheaderout
;
3629 httpheaderkeys
= httpheaderout
.keys();
3630 imax
= len(httpheaderkeys
);
3634 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
3636 except AttributeError:
3638 httpheaderout
= fix_header_names(httpheaderout
);
3639 if(isinstance(httpheadersentout
, list)):
3640 httpheadersentout
= dict(make_http_headers_from_list_to_dict(httpheadersentout
));
3641 httpheadersentout
= fix_header_names(httpheadersentout
);
3642 downloadsize
= httpheaderout
.get('Content-Length');
3643 if(downloadsize
is not None):
3644 downloadsize
= int(downloadsize
);
3645 if downloadsize
is None: downloadsize
= 0;
3648 log
.info("Downloading URL "+httpurl
);
3649 with
BytesIO() as strbuf
:
3651 databytes
= geturls_text
.read(buffersize
);
3652 if not databytes
: break;
3653 datasize
= len(databytes
);
3654 fulldatasize
= datasize
+ fulldatasize
;
3657 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3658 downloaddiff
= fulldatasize
- prevdownsize
;
3659 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3660 prevdownsize
= fulldatasize
;
3661 strbuf
.write(databytes
);
3663 returnval_content
= strbuf
.read();
3664 if(httpheaderout
.get("Content-Encoding")=="gzip"):
3666 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
3669 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
3671 returnval_content
= zlib
.decompress(returnval_content
);
3674 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
3676 returnval_content
= brotli
.decompress(returnval_content
);
3677 except brotli
.error
:
3679 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
3681 returnval_content
= zstandard
.decompress(returnval_content
);
3682 except zstandard
.error
:
3684 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
3686 returnval_content
= lzma
.decompress(returnval_content
);
3687 except zstandard
.error
:
3689 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
3691 returnval_content
= bz2
.decompress(returnval_content
);
3692 except zstandard
.error
:
3694 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "mechanize"};
3695 geturls_text
.close();
3698 if(not havemechanize
):
3699 def download_from_url_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3700 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
3704 def download_from_url_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3705 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
3706 exec_time_start
= time
.time();
3707 myhash
= hashlib
.new("sha1");
3708 if(sys
.version
[0]=="2"):
3709 myhash
.update(httpurl
);
3710 myhash
.update(str(buffersize
));
3711 myhash
.update(str(exec_time_start
));
3712 if(sys
.version
[0]>="3"):
3713 myhash
.update(httpurl
.encode('utf-8'));
3714 myhash
.update(str(buffersize
).encode('utf-8'));
3715 myhash
.update(str(exec_time_start
).encode('utf-8'));
3716 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
3718 sleep
= geturls_download_sleep
;
3721 pretmpfilename
= download_from_url_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
3722 if(not pretmpfilename
):
3724 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
3725 tmpfilename
= f
.name
;
3727 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3728 except AttributeError:
3730 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3735 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3736 f
.write(pretmpfilename
.get('Content'));
3738 exec_time_end
= time
.time();
3739 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
3740 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
3743 if(not havemechanize
):
3744 def download_from_url_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3745 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
3749 def download_from_url_to_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3750 global geturls_download_sleep
, havezstd
, havebrotli
;
3752 sleep
= geturls_download_sleep
;
3755 if(not outfile
=="-"):
3756 outpath
= outpath
.rstrip(os
.path
.sep
);
3757 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
3758 if(not os
.path
.exists(outpath
)):
3759 os
.makedirs(outpath
);
3760 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
3762 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
3764 pretmpfilename
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3765 if(not pretmpfilename
):
3767 tmpfilename
= pretmpfilename
.get('Filename');
3768 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3770 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
3771 exec_time_start
= time
.time();
3772 shutil
.move(tmpfilename
, filepath
);
3774 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
3775 except AttributeError:
3777 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3782 exec_time_end
= time
.time();
3783 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
3784 if(os
.path
.exists(tmpfilename
)):
3785 os
.remove(tmpfilename
);
3786 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3788 pretmpfilename
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
3789 tmpfilename
= pretmpfilename
.get('Filename');
3790 downloadsize
= int(os
.path
.getsize(tmpfilename
));
3793 exec_time_start
= time
.time();
3794 with
open(tmpfilename
, 'rb') as ft
:
3797 databytes
= ft
.read(buffersize
[1]);
3798 if not databytes
: break;
3799 datasize
= len(databytes
);
3800 fulldatasize
= datasize
+ fulldatasize
;
3803 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3804 downloaddiff
= fulldatasize
- prevdownsize
;
3805 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3806 prevdownsize
= fulldatasize
;
3809 fdata
= f
.getvalue();
3812 os
.remove(tmpfilename
);
3813 exec_time_end
= time
.time();
3814 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3815 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
3818 if(not havemechanize
):
3819 def download_from_url_to_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
3820 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
3824 def download_from_url_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3825 global geturls_download_sleep
, havezstd
, havebrotli
;
3827 sleep
= geturls_download_sleep
;
3830 urlparts
= urlparse
.urlparse(httpurl
);
3831 if(isinstance(httpheaders
, list)):
3832 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3833 httpheaders
= fix_header_names(httpheaders
);
3834 if(httpuseragent
is not None):
3835 if('User-Agent' in httpheaders
):
3836 httpheaders
['User-Agent'] = httpuseragent
;
3838 httpuseragent
.update({'User-Agent': httpuseragent
});
3839 if(httpreferer
is not None):
3840 if('Referer' in httpheaders
):
3841 httpheaders
['Referer'] = httpreferer
;
3843 httpuseragent
.update({'Referer': httpreferer
});
3844 if(urlparts
.username
is not None or urlparts
.password
is not None):
3845 if(sys
.version
[0]=="2"):
3846 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
3847 if(sys
.version
[0]>="3"):
3848 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
3849 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
3850 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
3851 if(isinstance(httpheaders
, dict)):
3852 httpheaders
= make_http_headers_from_dict_to_pycurl(httpheaders
);
3853 geturls_opener
.addheaders
= httpheaders
;
3855 if(postdata
is not None and not isinstance(postdata
, dict)):
3856 postdata
= urlencode(postdata
);
3857 retrieved_body
= BytesIO();
3858 retrieved_headers
= BytesIO();
3860 if(httpmethod
=="GET"):
3861 geturls_text
= pycurl
.Curl();
3862 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3863 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3864 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3865 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3866 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3867 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3868 geturls_text
.perform();
3869 elif(httpmethod
=="POST"):
3870 geturls_text
= pycurl
.Curl();
3871 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3872 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3873 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3874 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3875 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3876 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3877 geturls_text
.setopt(geturls_text
.POST
, True);
3878 geturls_text
.setopt(geturls_text
.POSTFIELDS
, postdata
);
3879 geturls_text
.perform();
3881 geturls_text
= pycurl
.Curl();
3882 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
3883 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
3884 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
3885 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
3886 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
3887 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
3888 geturls_text
.perform();
3889 retrieved_headers
.seek(0);
3890 if(sys
.version
[0]=="2"):
3891 pycurlhead
= retrieved_headers
.read();
3892 if(sys
.version
[0]>="3"):
3893 pycurlhead
= retrieved_headers
.read().decode('UTF-8');
3894 pyhttpverinfo
= re
.findall(r
'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead
.splitlines()[0].strip().rstrip('\r\n'))[0];
3895 pycurlheadersout
= make_http_headers_from_pycurl_to_dict(pycurlhead
);
3896 retrieved_body
.seek(0);
3897 except socket
.timeout
:
3898 log
.info("Error With URL "+httpurl
);
3900 except socket
.gaierror
:
3901 log
.info("Error With URL "+httpurl
);
3904 log
.info("Error With URL "+httpurl
);
3906 httpcodeout
= geturls_text
.getinfo(geturls_text
.HTTP_CODE
);
3907 httpcodereason
= http_status_to_reason(geturls_text
.getinfo(geturls_text
.HTTP_CODE
));
3908 httpversionout
= pyhttpverinfo
[0];
3909 httpmethodout
= httpmethod
;
3910 httpurlout
= geturls_text
.getinfo(geturls_text
.EFFECTIVE_URL
);
3911 httpheaderout
= pycurlheadersout
;
3912 httpheadersentout
= httpheaders
;
3913 if(isinstance(httpheaderout
, list)):
3914 httpheaderout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout
)));
3915 if(sys
.version
[0]=="2"):
3917 prehttpheaderout
= httpheaderout
;
3918 httpheaderkeys
= httpheaderout
.keys();
3919 imax
= len(httpheaderkeys
);
3923 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
3925 except AttributeError:
3927 httpheaderout
= fix_header_names(httpheaderout
);
3928 if(isinstance(httpheadersentout
, list)):
3929 httpheadersentout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout
)));
3930 httpheadersentout
= fix_header_names(httpheadersentout
);
3931 downloadsize
= httpheaderout
.get('Content-Length');
3932 if(downloadsize
is not None):
3933 downloadsize
= int(downloadsize
);
3934 if downloadsize
is None: downloadsize
= 0;
3937 log
.info("Downloading URL "+httpurl
);
3938 with
BytesIO() as strbuf
:
3940 databytes
= retrieved_body
.read(buffersize
);
3941 if not databytes
: break;
3942 datasize
= len(databytes
);
3943 fulldatasize
= datasize
+ fulldatasize
;
3946 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3947 downloaddiff
= fulldatasize
- prevdownsize
;
3948 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3949 prevdownsize
= fulldatasize
;
3950 strbuf
.write(databytes
);
3952 returnval_content
= strbuf
.read();
3953 if(httpheaderout
.get("Content-Encoding")=="gzip"):
3955 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
3958 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
3960 returnval_content
= zlib
.decompress(returnval_content
);
3963 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
3965 returnval_content
= brotli
.decompress(returnval_content
);
3966 except brotli
.error
:
3968 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
3970 returnval_content
= zstandard
.decompress(returnval_content
);
3971 except zstandard
.error
:
3973 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
3975 returnval_content
= lzma
.decompress(returnval_content
);
3976 except zstandard
.error
:
3978 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
3980 returnval_content
= bz2
.decompress(returnval_content
);
3981 except zstandard
.error
:
3983 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "pycurl"};
3984 geturls_text
.close();
3988 def download_from_url_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
3989 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
3993 def download_from_url_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
3994 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
3995 exec_time_start
= time
.time();
3996 myhash
= hashlib
.new("sha1");
3997 if(sys
.version
[0]=="2"):
3998 myhash
.update(httpurl
);
3999 myhash
.update(str(buffersize
));
4000 myhash
.update(str(exec_time_start
));
4001 if(sys
.version
[0]>="3"):
4002 myhash
.update(httpurl
.encode('utf-8'));
4003 myhash
.update(str(buffersize
).encode('utf-8'));
4004 myhash
.update(str(exec_time_start
).encode('utf-8'));
4005 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
4007 sleep
= geturls_download_sleep
;
4010 pretmpfilename
= download_from_url_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
4011 if(not pretmpfilename
):
4013 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
4014 tmpfilename
= f
.name
;
4016 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4017 except AttributeError:
4019 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4024 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4025 f
.write(pretmpfilename
.get('Content'));
4027 exec_time_end
= time
.time();
4028 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
4029 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
4033 def download_from_url_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4034 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
4038 def download_from_url_to_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4039 global geturls_download_sleep
, havezstd
, havebrotli
;
4041 sleep
= geturls_download_sleep
;
4044 if(not outfile
=="-"):
4045 outpath
= outpath
.rstrip(os
.path
.sep
);
4046 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
4047 if(not os
.path
.exists(outpath
)):
4048 os
.makedirs(outpath
);
4049 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
4051 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
4053 pretmpfilename
= download_from_url_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4054 if(not pretmpfilename
):
4056 tmpfilename
= pretmpfilename
.get('Filename');
4057 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4059 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
4060 exec_time_start
= time
.time();
4061 shutil
.move(tmpfilename
, filepath
);
4063 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4064 except AttributeError:
4066 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4071 exec_time_end
= time
.time();
4072 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
4073 if(os
.path
.exists(tmpfilename
)):
4074 os
.remove(tmpfilename
);
4075 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4077 pretmpfilename
= download_from_url_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4078 tmpfilename
= pretmpfilename
.get('Filename');
4079 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4082 exec_time_start
= time
.time();
4083 with
open(tmpfilename
, 'rb') as ft
:
4086 databytes
= ft
.read(buffersize
[1]);
4087 if not databytes
: break;
4088 datasize
= len(databytes
);
4089 fulldatasize
= datasize
+ fulldatasize
;
4092 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4093 downloaddiff
= fulldatasize
- prevdownsize
;
4094 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4095 prevdownsize
= fulldatasize
;
4098 fdata
= f
.getvalue();
4101 os
.remove(tmpfilename
);
4102 exec_time_end
= time
.time();
4103 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
4104 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4108 def download_from_url_to_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4109 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
4112 if(havepycurl
and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4113 def download_from_url_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4114 global geturls_download_sleep
, havezstd
, havebrotli
;
4116 sleep
= geturls_download_sleep
;
4119 urlparts
= urlparse
.urlparse(httpurl
);
4120 if(isinstance(httpheaders
, list)):
4121 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
4122 httpheaders
= fix_header_names(httpheaders
);
4123 if(httpuseragent
is not None):
4124 if('User-Agent' in httpheaders
):
4125 httpheaders
['User-Agent'] = httpuseragent
;
4127 httpuseragent
.update({'User-Agent': httpuseragent
});
4128 if(httpreferer
is not None):
4129 if('Referer' in httpheaders
):
4130 httpheaders
['Referer'] = httpreferer
;
4132 httpuseragent
.update({'Referer': httpreferer
});
4133 if(urlparts
.username
is not None or urlparts
.password
is not None):
4134 if(sys
.version
[0]=="2"):
4135 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
4136 if(sys
.version
[0]>="3"):
4137 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
4138 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
4139 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
4140 if(isinstance(httpheaders
, dict)):
4141 httpheaders
= make_http_headers_from_dict_to_pycurl(httpheaders
);
4142 geturls_opener
.addheaders
= httpheaders
;
4144 if(postdata
is not None and not isinstance(postdata
, dict)):
4145 postdata
= urlencode(postdata
);
4146 retrieved_body
= BytesIO();
4147 retrieved_headers
= BytesIO();
4149 if(httpmethod
=="GET"):
4150 geturls_text
= pycurl
.Curl();
4151 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
4152 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_2_0
);
4153 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
4154 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
4155 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
4156 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
4157 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
4158 geturls_text
.perform();
4159 elif(httpmethod
=="POST"):
4160 geturls_text
= pycurl
.Curl();
4161 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
4162 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_2_0
);
4163 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
4164 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
4165 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
4166 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
4167 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
4168 geturls_text
.setopt(geturls_text
.POST
, True);
4169 geturls_text
.setopt(geturls_text
.POSTFIELDS
, postdata
);
4170 geturls_text
.perform();
4172 geturls_text
= pycurl
.Curl();
4173 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
4174 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_2_0
);
4175 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
4176 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
4177 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
4178 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
4179 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
4180 geturls_text
.perform();
4181 retrieved_headers
.seek(0);
4182 if(sys
.version
[0]=="2"):
4183 pycurlhead
= retrieved_headers
.read();
4184 if(sys
.version
[0]>="3"):
4185 pycurlhead
= retrieved_headers
.read().decode('UTF-8');
4186 pyhttpverinfo
= re
.findall(r
'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead
.splitlines()[0].strip())[0];
4187 pycurlheadersout
= make_http_headers_from_pycurl_to_dict(pycurlhead
);
4188 retrieved_body
.seek(0);
4189 except socket
.timeout
:
4190 log
.info("Error With URL "+httpurl
);
4192 except socket
.gaierror
:
4193 log
.info("Error With URL "+httpurl
);
4196 log
.info("Error With URL "+httpurl
);
4198 httpcodeout
= geturls_text
.getinfo(geturls_text
.HTTP_CODE
);
4199 httpcodereason
= http_status_to_reason(geturls_text
.getinfo(geturls_text
.HTTP_CODE
));
4200 httpversionout
= pyhttpverinfo
[0];
4201 httpmethodout
= httpmethod
;
4202 httpurlout
= geturls_text
.getinfo(geturls_text
.EFFECTIVE_URL
);
4203 httpheaderout
= pycurlheadersout
;
4204 httpheadersentout
= httpheaders
;
4205 if(isinstance(httpheaderout
, list)):
4206 httpheaderout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout
)));
4207 if(sys
.version
[0]=="2"):
4209 prehttpheaderout
= httpheaderout
;
4210 httpheaderkeys
= httpheaderout
.keys();
4211 imax
= len(httpheaderkeys
);
4215 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
4217 except AttributeError:
4219 httpheaderout
= fix_header_names(httpheaderout
);
4220 if(isinstance(httpheadersentout
, list)):
4221 httpheadersentout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout
)));
4222 httpheadersentout
= fix_header_names(httpheadersentout
);
4223 downloadsize
= httpheaderout
.get('Content-Length');
4224 if(downloadsize
is not None):
4225 downloadsize
= int(downloadsize
);
4226 if downloadsize
is None: downloadsize
= 0;
4229 log
.info("Downloading URL "+httpurl
);
4230 with
BytesIO() as strbuf
:
4232 databytes
= retrieved_body
.read(buffersize
);
4233 if not databytes
: break;
4234 datasize
= len(databytes
);
4235 fulldatasize
= datasize
+ fulldatasize
;
4238 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4239 downloaddiff
= fulldatasize
- prevdownsize
;
4240 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4241 prevdownsize
= fulldatasize
;
4242 strbuf
.write(databytes
);
4244 returnval_content
= strbuf
.read();
4245 if(httpheaderout
.get("Content-Encoding")=="gzip"):
4247 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
4250 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
4252 returnval_content
= zlib
.decompress(returnval_content
);
4255 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
4257 returnval_content
= brotli
.decompress(returnval_content
);
4258 except brotli
.error
:
4260 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
4262 returnval_content
= zstandard
.decompress(returnval_content
);
4263 except zstandard
.error
:
4265 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
4267 returnval_content
= lzma
.decompress(returnval_content
);
4268 except zstandard
.error
:
4270 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
4272 returnval_content
= bz2
.decompress(returnval_content
);
4273 except zstandard
.error
:
4275 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "pycurl2"};
4276 geturls_text
.close();
4280 def download_from_url_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4281 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
4284 if(havepycurl
and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4285 def download_from_url_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4286 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
4289 if(havepycurl
and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4290 def download_from_url_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4291 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
4292 exec_time_start
= time
.time();
4293 myhash
= hashlib
.new("sha1");
4294 if(sys
.version
[0]=="2"):
4295 myhash
.update(httpurl
);
4296 myhash
.update(str(buffersize
));
4297 myhash
.update(str(exec_time_start
));
4298 if(sys
.version
[0]>="3"):
4299 myhash
.update(httpurl
.encode('utf-8'));
4300 myhash
.update(str(buffersize
).encode('utf-8'));
4301 myhash
.update(str(exec_time_start
).encode('utf-8'));
4302 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
4304 sleep
= geturls_download_sleep
;
4307 pretmpfilename
= download_from_url_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
4308 if(not pretmpfilename
):
4310 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
4311 tmpfilename
= f
.name
;
4313 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4314 except AttributeError:
4316 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4321 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4322 f
.write(pretmpfilename
.get('Content'));
4324 exec_time_end
= time
.time();
4325 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
4326 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
4330 def download_from_url_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4331 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
4334 if(havepycurl
and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4335 def download_from_url_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4336 returnval
= download_from_url_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
4339 if(havepycurl
and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4340 def download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4341 global geturls_download_sleep
, havezstd
, havebrotli
;
4343 sleep
= geturls_download_sleep
;
4346 if(not outfile
=="-"):
4347 outpath
= outpath
.rstrip(os
.path
.sep
);
4348 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
4349 if(not os
.path
.exists(outpath
)):
4350 os
.makedirs(outpath
);
4351 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
4353 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
4355 pretmpfilename
= download_from_url_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4356 if(not pretmpfilename
):
4358 tmpfilename
= pretmpfilename
.get('Filename');
4359 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4361 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
4362 exec_time_start
= time
.time();
4363 shutil
.move(tmpfilename
, filepath
);
4365 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4366 except AttributeError:
4368 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4373 exec_time_end
= time
.time();
4374 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
4375 if(os
.path
.exists(tmpfilename
)):
4376 os
.remove(tmpfilename
);
4377 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4379 pretmpfilename
= download_from_url_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4380 tmpfilename
= pretmpfilename
.get('Filename');
4381 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4384 exec_time_start
= time
.time();
4385 with
open(tmpfilename
, 'rb') as ft
:
4388 databytes
= ft
.read(buffersize
[1]);
4389 if not databytes
: break;
4390 datasize
= len(databytes
);
4391 fulldatasize
= datasize
+ fulldatasize
;
4394 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4395 downloaddiff
= fulldatasize
- prevdownsize
;
4396 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4397 prevdownsize
= fulldatasize
;
4400 fdata
= f
.getvalue();
4403 os
.remove(tmpfilename
);
4404 exec_time_end
= time
.time();
4405 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
4406 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4410 def download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4411 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
4414 if(havepycurl
and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4415 def download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4416 returnval
= download_from_url_to_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
4419 if(havepycurl
and hasattr(pycurl
, "CURL_HTTP_VERSION_3_0")):
4420 def download_from_url_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4421 global geturls_download_sleep
, havezstd
, havebrotli
;
4423 sleep
= geturls_download_sleep
;
4426 urlparts
= urlparse
.urlparse(httpurl
);
4427 if(isinstance(httpheaders
, list)):
4428 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
4429 httpheaders
= fix_header_names(httpheaders
);
4430 if(httpuseragent
is not None):
4431 if('User-Agent' in httpheaders
):
4432 httpheaders
['User-Agent'] = httpuseragent
;
4434 httpuseragent
.update({'User-Agent': httpuseragent
});
4435 if(httpreferer
is not None):
4436 if('Referer' in httpheaders
):
4437 httpheaders
['Referer'] = httpreferer
;
4439 httpuseragent
.update({'Referer': httpreferer
});
4440 if(urlparts
.username
is not None or urlparts
.password
is not None):
4441 if(sys
.version
[0]=="2"):
4442 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
));
4443 if(sys
.version
[0]>="3"):
4444 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
4445 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
4446 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
4447 if(isinstance(httpheaders
, dict)):
4448 httpheaders
= make_http_headers_from_dict_to_pycurl(httpheaders
);
4449 geturls_opener
.addheaders
= httpheaders
;
4451 if(postdata
is not None and not isinstance(postdata
, dict)):
4452 postdata
= urlencode(postdata
);
4453 retrieved_body
= BytesIO();
4454 retrieved_headers
= BytesIO();
4456 if(httpmethod
=="GET"):
4457 geturls_text
= pycurl
.Curl();
4458 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
4459 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_3_0
);
4460 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
4461 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
4462 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
4463 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
4464 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
4465 geturls_text
.perform();
4466 elif(httpmethod
=="POST"):
4467 geturls_text
= pycurl
.Curl();
4468 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
4469 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_3_0
);
4470 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
4471 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
4472 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
4473 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
4474 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
4475 geturls_text
.setopt(geturls_text
.POST
, True);
4476 geturls_text
.setopt(geturls_text
.POSTFIELDS
, postdata
);
4477 geturls_text
.perform();
4479 geturls_text
= pycurl
.Curl();
4480 geturls_text
.setopt(geturls_text
.URL
, httpurl
);
4481 geturls_text
.setopt(geturls_text
.HTTP_VERSION
, geturls_text
.CURL_HTTP_VERSION_3_0
);
4482 geturls_text
.setopt(geturls_text
.WRITEFUNCTION
, retrieved_body
.write
);
4483 geturls_text
.setopt(geturls_text
.HTTPHEADER
, httpheaders
);
4484 geturls_text
.setopt(geturls_text
.HEADERFUNCTION
, retrieved_headers
.write
);
4485 geturls_text
.setopt(geturls_text
.FOLLOWLOCATION
, True);
4486 geturls_text
.setopt(geturls_text
.TIMEOUT
, timeout
);
4487 geturls_text
.perform();
4488 retrieved_headers
.seek(0);
4489 if(sys
.version
[0]=="2"):
4490 pycurlhead
= retrieved_headers
.read();
4491 if(sys
.version
[0]>="3"):
4492 pycurlhead
= retrieved_headers
.read().decode('UTF-8');
4493 pyhttpverinfo
= re
.findall(r
'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead
.splitlines()[0].strip().rstrip('\r\n'))[0];
4494 pycurlheadersout
= make_http_headers_from_pycurl_to_dict(pycurlhead
);
4495 retrieved_body
.seek(0);
4496 except socket
.timeout
:
4497 log
.info("Error With URL "+httpurl
);
4499 except socket
.gaierror
:
4500 log
.info("Error With URL "+httpurl
);
4503 log
.info("Error With URL "+httpurl
);
4505 httpcodeout
= geturls_text
.getinfo(geturls_text
.HTTP_CODE
);
4506 httpcodereason
= http_status_to_reason(geturls_text
.getinfo(geturls_text
.HTTP_CODE
));
4507 httpversionout
= pyhttpverinfo
[0];
4508 httpmethodout
= httpmethod
;
4509 httpurlout
= geturls_text
.getinfo(geturls_text
.EFFECTIVE_URL
);
4510 httpheaderout
= pycurlheadersout
;
4511 httpheadersentout
= httpheaders
;
4512 if(isinstance(httpheaderout
, list)):
4513 httpheaderout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout
)));
4514 if(sys
.version
[0]=="2"):
4516 prehttpheaderout
= httpheaderout
;
4517 httpheaderkeys
= httpheaderout
.keys();
4518 imax
= len(httpheaderkeys
);
4522 httpheaderout
.update({httpheaderkeys
[ic
]: prehttpheaderout
[httpheaderkeys
[ic
]]});
4524 except AttributeError:
4526 httpheaderout
= fix_header_names(httpheaderout
);
4527 if(isinstance(httpheadersentout
, list)):
4528 httpheadersentout
= dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout
)));
4529 httpheadersentout
= fix_header_names(httpheadersentout
);
4530 downloadsize
= httpheaderout
.get('Content-Length');
4531 if(downloadsize
is not None):
4532 downloadsize
= int(downloadsize
);
4533 if downloadsize
is None: downloadsize
= 0;
4536 log
.info("Downloading URL "+httpurl
);
4537 with
BytesIO() as strbuf
:
4539 databytes
= retrieved_body
.read(buffersize
);
4540 if not databytes
: break;
4541 datasize
= len(databytes
);
4542 fulldatasize
= datasize
+ fulldatasize
;
4545 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4546 downloaddiff
= fulldatasize
- prevdownsize
;
4547 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4548 prevdownsize
= fulldatasize
;
4549 strbuf
.write(databytes
);
4551 returnval_content
= strbuf
.read();
4552 if(httpheaderout
.get("Content-Encoding")=="gzip"):
4554 returnval_content
= zlib
.decompress(returnval_content
, 16+zlib
.MAX_WBITS
);
4557 elif(httpheaderout
.get("Content-Encoding")=="deflate"):
4559 returnval_content
= zlib
.decompress(returnval_content
);
4562 elif(httpheaderout
.get("Content-Encoding")=="br" and havebrotli
):
4564 returnval_content
= brotli
.decompress(returnval_content
);
4565 except brotli
.error
:
4567 elif(httpheaderout
.get("Content-Encoding")=="zstd" and havezstd
):
4569 returnval_content
= zstandard
.decompress(returnval_content
);
4570 except zstandard
.error
:
4572 elif((httpheaderout
.get("Content-Encoding")=="lzma" or httpheaderout
.get("Content-Encoding")=="xz") and havelzma
):
4574 returnval_content
= lzma
.decompress(returnval_content
);
4575 except zstandard
.error
:
4577 elif(httpheaderout
.get("Content-Encoding")=="bzip2"):
4579 returnval_content
= bz2
.decompress(returnval_content
);
4580 except zstandard
.error
:
4582 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': httpheaderout
, 'Version': httpversionout
, 'Method': httpmethodout
, 'HeadersSent': httpheadersentout
, 'URL': httpurlout
, 'Code': httpcodeout
, 'Reason': httpcodereason
, 'HTTPLib': "pycurl3"};
4583 geturls_text
.close();
4587 def download_from_url_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4588 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
4591 if(havepycurl
and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4592 def download_from_url_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4593 returnval
= download_from_url_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
4596 if(havepycurl
and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4597 def download_from_url_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4598 returnval
= download_from_url_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
)
4601 if(havepycurl
and hasattr(pycurl
, "CURL_HTTP_VERSION_3_0")):
4602 def download_from_url_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4603 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
4604 exec_time_start
= time
.time();
4605 myhash
= hashlib
.new("sha1");
4606 if(sys
.version
[0]=="2"):
4607 myhash
.update(httpurl
);
4608 myhash
.update(str(buffersize
));
4609 myhash
.update(str(exec_time_start
));
4610 if(sys
.version
[0]>="3"):
4611 myhash
.update(httpurl
.encode('utf-8'));
4612 myhash
.update(str(buffersize
).encode('utf-8'));
4613 myhash
.update(str(exec_time_start
).encode('utf-8'));
4614 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
4616 sleep
= geturls_download_sleep
;
4619 pretmpfilename
= download_from_url_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
4620 if(not pretmpfilename
):
4622 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
4623 tmpfilename
= f
.name
;
4625 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4626 except AttributeError:
4628 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4633 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4634 f
.write(pretmpfilename
.get('Content'));
4636 exec_time_end
= time
.time();
4637 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
4638 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
4642 def download_from_url_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4643 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
4646 if(havepycurl
and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4647 def download_from_url_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4648 returnval
= download_from_url_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
4651 if(havepycurl
and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4652 def download_from_url_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4653 returnval
= download_from_url_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
, sleep
, timeout
)
4656 if(havepycurl
and hasattr(pycurl
, "CURL_HTTP_VERSION_3_0")):
4657 def download_from_url_to_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4658 global geturls_download_sleep
, havezstd
, havebrotli
;
4660 sleep
= geturls_download_sleep
;
4663 if(not outfile
=="-"):
4664 outpath
= outpath
.rstrip(os
.path
.sep
);
4665 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
4666 if(not os
.path
.exists(outpath
)):
4667 os
.makedirs(outpath
);
4668 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
4670 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
4672 pretmpfilename
= download_from_url_file_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4673 if(not pretmpfilename
):
4675 tmpfilename
= pretmpfilename
.get('Filename');
4676 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4678 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
4679 exec_time_start
= time
.time();
4680 shutil
.move(tmpfilename
, filepath
);
4682 os
.utime(filepath
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4683 except AttributeError:
4685 os
.utime(filepath
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4690 exec_time_end
= time
.time();
4691 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
4692 if(os
.path
.exists(tmpfilename
)):
4693 os
.remove(tmpfilename
);
4694 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4696 pretmpfilename
= download_from_url_file_with_pycurl3(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4697 tmpfilename
= pretmpfilename
.get('Filename');
4698 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4701 exec_time_start
= time
.time();
4702 with
open(tmpfilename
, 'rb') as ft
:
4705 databytes
= ft
.read(buffersize
[1]);
4706 if not databytes
: break;
4707 datasize
= len(databytes
);
4708 fulldatasize
= datasize
+ fulldatasize
;
4711 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4712 downloaddiff
= fulldatasize
- prevdownsize
;
4713 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4714 prevdownsize
= fulldatasize
;
4717 fdata
= f
.getvalue();
4720 os
.remove(tmpfilename
);
4721 exec_time_end
= time
.time();
4722 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
4723 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': httpmethod
, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4727 def download_from_url_to_file_with_pycurl3(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4728 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
4731 if(havepycurl
and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl
, "CURL_HTTP_VERSION_2_0")):
4732 def download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4733 returnval
= download_from_url_to_file_with_pycurl2(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
4736 if(havepycurl
and not hasattr(pycurl
, "CURL_HTTP_VERSION_2_0") and not hasattr(pycurl
, "CURL_HTTP_VERSION_3_0")):
4737 def download_from_url_to_file_with_pycurl(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4738 returnval
= download_from_url_to_file_with_pycurl(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
, timeout
)
4741 def download_file_from_ftp_file(url
):
4742 urlparts
= urlparse
.urlparse(url
);
4743 file_name
= os
.path
.basename(urlparts
.path
);
4744 file_dir
= os
.path
.dirname(urlparts
.path
);
4745 if(urlparts
.username
is not None):
4746 ftp_username
= urlparts
.username
;
4748 ftp_username
= "anonymous";
4749 if(urlparts
.password
is not None):
4750 ftp_password
= urlparts
.password
;
4751 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
4752 ftp_password
= "anonymous";
4755 if(urlparts
.scheme
=="ftp"):
4757 elif(urlparts
.scheme
=="ftps"):
4761 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
4763 ftp_port
= urlparts
.port
;
4764 if(urlparts
.port
is None):
4767 ftp
.connect(urlparts
.hostname
, ftp_port
);
4768 except socket
.gaierror
:
4769 log
.info("Error With URL "+httpurl
);
4771 except socket
.timeout
:
4772 log
.info("Error With URL "+httpurl
);
4774 ftp
.login(urlparts
.username
, urlparts
.password
);
4775 if(urlparts
.scheme
=="ftps"):
4777 ftpfile
= BytesIO();
4778 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
4779 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
4784 def download_file_from_ftp_string(url
):
4785 ftpfile
= download_file_from_ftp_file(url
);
4786 return ftpfile
.read();
4788 def download_from_url_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
4789 global geturls_download_sleep
, havezstd
, havebrotli
;
4791 sleep
= geturls_download_sleep
;
4794 urlparts
= urlparse
.urlparse(httpurl
);
4795 if(isinstance(httpheaders
, list)):
4796 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
4797 httpheaders
= fix_header_names(httpheaders
);
4798 if(httpuseragent
is not None):
4799 if('User-Agent' in httpheaders
):
4800 httpheaders
['User-Agent'] = httpuseragent
;
4802 httpuseragent
.update({'User-Agent': httpuseragent
});
4803 if(httpreferer
is not None):
4804 if('Referer' in httpheaders
):
4805 httpheaders
['Referer'] = httpreferer
;
4807 httpuseragent
.update({'Referer': httpreferer
});
4808 if(isinstance(httpheaders
, dict)):
4809 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
4811 geturls_text
= download_file_from_ftp_file(httpurl
);
4812 if(not geturls_text
):
4814 downloadsize
= None;
4815 if(downloadsize
is not None):
4816 downloadsize
= int(downloadsize
);
4817 if downloadsize
is None: downloadsize
= 0;
4820 log
.info("Downloading URL "+httpurl
);
4821 with
BytesIO() as strbuf
:
4823 databytes
= geturls_text
.read(buffersize
);
4824 if not databytes
: break;
4825 datasize
= len(databytes
);
4826 fulldatasize
= datasize
+ fulldatasize
;
4829 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4830 downloaddiff
= fulldatasize
- prevdownsize
;
4831 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4832 prevdownsize
= fulldatasize
;
4833 strbuf
.write(databytes
);
4835 returnval_content
= strbuf
.read();
4836 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
4837 geturls_text
.close();
4840 def download_from_url_file_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
4841 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
4842 exec_time_start
= time
.time();
4843 myhash
= hashlib
.new("sha1");
4844 if(sys
.version
[0]=="2"):
4845 myhash
.update(httpurl
);
4846 myhash
.update(str(buffersize
));
4847 myhash
.update(str(exec_time_start
));
4848 if(sys
.version
[0]>="3"):
4849 myhash
.update(httpurl
.encode('utf-8'));
4850 myhash
.update(str(buffersize
).encode('utf-8'));
4851 myhash
.update(str(exec_time_start
).encode('utf-8'));
4852 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
4854 sleep
= geturls_download_sleep
;
4857 pretmpfilename
= download_from_url_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
4858 if(not pretmpfilename
):
4860 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
4861 tmpfilename
= f
.name
;
4863 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
4864 except AttributeError:
4866 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4871 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4872 f
.write(pretmpfilename
.get('Content'));
4874 exec_time_end
= time
.time();
4875 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
4876 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
4879 def download_from_url_to_file_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
4880 global geturls_download_sleep
, havezstd
, havebrotli
;
4882 sleep
= geturls_download_sleep
;
4885 if(not outfile
=="-"):
4886 outpath
= outpath
.rstrip(os
.path
.sep
);
4887 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
4888 if(not os
.path
.exists(outpath
)):
4889 os
.makedirs(outpath
);
4890 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
4892 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
4894 pretmpfilename
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4895 if(not pretmpfilename
):
4897 tmpfilename
= pretmpfilename
.get('Filename');
4898 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4900 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
4901 exec_time_start
= time
.time();
4902 shutil
.move(tmpfilename
, filepath
);
4903 exec_time_end
= time
.time();
4904 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
4905 if(os
.path
.exists(tmpfilename
)):
4906 os
.remove(tmpfilename
);
4907 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': None, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4909 pretmpfilename
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
4910 tmpfilename
= pretmpfilename
.get('Filename');
4911 downloadsize
= int(os
.path
.getsize(tmpfilename
));
4914 exec_time_start
= time
.time();
4915 with
open(tmpfilename
, 'rb') as ft
:
4918 databytes
= ft
.read(buffersize
[1]);
4919 if not databytes
: break;
4920 datasize
= len(databytes
);
4921 fulldatasize
= datasize
+ fulldatasize
;
4924 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
4925 downloaddiff
= fulldatasize
- prevdownsize
;
4926 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
4927 prevdownsize
= fulldatasize
;
4930 fdata
= f
.getvalue();
4933 os
.remove(tmpfilename
);
4934 exec_time_end
= time
.time();
4935 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
4936 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': None, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
4939 def upload_file_to_ftp_file(ftpfile
, url
):
4940 urlparts
= urlparse
.urlparse(url
);
4941 file_name
= os
.path
.basename(urlparts
.path
);
4942 file_dir
= os
.path
.dirname(urlparts
.path
);
4943 if(urlparts
.username
is not None):
4944 ftp_username
= urlparts
.username
;
4946 ftp_username
= "anonymous";
4947 if(urlparts
.password
is not None):
4948 ftp_password
= urlparts
.password
;
4949 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
4950 ftp_password
= "anonymous";
4953 if(urlparts
.scheme
=="ftp"):
4955 elif(urlparts
.scheme
=="ftps"):
4959 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
4961 ftp_port
= urlparts
.port
;
4962 if(urlparts
.port
is None):
4965 ftp
.connect(urlparts
.hostname
, ftp_port
);
4966 except socket
.gaierror
:
4967 log
.info("Error With URL "+httpurl
);
4969 except socket
.timeout
:
4970 log
.info("Error With URL "+httpurl
);
4972 ftp
.login(urlparts
.username
, urlparts
.password
);
4973 if(urlparts
.scheme
=="ftps"):
4975 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
4980 def upload_file_to_ftp_string(ftpstring
, url
):
4981 ftpfileo
= BytesIO(ftpstring
);
4982 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
4987 def download_file_from_sftp_file(url
):
4988 urlparts
= urlparse
.urlparse(url
);
4989 file_name
= os
.path
.basename(urlparts
.path
);
4990 file_dir
= os
.path
.dirname(urlparts
.path
);
4991 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
4993 sftp_port
= urlparts
.port
;
4994 if(urlparts
.port
is None):
4997 sftp_port
= urlparts
.port
;
4998 if(urlparts
.username
is not None):
4999 sftp_username
= urlparts
.username
;
5001 sftp_username
= "anonymous";
5002 if(urlparts
.password
is not None):
5003 sftp_password
= urlparts
.password
;
5004 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
5005 sftp_password
= "anonymous";
5008 if(urlparts
.scheme
!="sftp"):
5010 ssh
= paramiko
.SSHClient();
5011 ssh
.load_system_host_keys();
5012 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
5014 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
5015 except paramiko
.ssh_exception
.SSHException
:
5017 except socket
.gaierror
:
5018 log
.info("Error With URL "+httpurl
);
5020 except socket
.timeout
:
5021 log
.info("Error With URL "+httpurl
);
5023 sftp
= ssh
.open_sftp();
5024 sftpfile
= BytesIO();
5025 sftp
.getfo(urlparts
.path
, sftpfile
);
5028 sftpfile
.seek(0, 0);
5031 def download_file_from_sftp_file(url
):
5035 def download_file_from_sftp_string(url
):
5036 sftpfile
= download_file_from_sftp_file(url
);
5037 return sftpfile
.read();
5039 def download_file_from_ftp_string(url
):
5043 def download_from_url_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
5044 global geturls_download_sleep
, havezstd
, havebrotli
;
5046 sleep
= geturls_download_sleep
;
5049 urlparts
= urlparse
.urlparse(httpurl
);
5050 if(isinstance(httpheaders
, list)):
5051 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
5052 httpheaders
= fix_header_names(httpheaders
);
5053 if(httpuseragent
is not None):
5054 if('User-Agent' in httpheaders
):
5055 httpheaders
['User-Agent'] = httpuseragent
;
5057 httpuseragent
.update({'User-Agent': httpuseragent
});
5058 if(httpreferer
is not None):
5059 if('Referer' in httpheaders
):
5060 httpheaders
['Referer'] = httpreferer
;
5062 httpuseragent
.update({'Referer': httpreferer
});
5063 if(isinstance(httpheaders
, dict)):
5064 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
5066 geturls_text
= download_file_from_sftp_file(httpurl
);
5067 if(not geturls_text
):
5069 downloadsize
= None;
5070 if(downloadsize
is not None):
5071 downloadsize
= int(downloadsize
);
5072 if downloadsize
is None: downloadsize
= 0;
5075 log
.info("Downloading URL "+httpurl
);
5076 with
BytesIO() as strbuf
:
5078 databytes
= geturls_text
.read(buffersize
);
5079 if not databytes
: break;
5080 datasize
= len(databytes
);
5081 fulldatasize
= datasize
+ fulldatasize
;
5084 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
5085 downloaddiff
= fulldatasize
- prevdownsize
;
5086 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
5087 prevdownsize
= fulldatasize
;
5088 strbuf
.write(databytes
);
5090 returnval_content
= strbuf
.read();
5091 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
5092 geturls_text
.close();
5095 if(not haveparamiko
):
5096 def download_from_url_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
5100 def download_from_url_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
5101 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
5102 exec_time_start
= time
.time();
5103 myhash
= hashlib
.new("sha1");
5104 if(sys
.version
[0]=="2"):
5105 myhash
.update(httpurl
);
5106 myhash
.update(str(buffersize
));
5107 myhash
.update(str(exec_time_start
));
5108 if(sys
.version
[0]>="3"):
5109 myhash
.update(httpurl
.encode('utf-8'));
5110 myhash
.update(str(buffersize
).encode('utf-8'));
5111 myhash
.update(str(exec_time_start
).encode('utf-8'));
5112 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
5114 sleep
= geturls_download_sleep
;
5117 pretmpfilename
= download_from_url_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
5118 if(not pretmpfilename
):
5120 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
5121 tmpfilename
= f
.name
;
5123 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
5124 except AttributeError:
5126 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
5131 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
5132 f
.write(pretmpfilename
.get('Content'));
5134 exec_time_end
= time
.time();
5135 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
5136 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
5139 if(not haveparamiko
):
5140 def download_from_url_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
5144 def download_from_url_to_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
5145 global geturls_download_sleep
, havezstd
, havebrotli
;
5147 sleep
= geturls_download_sleep
;
5150 if(not outfile
=="-"):
5151 outpath
= outpath
.rstrip(os
.path
.sep
);
5152 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
5153 if(not os
.path
.exists(outpath
)):
5154 os
.makedirs(outpath
);
5155 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
5157 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
5159 pretmpfilename
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
5160 if(not pretmpfilename
):
5162 tmpfilename
= pretmpfilename
.get('Filename');
5163 downloadsize
= int(os
.path
.getsize(tmpfilename
));
5165 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
5166 exec_time_start
= time
.time();
5167 shutil
.move(tmpfilename
, filepath
);
5168 exec_time_end
= time
.time();
5169 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
5170 if(os
.path
.exists(tmpfilename
)):
5171 os
.remove(tmpfilename
);
5172 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': None, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
5174 pretmpfilename
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
5175 tmpfilename
= pretmpfilename
.get('Filename');
5176 downloadsize
= int(os
.path
.getsize(tmpfilename
));
5179 exec_time_start
= time
.time();
5180 with
open(tmpfilename
, 'rb') as ft
:
5183 databytes
= ft
.read(buffersize
[1]);
5184 if not databytes
: break;
5185 datasize
= len(databytes
);
5186 fulldatasize
= datasize
+ fulldatasize
;
5189 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
5190 downloaddiff
= fulldatasize
- prevdownsize
;
5191 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
5192 prevdownsize
= fulldatasize
;
5195 fdata
= f
.getvalue();
5198 os
.remove(tmpfilename
);
5199 exec_time_end
= time
.time();
5200 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
5201 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': None, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
5204 if(not haveparamiko
):
5205 def download_from_url_to_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpuseragent
=None, httpreferer
=None, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
5209 def upload_file_to_sftp_file(sftpfile
, url
):
5210 urlparts
= urlparse
.urlparse(url
);
5211 file_name
= os
.path
.basename(urlparts
.path
);
5212 file_dir
= os
.path
.dirname(urlparts
.path
);
5213 sftp_port
= urlparts
.port
;
5214 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
5216 if(urlparts
.port
is None):
5219 sftp_port
= urlparts
.port
;
5220 if(urlparts
.username
is not None):
5221 sftp_username
= urlparts
.username
;
5223 sftp_username
= "anonymous";
5224 if(urlparts
.password
is not None):
5225 sftp_password
= urlparts
.password
;
5226 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
5227 sftp_password
= "anonymous";
5230 if(urlparts
.scheme
!="sftp"):
5232 ssh
= paramiko
.SSHClient();
5233 ssh
.load_system_host_keys();
5234 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
5236 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
5237 except paramiko
.ssh_exception
.SSHException
:
5239 except socket
.gaierror
:
5240 log
.info("Error With URL "+httpurl
);
5242 except socket
.timeout
:
5243 log
.info("Error With URL "+httpurl
);
5245 sftp
= ssh
.open_sftp();
5246 sftp
.putfo(sftpfile
, urlparts
.path
);
5249 sftpfile
.seek(0, 0);
5252 def upload_file_to_sftp_file(sftpfile
, url
):
5256 def upload_file_to_sftp_string(sftpstring
, url
):
5257 sftpfileo
= BytesIO(sftpstring
);
5258 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
5262 def upload_file_to_sftp_string(url
):
5267 def download_file_from_pysftp_file(url
):
5268 urlparts
= urlparse
.urlparse(url
);
5269 file_name
= os
.path
.basename(urlparts
.path
);
5270 file_dir
= os
.path
.dirname(urlparts
.path
);
5271 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
5273 sftp_port
= urlparts
.port
;
5274 if(urlparts
.port
is None):
5277 sftp_port
= urlparts
.port
;
5278 if(urlparts
.username
is not None):
5279 sftp_username
= urlparts
.username
;
5281 sftp_username
= "anonymous";
5282 if(urlparts
.password
is not None):
5283 sftp_password
= urlparts
.password
;
5284 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
5285 sftp_password
= "anonymous";
5288 if(urlparts
.scheme
!="sftp"):
5291 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
5292 except paramiko
.ssh_exception
.SSHException
:
5294 except socket
.gaierror
:
5295 log
.info("Error With URL "+httpurl
);
5297 except socket
.timeout
:
5298 log
.info("Error With URL "+httpurl
);
5300 sftp
= ssh
.open_sftp();
5301 sftpfile
= BytesIO();
5302 sftp
.getfo(urlparts
.path
, sftpfile
);
5305 sftpfile
.seek(0, 0);
5308 def download_file_from_pysftp_file(url
):
5312 def download_file_from_pysftp_string(url
):
5313 sftpfile
= download_file_from_pysftp_file(url
);
5314 return sftpfile
.read();
5316 def download_file_from_ftp_string(url
):
5320 def download_from_url_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
5321 global geturls_download_sleep
, havezstd
, havebrotli
;
5323 sleep
= geturls_download_sleep
;
5326 urlparts
= urlparse
.urlparse(httpurl
);
5327 if(isinstance(httpheaders
, list)):
5328 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
5329 httpheaders
= fix_header_names(httpheaders
);
5330 if(isinstance(httpheaders
, dict)):
5331 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
5333 geturls_text
= download_file_from_pysftp_file(httpurl
);
5334 if(not geturls_text
):
5336 downloadsize
= None;
5337 if(downloadsize
is not None):
5338 downloadsize
= int(downloadsize
);
5339 if downloadsize
is None: downloadsize
= 0;
5342 log
.info("Downloading URL "+httpurl
);
5343 with
BytesIO() as strbuf
:
5345 databytes
= geturls_text
.read(buffersize
);
5346 if not databytes
: break;
5347 datasize
= len(databytes
);
5348 fulldatasize
= datasize
+ fulldatasize
;
5351 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
5352 downloaddiff
= fulldatasize
- prevdownsize
;
5353 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
5354 prevdownsize
= fulldatasize
;
5355 strbuf
.write(databytes
);
5357 returnval_content
= strbuf
.read();
5358 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Contentsize': fulldatasize
, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize
, 2, "IEC"), 'SI': get_readable_size(fulldatasize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
5359 geturls_text
.close();
5363 def download_from_url_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1, timeout
=10):
5367 def download_from_url_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
5368 global geturls_download_sleep
, havezstd
, havebrotli
, tmpfileprefix
, tmpfilesuffix
;
5369 exec_time_start
= time
.time();
5370 myhash
= hashlib
.new("sha1");
5371 if(sys
.version
[0]=="2"):
5372 myhash
.update(httpurl
);
5373 myhash
.update(str(buffersize
));
5374 myhash
.update(str(exec_time_start
));
5375 if(sys
.version
[0]>="3"):
5376 myhash
.update(httpurl
.encode('utf-8'));
5377 myhash
.update(str(buffersize
).encode('utf-8'));
5378 myhash
.update(str(exec_time_start
).encode('utf-8'));
5379 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
5381 sleep
= geturls_download_sleep
;
5384 pretmpfilename
= download_from_url_with_pysftp(httpurl
, httpheaders
, httpuseragent
, httpreferer
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
, timeout
);
5385 if(not pretmpfilename
):
5387 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
5388 tmpfilename
= f
.name
;
5390 os
.utime(tmpfilename
, (time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple()), time
.mktime(email
.utils
.parsedate_to_datetime(pretmpfilename
.get('Headers').get('Last-Modified')).timetuple())));
5391 except AttributeError:
5393 os
.utime(tmpfilename
, (time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time
.mktime(datetime
.datetime
.strptime(pretmpfilename
.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
5398 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': pretmpfilename
.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename
.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
5399 f
.write(pretmpfilename
.get('Content'));
5401 exec_time_end
= time
.time();
5402 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
5403 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'FilesizeAlt': {'IEC': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "IEC"), 'SI': get_readable_size(os
.path
.getsize(tmpfilename
), 2, "SI")}, 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
5407 def download_from_url_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, ranges
=[None, None], buffersize
=524288, sleep
=-1, timeout
=10):
5411 def download_from_url_to_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
5412 global geturls_download_sleep
, havezstd
, havebrotli
;
5414 sleep
= geturls_download_sleep
;
5417 if(not outfile
=="-"):
5418 outpath
= outpath
.rstrip(os
.path
.sep
);
5419 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
5420 if(not os
.path
.exists(outpath
)):
5421 os
.makedirs(outpath
);
5422 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
5424 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
5426 pretmpfilename
= download_from_url_file_with_pysftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
5427 if(not pretmpfilename
):
5429 tmpfilename
= pretmpfilename
.get('Filename');
5430 downloadsize
= int(os
.path
.getsize(tmpfilename
));
5432 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
5433 exec_time_start
= time
.time();
5434 shutil
.move(tmpfilename
, filepath
);
5435 exec_time_end
= time
.time();
5436 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
5437 if(os
.path
.exists(tmpfilename
)):
5438 os
.remove(tmpfilename
);
5439 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': None, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
5441 pretmpfilename
= download_from_url_file_with_pysftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, ranges
, buffersize
[0], sleep
, timeout
);
5442 tmpfilename
= pretmpfilename
.get('Filename');
5443 downloadsize
= int(os
.path
.getsize(tmpfilename
));
5446 exec_time_start
= time
.time();
5447 with
open(tmpfilename
, 'rb') as ft
:
5450 databytes
= ft
.read(buffersize
[1]);
5451 if not databytes
: break;
5452 datasize
= len(databytes
);
5453 fulldatasize
= datasize
+ fulldatasize
;
5456 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
5457 downloaddiff
= fulldatasize
- prevdownsize
;
5458 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
5459 prevdownsize
= fulldatasize
;
5462 fdata
= f
.getvalue();
5465 os
.remove(tmpfilename
);
5466 exec_time_end
= time
.time();
5467 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
5468 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename
.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
.get('Headers'), 'Version': pretmpfilename
.get('Version'), 'Method': pretmpfilename
.get('Method'), 'Method': None, 'HeadersSent': pretmpfilename
.get('HeadersSent'), 'URL': pretmpfilename
.get('URL'), 'Code': pretmpfilename
.get('Code'), 'Reason': pretmpfilename
.get('Reason'), 'HTTPLib': pretmpfilename
.get('HTTPLib')};
5472 def download_from_url_to_file_with_pysftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), ranges
=[None, None], buffersize
=[524288, 524288], sleep
=-1, timeout
=10):
5476 def upload_file_to_pysftp_file(sftpfile
, url
):
5477 urlparts
= urlparse
.urlparse(url
);
5478 file_name
= os
.path
.basename(urlparts
.path
);
5479 file_dir
= os
.path
.dirname(urlparts
.path
);
5480 sftp_port
= urlparts
.port
;
5481 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
5483 if(urlparts
.port
is None):
5486 sftp_port
= urlparts
.port
;
5487 if(urlparts
.username
is not None):
5488 sftp_username
= urlparts
.username
;
5490 sftp_username
= "anonymous";
5491 if(urlparts
.password
is not None):
5492 sftp_password
= urlparts
.password
;
5493 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
5494 sftp_password
= "anonymous";
5497 if(urlparts
.scheme
!="sftp"):
5500 pysftp
.Connection(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
5501 except paramiko
.ssh_exception
.SSHException
:
5503 except socket
.gaierror
:
5504 log
.info("Error With URL "+httpurl
);
5506 except socket
.timeout
:
5507 log
.info("Error With URL "+httpurl
);
5509 sftp
= ssh
.open_sftp();
5510 sftp
.putfo(sftpfile
, urlparts
.path
);
5513 sftpfile
.seek(0, 0);
5516 def upload_file_to_pysftp_file(sftpfile
, url
):
5520 def upload_file_to_pysftp_string(sftpstring
, url
):
5521 sftpfileo
= BytesIO(sftpstring
);
5522 sftpfile
= upload_file_to_pysftp_files(ftpfileo
, url
);
5526 def upload_file_to_pysftp_string(url
):