?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyWWW-Get.git] / pywwwgetold.py
blob346873973981d4271d1530c99011ffb8047a7d18
1 #!/usr/bin/env python
2
3 '''
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the Revised BSD License.
6
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 Revised BSD License for more details.
11
12 Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
13 Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
14
15 $FileInfo: pywwwgetold.py - Last Update: 10/5/2023 Ver. 2.0.2 RC 1 - Author: cooldude2k $
16 '''
17
18 from __future__ import division, absolute_import, print_function;
19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, zlib, time, argparse, cgi, subprocess, socket, email.utils, datetime, time;
20 import logging as log;
21 from ftplib import FTP, FTP_TLS;
22 from base64 import b64encode;
23 haverequests = False;
24 try:
25 import requests;
26 haverequests = True;
27 except ImportError:
28 haverequests = False;
29 havemechanize = False;
30 try:
31 import mechanize;
32 havemechanize = True;
33 except ImportError:
34 havemechanize = False;
35 havepycurl = False;
36 try:
37 import pycurl;
38 havepycurl = True;
39 except ImportError:
40 havepycurl = False;
41 haveparamiko = False;
42 try:
43 import paramiko;
44 haveparamiko = True;
45 except ImportError:
46 haveparamiko = False;
47 havepysftp = False;
48 try:
49 import pysftp;
50 havepysftp = True;
51 except ImportError:
52 havepysftp = False;
53 haveurllib3 = False;
54 try:
55 import urllib3;
56 haveurllib3 = True;
57 except ImportError:
58 haveurllib3 = False;
59 havehttplib2 = False;
60 try:
61 import httplib2;
62 from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
63 havehttplib2 = True;
64 except ImportError:
65 havehttplib2 = False;
66 havehttpx = False;
67 try:
68 import httpx;
69 havehttpx = True;
70 except ImportError:
71 havehttpx = False;
72 havehttpcore = False;
73 try:
74 import httpcore;
75 havehttpcore = True;
76 except ImportError:
77 havehttpcore = False;
78 havebrotli = False;
79 try:
80 import brotli;
81 havebrotli = True;
82 except ImportError:
83 havebrotli = False;
84 havezstd = False;
85 try:
86 import zstandard;
87 havezstd = True;
88 except ImportError:
89 havezstd = False;
90 if(sys.version[0]=="2"):
91 try:
92 from io import StringIO, BytesIO;
93 except ImportError:
94 try:
95 from cStringIO import StringIO;
96 from cStringIO import StringIO as BytesIO;
97 except ImportError:
98 from StringIO import StringIO;
99 from StringIO import StringIO as BytesIO;
100 # From http://python-future.org/compatible_idioms.html
101 from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
102 from urllib import urlencode;
103 from urllib import urlopen as urlopenalt;
104 from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
105 import urlparse, cookielib;
106 from httplib import HTTPConnection, HTTPSConnection;
107 if(sys.version[0]>="3"):
108 from io import StringIO, BytesIO;
109 # From http://python-future.org/compatible_idioms.html
110 from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
111 from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
112 from urllib.error import HTTPError, URLError;
113 import urllib.parse as urlparse;
114 import http.cookiejar as cookielib;
115 from http.client import HTTPConnection, HTTPSConnection;
116
117 __program_name__ = "PyWWW-Get";
118 __program_alt_name__ = "PyWWWGet";
119 __program_small_name__ = "wwwget";
120 __project__ = __program_name__;
121 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
122 __version_info__ = (2, 0, 2, "RC 1", 1);
123 __version_date_info__ = (2023, 10, 5, "RC 1", 1);
124 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
125 __revision__ = __version_info__[3];
126 __revision_id__ = "$Id$";
127 if(__version_info__[4] is not None):
128 __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
129 if(__version_info__[4] is None):
130 __version_date_plusrc__ = __version_date__;
131 if(__version_info__[3] is not None):
132 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
133 if(__version_info__[3] is None):
134 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
135
136 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
137 tmpfilesuffix = "-";
138 pytempdir = tempfile.gettempdir();
139
140 PyBitness = platform.architecture();
141 if(PyBitness=="32bit" or PyBitness=="32"):
142 PyBitness = "32";
143 elif(PyBitness=="64bit" or PyBitness=="64"):
144 PyBitness = "64";
145 else:
146 PyBitness = "32";
147
148 compression_supported = "gzip, deflate";
149 if(havebrotli and not havezstd):
150 compression_supported = "gzip, deflate, br";
151 elif(not havebrotli and havezstd):
152 compression_supported = "gzip, deflate, zstd";
153 elif(havebrotli and havezstd):
154 compression_supported = "gzip, deflate, zstd, br";
155 else:
156 compression_supported = "gzip, deflate";
157
158 geturls_cj = cookielib.CookieJar();
159 windowsNT4_ua_string = "Windows NT 4.0";
160 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
161 windows2k_ua_string = "Windows NT 5.0";
162 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
163 windowsXP_ua_string = "Windows NT 5.1";
164 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
165 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
166 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
167 windows7_ua_string = "Windows NT 6.1; Win64; x64";
168 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
169 windows8_ua_string = "Windows NT 6.2; Win64; x64";
170 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
171 windows81_ua_string = "Windows NT 6.3; Win64; x64";
172 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
173 windows10_ua_string = "Windows NT 10.0; Win64; x64";
174 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
175 windows11_ua_string = "Windows NT 11.0; Win64; x64";
176 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
177 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
178 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
179 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
180 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
181 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
182 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
183 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
184 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
185 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
186 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
187 if(platform.python_implementation()!=""):
188 py_implementation = platform.python_implementation();
189 if(platform.python_implementation()==""):
190 py_implementation = "Python";
191 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
192 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
193 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
194 geturls_ua = geturls_ua_firefox_windows7;
195 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
196 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
197 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
198 geturls_headers_chrome_windows7.update(windows7_ua_addon);
199 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
200 geturls_headers_chromium_windows7.update(windows7_ua_addon);
201 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
202 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
203 geturls_headers_opera_windows7.update(windows7_ua_addon);
204 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
205 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
206 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
207 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
208 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
209 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
210 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
211 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
212 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 geturls_headers = geturls_headers_firefox_windows7;
214 geturls_download_sleep = 0;
215
216 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
217 if(outtype=="print" and dbgenable):
218 print(dbgtxt);
219 return True;
220 elif(outtype=="log" and dbgenable):
221 logging.info(dbgtxt);
222 return True;
223 elif(outtype=="warning" and dbgenable):
224 logging.warning(dbgtxt);
225 return True;
226 elif(outtype=="error" and dbgenable):
227 logging.error(dbgtxt);
228 return True;
229 elif(outtype=="critical" and dbgenable):
230 logging.critical(dbgtxt);
231 return True;
232 elif(outtype=="exception" and dbgenable):
233 logging.exception(dbgtxt);
234 return True;
235 elif(outtype=="logalt" and dbgenable):
236 logging.log(dgblevel, dbgtxt);
237 return True;
238 elif(outtype=="debug" and dbgenable):
239 logging.debug(dbgtxt);
240 return True;
241 elif(not dbgenable):
242 return True;
243 else:
244 return False;
245 return False;
246
247 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
248 dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
249 if(not dbgout):
250 return False;
251 return dbgtxt;
252
253 def add_url_param(url, **params):
254 n=3;
255 parts = list(urlparse.urlsplit(url));
256 d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
257 d.update(params);
258 parts[n]=urlencode(d);
259 return urlparse.urlunsplit(parts);
260
261 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
262 def which_exec(execfile):
263 for path in os.environ["PATH"].split(":"):
264 if os.path.exists(path + "/" + execfile):
265 return path + "/" + execfile;
266
267 def listize(varlist):
268 il = 0;
269 ix = len(varlist);
270 ilx = 1;
271 newlistreg = {};
272 newlistrev = {};
273 newlistfull = {};
274 while(il < ix):
275 newlistreg.update({ilx: varlist[il]});
276 newlistrev.update({varlist[il]: ilx});
277 ilx = ilx + 1;
278 il = il + 1;
279 newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
280 return newlistfull;
281
282 def twolistize(varlist):
283 il = 0;
284 ix = len(varlist);
285 ilx = 1;
286 newlistnamereg = {};
287 newlistnamerev = {};
288 newlistdescreg = {};
289 newlistdescrev = {};
290 newlistfull = {};
291 while(il < ix):
292 newlistnamereg.update({ilx: varlist[il][0].strip()});
293 newlistnamerev.update({varlist[il][0].strip(): ilx});
294 newlistdescreg.update({ilx: varlist[il][1].strip()});
295 newlistdescrev.update({varlist[il][1].strip(): ilx});
296 ilx = ilx + 1;
297 il = il + 1;
298 newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
299 newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
300 newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
301 return newlistfull;
302
303 def arglistize(proexec, *varlist):
304 il = 0;
305 ix = len(varlist);
306 ilx = 1;
307 newarglist = [proexec];
308 while(il < ix):
309 if varlist[il][0] is not None:
310 newarglist.append(varlist[il][0]);
311 if varlist[il][1] is not None:
312 newarglist.append(varlist[il][1]);
313 il = il + 1;
314 return newarglist;
315
316 def fix_header_names(header_dict):
317 if(sys.version[0]=="2"):
318 header_dict = {k.title(): v for k, v in header_dict.iteritems()};
319 if(sys.version[0]>="3"):
320 header_dict = {k.title(): v for k, v in header_dict.items()};
321 return header_dict;
322
323 # hms_string by ArcGIS Python Recipes
324 # https://arcpy.wordpress.com/2012/04/20/146/
325 def hms_string(sec_elapsed):
326 h = int(sec_elapsed / (60 * 60));
327 m = int((sec_elapsed % (60 * 60)) / 60);
328 s = sec_elapsed % 60.0;
329 return "{}:{:>02}:{:>05.2f}".format(h, m, s);
330
331 # get_readable_size by Lipis
332 # http://stackoverflow.com/posts/14998888/revisions
333 def get_readable_size(bytes, precision=1, unit="IEC"):
334 unit = unit.upper();
335 if(unit!="IEC" and unit!="SI"):
336 unit = "IEC";
337 if(unit=="IEC"):
338 units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
339 unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
340 unitsize = 1024.0;
341 if(unit=="SI"):
342 units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
343 unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
344 unitsize = 1000.0;
345 return_val = {};
346 orgbytes = bytes;
347 for unit in units:
348 if abs(bytes) < unitsize:
349 strformat = "%3."+str(precision)+"f%s";
350 pre_return_val = (strformat % (bytes, unit));
351 pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
352 pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
353 alt_return_val = pre_return_val.split();
354 return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
355 return return_val;
356 bytes /= unitsize;
357 strformat = "%."+str(precision)+"f%s";
358 pre_return_val = (strformat % (bytes, "YiB"));
359 pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
360 pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
361 alt_return_val = pre_return_val.split();
362 return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
363 return return_val;
364
365 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
366 unit = unit.upper();
367 usehashtypes = usehashtypes.lower();
368 getfilesize = os.path.getsize(infile);
369 return_val = get_readable_size(getfilesize, precision, unit);
370 if(usehashes):
371 hashtypelist = usehashtypes.split(",");
372 openfile = open(infile, "rb");
373 filecontents = openfile.read();
374 openfile.close();
375 listnumcount = 0;
376 listnumend = len(hashtypelist);
377 while(listnumcount < listnumend):
378 hashtypelistlow = hashtypelist[listnumcount].strip();
379 hashtypelistup = hashtypelistlow.upper();
380 filehash = hashlib.new(hashtypelistup);
381 filehash.update(filecontents);
382 filegethash = filehash.hexdigest();
383 return_val.update({hashtypelistup: filegethash});
384 listnumcount += 1;
385 return return_val;
386
387 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
388 unit = unit.upper();
389 usehashtypes = usehashtypes.lower();
390 getfilesize = len(instring);
391 return_val = get_readable_size(getfilesize, precision, unit);
392 if(usehashes):
393 hashtypelist = usehashtypes.split(",");
394 listnumcount = 0;
395 listnumend = len(hashtypelist);
396 while(listnumcount < listnumend):
397 hashtypelistlow = hashtypelist[listnumcount].strip();
398 hashtypelistup = hashtypelistlow.upper();
399 filehash = hashlib.new(hashtypelistup);
400 if(sys.version[0]=="2"):
401 filehash.update(instring);
402 if(sys.version[0]>="3"):
403 filehash.update(instring.encode('utf-8'));
404 filegethash = filehash.hexdigest();
405 return_val.update({hashtypelistup: filegethash});
406 listnumcount += 1;
407 return return_val;
408
409 def http_status_to_reason(code):
410 reasons = {
411 100: 'Continue',
412 101: 'Switching Protocols',
413 102: 'Processing',
414 200: 'OK',
415 201: 'Created',
416 202: 'Accepted',
417 203: 'Non-Authoritative Information',
418 204: 'No Content',
419 205: 'Reset Content',
420 206: 'Partial Content',
421 207: 'Multi-Status',
422 208: 'Already Reported',
423 226: 'IM Used',
424 300: 'Multiple Choices',
425 301: 'Moved Permanently',
426 302: 'Found',
427 303: 'See Other',
428 304: 'Not Modified',
429 305: 'Use Proxy',
430 307: 'Temporary Redirect',
431 308: 'Permanent Redirect',
432 400: 'Bad Request',
433 401: 'Unauthorized',
434 402: 'Payment Required',
435 403: 'Forbidden',
436 404: 'Not Found',
437 405: 'Method Not Allowed',
438 406: 'Not Acceptable',
439 407: 'Proxy Authentication Required',
440 408: 'Request Timeout',
441 409: 'Conflict',
442 410: 'Gone',
443 411: 'Length Required',
444 412: 'Precondition Failed',
445 413: 'Payload Too Large',
446 414: 'URI Too Long',
447 415: 'Unsupported Media Type',
448 416: 'Range Not Satisfiable',
449 417: 'Expectation Failed',
450 421: 'Misdirected Request',
451 422: 'Unprocessable Entity',
452 423: 'Locked',
453 424: 'Failed Dependency',
454 426: 'Upgrade Required',
455 428: 'Precondition Required',
456 429: 'Too Many Requests',
457 431: 'Request Header Fields Too Large',
458 451: 'Unavailable For Legal Reasons',
459 500: 'Internal Server Error',
460 501: 'Not Implemented',
461 502: 'Bad Gateway',
462 503: 'Service Unavailable',
463 504: 'Gateway Timeout',
464 505: 'HTTP Version Not Supported',
465 506: 'Variant Also Negotiates',
466 507: 'Insufficient Storage',
467 508: 'Loop Detected',
468 510: 'Not Extended',
469 511: 'Network Authentication Required'
470 }
471 return reasons.get(code, 'Unknown Status Code');
472
473 def ftp_status_to_reason(code):
474 reasons = {
475 110: 'Restart marker reply',
476 120: 'Service ready in nnn minutes',
477 125: 'Data connection already open; transfer starting',
478 150: 'File status okay; about to open data connection',
479 200: 'Command okay',
480 202: 'Command not implemented, superfluous at this site',
481 211: 'System status, or system help reply',
482 212: 'Directory status',
483 213: 'File status',
484 214: 'Help message',
485 215: 'NAME system type',
486 220: 'Service ready for new user',
487 221: 'Service closing control connection',
488 225: 'Data connection open; no transfer in progress',
489 226: 'Closing data connection',
490 227: 'Entering Passive Mode',
491 230: 'User logged in, proceed',
492 250: 'Requested file action okay, completed',
493 257: '"PATHNAME" created',
494 331: 'User name okay, need password',
495 332: 'Need account for login',
496 350: 'Requested file action pending further information',
497 421: 'Service not available, closing control connection',
498 425: 'Can\'t open data connection',
499 426: 'Connection closed; transfer aborted',
500 450: 'Requested file action not taken',
501 451: 'Requested action aborted. Local error in processing',
502 452: 'Requested action not taken. Insufficient storage space in system',
503 500: 'Syntax error, command unrecognized',
504 501: 'Syntax error in parameters or arguments',
505 502: 'Command not implemented',
506 503: 'Bad sequence of commands',
507 504: 'Command not implemented for that parameter',
508 530: 'Not logged in',
509 532: 'Need account for storing files',
510 550: 'Requested action not taken. File unavailable',
511 551: 'Requested action aborted. Page type unknown',
512 552: 'Requested file action aborted. Exceeded storage allocation',
513 553: 'Requested action not taken. File name not allowed'
514 };
515 return reasons.get(code, 'Unknown Status Code');
516
517 def sftp_status_to_reason(code):
518 reasons = {
519 0: 'SSH_FX_OK',
520 1: 'SSH_FX_EOF',
521 2: 'SSH_FX_NO_SUCH_FILE',
522 3: 'SSH_FX_PERMISSION_DENIED',
523 4: 'SSH_FX_FAILURE',
524 5: 'SSH_FX_BAD_MESSAGE',
525 6: 'SSH_FX_NO_CONNECTION',
526 7: 'SSH_FX_CONNECTION_LOST',
527 8: 'SSH_FX_OP_UNSUPPORTED'
528 };
529 return reasons.get(code, 'Unknown Status Code');
530
531 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
532 if isinstance(headers, dict):
533 returnval = [];
534 if(sys.version[0]=="2"):
535 for headkey, headvalue in headers.iteritems():
536 returnval.append((headkey, headvalue));
537 if(sys.version[0]>="3"):
538 for headkey, headvalue in headers.items():
539 returnval.append((headkey, headvalue));
540 elif isinstance(headers, list):
541 returnval = headers;
542 else:
543 returnval = False;
544 return returnval;
545
546 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
547 if isinstance(headers, dict):
548 returnval = [];
549 if(sys.version[0]=="2"):
550 for headkey, headvalue in headers.iteritems():
551 returnval.append(headkey+": "+headvalue);
552 if(sys.version[0]>="3"):
553 for headkey, headvalue in headers.items():
554 returnval.append(headkey+": "+headvalue);
555 elif isinstance(headers, list):
556 returnval = headers;
557 else:
558 returnval = False;
559 return returnval;
560
561 def make_http_headers_from_pycurl_to_dict(headers):
562 header_dict = {};
563 headers = headers.strip().split('\r\n');
564 for header in headers:
565 parts = header.split(': ', 1)
566 if(len(parts) == 2):
567 key, value = parts;
568 header_dict[key.title()] = value;
569 return header_dict;
570
571 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
572 if isinstance(headers, list):
573 returnval = {};
574 mli = 0;
575 mlil = len(headers);
576 while(mli<mlil):
577 returnval.update({headers[mli][0]: headers[mli][1]});
578 mli = mli + 1;
579 elif isinstance(headers, dict):
580 returnval = headers;
581 else:
582 returnval = False;
583 return returnval;
584
585 def get_httplib_support(checkvalue=None):
586 global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
587 returnval = [];
588 returnval.append("ftp");
589 returnval.append("httplib");
590 if(havehttplib2):
591 returnval.append("httplib2");
592 returnval.append("urllib");
593 if(haveurllib3):
594 returnval.append("urllib3");
595 returnval.append("request3");
596 returnval.append("request");
597 if(haverequests):
598 returnval.append("requests");
599 if(havehttpx):
600 returnval.append("httpx");
601 returnval.append("httpx2");
602 if(havemechanize):
603 returnval.append("mechanize");
604 if(havepycurl):
605 returnval.append("pycurl");
606 returnval.append("pycurl2");
607 returnval.append("pycurl3");
608 if(haveparamiko):
609 returnval.append("sftp");
610 if(havepysftp):
611 returnval.append("pysftp");
612 if(not checkvalue is None):
613 if(checkvalue=="urllib1" or checkvalue=="urllib2"):
614 checkvalue = "urllib";
615 if(checkvalue=="httplib1"):
616 checkvalue = "httplib";
617 if(checkvalue in returnval):
618 returnval = True;
619 else:
620 returnval = False;
621 return returnval;
622
623 def check_httplib_support(checkvalue="urllib"):
624 if(checkvalue=="urllib1" or checkvalue=="urllib2"):
625 checkvalue = "urllib";
626 if(checkvalue=="httplib1"):
627 checkvalue = "httplib";
628 returnval = get_httplib_support(checkvalue);
629 return returnval;
630
631 def get_httplib_support_list():
632 returnval = get_httplib_support(None);
633 return returnval;
634
635 def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
636 global geturls_download_sleep, havezstd, havebrotli, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
637 if(sleep<0):
638 sleep = geturls_download_sleep;
639 if(timeout<=0):
640 timeout = 10;
641 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
642 httplibuse = "urllib";
643 if(httplibuse=="httplib1"):
644 httplibuse = "httplib";
645 if(not haverequests and httplibuse=="requests"):
646 httplibuse = "urllib";
647 if(not havehttpx and httplibuse=="httpx"):
648 httplibuse = "urllib";
649 if(not havehttpx and httplibuse=="httpx2"):
650 httplibuse = "urllib";
651 if(not havehttpcore and httplibuse=="httpcore"):
652 httplibuse = "urllib";
653 if(not havehttpcore and httplibuse=="httpcore2"):
654 httplibuse = "urllib";
655 if(not havemechanize and httplibuse=="mechanize"):
656 httplibuse = "urllib";
657 if(not havepycurl and httplibuse=="pycurl"):
658 httplibuse = "urllib";
659 if(not havepycurl and httplibuse=="pycurl2"):
660 httplibuse = "urllib";
661 if(not havepycurl and httplibuse=="pycurl3"):
662 httplibuse = "urllib";
663 if(not havehttplib2 and httplibuse=="httplib2"):
664 httplibuse = "httplib";
665 if(not haveparamiko and httplibuse=="sftp"):
666 httplibuse = "ftp";
667 if(not havepysftp and httplibuse=="pysftp"):
668 httplibuse = "ftp";
669 if(httplibuse=="urllib" or httplibuse=="request"):
670 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
671 elif(httplibuse=="request"):
672 returnval = download_from_url_with_request(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
673 elif(httplibuse=="request3"):
674 returnval = download_from_url_with_request3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
675 elif(httplibuse=="httplib"):
676 returnval = download_from_url_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
677 elif(httplibuse=="httplib2"):
678 returnval = download_from_url_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
679 elif(httplibuse=="urllib3" or httplibuse=="request3"):
680 returnval = download_from_url_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
681 elif(httplibuse=="requests"):
682 returnval = download_from_url_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
683 elif(httplibuse=="httpx"):
684 returnval = download_from_url_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
685 elif(httplibuse=="httpx2"):
686 returnval = download_from_url_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
687 elif(httplibuse=="httpcore"):
688 returnval = download_from_url_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
689 elif(httplibuse=="httpcore2"):
690 returnval = download_from_url_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
691 elif(httplibuse=="mechanize"):
692 returnval = download_from_url_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
693 elif(httplibuse=="pycurl"):
694 returnval = download_from_url_with_pycurl(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
695 elif(httplibuse=="pycurl2"):
696 returnval = download_from_url_with_pycurl2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
697 elif(httplibuse=="pycurl3"):
698 returnval = download_from_url_with_pycurl3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
699 elif(httplibuse=="ftp"):
700 returnval = download_from_url_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
701 elif(httplibuse=="sftp"):
702 returnval = download_from_url_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
703 elif(httplibuse=="pysftp"):
704 returnval = download_from_url_with_pysftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
705 else:
706 returnval = False;
707 return returnval;
708
709 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
710 global geturls_download_sleep, havezstd, havebrotli, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
711 if(sleep<0):
712 sleep = geturls_download_sleep;
713 if(timeout<=0):
714 timeout = 10;
715 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
716 httplibuse = "urllib";
717 if(httplibuse=="httplib1"):
718 httplibuse = "httplib";
719 if(not haverequests and httplibuse=="requests"):
720 httplibuse = "urllib";
721 if(not havehttpx and httplibuse=="httpx"):
722 httplibuse = "urllib";
723 if(not havehttpx and httplibuse=="httpx2"):
724 httplibuse = "urllib";
725 if(not havehttpcore and httplibuse=="httpcore"):
726 httplibuse = "urllib";
727 if(not havehttpcore and httplibuse=="httpcore2"):
728 httplibuse = "urllib";
729 if(not havemechanize and httplibuse=="mechanize"):
730 httplibuse = "urllib";
731 if(not havepycurl and httplibuse=="pycurl"):
732 httplibuse = "urllib";
733 if(not havepycurl and httplibuse=="pycurl2"):
734 httplibuse = "urllib";
735 if(not havepycurl and httplibuse=="pycurl3"):
736 httplibuse = "urllib";
737 if(not havehttplib2 and httplibuse=="httplib2"):
738 httplibuse = "httplib";
739 if(not haveparamiko and httplibuse=="sftp"):
740 httplibuse = "ftp";
741 if(not haveparamiko and httplibuse=="pysftp"):
742 httplibuse = "ftp";
743 if(httplibuse=="urllib" or httplibuse=="request"):
744 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
745 elif(httplibuse=="request"):
746 returnval = download_from_url_file_with_request(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
747 elif(httplibuse=="request3"):
748 returnval = download_from_url_file_with_request3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
749 elif(httplibuse=="httplib"):
750 returnval = download_from_url_file_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
751 elif(httplibuse=="httplib2"):
752 returnval = download_from_url_file_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
753 elif(httplibuse=="urllib3" or httplibuse=="request3"):
754 returnval = download_from_url_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
755 elif(httplibuse=="requests"):
756 returnval = download_from_url_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
757 elif(httplibuse=="httpx"):
758 returnval = download_from_url_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
759 elif(httplibuse=="httpx2"):
760 returnval = download_from_url_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
761 elif(httplibuse=="httpcore"):
762 returnval = download_from_url_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
763 elif(httplibuse=="httpcore2"):
764 returnval = download_from_url_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
765 elif(httplibuse=="mechanize"):
766 returnval = download_from_url_file_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
767 elif(httplibuse=="pycurl"):
768 returnval = download_from_url_file_with_pycurl(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
769 elif(httplibuse=="pycurl2"):
770 returnval = download_from_url_file_with_pycurl2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
771 elif(httplibuse=="pycurl3"):
772 returnval = download_from_url_file_with_pycurl3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
773 elif(httplibuse=="ftp"):
774 returnval = download_from_url_file_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
775 elif(httplibuse=="sftp"):
776 returnval = download_from_url_file_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
777 elif(httplibuse=="pysftp"):
778 returnval = download_from_url_file_with_pysftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
779 else:
780 returnval = False;
781 return returnval;
782
783 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
784 global geturls_download_sleep, havezstd, havebrotli, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
785 if(sleep<0):
786 sleep = geturls_download_sleep;
787 if(timeout<=0):
788 timeout = 10;
789 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
790 httplibuse = "urllib";
791 if(httplibuse=="httplib1"):
792 httplibuse = "httplib";
793 if(not haverequests and httplibuse=="requests"):
794 httplibuse = "urllib";
795 if(not havehttpx and httplibuse=="httpx"):
796 httplibuse = "urllib";
797 if(not havehttpx and httplibuse=="httpx2"):
798 httplibuse = "urllib";
799 if(not havehttpcore and httplibuse=="httpcore"):
800 httplibuse = "urllib";
801 if(not havehttpcore and httplibuse=="httpcore2"):
802 httplibuse = "urllib";
803 if(not havemechanize and httplibuse=="mechanize"):
804 httplibuse = "urllib";
805 if(not havepycurl and httplibuse=="pycurl"):
806 httplibuse = "urllib";
807 if(not havepycurl and httplibuse=="pycurl2"):
808 httplibuse = "urllib";
809 if(not havepycurl and httplibuse=="pycurl3"):
810 httplibuse = "urllib";
811 if(not havehttplib2 and httplibuse=="httplib2"):
812 httplibuse = "httplib";
813 if(not haveparamiko and httplibuse=="sftp"):
814 httplibuse = "ftp";
815 if(not havepysftp and httplibuse=="pysftp"):
816 httplibuse = "ftp";
817 if(httplibuse=="urllib" or httplibuse=="request"):
818 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
819 elif(httplibuse=="request"):
820 returnval = download_from_url_to_file_with_request(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
821 elif(httplibuse=="request3"):
822 returnval = download_from_url_to_file_with_request3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
823 elif(httplibuse=="httplib"):
824 returnval = download_from_url_to_file_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
825 elif(httplibuse=="httplib2"):
826 returnval = download_from_url_to_file_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
827 elif(httplibuse=="urllib3" or httplibuse=="request3"):
828 returnval = download_from_url_to_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
829 elif(httplibuse=="requests"):
830 returnval = download_from_url_to_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
831 elif(httplibuse=="httpx"):
832 returnval = download_from_url_to_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
833 elif(httplibuse=="httpx2"):
834 returnval = download_from_url_to_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
835 elif(httplibuse=="httpcore"):
836 returnval = download_from_url_to_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
837 elif(httplibuse=="httpcore2"):
838 returnval = download_from_url_to_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout);
839 elif(httplibuse=="mechanize"):
840 returnval = download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
841 elif(httplibuse=="pycurl"):
842 returnval = download_from_url_to_file_with_pycurl(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
843 elif(httplibuse=="pycurl2"):
844 returnval = download_from_url_to_file_with_pycurl2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
845 elif(httplibuse=="pycurl3"):
846 returnval = download_from_url_to_file_with_pycurl3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
847 elif(httplibuse=="ftp"):
848 returnval = download_from_url_to_file_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
849 elif(httplibuse=="sftp"):
850 returnval = download_from_url_to_file_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
851 elif(httplibuse=="pysftp"):
852 returnval = download_from_url_to_file_with_pysftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, ranges, buffersize, sleep, timeout);
853 else:
854 returnval = False;
855 return returnval;
856
857 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
858 global geturls_download_sleep, havezstd, havebrotli;
859 if(sleep<0):
860 sleep = geturls_download_sleep;
861 if(timeout<=0):
862 timeout = 10;
863 urlparts = urlparse.urlparse(httpurl);
864 if(isinstance(httpheaders, list)):
865 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
866 httpheaders = fix_header_names(httpheaders);
867 if(httpuseragent is not None):
868 if('User-Agent' in httpheaders):
869 httpheaders['User-Agent'] = httpuseragent;
870 else:
871 httpuseragent.update({'User-Agent': httpuseragent});
872 if(httpreferer is not None):
873 if('Referer' in httpheaders):
874 httpheaders['Referer'] = httpreferer;
875 else:
876 httpuseragent.update({'Referer': httpreferer});
877 if(urlparts.username is not None or urlparts.password is not None):
878 if(sys.version[0]=="2"):
879 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
880 if(sys.version[0]>="3"):
881 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
882 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
883 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
884 if(isinstance(httpheaders, dict)):
885 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
886 geturls_opener.addheaders = httpheaders;
887 time.sleep(sleep);
888 if(postdata is not None and not isinstance(postdata, dict)):
889 postdata = urlencode(postdata);
890 try:
891 geturls_request = Request(httpurl);
892 if(httpmethod=="GET"):
893 geturls_text = geturls_opener.open(geturls_request);
894 elif(httpmethod=="POST"):
895 geturls_text = geturls_opener.open(geturls_request, data=postdata);
896 else:
897 geturls_text = geturls_opener.open(geturls_request);
898 except HTTPError as geturls_text_error:
899 geturls_text = geturls_text_error;
900 log.info("Error With URL "+httpurl);
901 except URLError:
902 log.info("Error With URL "+httpurl);
903 return False;
904 except socket.timeout:
905 log.info("Error With URL "+httpurl);
906 return False;
907 httpcodeout = geturls_text.getcode();
908 try:
909 httpcodereason = geturls_text.reason;
910 except AttributeError:
911 httpcodereason = http_status_to_reason(geturls_text.getcode());
912 try:
913 httpversionout = geturls_text.version;
914 except AttributeError:
915 httpversionout = "1.1";
916 httpmethodout = geturls_request.get_method();
917 httpurlout = geturls_text.geturl();
918 httpheaderout = geturls_text.info();
919 httpheadersentout = httpheaders;
920 if(isinstance(httpheaderout, list)):
921 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
922 httpheaderout = fix_header_names(httpheaderout);
923 if(sys.version[0]=="2"):
924 try:
925 prehttpheaderout = httpheaderout;
926 httpheaderkeys = httpheaderout.keys();
927 imax = len(httpheaderkeys);
928 ic = 0;
929 httpheaderout = {};
930 while(ic < imax):
931 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
932 ic += 1;
933 except AttributeError:
934 pass;
935 if(isinstance(httpheadersentout, list)):
936 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
937 httpheadersentout = fix_header_names(httpheadersentout);
938 log.info("Downloading URL "+httpurl);
939 downloadsize = httpheaderout.get('Content-Length');
940 if(downloadsize is not None):
941 downloadsize = int(downloadsize);
942 if downloadsize is None: downloadsize = 0;
943 fulldatasize = 0;
944 prevdownsize = 0;
945 log.info("Downloading URL "+httpurl);
946 with BytesIO() as strbuf:
947 while True:
948 databytes = geturls_text.read(buffersize);
949 if not databytes: break;
950 datasize = len(databytes);
951 fulldatasize = datasize + fulldatasize;
952 percentage = "";
953 if(downloadsize>0):
954 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
955 downloaddiff = fulldatasize - prevdownsize;
956 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
957 prevdownsize = fulldatasize;
958 strbuf.write(databytes);
959 strbuf.seek(0);
960 returnval_content = strbuf.read();
961 if(httpheaderout.get("Content-Encoding")=="gzip"):
962 try:
963 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
964 except zlib.error:
965 pass;
966 elif(httpheaderout.get("Content-Encoding")=="deflate"):
967 try:
968 returnval_content = zlib.decompress(returnval_content);
969 except zlib.error:
970 pass;
971 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
972 try:
973 returnval_content = brotli.decompress(returnval_content);
974 except brotli.error:
975 pass;
976 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
977 try:
978 returnval_content = zstandard.decompress(returnval_content);
979 except zstandard.error:
980 pass;
981 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
982 geturls_text.close();
983 return returnval;
984
985 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
986 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
987 exec_time_start = time.time();
988 myhash = hashlib.new("sha1");
989 if(sys.version[0]=="2"):
990 myhash.update(httpurl);
991 myhash.update(str(buffersize));
992 myhash.update(str(exec_time_start));
993 if(sys.version[0]>="3"):
994 myhash.update(httpurl.encode('utf-8'));
995 myhash.update(str(buffersize).encode('utf-8'));
996 myhash.update(str(exec_time_start).encode('utf-8'));
997 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
998 if(sleep<0):
999 sleep = geturls_download_sleep;
1000 if(timeout<=0):
1001 timeout = 10;
1002 pretmpfilename = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
1003 if(not pretmpfilename):
1004 return False;
1005 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1006 tmpfilename = f.name;
1007 try:
1008 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1009 except AttributeError:
1010 try:
1011 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1012 except ValueError:
1013 pass;
1014 except ValueError:
1015 pass;
1016 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
1017 f.write(pretmpfilename['Content']);
1018 f.close();
1019 exec_time_end = time.time();
1020 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1021 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1022 return returnval;
1023
1024 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1025 global geturls_download_sleep, havezstd, havebrotli;
1026 if(sleep<0):
1027 sleep = geturls_download_sleep;
1028 if(timeout<=0):
1029 timeout = 10;
1030 if(not outfile=="-"):
1031 outpath = outpath.rstrip(os.path.sep);
1032 filepath = os.path.realpath(outpath+os.path.sep+outfile);
1033 if(not os.path.exists(outpath)):
1034 os.makedirs(outpath);
1035 if(os.path.exists(outpath) and os.path.isfile(outpath)):
1036 return False;
1037 if(os.path.exists(filepath) and os.path.isdir(filepath)):
1038 return False;
1039 pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
1040 if(not pretmpfilename):
1041 return False;
1042 tmpfilename = pretmpfilename['Filename'];
1043 downloadsize = int(os.path.getsize(tmpfilename));
1044 fulldatasize = 0;
1045 log.info("Moving file "+tmpfilename+" to "+filepath);
1046 exec_time_start = time.time();
1047 shutil.move(tmpfilename, filepath);
1048 try:
1049 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1050 except AttributeError:
1051 try:
1052 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1053 except ValueError:
1054 pass;
1055 except ValueError:
1056 pass;
1057 exec_time_end = time.time();
1058 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1059 if(os.path.exists(tmpfilename)):
1060 os.remove(tmpfilename);
1061 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1062 if(outfile=="-"):
1063 pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
1064 tmpfilename = pretmpfilename['Filename'];
1065 downloadsize = int(os.path.getsize(tmpfilename));
1066 fulldatasize = 0;
1067 prevdownsize = 0;
1068 exec_time_start = time.time();
1069 with open(tmpfilename, 'rb') as ft:
1070 f = BytesIO();
1071 while True:
1072 databytes = ft.read(buffersize[1]);
1073 if not databytes: break;
1074 datasize = len(databytes);
1075 fulldatasize = datasize + fulldatasize;
1076 percentage = "";
1077 if(downloadsize>0):
1078 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1079 downloaddiff = fulldatasize - prevdownsize;
1080 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1081 prevdownsize = fulldatasize;
1082 f.write(databytes);
1083 f.seek(0);
1084 fdata = f.getvalue();
1085 f.close();
1086 ft.close();
1087 os.remove(tmpfilename);
1088 exec_time_end = time.time();
1089 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1090 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1091 return returnval;
1092
1093 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1094 global geturls_download_sleep, havezstd, havebrotli, havezstd, havebrotli;
1095 if(sleep<0):
1096 sleep = geturls_download_sleep;
1097 if(timeout<=0):
1098 timeout = 10;
1099 urlparts = urlparse.urlparse(httpurl);
1100 if(isinstance(httpheaders, list)):
1101 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1102 httpheaders = fix_header_names(httpheaders);
1103 if(httpuseragent is not None):
1104 if('User-Agent' in httpheaders):
1105 httpheaders['User-Agent'] = httpuseragent;
1106 else:
1107 httpuseragent.update({'User-Agent': httpuseragent});
1108 if(httpreferer is not None):
1109 if('Referer' in httpheaders):
1110 httpheaders['Referer'] = httpreferer;
1111 else:
1112 httpuseragent.update({'Referer': httpreferer});
1113 if(urlparts.username is not None or urlparts.password is not None):
1114 if(sys.version[0]=="2"):
1115 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
1116 if(sys.version[0]>="3"):
1117 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1118 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1119 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1120 geturls_opener.addheaders = httpheaders;
1121 time.sleep(sleep);
1122 if(urlparts[0]=="http"):
1123 httpconn = HTTPConnection(urlparts[1], timeout=timeout);
1124 elif(urlparts[0]=="https"):
1125 httpconn = HTTPSConnection(urlparts[1], timeout=timeout);
1126 else:
1127 return False;
1128 if(postdata is not None and not isinstance(postdata, dict)):
1129 postdata = urlencode(postdata);
1130 try:
1131 if(httpmethod=="GET"):
1132 httpconn.request("GET", urlparts[2], headers=httpheaders);
1133 elif(httpmethod=="POST"):
1134 httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1135 else:
1136 httpconn.request("GET", urlparts[2], headers=httpheaders);
1137 except socket.timeout:
1138 log.info("Error With URL "+httpurl);
1139 return False;
1140 except socket.gaierror:
1141 log.info("Error With URL "+httpurl);
1142 return False;
1143 except BlockingIOError:
1144 log.info("Error With URL "+httpurl);
1145 return False;
1146 geturls_text = httpconn.getresponse();
1147 httpcodeout = geturls_text.status;
1148 httpcodereason = geturls_text.reason;
1149 if(geturls_text.version=="10"):
1150 httpversionout = "1.0";
1151 else:
1152 httpversionout = "1.1";
1153 httpmethodout = geturls_text._method;
1154 httpurlout = httpurl;
1155 httpheaderout = geturls_text.getheaders();
1156 httpheadersentout = httpheaders;
1157 if(isinstance(httpheaderout, list)):
1158 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1159 if(sys.version[0]=="2"):
1160 try:
1161 prehttpheaderout = httpheaderout;
1162 httpheaderkeys = httpheaderout.keys();
1163 imax = len(httpheaderkeys);
1164 ic = 0;
1165 httpheaderout = {};
1166 while(ic < imax):
1167 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1168 ic += 1;
1169 except AttributeError:
1170 pass;
1171 httpheaderout = fix_header_names(httpheaderout);
1172 if(isinstance(httpheadersentout, list)):
1173 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1174 httpheadersentout = fix_header_names(httpheadersentout);
1175 log.info("Downloading URL "+httpurl);
1176 downloadsize = httpheaderout.get('Content-Length');
1177 if(downloadsize is not None):
1178 downloadsize = int(downloadsize);
1179 if downloadsize is None: downloadsize = 0;
1180 fulldatasize = 0;
1181 prevdownsize = 0;
1182 log.info("Downloading URL "+httpurl);
1183 with BytesIO() as strbuf:
1184 while True:
1185 databytes = geturls_text.read(buffersize);
1186 if not databytes: break;
1187 datasize = len(databytes);
1188 fulldatasize = datasize + fulldatasize;
1189 percentage = "";
1190 if(downloadsize>0):
1191 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1192 downloaddiff = fulldatasize - prevdownsize;
1193 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1194 prevdownsize = fulldatasize;
1195 strbuf.write(databytes);
1196 strbuf.seek(0);
1197 returnval_content = strbuf.read();
1198 if(httpheaderout.get("Content-Encoding")=="gzip"):
1199 try:
1200 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1201 except zlib.error:
1202 pass;
1203 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1204 try:
1205 returnval_content = zlib.decompress(returnval_content);
1206 except zlib.error:
1207 pass;
1208 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1209 try:
1210 returnval_content = brotli.decompress(returnval_content);
1211 except brotli.error:
1212 pass;
1213 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1214 try:
1215 returnval_content = zstandard.decompress(returnval_content);
1216 except zstandard.error:
1217 pass;
1218 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1219 geturls_text.close();
1220 return returnval;
1221
1222 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1223 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
1224 exec_time_start = time.time();
1225 myhash = hashlib.new("sha1");
1226 if(sys.version[0]=="2"):
1227 myhash.update(httpurl);
1228 myhash.update(str(buffersize));
1229 myhash.update(str(exec_time_start));
1230 if(sys.version[0]>="3"):
1231 myhash.update(httpurl.encode('utf-8'));
1232 myhash.update(str(buffersize).encode('utf-8'));
1233 myhash.update(str(exec_time_start).encode('utf-8'));
1234 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1235 if(sleep<0):
1236 sleep = geturls_download_sleep;
1237 if(timeout<=0):
1238 timeout = 10;
1239 pretmpfilename = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
1240 if(not pretmpfilename):
1241 return False;
1242 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1243 tmpfilename = f.name;
1244 try:
1245 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1246 except AttributeError:
1247 try:
1248 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1249 except ValueError:
1250 pass;
1251 except ValueError:
1252 pass;
1253 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
1254 f.write(pretmpfilename['Content']);
1255 f.close();
1256 exec_time_end = time.time();
1257 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1258 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1259 return returnval;
1260
1261 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1262 global geturls_download_sleep, havezstd, havebrotli;
1263 if(sleep<0):
1264 sleep = geturls_download_sleep;
1265 if(timeout<=0):
1266 timeout = 10;
1267 if(not outfile=="-"):
1268 outpath = outpath.rstrip(os.path.sep);
1269 filepath = os.path.realpath(outpath+os.path.sep+outfile);
1270 if(not os.path.exists(outpath)):
1271 os.makedirs(outpath);
1272 if(os.path.exists(outpath) and os.path.isfile(outpath)):
1273 return False;
1274 if(os.path.exists(filepath) and os.path.isdir(filepath)):
1275 return False;
1276 pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
1277 if(not pretmpfilename):
1278 return False;
1279 tmpfilename = pretmpfilename['Filename'];
1280 downloadsize = int(os.path.getsize(tmpfilename));
1281 fulldatasize = 0;
1282 log.info("Moving file "+tmpfilename+" to "+filepath);
1283 exec_time_start = time.time();
1284 shutil.move(tmpfilename, filepath);
1285 try:
1286 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1287 except AttributeError:
1288 try:
1289 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1290 except ValueError:
1291 pass;
1292 except ValueError:
1293 pass;
1294 exec_time_end = time.time();
1295 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1296 if(os.path.exists(tmpfilename)):
1297 os.remove(tmpfilename);
1298 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1299 if(outfile=="-"):
1300 pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
1301 tmpfilename = pretmpfilename['Filename'];
1302 downloadsize = int(os.path.getsize(tmpfilename));
1303 fulldatasize = 0;
1304 prevdownsize = 0;
1305 exec_time_start = time.time();
1306 with open(tmpfilename, 'rb') as ft:
1307 f = BytesIO();
1308 while True:
1309 databytes = ft.read(buffersize[1]);
1310 if not databytes: break;
1311 datasize = len(databytes);
1312 fulldatasize = datasize + fulldatasize;
1313 percentage = "";
1314 if(downloadsize>0):
1315 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1316 downloaddiff = fulldatasize - prevdownsize;
1317 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1318 prevdownsize = fulldatasize;
1319 f.write(databytes);
1320 f.seek(0);
1321 fdata = f.getvalue();
1322 f.close();
1323 ft.close();
1324 os.remove(tmpfilename);
1325 exec_time_end = time.time();
1326 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1327 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1328 return returnval;
1329
1330 if(havehttplib2):
1331 def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1332 global geturls_download_sleep, havezstd, havebrotli;
1333 if(sleep<0):
1334 sleep = geturls_download_sleep;
1335 if(timeout<=0):
1336 timeout = 10;
1337 urlparts = urlparse.urlparse(httpurl);
1338 if(isinstance(httpheaders, list)):
1339 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1340 httpheaders = fix_header_names(httpheaders);
1341 if(httpuseragent is not None):
1342 if('User-Agent' in httpheaders):
1343 httpheaders['User-Agent'] = httpuseragent;
1344 else:
1345 httpuseragent.update({'User-Agent': httpuseragent});
1346 if(httpreferer is not None):
1347 if('Referer' in httpheaders):
1348 httpheaders['Referer'] = httpreferer;
1349 else:
1350 httpuseragent.update({'Referer': httpreferer});
1351 if(urlparts.username is not None or urlparts.password is not None):
1352 if(sys.version[0]=="2"):
1353 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
1354 if(sys.version[0]>="3"):
1355 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1356 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1357 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1358 geturls_opener.addheaders = httpheaders;
1359 time.sleep(sleep);
1360 if(urlparts[0]=="http"):
1361 httpconn = HTTPConnectionWithTimeout(urlparts[1], timeout=timeout);
1362 elif(urlparts[0]=="https"):
1363 httpconn = HTTPSConnectionWithTimeout(urlparts[1], timeout=timeout);
1364 else:
1365 return False;
1366 if(postdata is not None and not isinstance(postdata, dict)):
1367 postdata = urlencode(postdata);
1368 try:
1369 if(httpmethod=="GET"):
1370 httpconn.request("GET", urlparts[2], headers=httpheaders);
1371 elif(httpmethod=="POST"):
1372 httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1373 else:
1374 httpconn.request("GET", urlparts[2], headers=httpheaders);
1375 except socket.timeout:
1376 log.info("Error With URL "+httpurl);
1377 return False;
1378 except socket.gaierror:
1379 log.info("Error With URL "+httpurl);
1380 return False;
1381 except BlockingIOError:
1382 log.info("Error With URL "+httpurl);
1383 return False;
1384 geturls_text = httpconn.getresponse();
1385 httpcodeout = geturls_text.status;
1386 httpcodereason = geturls_text.reason;
1387 if(geturls_text.version=="10"):
1388 httpversionout = "1.0";
1389 else:
1390 httpversionout = "1.1";
1391 httpmethodout = httpmethod;
1392 httpurlout = httpurl;
1393 httpheaderout = geturls_text.getheaders();
1394 httpheadersentout = httpheaders;
1395 if(isinstance(httpheaderout, list)):
1396 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1397 if(sys.version[0]=="2"):
1398 try:
1399 prehttpheaderout = httpheaderout;
1400 httpheaderkeys = httpheaderout.keys();
1401 imax = len(httpheaderkeys);
1402 ic = 0;
1403 httpheaderout = {};
1404 while(ic < imax):
1405 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1406 ic += 1;
1407 except AttributeError:
1408 pass;
1409 httpheaderout = fix_header_names(httpheaderout);
1410 if(isinstance(httpheadersentout, list)):
1411 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1412 httpheadersentout = fix_header_names(httpheadersentout);
1413 log.info("Downloading URL "+httpurl);
1414 downloadsize = httpheaderout.get('Content-Length');
1415 if(downloadsize is not None):
1416 downloadsize = int(downloadsize);
1417 if downloadsize is None: downloadsize = 0;
1418 fulldatasize = 0;
1419 prevdownsize = 0;
1420 log.info("Downloading URL "+httpurl);
1421 with BytesIO() as strbuf:
1422 while True:
1423 databytes = geturls_text.read(buffersize);
1424 if not databytes: break;
1425 datasize = len(databytes);
1426 fulldatasize = datasize + fulldatasize;
1427 percentage = "";
1428 if(downloadsize>0):
1429 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1430 downloaddiff = fulldatasize - prevdownsize;
1431 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1432 prevdownsize = fulldatasize;
1433 strbuf.write(databytes);
1434 strbuf.seek(0);
1435 returnval_content = strbuf.read();
1436 if(httpheaderout.get("Content-Encoding")=="gzip"):
1437 try:
1438 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1439 except zlib.error:
1440 pass;
1441 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1442 try:
1443 returnval_content = zlib.decompress(returnval_content);
1444 except zlib.error:
1445 pass;
1446 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1447 try:
1448 returnval_content = brotli.decompress(returnval_content);
1449 except brotli.error:
1450 pass;
1451 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1452 try:
1453 returnval_content = zstandard.decompress(returnval_content);
1454 except zstandard.error:
1455 pass;
1456 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1457 geturls_text.close();
1458 return returnval;
1459
1460 if(not havehttplib2):
1461 def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1462 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
1463 return returnval;
1464
1465 if(havehttplib2):
1466 def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1467 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
1468 exec_time_start = time.time();
1469 myhash = hashlib.new("sha1");
1470 if(sys.version[0]=="2"):
1471 myhash.update(httpurl);
1472 myhash.update(str(buffersize));
1473 myhash.update(str(exec_time_start));
1474 if(sys.version[0]>="3"):
1475 myhash.update(httpurl.encode('utf-8'));
1476 myhash.update(str(buffersize).encode('utf-8'));
1477 myhash.update(str(exec_time_start).encode('utf-8'));
1478 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1479 if(sleep<0):
1480 sleep = geturls_download_sleep;
1481 if(timeout<=0):
1482 timeout = 10;
1483 pretmpfilename = download_from_url_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
1484 if(not pretmpfilename):
1485 return False;
1486 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1487 tmpfilename = f.name;
1488 try:
1489 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1490 except AttributeError:
1491 try:
1492 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1493 except ValueError:
1494 pass;
1495 except ValueError:
1496 pass;
1497 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
1498 f.write(pretmpfilename['Content']);
1499 f.close();
1500 exec_time_end = time.time();
1501 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1502 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1503 return returnval;
1504
1505 if(not havehttplib2):
1506 def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1507 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
1508 return returnval;
1509
1510 if(havehttplib2):
1511 def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1512 global geturls_download_sleep, havezstd, havebrotli;
1513 if(sleep<0):
1514 sleep = geturls_download_sleep;
1515 if(timeout<=0):
1516 timeout = 10;
1517 if(not outfile=="-"):
1518 outpath = outpath.rstrip(os.path.sep);
1519 filepath = os.path.realpath(outpath+os.path.sep+outfile);
1520 if(not os.path.exists(outpath)):
1521 os.makedirs(outpath);
1522 if(os.path.exists(outpath) and os.path.isfile(outpath)):
1523 return False;
1524 if(os.path.exists(filepath) and os.path.isdir(filepath)):
1525 return False;
1526 pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
1527 if(not pretmpfilename):
1528 return False;
1529 tmpfilename = pretmpfilename['Filename'];
1530 downloadsize = int(os.path.getsize(tmpfilename));
1531 fulldatasize = 0;
1532 log.info("Moving file "+tmpfilename+" to "+filepath);
1533 exec_time_start = time.time();
1534 shutil.move(tmpfilename, filepath);
1535 try:
1536 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1537 except AttributeError:
1538 try:
1539 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1540 except ValueError:
1541 pass;
1542 except ValueError:
1543 pass;
1544 exec_time_end = time.time();
1545 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1546 if(os.path.exists(tmpfilename)):
1547 os.remove(tmpfilename);
1548 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1549 if(outfile=="-"):
1550 pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
1551 tmpfilename = pretmpfilename['Filename'];
1552 downloadsize = int(os.path.getsize(tmpfilename));
1553 fulldatasize = 0;
1554 prevdownsize = 0;
1555 exec_time_start = time.time();
1556 with open(tmpfilename, 'rb') as ft:
1557 f = BytesIO();
1558 while True:
1559 databytes = ft.read(buffersize[1]);
1560 if not databytes: break;
1561 datasize = len(databytes);
1562 fulldatasize = datasize + fulldatasize;
1563 percentage = "";
1564 if(downloadsize>0):
1565 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1566 downloaddiff = fulldatasize - prevdownsize;
1567 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1568 prevdownsize = fulldatasize;
1569 f.write(databytes);
1570 f.seek(0);
1571 fdata = f.getvalue();
1572 f.close();
1573 ft.close();
1574 os.remove(tmpfilename);
1575 exec_time_end = time.time();
1576 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1577 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1578 return returnval;
1579
1580 if(not havehttplib2):
1581 def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1582 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
1583 return returnval;
1584
1585 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1586 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
1587 return returnval;
1588
1589 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1590 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
1591 return returnval;
1592
1593 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1594 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
1595 return returnval;
1596
1597 if(haverequests):
1598 def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1599 global geturls_download_sleep, havezstd, havebrotli;
1600 if(sleep<0):
1601 sleep = geturls_download_sleep;
1602 if(timeout<=0):
1603 timeout = 10;
1604 urlparts = urlparse.urlparse(httpurl);
1605 if(isinstance(httpheaders, list)):
1606 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1607 httpheaders = fix_header_names(httpheaders);
1608 if(httpuseragent is not None):
1609 if('User-Agent' in httpheaders):
1610 httpheaders['User-Agent'] = httpuseragent;
1611 else:
1612 httpuseragent.update({'User-Agent': httpuseragent});
1613 if(httpreferer is not None):
1614 if('Referer' in httpheaders):
1615 httpheaders['Referer'] = httpreferer;
1616 else:
1617 httpuseragent.update({'Referer': httpreferer});
1618 if(urlparts.username is not None or urlparts.password is not None):
1619 if(sys.version[0]=="2"):
1620 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
1621 if(sys.version[0]>="3"):
1622 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1623 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1624 time.sleep(sleep);
1625 if(postdata is not None and not isinstance(postdata, dict)):
1626 postdata = urlencode(postdata);
1627 try:
1628 reqsession = requests.Session();
1629 if(httpmethod=="GET"):
1630 geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
1631 elif(httpmethod=="POST"):
1632 geturls_text = reqsession.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie, stream=True);
1633 else:
1634 geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
1635 except requests.exceptions.ConnectTimeout:
1636 log.info("Error With URL "+httpurl);
1637 return False;
1638 except requests.exceptions.ConnectError:
1639 log.info("Error With URL "+httpurl);
1640 return False;
1641 except socket.timeout:
1642 log.info("Error With URL "+httpurl);
1643 return False;
1644 httpcodeout = geturls_text.status_code;
1645 httpcodereason = geturls_text.reason;
1646 if(geturls_text.raw.version=="10"):
1647 httpversionout = "1.0";
1648 else:
1649 httpversionout = "1.1";
1650 httpmethodout = httpmethod;
1651 httpurlout = geturls_text.url;
1652 httpheaderout = geturls_text.headers;
1653 httpheadersentout = geturls_text.request.headers;
1654 if(isinstance(httpheaderout, list)):
1655 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1656 if(sys.version[0]=="2"):
1657 try:
1658 prehttpheaderout = httpheaderout;
1659 httpheaderkeys = httpheaderout.keys();
1660 imax = len(httpheaderkeys);
1661 ic = 0;
1662 httpheaderout = {};
1663 while(ic < imax):
1664 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1665 ic += 1;
1666 except AttributeError:
1667 pass;
1668 httpheaderout = fix_header_names(httpheaderout);
1669 if(isinstance(httpheadersentout, list)):
1670 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1671 httpheadersentout = fix_header_names(httpheadersentout);
1672 log.info("Downloading URL "+httpurl);
1673 downloadsize = httpheaderout.get('Content-Length');
1674 if(downloadsize is not None):
1675 downloadsize = int(downloadsize);
1676 if downloadsize is None: downloadsize = 0;
1677 fulldatasize = 0;
1678 prevdownsize = 0;
1679 log.info("Downloading URL "+httpurl);
1680 with BytesIO() as strbuf:
1681 while True:
1682 databytes = geturls_text.raw.read(buffersize);
1683 if not databytes: break;
1684 datasize = len(databytes);
1685 fulldatasize = datasize + fulldatasize;
1686 percentage = "";
1687 if(downloadsize>0):
1688 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1689 downloaddiff = fulldatasize - prevdownsize;
1690 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1691 prevdownsize = fulldatasize;
1692 strbuf.write(databytes);
1693 strbuf.seek(0);
1694 returnval_content = strbuf.read();
1695 if(httpheaderout.get("Content-Encoding")=="gzip"):
1696 try:
1697 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1698 except zlib.error:
1699 pass;
1700 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1701 try:
1702 returnval_content = zlib.decompress(returnval_content);
1703 except zlib.error:
1704 pass;
1705 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1706 try:
1707 returnval_content = brotli.decompress(returnval_content);
1708 except brotli.error:
1709 pass;
1710 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1711 try:
1712 returnval_content = zstandard.decompress(returnval_content);
1713 except zstandard.error:
1714 pass;
1715 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1716 geturls_text.close();
1717 return returnval;
1718
1719 if(not haverequests):
1720 def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1721 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
1722 return returnval;
1723
1724 if(haverequests):
1725 def download_from_url_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1726 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
1727 exec_time_start = time.time();
1728 myhash = hashlib.new("sha1");
1729 if(sys.version[0]=="2"):
1730 myhash.update(httpurl);
1731 myhash.update(str(buffersize));
1732 myhash.update(str(exec_time_start));
1733 if(sys.version[0]>="3"):
1734 myhash.update(httpurl.encode('utf-8'));
1735 myhash.update(str(buffersize).encode('utf-8'));
1736 myhash.update(str(exec_time_start).encode('utf-8'));
1737 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1738 if(sleep<0):
1739 sleep = geturls_download_sleep;
1740 if(timeout<=0):
1741 timeout = 10;
1742 pretmpfilename = download_from_url_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
1743 if(not pretmpfilename):
1744 return False;
1745 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1746 tmpfilename = f.name;
1747 try:
1748 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1749 except AttributeError:
1750 try:
1751 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1752 except ValueError:
1753 pass;
1754 except ValueError:
1755 pass;
1756 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
1757 f.write(pretmpfilename['Content']);
1758 f.close();
1759 exec_time_end = time.time();
1760 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1761 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1762 return returnval;
1763
1764 if(not haverequests):
1765 def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1766 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
1767 return returnval;
1768
1769 if(haverequests):
1770 def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1771 global geturls_download_sleep, havezstd, havebrotli;
1772 if(sleep<0):
1773 sleep = geturls_download_sleep;
1774 if(timeout<=0):
1775 timeout = 10;
1776 if(not outfile=="-"):
1777 outpath = outpath.rstrip(os.path.sep);
1778 filepath = os.path.realpath(outpath+os.path.sep+outfile);
1779 if(not os.path.exists(outpath)):
1780 os.makedirs(outpath);
1781 if(os.path.exists(outpath) and os.path.isfile(outpath)):
1782 return False;
1783 if(os.path.exists(filepath) and os.path.isdir(filepath)):
1784 return False;
1785 pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
1786 if(not pretmpfilename):
1787 return False;
1788 tmpfilename = pretmpfilename['Filename'];
1789 downloadsize = int(os.path.getsize(tmpfilename));
1790 fulldatasize = 0;
1791 log.info("Moving file "+tmpfilename+" to "+filepath);
1792 exec_time_start = time.time();
1793 shutil.move(tmpfilename, filepath);
1794 try:
1795 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1796 except AttributeError:
1797 try:
1798 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1799 except ValueError:
1800 pass;
1801 except ValueError:
1802 pass;
1803 exec_time_end = time.time();
1804 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1805 if(os.path.exists(tmpfilename)):
1806 os.remove(tmpfilename);
1807 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1808 if(outfile=="-"):
1809 pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
1810 tmpfilename = pretmpfilename['Filename'];
1811 downloadsize = int(os.path.getsize(tmpfilename));
1812 fulldatasize = 0;
1813 prevdownsize = 0;
1814 exec_time_start = time.time();
1815 with open(tmpfilename, 'rb') as ft:
1816 f = BytesIO();
1817 while True:
1818 databytes = ft.read(buffersize[1]);
1819 if not databytes: break;
1820 datasize = len(databytes);
1821 fulldatasize = datasize + fulldatasize;
1822 percentage = "";
1823 if(downloadsize>0):
1824 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1825 downloaddiff = fulldatasize - prevdownsize;
1826 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1827 prevdownsize = fulldatasize;
1828 f.write(databytes);
1829 f.seek(0);
1830 fdata = f.getvalue();
1831 f.close();
1832 ft.close();
1833 os.remove(tmpfilename);
1834 exec_time_end = time.time();
1835 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1836 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1837 return returnval;
1838
1839 if(not haverequests):
1840 def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1841 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
1842 return returnval;
1843
1844 if(havehttpx):
1845 def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1846 global geturls_download_sleep, havezstd, havebrotli;
1847 if(sleep<0):
1848 sleep = geturls_download_sleep;
1849 if(timeout<=0):
1850 timeout = 10;
1851 urlparts = urlparse.urlparse(httpurl);
1852 if(isinstance(httpheaders, list)):
1853 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1854 httpheaders = fix_header_names(httpheaders);
1855 if(httpuseragent is not None):
1856 if('User-Agent' in httpheaders):
1857 httpheaders['User-Agent'] = httpuseragent;
1858 else:
1859 httpuseragent.update({'User-Agent': httpuseragent});
1860 if(httpreferer is not None):
1861 if('Referer' in httpheaders):
1862 httpheaders['Referer'] = httpreferer;
1863 else:
1864 httpuseragent.update({'Referer': httpreferer});
1865 if(urlparts.username is not None or urlparts.password is not None):
1866 if(sys.version[0]=="2"):
1867 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
1868 if(sys.version[0]>="3"):
1869 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1870 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1871 time.sleep(sleep);
1872 if(postdata is not None and not isinstance(postdata, dict)):
1873 postdata = urlencode(postdata);
1874 try:
1875 if(httpmethod=="GET"):
1876 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1877 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
1878 elif(httpmethod=="POST"):
1879 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1880 geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
1881 else:
1882 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1883 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
1884 except httpx.ConnectTimeout:
1885 log.info("Error With URL "+httpurl);
1886 return False;
1887 except httpx.ConnectError:
1888 log.info("Error With URL "+httpurl);
1889 return False;
1890 except socket.timeout:
1891 log.info("Error With URL "+httpurl);
1892 return False;
1893 httpcodeout = geturls_text.status_code;
1894 try:
1895 httpcodereason = geturls_text.reason_phrase;
1896 except:
1897 httpcodereason = http_status_to_reason(geturls_text.status_code);
1898 httpversionout = geturls_text.http_version;
1899 httpmethodout = httpmethod;
1900 httpurlout = str(geturls_text.url);
1901 httpheaderout = geturls_text.headers;
1902 httpheadersentout = geturls_text.request.headers;
1903 if(isinstance(httpheaderout, list)):
1904 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1905 if(sys.version[0]=="2"):
1906 try:
1907 prehttpheaderout = httpheaderout;
1908 httpheaderkeys = httpheaderout.keys();
1909 imax = len(httpheaderkeys);
1910 ic = 0;
1911 httpheaderout = {};
1912 while(ic < imax):
1913 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1914 ic += 1;
1915 except AttributeError:
1916 pass;
1917 httpheaderout = fix_header_names(httpheaderout);
1918 if(isinstance(httpheadersentout, list)):
1919 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1920 httpheadersentout = fix_header_names(httpheadersentout);
1921 log.info("Downloading URL "+httpurl);
1922 downloadsize = httpheaderout.get('Content-Length');
1923 if(downloadsize is not None):
1924 downloadsize = int(downloadsize);
1925 if downloadsize is None: downloadsize = 0;
1926 fulldatasize = 0;
1927 prevdownsize = 0;
1928 log.info("Downloading URL "+httpurl);
1929 with BytesIO() as strbuf:
1930 while True:
1931 databytes = geturls_text.read(buffersize);
1932 if not databytes: break;
1933 datasize = len(databytes);
1934 fulldatasize = datasize + fulldatasize;
1935 percentage = "";
1936 if(downloadsize>0):
1937 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1938 downloaddiff = fulldatasize - prevdownsize;
1939 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1940 prevdownsize = fulldatasize;
1941 strbuf.write(databytes);
1942 strbuf.seek(0);
1943 returnval_content = strbuf.read();
1944 if(httpheaderout.get("Content-Encoding")=="gzip"):
1945 try:
1946 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1947 except zlib.error:
1948 pass;
1949 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1950 try:
1951 returnval_content = zlib.decompress(returnval_content);
1952 except zlib.error:
1953 pass;
1954 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1955 try:
1956 returnval_content = brotli.decompress(returnval_content);
1957 except brotli.error:
1958 pass;
1959 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1960 try:
1961 returnval_content = zstandard.decompress(returnval_content);
1962 except zstandard.error:
1963 pass;
1964 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1965 geturls_text.close();
1966 return returnval;
1967
1968 if(not havehttpx):
1969 def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1970 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
1971 return returnval;
1972
1973 if(havehttpx):
1974 def download_from_url_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1975 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
1976 exec_time_start = time.time();
1977 myhash = hashlib.new("sha1");
1978 if(sys.version[0]=="2"):
1979 myhash.update(httpurl);
1980 myhash.update(str(buffersize));
1981 myhash.update(str(exec_time_start));
1982 if(sys.version[0]>="3"):
1983 myhash.update(httpurl.encode('utf-8'));
1984 myhash.update(str(buffersize).encode('utf-8'));
1985 myhash.update(str(exec_time_start).encode('utf-8'));
1986 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1987 if(sleep<0):
1988 sleep = geturls_download_sleep;
1989 if(timeout<=0):
1990 timeout = 10;
1991 pretmpfilename = download_from_url_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
1992 if(not pretmpfilename):
1993 return False;
1994 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1995 tmpfilename = f.name;
1996 try:
1997 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1998 except AttributeError:
1999 try:
2000 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2001 except ValueError:
2002 pass;
2003 except ValueError:
2004 pass;
2005 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
2006 f.write(pretmpfilename['Content']);
2007 f.close();
2008 exec_time_end = time.time();
2009 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2010 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2011 return returnval;
2012
2013 if(not havehttpx):
2014 def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
2015 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
2016 return returnval;
2017
2018 if(havehttpx):
2019 def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2020 global geturls_download_sleep, havezstd, havebrotli;
2021 if(sleep<0):
2022 sleep = geturls_download_sleep;
2023 if(timeout<=0):
2024 timeout = 10;
2025 if(not outfile=="-"):
2026 outpath = outpath.rstrip(os.path.sep);
2027 filepath = os.path.realpath(outpath+os.path.sep+outfile);
2028 if(not os.path.exists(outpath)):
2029 os.makedirs(outpath);
2030 if(os.path.exists(outpath) and os.path.isfile(outpath)):
2031 return False;
2032 if(os.path.exists(filepath) and os.path.isdir(filepath)):
2033 return False;
2034 pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
2035 if(not pretmpfilename):
2036 return False;
2037 tmpfilename = pretmpfilename['Filename'];
2038 downloadsize = int(os.path.getsize(tmpfilename));
2039 fulldatasize = 0;
2040 log.info("Moving file "+tmpfilename+" to "+filepath);
2041 exec_time_start = time.time();
2042 shutil.move(tmpfilename, filepath);
2043 try:
2044 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2045 except AttributeError:
2046 try:
2047 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2048 except ValueError:
2049 pass;
2050 except ValueError:
2051 pass;
2052 exec_time_end = time.time();
2053 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2054 if(os.path.exists(tmpfilename)):
2055 os.remove(tmpfilename);
2056 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2057 if(outfile=="-"):
2058 pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
2059 tmpfilename = pretmpfilename['Filename'];
2060 downloadsize = int(os.path.getsize(tmpfilename));
2061 fulldatasize = 0;
2062 prevdownsize = 0;
2063 exec_time_start = time.time();
2064 with open(tmpfilename, 'rb') as ft:
2065 f = BytesIO();
2066 while True:
2067 databytes = ft.read(buffersize[1]);
2068 if not databytes: break;
2069 datasize = len(databytes);
2070 fulldatasize = datasize + fulldatasize;
2071 percentage = "";
2072 if(downloadsize>0):
2073 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2074 downloaddiff = fulldatasize - prevdownsize;
2075 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2076 prevdownsize = fulldatasize;
2077 f.write(databytes);
2078 f.seek(0);
2079 fdata = f.getvalue();
2080 f.close();
2081 ft.close();
2082 os.remove(tmpfilename);
2083 exec_time_end = time.time();
2084 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2085 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2086 return returnval;
2087
2088 if(not havehttpx):
2089 def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2090 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
2091 return returnval;
2092
2093 if(havehttpx):
2094 def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2095 global geturls_download_sleep, havezstd, havebrotli;
2096 if(sleep<0):
2097 sleep = geturls_download_sleep;
2098 if(timeout<=0):
2099 timeout = 10;
2100 urlparts = urlparse.urlparse(httpurl);
2101 if(isinstance(httpheaders, list)):
2102 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2103 httpheaders = fix_header_names(httpheaders);
2104 if(httpuseragent is not None):
2105 if('User-Agent' in httpheaders):
2106 httpheaders['User-Agent'] = httpuseragent;
2107 else:
2108 httpuseragent.update({'User-Agent': httpuseragent});
2109 if(httpreferer is not None):
2110 if('Referer' in httpheaders):
2111 httpheaders['Referer'] = httpreferer;
2112 else:
2113 httpuseragent.update({'Referer': httpreferer});
2114 if(urlparts.username is not None or urlparts.password is not None):
2115 if(sys.version[0]=="2"):
2116 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
2117 if(sys.version[0]>="3"):
2118 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2119 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2120 time.sleep(sleep);
2121 if(postdata is not None and not isinstance(postdata, dict)):
2122 postdata = urlencode(postdata);
2123 try:
2124 if(httpmethod=="GET"):
2125 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2126 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
2127 elif(httpmethod=="POST"):
2128 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2129 geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
2130 else:
2131 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2132 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
2133 except httpx.ConnectTimeout:
2134 log.info("Error With URL "+httpurl);
2135 return False;
2136 except httpx.ConnectError:
2137 log.info("Error With URL "+httpurl);
2138 return False;
2139 except socket.timeout:
2140 log.info("Error With URL "+httpurl);
2141 return False;
2142 httpcodeout = geturls_text.status_code;
2143 try:
2144 httpcodereason = geturls_text.reason_phrase;
2145 except:
2146 httpcodereason = http_status_to_reason(geturls_text.status_code);
2147 httpversionout = geturls_text.http_version;
2148 httpmethodout = httpmethod;
2149 httpurlout = str(geturls_text.url);
2150 httpheaderout = geturls_text.headers;
2151 httpheadersentout = geturls_text.request.headers;
2152 if(isinstance(httpheaderout, list)):
2153 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2154 if(sys.version[0]=="2"):
2155 try:
2156 prehttpheaderout = httpheaderout;
2157 httpheaderkeys = httpheaderout.keys();
2158 imax = len(httpheaderkeys);
2159 ic = 0;
2160 httpheaderout = {};
2161 while(ic < imax):
2162 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2163 ic += 1;
2164 except AttributeError:
2165 pass;
2166 httpheaderout = fix_header_names(httpheaderout);
2167 if(isinstance(httpheadersentout, list)):
2168 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2169 httpheadersentout = fix_header_names(httpheadersentout);
2170 log.info("Downloading URL "+httpurl);
2171 downloadsize = httpheaderout.get('Content-Length');
2172 if(downloadsize is not None):
2173 downloadsize = int(downloadsize);
2174 if downloadsize is None: downloadsize = 0;
2175 fulldatasize = 0;
2176 prevdownsize = 0;
2177 log.info("Downloading URL "+httpurl);
2178 with BytesIO() as strbuf:
2179 while True:
2180 databytes = geturls_text.read(buffersize);
2181 if not databytes: break;
2182 datasize = len(databytes);
2183 fulldatasize = datasize + fulldatasize;
2184 percentage = "";
2185 if(downloadsize>0):
2186 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2187 downloaddiff = fulldatasize - prevdownsize;
2188 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2189 prevdownsize = fulldatasize;
2190 strbuf.write(databytes);
2191 strbuf.seek(0);
2192 returnval_content = strbuf.read();
2193 if(httpheaderout.get("Content-Encoding")=="gzip"):
2194 try:
2195 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
2196 except zlib.error:
2197 pass;
2198 elif(httpheaderout.get("Content-Encoding")=="deflate"):
2199 try:
2200 returnval_content = zlib.decompress(returnval_content);
2201 except zlib.error:
2202 pass;
2203 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
2204 try:
2205 returnval_content = brotli.decompress(returnval_content);
2206 except brotli.error:
2207 pass;
2208 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
2209 try:
2210 returnval_content = zstandard.decompress(returnval_content);
2211 except zstandard.error:
2212 pass;
2213 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2214 geturls_text.close();
2215 return returnval;
2216
2217 if(not havehttpx):
2218 def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2219 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
2220 return returnval;
2221
2222 if(havehttpx):
2223 def download_from_url_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
2224 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
2225 exec_time_start = time.time();
2226 myhash = hashlib.new("sha1");
2227 if(sys.version[0]=="2"):
2228 myhash.update(httpurl);
2229 myhash.update(str(buffersize));
2230 myhash.update(str(exec_time_start));
2231 if(sys.version[0]>="3"):
2232 myhash.update(httpurl.encode('utf-8'));
2233 myhash.update(str(buffersize).encode('utf-8'));
2234 myhash.update(str(exec_time_start).encode('utf-8'));
2235 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2236 if(sleep<0):
2237 sleep = geturls_download_sleep;
2238 if(timeout<=0):
2239 timeout = 10;
2240 pretmpfilename = download_from_url_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
2241 if(not pretmpfilename):
2242 return False;
2243 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2244 tmpfilename = f.name;
2245 try:
2246 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2247 except AttributeError:
2248 try:
2249 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2250 except ValueError:
2251 pass;
2252 except ValueError:
2253 pass;
2254 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
2255 f.write(pretmpfilename['Content']);
2256 f.close();
2257 exec_time_end = time.time();
2258 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2259 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2260 return returnval;
2261
2262 if(not havehttpx):
2263 def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
2264 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
2265 return returnval;
2266
2267 if(havehttpx):
2268 def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2269 global geturls_download_sleep, havezstd, havebrotli;
2270 if(sleep<0):
2271 sleep = geturls_download_sleep;
2272 if(timeout<=0):
2273 timeout = 10;
2274 if(not outfile=="-"):
2275 outpath = outpath.rstrip(os.path.sep);
2276 filepath = os.path.realpath(outpath+os.path.sep+outfile);
2277 if(not os.path.exists(outpath)):
2278 os.makedirs(outpath);
2279 if(os.path.exists(outpath) and os.path.isfile(outpath)):
2280 return False;
2281 if(os.path.exists(filepath) and os.path.isdir(filepath)):
2282 return False;
2283 pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
2284 if(not pretmpfilename):
2285 return False;
2286 tmpfilename = pretmpfilename['Filename'];
2287 downloadsize = int(os.path.getsize(tmpfilename));
2288 fulldatasize = 0;
2289 log.info("Moving file "+tmpfilename+" to "+filepath);
2290 exec_time_start = time.time();
2291 shutil.move(tmpfilename, filepath);
2292 try:
2293 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2294 except AttributeError:
2295 try:
2296 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2297 except ValueError:
2298 pass;
2299 except ValueError:
2300 pass;
2301 exec_time_end = time.time();
2302 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2303 if(os.path.exists(tmpfilename)):
2304 os.remove(tmpfilename);
2305 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2306 if(outfile=="-"):
2307 pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
2308 tmpfilename = pretmpfilename['Filename'];
2309 downloadsize = int(os.path.getsize(tmpfilename));
2310 fulldatasize = 0;
2311 prevdownsize = 0;
2312 exec_time_start = time.time();
2313 with open(tmpfilename, 'rb') as ft:
2314 f = BytesIO();
2315 while True:
2316 databytes = ft.read(buffersize[1]);
2317 if not databytes: break;
2318 datasize = len(databytes);
2319 fulldatasize = datasize + fulldatasize;
2320 percentage = "";
2321 if(downloadsize>0):
2322 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2323 downloaddiff = fulldatasize - prevdownsize;
2324 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2325 prevdownsize = fulldatasize;
2326 f.write(databytes);
2327 f.seek(0);
2328 fdata = f.getvalue();
2329 f.close();
2330 ft.close();
2331 os.remove(tmpfilename);
2332 exec_time_end = time.time();
2333 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2334 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2335 return returnval;
2336
2337 if(not havehttpx):
2338 def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2339 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
2340 return returnval;
2341
2342 if(havehttpcore):
2343 def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2344 global geturls_download_sleep, havezstd, havebrotli;
2345 if(sleep<0):
2346 sleep = geturls_download_sleep;
2347 if(timeout<=0):
2348 timeout = 10;
2349 urlparts = urlparse.urlparse(httpurl);
2350 if(isinstance(httpheaders, list)):
2351 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2352 httpheaders = fix_header_names(httpheaders);
2353 if(httpuseragent is not None):
2354 if('User-Agent' in httpheaders):
2355 httpheaders['User-Agent'] = httpuseragent;
2356 else:
2357 httpuseragent.update({'User-Agent': httpuseragent});
2358 if(httpreferer is not None):
2359 if('Referer' in httpheaders):
2360 httpheaders['Referer'] = httpreferer;
2361 else:
2362 httpuseragent.update({'Referer': httpreferer});
2363 if(urlparts.username is not None or urlparts.password is not None):
2364 if(sys.version[0]=="2"):
2365 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
2366 if(sys.version[0]>="3"):
2367 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2368 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2369 time.sleep(sleep);
2370 if(postdata is not None and not isinstance(postdata, dict)):
2371 postdata = urlencode(postdata);
2372 try:
2373 if(httpmethod=="GET"):
2374 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
2375 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
2376 elif(httpmethod=="POST"):
2377 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
2378 geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
2379 else:
2380 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
2381 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
2382 except httpcore.ConnectTimeout:
2383 log.info("Error With URL "+httpurl);
2384 return False;
2385 except httpcore.ConnectError:
2386 log.info("Error With URL "+httpurl);
2387 return False;
2388 except socket.timeout:
2389 log.info("Error With URL "+httpurl);
2390 return False;
2391 httpcodeout = geturls_text.status;
2392 httpcodereason = http_status_to_reason(geturls_text.status);
2393 httpversionout = "1.1";
2394 httpmethodout = httpmethod;
2395 httpurlout = str(httpurl);
2396 httpheaderout = geturls_text.headers;
2397 httpheadersentout = httpheaders;
2398 if(isinstance(httpheaderout, list)):
2399 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2400 if(sys.version[0]=="2"):
2401 try:
2402 prehttpheaderout = httpheaderout;
2403 httpheaderkeys = httpheaderout.keys();
2404 imax = len(httpheaderkeys);
2405 ic = 0;
2406 httpheaderout = {};
2407 while(ic < imax):
2408 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2409 ic += 1;
2410 except AttributeError:
2411 pass;
2412 httpheaderout = fix_header_names(httpheaderout);
2413 if(isinstance(httpheadersentout, list)):
2414 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2415 httpheadersentout = fix_header_names(httpheadersentout);
2416 log.info("Downloading URL "+httpurl);
2417 downloadsize = httpheaderout.get('Content-Length');
2418 if(downloadsize is not None):
2419 downloadsize = int(downloadsize);
2420 if downloadsize is None: downloadsize = 0;
2421 fulldatasize = 0;
2422 prevdownsize = 0;
2423 log.info("Downloading URL "+httpurl);
2424 with BytesIO() as strbuf:
2425 while True:
2426 databytes = geturls_text.read(buffersize);
2427 if not databytes: break;
2428 datasize = len(databytes);
2429 fulldatasize = datasize + fulldatasize;
2430 percentage = "";
2431 if(downloadsize>0):
2432 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2433 downloaddiff = fulldatasize - prevdownsize;
2434 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2435 prevdownsize = fulldatasize;
2436 strbuf.write(databytes);
2437 strbuf.seek(0);
2438 returnval_content = strbuf.read();
2439 if(httpheaderout.get("Content-Encoding")=="gzip"):
2440 try:
2441 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
2442 except zlib.error:
2443 pass;
2444 elif(httpheaderout.get("Content-Encoding")=="deflate"):
2445 try:
2446 returnval_content = zlib.decompress(returnval_content);
2447 except zlib.error:
2448 pass;
2449 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
2450 try:
2451 returnval_content = brotli.decompress(returnval_content);
2452 except brotli.error:
2453 pass;
2454 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
2455 try:
2456 returnval_content = zstandard.decompress(returnval_content);
2457 except zstandard.error:
2458 pass;
2459 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2460 geturls_text.close();
2461 return returnval;
2462
2463 if(not havehttpcore):
2464 def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2465 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
2466 return returnval;
2467
2468 if(havehttpcore):
2469 def download_from_url_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
2470 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
2471 exec_time_start = time.time();
2472 myhash = hashlib.new("sha1");
2473 if(sys.version[0]=="2"):
2474 myhash.update(httpurl);
2475 myhash.update(str(buffersize));
2476 myhash.update(str(exec_time_start));
2477 if(sys.version[0]>="3"):
2478 myhash.update(httpurl.encode('utf-8'));
2479 myhash.update(str(buffersize).encode('utf-8'));
2480 myhash.update(str(exec_time_start).encode('utf-8'));
2481 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2482 if(sleep<0):
2483 sleep = geturls_download_sleep;
2484 if(timeout<=0):
2485 timeout = 10;
2486 pretmpfilename = download_from_url_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
2487 if(not pretmpfilename):
2488 return False;
2489 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2490 tmpfilename = f.name;
2491 try:
2492 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2493 except AttributeError:
2494 try:
2495 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2496 except ValueError:
2497 pass;
2498 except ValueError:
2499 pass;
2500 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
2501 f.write(pretmpfilename['Content']);
2502 f.close();
2503 exec_time_end = time.time();
2504 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2505 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2506 return returnval;
2507
2508 if(not havehttpcore):
2509 def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
2510 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
2511 return returnval;
2512
2513 if(havehttpcore):
2514 def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2515 global geturls_download_sleep, havezstd, havebrotli;
2516 if(sleep<0):
2517 sleep = geturls_download_sleep;
2518 if(timeout<=0):
2519 timeout = 10;
2520 if(not outfile=="-"):
2521 outpath = outpath.rstrip(os.path.sep);
2522 filepath = os.path.realpath(outpath+os.path.sep+outfile);
2523 if(not os.path.exists(outpath)):
2524 os.makedirs(outpath);
2525 if(os.path.exists(outpath) and os.path.isfile(outpath)):
2526 return False;
2527 if(os.path.exists(filepath) and os.path.isdir(filepath)):
2528 return False;
2529 pretmpfilename = download_from_url_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
2530 if(not pretmpfilename):
2531 return False;
2532 tmpfilename = pretmpfilename['Filename'];
2533 downloadsize = int(os.path.getsize(tmpfilename));
2534 fulldatasize = 0;
2535 log.info("Moving file "+tmpfilename+" to "+filepath);
2536 exec_time_start = time.time();
2537 shutil.move(tmpfilename, filepath);
2538 try:
2539 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2540 except AttributeError:
2541 try:
2542 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2543 except ValueError:
2544 pass;
2545 except ValueError:
2546 pass;
2547 exec_time_end = time.time();
2548 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2549 if(os.path.exists(tmpfilename)):
2550 os.remove(tmpfilename);
2551 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2552 if(outfile=="-"):
2553 pretmpfilename = download_from_url_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
2554 tmpfilename = pretmpfilename['Filename'];
2555 downloadsize = int(os.path.getsize(tmpfilename));
2556 fulldatasize = 0;
2557 prevdownsize = 0;
2558 exec_time_start = time.time();
2559 with open(tmpfilename, 'rb') as ft:
2560 f = BytesIO();
2561 while True:
2562 databytes = ft.read(buffersize[1]);
2563 if not databytes: break;
2564 datasize = len(databytes);
2565 fulldatasize = datasize + fulldatasize;
2566 percentage = "";
2567 if(downloadsize>0):
2568 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2569 downloaddiff = fulldatasize - prevdownsize;
2570 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2571 prevdownsize = fulldatasize;
2572 f.write(databytes);
2573 f.seek(0);
2574 fdata = f.getvalue();
2575 f.close();
2576 ft.close();
2577 os.remove(tmpfilename);
2578 exec_time_end = time.time();
2579 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2580 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2581 return returnval;
2582
2583 if(not havehttpcore):
2584 def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2585 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
2586 return returnval;
2587
2588 if(havehttpcore):
2589 def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2590 global geturls_download_sleep, havezstd, havebrotli;
2591 if(sleep<0):
2592 sleep = geturls_download_sleep;
2593 if(timeout<=0):
2594 timeout = 10;
2595 urlparts = urlparse.urlparse(httpurl);
2596 if(isinstance(httpheaders, list)):
2597 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2598 httpheaders = fix_header_names(httpheaders);
2599 if(httpuseragent is not None):
2600 if('User-Agent' in httpheaders):
2601 httpheaders['User-Agent'] = httpuseragent;
2602 else:
2603 httpuseragent.update({'User-Agent': httpuseragent});
2604 if(httpreferer is not None):
2605 if('Referer' in httpheaders):
2606 httpheaders['Referer'] = httpreferer;
2607 else:
2608 httpuseragent.update({'Referer': httpreferer});
2609 if(urlparts.username is not None or urlparts.password is not None):
2610 if(sys.version[0]=="2"):
2611 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
2612 if(sys.version[0]>="3"):
2613 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2614 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2615 time.sleep(sleep);
2616 if(postdata is not None and not isinstance(postdata, dict)):
2617 postdata = urlencode(postdata);
2618 try:
2619 if(httpmethod=="GET"):
2620 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
2621 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
2622 elif(httpmethod=="POST"):
2623 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
2624 geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
2625 else:
2626 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
2627 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
2628 except httpcore.ConnectTimeout:
2629 log.info("Error With URL "+httpurl);
2630 return False;
2631 except httpcore.ConnectError:
2632 log.info("Error With URL "+httpurl);
2633 return False;
2634 except socket.timeout:
2635 log.info("Error With URL "+httpurl);
2636 return False;
2637 httpcodeout = geturls_text.status;
2638 httpcodereason = http_status_to_reason(geturls_text.status);
2639 httpversionout = "1.1";
2640 httpmethodout = httpmethod;
2641 httpurlout = str(httpurl);
2642 httpheaderout = geturls_text.headers;
2643 httpheadersentout = httpheaders;
2644 if(isinstance(httpheaderout, list)):
2645 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2646 if(sys.version[0]=="2"):
2647 try:
2648 prehttpheaderout = httpheaderout;
2649 httpheaderkeys = httpheaderout.keys();
2650 imax = len(httpheaderkeys);
2651 ic = 0;
2652 httpheaderout = {};
2653 while(ic < imax):
2654 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2655 ic += 1;
2656 except AttributeError:
2657 pass;
2658 httpheaderout = fix_header_names(httpheaderout);
2659 if(isinstance(httpheadersentout, list)):
2660 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2661 httpheadersentout = fix_header_names(httpheadersentout);
2662 log.info("Downloading URL "+httpurl);
2663 downloadsize = httpheaderout.get('Content-Length');
2664 if(downloadsize is not None):
2665 downloadsize = int(downloadsize);
2666 if downloadsize is None: downloadsize = 0;
2667 fulldatasize = 0;
2668 prevdownsize = 0;
2669 log.info("Downloading URL "+httpurl);
2670 with BytesIO() as strbuf:
2671 while True:
2672 databytes = geturls_text.read(buffersize);
2673 if not databytes: break;
2674 datasize = len(databytes);
2675 fulldatasize = datasize + fulldatasize;
2676 percentage = "";
2677 if(downloadsize>0):
2678 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2679 downloaddiff = fulldatasize - prevdownsize;
2680 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2681 prevdownsize = fulldatasize;
2682 strbuf.write(databytes);
2683 strbuf.seek(0);
2684 returnval_content = strbuf.read();
2685 if(httpheaderout.get("Content-Encoding")=="gzip"):
2686 try:
2687 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
2688 except zlib.error:
2689 pass;
2690 elif(httpheaderout.get("Content-Encoding")=="deflate"):
2691 try:
2692 returnval_content = zlib.decompress(returnval_content);
2693 except zlib.error:
2694 pass;
2695 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
2696 try:
2697 returnval_content = brotli.decompress(returnval_content);
2698 except brotli.error:
2699 pass;
2700 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
2701 try:
2702 returnval_content = zstandard.decompress(returnval_content);
2703 except zstandard.error:
2704 pass;
2705 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2706 geturls_text.close();
2707 return returnval;
2708
2709 if(not havehttpcore):
2710 def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2711 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
2712 return returnval;
2713
2714 if(havehttpcore):
2715 def download_from_url_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
2716 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
2717 exec_time_start = time.time();
2718 myhash = hashlib.new("sha1");
2719 if(sys.version[0]=="2"):
2720 myhash.update(httpurl);
2721 myhash.update(str(buffersize));
2722 myhash.update(str(exec_time_start));
2723 if(sys.version[0]>="3"):
2724 myhash.update(httpurl.encode('utf-8'));
2725 myhash.update(str(buffersize).encode('utf-8'));
2726 myhash.update(str(exec_time_start).encode('utf-8'));
2727 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2728 if(sleep<0):
2729 sleep = geturls_download_sleep;
2730 if(timeout<=0):
2731 timeout = 10;
2732 pretmpfilename = download_from_url_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
2733 if(not pretmpfilename):
2734 return False;
2735 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2736 tmpfilename = f.name;
2737 try:
2738 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2739 except AttributeError:
2740 try:
2741 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2742 except ValueError:
2743 pass;
2744 except ValueError:
2745 pass;
2746 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
2747 f.write(pretmpfilename['Content']);
2748 f.close();
2749 exec_time_end = time.time();
2750 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2751 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2752 return returnval;
2753
2754 if(not havehttpcore):
2755 def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
2756 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
2757 return returnval;
2758
2759 if(havehttpcore):
2760 def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2761 global geturls_download_sleep, havezstd, havebrotli;
2762 if(sleep<0):
2763 sleep = geturls_download_sleep;
2764 if(timeout<=0):
2765 timeout = 10;
2766 if(not outfile=="-"):
2767 outpath = outpath.rstrip(os.path.sep);
2768 filepath = os.path.realpath(outpath+os.path.sep+outfile);
2769 if(not os.path.exists(outpath)):
2770 os.makedirs(outpath);
2771 if(os.path.exists(outpath) and os.path.isfile(outpath)):
2772 return False;
2773 if(os.path.exists(filepath) and os.path.isdir(filepath)):
2774 return False;
2775 pretmpfilename = download_from_url_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
2776 if(not pretmpfilename):
2777 return False;
2778 tmpfilename = pretmpfilename['Filename'];
2779 downloadsize = int(os.path.getsize(tmpfilename));
2780 fulldatasize = 0;
2781 log.info("Moving file "+tmpfilename+" to "+filepath);
2782 exec_time_start = time.time();
2783 shutil.move(tmpfilename, filepath);
2784 try:
2785 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2786 except AttributeError:
2787 try:
2788 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2789 except ValueError:
2790 pass;
2791 except ValueError:
2792 pass;
2793 exec_time_end = time.time();
2794 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2795 if(os.path.exists(tmpfilename)):
2796 os.remove(tmpfilename);
2797 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2798 if(outfile=="-"):
2799 pretmpfilename = download_from_url_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
2800 tmpfilename = pretmpfilename['Filename'];
2801 downloadsize = int(os.path.getsize(tmpfilename));
2802 fulldatasize = 0;
2803 prevdownsize = 0;
2804 exec_time_start = time.time();
2805 with open(tmpfilename, 'rb') as ft:
2806 f = BytesIO();
2807 while True:
2808 databytes = ft.read(buffersize[1]);
2809 if not databytes: break;
2810 datasize = len(databytes);
2811 fulldatasize = datasize + fulldatasize;
2812 percentage = "";
2813 if(downloadsize>0):
2814 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2815 downloaddiff = fulldatasize - prevdownsize;
2816 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2817 prevdownsize = fulldatasize;
2818 f.write(databytes);
2819 f.seek(0);
2820 fdata = f.getvalue();
2821 f.close();
2822 ft.close();
2823 os.remove(tmpfilename);
2824 exec_time_end = time.time();
2825 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2826 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2827 return returnval;
2828
2829 if(not havehttpx):
2830 def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2831 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
2832 return returnval;
2833
2834 if(haveurllib3):
2835 def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2836 returnval = download_from_url_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
2837 return returnval;
2838
2839 if(not haveurllib3):
2840 def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2841 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
2842 return returnval;
2843
2844 if(haveurllib3):
2845 def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
2846 returnval = download_from_url_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
2847 return returnval;
2848
2849 if(not haveurllib3):
2850 def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
2851 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
2852 return returnval;
2853
2854 if(haveurllib3):
2855 def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2856 returnval = download_from_url_to_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
2857 return returnval;
2858
2859 if(not haveurllib3):
2860 def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2861 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
2862 return returnval;
2863
2864 if(haveurllib3):
2865 def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2866 global geturls_download_sleep, havezstd, havebrotli;
2867 if(sleep<0):
2868 sleep = geturls_download_sleep;
2869 if(timeout<=0):
2870 timeout = 10;
2871 urlparts = urlparse.urlparse(httpurl);
2872 if(isinstance(httpheaders, list)):
2873 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2874 httpheaders = fix_header_names(httpheaders);
2875 if(httpuseragent is not None):
2876 if('User-Agent' in httpheaders):
2877 httpheaders['User-Agent'] = httpuseragent;
2878 else:
2879 httpuseragent.update({'User-Agent': httpuseragent});
2880 if(httpreferer is not None):
2881 if('Referer' in httpheaders):
2882 httpheaders['Referer'] = httpreferer;
2883 else:
2884 httpuseragent.update({'Referer': httpreferer});
2885 if(urlparts.username is not None or urlparts.password is not None):
2886 if(sys.version[0]=="2"):
2887 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
2888 if(sys.version[0]>="3"):
2889 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2890 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2891 time.sleep(sleep);
2892 timeout = urllib3.util.Timeout(connect=timeout, read=timeout);
2893 urllib_pool = urllib3.PoolManager(headers=httpheaders, timeout=timeout);
2894 if(postdata is not None and not isinstance(postdata, dict)):
2895 postdata = urlencode(postdata);
2896 try:
2897 if(httpmethod=="GET"):
2898 geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2899 elif(httpmethod=="POST"):
2900 geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2901 else:
2902 geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2903 except urllib3.exceptions.ConnectTimeoutError:
2904 log.info("Error With URL "+httpurl);
2905 return False;
2906 except urllib3.exceptions.ConnectError:
2907 log.info("Error With URL "+httpurl);
2908 return False;
2909 except urllib3.exceptions.MaxRetryError:
2910 log.info("Error With URL "+httpurl);
2911 return False;
2912 except socket.timeout:
2913 log.info("Error With URL "+httpurl);
2914 return False;
2915 except ValueError:
2916 log.info("Error With URL "+httpurl);
2917 return False;
2918 httpcodeout = geturls_text.status;
2919 httpcodereason = geturls_text.reason;
2920 if(geturls_text.version=="10"):
2921 httpversionout = "1.0";
2922 else:
2923 httpversionout = "1.1";
2924 httpmethodout = httpmethod;
2925 httpurlout = geturls_text.geturl();
2926 httpheaderout = geturls_text.info();
2927 httpheadersentout = httpheaders;
2928 if(isinstance(httpheaderout, list)):
2929 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2930 if(sys.version[0]=="2"):
2931 try:
2932 prehttpheaderout = httpheaderout;
2933 httpheaderkeys = httpheaderout.keys();
2934 imax = len(httpheaderkeys);
2935 ic = 0;
2936 httpheaderout = {};
2937 while(ic < imax):
2938 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2939 ic += 1;
2940 except AttributeError:
2941 pass;
2942 httpheaderout = fix_header_names(httpheaderout);
2943 if(isinstance(httpheadersentout, list)):
2944 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2945 httpheadersentout = fix_header_names(httpheadersentout);
2946 log.info("Downloading URL "+httpurl);
2947 downloadsize = httpheaderout.get('Content-Length');
2948 if(downloadsize is not None):
2949 downloadsize = int(downloadsize);
2950 if downloadsize is None: downloadsize = 0;
2951 fulldatasize = 0;
2952 prevdownsize = 0;
2953 log.info("Downloading URL "+httpurl);
2954 with BytesIO() as strbuf:
2955 while True:
2956 databytes = geturls_text.read(buffersize);
2957 if not databytes: break;
2958 datasize = len(databytes);
2959 fulldatasize = datasize + fulldatasize;
2960 percentage = "";
2961 if(downloadsize>0):
2962 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2963 downloaddiff = fulldatasize - prevdownsize;
2964 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2965 prevdownsize = fulldatasize;
2966 strbuf.write(databytes);
2967 strbuf.seek(0);
2968 returnval_content = strbuf.read();
2969 if(httpheaderout.get("Content-Encoding")=="gzip"):
2970 try:
2971 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
2972 except zlib.error:
2973 pass;
2974 elif(httpheaderout.get("Content-Encoding")=="deflate"):
2975 try:
2976 returnval_content = zlib.decompress(returnval_content);
2977 except zlib.error:
2978 pass;
2979 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
2980 try:
2981 returnval_content = brotli.decompress(returnval_content);
2982 except brotli.error:
2983 pass;
2984 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
2985 try:
2986 returnval_content = zstandard.decompress(returnval_content);
2987 except zstandard.error:
2988 pass;
2989 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2990 geturls_text.close();
2991 return returnval;
2992
2993 if(not haveurllib3):
2994 def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
2995 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
2996 return returnval;
2997
2998 if(haveurllib3):
2999 def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
3000 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
3001 exec_time_start = time.time();
3002 myhash = hashlib.new("sha1");
3003 if(sys.version[0]=="2"):
3004 myhash.update(httpurl);
3005 myhash.update(str(buffersize));
3006 myhash.update(str(exec_time_start));
3007 if(sys.version[0]>="3"):
3008 myhash.update(httpurl.encode('utf-8'));
3009 myhash.update(str(buffersize).encode('utf-8'));
3010 myhash.update(str(exec_time_start).encode('utf-8'));
3011 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
3012 if(sleep<0):
3013 sleep = geturls_download_sleep;
3014 if(timeout<=0):
3015 timeout = 10;
3016 pretmpfilename = download_from_url_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
3017 if(not pretmpfilename):
3018 return False;
3019 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
3020 tmpfilename = f.name;
3021 try:
3022 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3023 except AttributeError:
3024 try:
3025 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3026 except ValueError:
3027 pass;
3028 except ValueError:
3029 pass;
3030 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
3031 f.write(pretmpfilename['Content']);
3032 f.close();
3033 exec_time_end = time.time();
3034 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
3035 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
3036 return returnval;
3037
3038 if(not haveurllib3):
3039 def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
3040 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
3041 return returnval;
3042
3043 if(haveurllib3):
3044 def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
3045 global geturls_download_sleep, havezstd, havebrotli;
3046 if(sleep<0):
3047 sleep = geturls_download_sleep;
3048 if(timeout<=0):
3049 timeout = 10;
3050 if(not outfile=="-"):
3051 outpath = outpath.rstrip(os.path.sep);
3052 filepath = os.path.realpath(outpath+os.path.sep+outfile);
3053 if(not os.path.exists(outpath)):
3054 os.makedirs(outpath);
3055 if(os.path.exists(outpath) and os.path.isfile(outpath)):
3056 return False;
3057 if(os.path.exists(filepath) and os.path.isdir(filepath)):
3058 return False;
3059 pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
3060 if(not pretmpfilename):
3061 return False;
3062 tmpfilename = pretmpfilename['Filename'];
3063 downloadsize = int(os.path.getsize(tmpfilename));
3064 fulldatasize = 0;
3065 log.info("Moving file "+tmpfilename+" to "+filepath);
3066 exec_time_start = time.time();
3067 shutil.move(tmpfilename, filepath);
3068 try:
3069 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3070 except AttributeError:
3071 try:
3072 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3073 except ValueError:
3074 pass;
3075 except ValueError:
3076 pass;
3077 exec_time_end = time.time();
3078 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
3079 if(os.path.exists(tmpfilename)):
3080 os.remove(tmpfilename);
3081 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3082 if(outfile=="-"):
3083 pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
3084 tmpfilename = pretmpfilename['Filename'];
3085 downloadsize = int(os.path.getsize(tmpfilename));
3086 fulldatasize = 0;
3087 prevdownsize = 0;
3088 exec_time_start = time.time();
3089 with open(tmpfilename, 'rb') as ft:
3090 f = BytesIO();
3091 while True:
3092 databytes = ft.read(buffersize[1]);
3093 if not databytes: break;
3094 datasize = len(databytes);
3095 fulldatasize = datasize + fulldatasize;
3096 percentage = "";
3097 if(downloadsize>0):
3098 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3099 downloaddiff = fulldatasize - prevdownsize;
3100 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3101 prevdownsize = fulldatasize;
3102 f.write(databytes);
3103 f.seek(0);
3104 fdata = f.getvalue();
3105 f.close();
3106 ft.close();
3107 os.remove(tmpfilename);
3108 exec_time_end = time.time();
3109 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3110 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3111 return returnval;
3112
3113 if(not haveurllib3):
3114 def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
3115 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
3116 return returnval;
3117
3118 if(havemechanize):
3119 def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
3120 global geturls_download_sleep, havezstd, havebrotli;
3121 if(sleep<0):
3122 sleep = geturls_download_sleep;
3123 if(timeout<=0):
3124 timeout = 10;
3125 urlparts = urlparse.urlparse(httpurl);
3126 if(isinstance(httpheaders, list)):
3127 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3128 httpheaders = fix_header_names(httpheaders);
3129 if(httpuseragent is not None):
3130 if('User-Agent' in httpheaders):
3131 httpheaders['User-Agent'] = httpuseragent;
3132 else:
3133 httpuseragent.update({'User-Agent': httpuseragent});
3134 if(httpreferer is not None):
3135 if('Referer' in httpheaders):
3136 httpheaders['Referer'] = httpreferer;
3137 else:
3138 httpuseragent.update({'Referer': httpreferer});
3139 if(urlparts.username is not None or urlparts.password is not None):
3140 if(sys.version[0]=="2"):
3141 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
3142 if(sys.version[0]>="3"):
3143 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3144 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3145 geturls_opener = mechanize.Browser();
3146 if(isinstance(httpheaders, dict)):
3147 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
3148 time.sleep(sleep);
3149 geturls_opener.addheaders = httpheaders;
3150 geturls_opener.set_cookiejar(httpcookie);
3151 geturls_opener.set_handle_robots(False);
3152 if(postdata is not None and not isinstance(postdata, dict)):
3153 postdata = urlencode(postdata);
3154 try:
3155 if(httpmethod=="GET"):
3156 geturls_text = geturls_opener.open(httpurl);
3157 elif(httpmethod=="POST"):
3158 geturls_text = geturls_opener.open(httpurl, data=postdata);
3159 else:
3160 geturls_text = geturls_opener.open(httpurl);
3161 except mechanize.HTTPError as geturls_text_error:
3162 geturls_text = geturls_text_error;
3163 log.info("Error With URL "+httpurl);
3164 except URLError:
3165 log.info("Error With URL "+httpurl);
3166 return False;
3167 except socket.timeout:
3168 log.info("Error With URL "+httpurl);
3169 return False;
3170 httpcodeout = geturls_text.code;
3171 httpcodereason = geturls_text.msg;
3172 httpversionout = "1.1";
3173 httpmethodout = httpmethod;
3174 httpurlout = geturls_text.geturl();
3175 httpheaderout = geturls_text.info();
3176 reqhead = geturls_opener.request;
3177 httpheadersentout = reqhead.header_items();
3178 if(isinstance(httpheaderout, list)):
3179 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
3180 if(sys.version[0]=="2"):
3181 try:
3182 prehttpheaderout = httpheaderout;
3183 httpheaderkeys = httpheaderout.keys();
3184 imax = len(httpheaderkeys);
3185 ic = 0;
3186 httpheaderout = {};
3187 while(ic < imax):
3188 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
3189 ic += 1;
3190 except AttributeError:
3191 pass;
3192 httpheaderout = fix_header_names(httpheaderout);
3193 if(isinstance(httpheadersentout, list)):
3194 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
3195 httpheadersentout = fix_header_names(httpheadersentout);
3196 log.info("Downloading URL "+httpurl);
3197 downloadsize = httpheaderout.get('Content-Length');
3198 if(downloadsize is not None):
3199 downloadsize = int(downloadsize);
3200 if downloadsize is None: downloadsize = 0;
3201 fulldatasize = 0;
3202 prevdownsize = 0;
3203 log.info("Downloading URL "+httpurl);
3204 with BytesIO() as strbuf:
3205 while True:
3206 databytes = geturls_text.read(buffersize);
3207 if not databytes: break;
3208 datasize = len(databytes);
3209 fulldatasize = datasize + fulldatasize;
3210 percentage = "";
3211 if(downloadsize>0):
3212 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3213 downloaddiff = fulldatasize - prevdownsize;
3214 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3215 prevdownsize = fulldatasize;
3216 strbuf.write(databytes);
3217 strbuf.seek(0);
3218 returnval_content = strbuf.read();
3219 if(httpheaderout.get("Content-Encoding")=="gzip"):
3220 try:
3221 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
3222 except zlib.error:
3223 pass;
3224 elif(httpheaderout.get("Content-Encoding")=="deflate"):
3225 try:
3226 returnval_content = zlib.decompress(returnval_content);
3227 except zlib.error:
3228 pass;
3229 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
3230 try:
3231 returnval_content = brotli.decompress(returnval_content);
3232 except brotli.error:
3233 pass;
3234 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
3235 try:
3236 returnval_content = zstandard.decompress(returnval_content);
3237 except zstandard.error:
3238 pass;
3239 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
3240 geturls_text.close();
3241 return returnval;
3242
3243 if(not havemechanize):
3244 def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
3245 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
3246 return returnval;
3247
3248 if(havemechanize):
3249 def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
3250 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
3251 exec_time_start = time.time();
3252 myhash = hashlib.new("sha1");
3253 if(sys.version[0]=="2"):
3254 myhash.update(httpurl);
3255 myhash.update(str(buffersize));
3256 myhash.update(str(exec_time_start));
3257 if(sys.version[0]>="3"):
3258 myhash.update(httpurl.encode('utf-8'));
3259 myhash.update(str(buffersize).encode('utf-8'));
3260 myhash.update(str(exec_time_start).encode('utf-8'));
3261 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
3262 if(sleep<0):
3263 sleep = geturls_download_sleep;
3264 if(timeout<=0):
3265 timeout = 10;
3266 pretmpfilename = download_from_url_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
3267 if(not pretmpfilename):
3268 return False;
3269 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
3270 tmpfilename = f.name;
3271 try:
3272 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3273 except AttributeError:
3274 try:
3275 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3276 except ValueError:
3277 pass;
3278 except ValueError:
3279 pass;
3280 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
3281 f.write(pretmpfilename['Content']);
3282 f.close();
3283 exec_time_end = time.time();
3284 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
3285 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
3286 return returnval;
3287
3288 if(not havemechanize):
3289 def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
3290 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
3291 return returnval;
3292
3293 if(havemechanize):
3294 def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
3295 global geturls_download_sleep, havezstd, havebrotli;
3296 if(sleep<0):
3297 sleep = geturls_download_sleep;
3298 if(timeout<=0):
3299 timeout = 10;
3300 if(not outfile=="-"):
3301 outpath = outpath.rstrip(os.path.sep);
3302 filepath = os.path.realpath(outpath+os.path.sep+outfile);
3303 if(not os.path.exists(outpath)):
3304 os.makedirs(outpath);
3305 if(os.path.exists(outpath) and os.path.isfile(outpath)):
3306 return False;
3307 if(os.path.exists(filepath) and os.path.isdir(filepath)):
3308 return False;
3309 pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
3310 if(not pretmpfilename):
3311 return False;
3312 tmpfilename = pretmpfilename['Filename'];
3313 downloadsize = int(os.path.getsize(tmpfilename));
3314 fulldatasize = 0;
3315 log.info("Moving file "+tmpfilename+" to "+filepath);
3316 exec_time_start = time.time();
3317 shutil.move(tmpfilename, filepath);
3318 try:
3319 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3320 except AttributeError:
3321 try:
3322 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3323 except ValueError:
3324 pass;
3325 except ValueError:
3326 pass;
3327 exec_time_end = time.time();
3328 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
3329 if(os.path.exists(tmpfilename)):
3330 os.remove(tmpfilename);
3331 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3332 if(outfile=="-"):
3333 pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
3334 tmpfilename = pretmpfilename['Filename'];
3335 downloadsize = int(os.path.getsize(tmpfilename));
3336 fulldatasize = 0;
3337 prevdownsize = 0;
3338 exec_time_start = time.time();
3339 with open(tmpfilename, 'rb') as ft:
3340 f = BytesIO();
3341 while True:
3342 databytes = ft.read(buffersize[1]);
3343 if not databytes: break;
3344 datasize = len(databytes);
3345 fulldatasize = datasize + fulldatasize;
3346 percentage = "";
3347 if(downloadsize>0):
3348 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3349 downloaddiff = fulldatasize - prevdownsize;
3350 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3351 prevdownsize = fulldatasize;
3352 f.write(databytes);
3353 f.seek(0);
3354 fdata = f.getvalue();
3355 f.close();
3356 ft.close();
3357 os.remove(tmpfilename);
3358 exec_time_end = time.time();
3359 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3360 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3361 return returnval;
3362
3363 if(not havemechanize):
3364 def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
3365 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
3366 return returnval;
3367
3368 if(havepycurl):
3369 def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
3370 global geturls_download_sleep, havezstd, havebrotli;
3371 if(sleep<0):
3372 sleep = geturls_download_sleep;
3373 if(timeout<=0):
3374 timeout = 10;
3375 urlparts = urlparse.urlparse(httpurl);
3376 if(isinstance(httpheaders, list)):
3377 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3378 httpheaders = fix_header_names(httpheaders);
3379 if(httpuseragent is not None):
3380 if('User-Agent' in httpheaders):
3381 httpheaders['User-Agent'] = httpuseragent;
3382 else:
3383 httpuseragent.update({'User-Agent': httpuseragent});
3384 if(httpreferer is not None):
3385 if('Referer' in httpheaders):
3386 httpheaders['Referer'] = httpreferer;
3387 else:
3388 httpuseragent.update({'Referer': httpreferer});
3389 if(urlparts.username is not None or urlparts.password is not None):
3390 if(sys.version[0]=="2"):
3391 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
3392 if(sys.version[0]>="3"):
3393 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3394 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3395 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
3396 if(isinstance(httpheaders, dict)):
3397 httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
3398 geturls_opener.addheaders = httpheaders;
3399 time.sleep(sleep);
3400 if(postdata is not None and not isinstance(postdata, dict)):
3401 postdata = urlencode(postdata);
3402 retrieved_body = BytesIO();
3403 retrieved_headers = BytesIO();
3404 try:
3405 if(httpmethod=="GET"):
3406 geturls_text = pycurl.Curl();
3407 geturls_text.setopt(geturls_text.URL, httpurl);
3408 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
3409 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
3410 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
3411 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
3412 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
3413 geturls_text.perform();
3414 elif(httpmethod=="POST"):
3415 geturls_text = pycurl.Curl();
3416 geturls_text.setopt(geturls_text.URL, httpurl);
3417 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
3418 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
3419 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
3420 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
3421 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
3422 geturls_text.setopt(geturls_text.POST, True);
3423 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
3424 geturls_text.perform();
3425 else:
3426 geturls_text = pycurl.Curl();
3427 geturls_text.setopt(geturls_text.URL, httpurl);
3428 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
3429 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
3430 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
3431 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
3432 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
3433 geturls_text.perform();
3434 retrieved_headers.seek(0);
3435 if(sys.version[0]=="2"):
3436 pycurlhead = retrieved_headers.read();
3437 if(sys.version[0]>="3"):
3438 pycurlhead = retrieved_headers.read().decode('UTF-8');
3439 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
3440 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
3441 retrieved_body.seek(0);
3442 except socket.timeout:
3443 log.info("Error With URL "+httpurl);
3444 return False;
3445 except socket.gaierror:
3446 log.info("Error With URL "+httpurl);
3447 return False;
3448 except ValueError:
3449 log.info("Error With URL "+httpurl);
3450 return False;
3451 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
3452 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
3453 httpversionout = pyhttpverinfo[0];
3454 httpmethodout = httpmethod;
3455 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
3456 httpheaderout = pycurlheadersout;
3457 httpheadersentout = httpheaders;
3458 if(isinstance(httpheaderout, list)):
3459 httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
3460 if(sys.version[0]=="2"):
3461 try:
3462 prehttpheaderout = httpheaderout;
3463 httpheaderkeys = httpheaderout.keys();
3464 imax = len(httpheaderkeys);
3465 ic = 0;
3466 httpheaderout = {};
3467 while(ic < imax):
3468 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
3469 ic += 1;
3470 except AttributeError:
3471 pass;
3472 httpheaderout = fix_header_names(httpheaderout);
3473 if(isinstance(httpheadersentout, list)):
3474 httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
3475 httpheadersentout = fix_header_names(httpheadersentout);
3476 log.info("Downloading URL "+httpurl);
3477 downloadsize = httpheaderout.get('Content-Length');
3478 if(downloadsize is not None):
3479 downloadsize = int(downloadsize);
3480 if downloadsize is None: downloadsize = 0;
3481 fulldatasize = 0;
3482 prevdownsize = 0;
3483 log.info("Downloading URL "+httpurl);
3484 with BytesIO() as strbuf:
3485 while True:
3486 databytes = retrieved_body.read(buffersize);
3487 if not databytes: break;
3488 datasize = len(databytes);
3489 fulldatasize = datasize + fulldatasize;
3490 percentage = "";
3491 if(downloadsize>0):
3492 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3493 downloaddiff = fulldatasize - prevdownsize;
3494 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3495 prevdownsize = fulldatasize;
3496 strbuf.write(databytes);
3497 strbuf.seek(0);
3498 returnval_content = strbuf.read();
3499 if(httpheaderout.get("Content-Encoding")=="gzip"):
3500 try:
3501 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
3502 except zlib.error:
3503 pass;
3504 elif(httpheaderout.get("Content-Encoding")=="deflate"):
3505 try:
3506 returnval_content = zlib.decompress(returnval_content);
3507 except zlib.error:
3508 pass;
3509 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
3510 try:
3511 returnval_content = brotli.decompress(returnval_content);
3512 except brotli.error:
3513 pass;
3514 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
3515 try:
3516 returnval_content = zstandard.decompress(returnval_content);
3517 except zstandard.error:
3518 pass;
3519 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
3520 geturls_text.close();
3521 return returnval;
3522
3523 if(not havepycurl):
3524 def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
3525 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
3526 return returnval;
3527
3528 if(havepycurl):
3529 def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
3530 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
3531 exec_time_start = time.time();
3532 myhash = hashlib.new("sha1");
3533 if(sys.version[0]=="2"):
3534 myhash.update(httpurl);
3535 myhash.update(str(buffersize));
3536 myhash.update(str(exec_time_start));
3537 if(sys.version[0]>="3"):
3538 myhash.update(httpurl.encode('utf-8'));
3539 myhash.update(str(buffersize).encode('utf-8'));
3540 myhash.update(str(exec_time_start).encode('utf-8'));
3541 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
3542 if(sleep<0):
3543 sleep = geturls_download_sleep;
3544 if(timeout<=0):
3545 timeout = 10;
3546 pretmpfilename = download_from_url_with_pycurl(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
3547 if(not pretmpfilename):
3548 return False;
3549 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
3550 tmpfilename = f.name;
3551 try:
3552 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3553 except AttributeError:
3554 try:
3555 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3556 except ValueError:
3557 pass;
3558 except ValueError:
3559 pass;
3560 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
3561 f.write(pretmpfilename['Content']);
3562 f.close();
3563 exec_time_end = time.time();
3564 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
3565 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
3566 return returnval;
3567
3568 if(not havepycurl):
3569 def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
3570 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
3571 return returnval;
3572
3573 if(havepycurl):
3574 def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
3575 global geturls_download_sleep, havezstd, havebrotli;
3576 if(sleep<0):
3577 sleep = geturls_download_sleep;
3578 if(timeout<=0):
3579 timeout = 10;
3580 if(not outfile=="-"):
3581 outpath = outpath.rstrip(os.path.sep);
3582 filepath = os.path.realpath(outpath+os.path.sep+outfile);
3583 if(not os.path.exists(outpath)):
3584 os.makedirs(outpath);
3585 if(os.path.exists(outpath) and os.path.isfile(outpath)):
3586 return False;
3587 if(os.path.exists(filepath) and os.path.isdir(filepath)):
3588 return False;
3589 pretmpfilename = download_from_url_file_with_pycurl(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
3590 if(not pretmpfilename):
3591 return False;
3592 tmpfilename = pretmpfilename['Filename'];
3593 downloadsize = int(os.path.getsize(tmpfilename));
3594 fulldatasize = 0;
3595 log.info("Moving file "+tmpfilename+" to "+filepath);
3596 exec_time_start = time.time();
3597 shutil.move(tmpfilename, filepath);
3598 try:
3599 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3600 except AttributeError:
3601 try:
3602 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3603 except ValueError:
3604 pass;
3605 except ValueError:
3606 pass;
3607 exec_time_end = time.time();
3608 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
3609 if(os.path.exists(tmpfilename)):
3610 os.remove(tmpfilename);
3611 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3612 if(outfile=="-"):
3613 pretmpfilename = download_from_url_file_with_pycurl(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
3614 tmpfilename = pretmpfilename['Filename'];
3615 downloadsize = int(os.path.getsize(tmpfilename));
3616 fulldatasize = 0;
3617 prevdownsize = 0;
3618 exec_time_start = time.time();
3619 with open(tmpfilename, 'rb') as ft:
3620 f = BytesIO();
3621 while True:
3622 databytes = ft.read(buffersize[1]);
3623 if not databytes: break;
3624 datasize = len(databytes);
3625 fulldatasize = datasize + fulldatasize;
3626 percentage = "";
3627 if(downloadsize>0):
3628 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3629 downloaddiff = fulldatasize - prevdownsize;
3630 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3631 prevdownsize = fulldatasize;
3632 f.write(databytes);
3633 f.seek(0);
3634 fdata = f.getvalue();
3635 f.close();
3636 ft.close();
3637 os.remove(tmpfilename);
3638 exec_time_end = time.time();
3639 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3640 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3641 return returnval;
3642
3643 if(not havepycurl):
3644 def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
3645 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
3646 return returnval;
3647
3648 if(havepycurl):
3649 def download_from_url_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
3650 global geturls_download_sleep, havezstd, havebrotli;
3651 if(sleep<0):
3652 sleep = geturls_download_sleep;
3653 if(timeout<=0):
3654 timeout = 10;
3655 urlparts = urlparse.urlparse(httpurl);
3656 if(isinstance(httpheaders, list)):
3657 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3658 httpheaders = fix_header_names(httpheaders);
3659 if(httpuseragent is not None):
3660 if('User-Agent' in httpheaders):
3661 httpheaders['User-Agent'] = httpuseragent;
3662 else:
3663 httpuseragent.update({'User-Agent': httpuseragent});
3664 if(httpreferer is not None):
3665 if('Referer' in httpheaders):
3666 httpheaders['Referer'] = httpreferer;
3667 else:
3668 httpuseragent.update({'Referer': httpreferer});
3669 if(urlparts.username is not None or urlparts.password is not None):
3670 if(sys.version[0]=="2"):
3671 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
3672 if(sys.version[0]>="3"):
3673 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3674 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3675 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
3676 if(isinstance(httpheaders, dict)):
3677 httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
3678 geturls_opener.addheaders = httpheaders;
3679 time.sleep(sleep);
3680 if(postdata is not None and not isinstance(postdata, dict)):
3681 postdata = urlencode(postdata);
3682 retrieved_body = BytesIO();
3683 retrieved_headers = BytesIO();
3684 try:
3685 if(httpmethod=="GET"):
3686 geturls_text = pycurl.Curl();
3687 geturls_text.setopt(geturls_text.URL, httpurl);
3688 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
3689 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
3690 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
3691 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
3692 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
3693 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
3694 geturls_text.perform();
3695 elif(httpmethod=="POST"):
3696 geturls_text = pycurl.Curl();
3697 geturls_text.setopt(geturls_text.URL, httpurl);
3698 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
3699 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
3700 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
3701 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
3702 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
3703 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
3704 geturls_text.setopt(geturls_text.POST, True);
3705 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
3706 geturls_text.perform();
3707 else:
3708 geturls_text = pycurl.Curl();
3709 geturls_text.setopt(geturls_text.URL, httpurl);
3710 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
3711 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
3712 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
3713 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
3714 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
3715 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
3716 geturls_text.perform();
3717 retrieved_headers.seek(0);
3718 if(sys.version[0]=="2"):
3719 pycurlhead = retrieved_headers.read();
3720 if(sys.version[0]>="3"):
3721 pycurlhead = retrieved_headers.read().decode('UTF-8');
3722 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip())[0];
3723 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
3724 retrieved_body.seek(0);
3725 except socket.timeout:
3726 log.info("Error With URL "+httpurl);
3727 return False;
3728 except socket.gaierror:
3729 log.info("Error With URL "+httpurl);
3730 return False;
3731 except ValueError:
3732 log.info("Error With URL "+httpurl);
3733 return False;
3734 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
3735 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
3736 httpversionout = pyhttpverinfo[0];
3737 httpmethodout = httpmethod;
3738 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
3739 httpheaderout = pycurlheadersout;
3740 httpheadersentout = httpheaders;
3741 if(isinstance(httpheaderout, list)):
3742 httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
3743 if(sys.version[0]=="2"):
3744 try:
3745 prehttpheaderout = httpheaderout;
3746 httpheaderkeys = httpheaderout.keys();
3747 imax = len(httpheaderkeys);
3748 ic = 0;
3749 httpheaderout = {};
3750 while(ic < imax):
3751 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
3752 ic += 1;
3753 except AttributeError:
3754 pass;
3755 httpheaderout = fix_header_names(httpheaderout);
3756 if(isinstance(httpheadersentout, list)):
3757 httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
3758 httpheadersentout = fix_header_names(httpheadersentout);
3759 log.info("Downloading URL "+httpurl);
3760 downloadsize = httpheaderout.get('Content-Length');
3761 if(downloadsize is not None):
3762 downloadsize = int(downloadsize);
3763 if downloadsize is None: downloadsize = 0;
3764 fulldatasize = 0;
3765 prevdownsize = 0;
3766 log.info("Downloading URL "+httpurl);
3767 with BytesIO() as strbuf:
3768 while True:
3769 databytes = retrieved_body.read(buffersize);
3770 if not databytes: break;
3771 datasize = len(databytes);
3772 fulldatasize = datasize + fulldatasize;
3773 percentage = "";
3774 if(downloadsize>0):
3775 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3776 downloaddiff = fulldatasize - prevdownsize;
3777 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3778 prevdownsize = fulldatasize;
3779 strbuf.write(databytes);
3780 strbuf.seek(0);
3781 returnval_content = strbuf.read();
3782 if(httpheaderout.get("Content-Encoding")=="gzip"):
3783 try:
3784 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
3785 except zlib.error:
3786 pass;
3787 elif(httpheaderout.get("Content-Encoding")=="deflate"):
3788 try:
3789 returnval_content = zlib.decompress(returnval_content);
3790 except zlib.error:
3791 pass;
3792 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
3793 try:
3794 returnval_content = brotli.decompress(returnval_content);
3795 except brotli.error:
3796 pass;
3797 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
3798 try:
3799 returnval_content = zstandard.decompress(returnval_content);
3800 except zstandard.error:
3801 pass;
3802 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
3803 geturls_text.close();
3804 return returnval;
3805
3806 if(not havepycurl):
3807 def download_from_url_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
3808 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
3809 return returnval;
3810
3811 if(havepycurl):
3812 def download_from_url_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
3813 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
3814 exec_time_start = time.time();
3815 myhash = hashlib.new("sha1");
3816 if(sys.version[0]=="2"):
3817 myhash.update(httpurl);
3818 myhash.update(str(buffersize));
3819 myhash.update(str(exec_time_start));
3820 if(sys.version[0]>="3"):
3821 myhash.update(httpurl.encode('utf-8'));
3822 myhash.update(str(buffersize).encode('utf-8'));
3823 myhash.update(str(exec_time_start).encode('utf-8'));
3824 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
3825 if(sleep<0):
3826 sleep = geturls_download_sleep;
3827 if(timeout<=0):
3828 timeout = 10;
3829 pretmpfilename = download_from_url_with_pycurl2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
3830 if(not pretmpfilename):
3831 return False;
3832 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
3833 tmpfilename = f.name;
3834 try:
3835 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3836 except AttributeError:
3837 try:
3838 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3839 except ValueError:
3840 pass;
3841 except ValueError:
3842 pass;
3843 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
3844 f.write(pretmpfilename['Content']);
3845 f.close();
3846 exec_time_end = time.time();
3847 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
3848 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
3849 return returnval;
3850
3851 if(not havepycurl):
3852 def download_from_url_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
3853 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
3854 return returnval;
3855
3856 if(havepycurl):
3857 def download_from_url_to_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
3858 global geturls_download_sleep, havezstd, havebrotli;
3859 if(sleep<0):
3860 sleep = geturls_download_sleep;
3861 if(timeout<=0):
3862 timeout = 10;
3863 if(not outfile=="-"):
3864 outpath = outpath.rstrip(os.path.sep);
3865 filepath = os.path.realpath(outpath+os.path.sep+outfile);
3866 if(not os.path.exists(outpath)):
3867 os.makedirs(outpath);
3868 if(os.path.exists(outpath) and os.path.isfile(outpath)):
3869 return False;
3870 if(os.path.exists(filepath) and os.path.isdir(filepath)):
3871 return False;
3872 pretmpfilename = download_from_url_file_with_pycurl2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
3873 if(not pretmpfilename):
3874 return False;
3875 tmpfilename = pretmpfilename['Filename'];
3876 downloadsize = int(os.path.getsize(tmpfilename));
3877 fulldatasize = 0;
3878 log.info("Moving file "+tmpfilename+" to "+filepath);
3879 exec_time_start = time.time();
3880 shutil.move(tmpfilename, filepath);
3881 try:
3882 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3883 except AttributeError:
3884 try:
3885 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3886 except ValueError:
3887 pass;
3888 except ValueError:
3889 pass;
3890 exec_time_end = time.time();
3891 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
3892 if(os.path.exists(tmpfilename)):
3893 os.remove(tmpfilename);
3894 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3895 if(outfile=="-"):
3896 pretmpfilename = download_from_url_file_with_pycurl2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
3897 tmpfilename = pretmpfilename['Filename'];
3898 downloadsize = int(os.path.getsize(tmpfilename));
3899 fulldatasize = 0;
3900 prevdownsize = 0;
3901 exec_time_start = time.time();
3902 with open(tmpfilename, 'rb') as ft:
3903 f = BytesIO();
3904 while True:
3905 databytes = ft.read(buffersize[1]);
3906 if not databytes: break;
3907 datasize = len(databytes);
3908 fulldatasize = datasize + fulldatasize;
3909 percentage = "";
3910 if(downloadsize>0):
3911 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3912 downloaddiff = fulldatasize - prevdownsize;
3913 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3914 prevdownsize = fulldatasize;
3915 f.write(databytes);
3916 f.seek(0);
3917 fdata = f.getvalue();
3918 f.close();
3919 ft.close();
3920 os.remove(tmpfilename);
3921 exec_time_end = time.time();
3922 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3923 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3924 return returnval;
3925
3926 if(not havepycurl):
3927 def download_from_url_to_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
3928 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
3929 return returnval;
3930
3931 if(havepycurl):
3932 def download_from_url_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
3933 global geturls_download_sleep, havezstd, havebrotli;
3934 if(sleep<0):
3935 sleep = geturls_download_sleep;
3936 if(timeout<=0):
3937 timeout = 10;
3938 urlparts = urlparse.urlparse(httpurl);
3939 if(isinstance(httpheaders, list)):
3940 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3941 httpheaders = fix_header_names(httpheaders);
3942 if(httpuseragent is not None):
3943 if('User-Agent' in httpheaders):
3944 httpheaders['User-Agent'] = httpuseragent;
3945 else:
3946 httpuseragent.update({'User-Agent': httpuseragent});
3947 if(httpreferer is not None):
3948 if('Referer' in httpheaders):
3949 httpheaders['Referer'] = httpreferer;
3950 else:
3951 httpuseragent.update({'Referer': httpreferer});
3952 if(urlparts.username is not None or urlparts.password is not None):
3953 if(sys.version[0]=="2"):
3954 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
3955 if(sys.version[0]>="3"):
3956 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3957 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3958 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
3959 if(isinstance(httpheaders, dict)):
3960 httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
3961 geturls_opener.addheaders = httpheaders;
3962 time.sleep(sleep);
3963 if(postdata is not None and not isinstance(postdata, dict)):
3964 postdata = urlencode(postdata);
3965 retrieved_body = BytesIO();
3966 retrieved_headers = BytesIO();
3967 try:
3968 if(httpmethod=="GET"):
3969 geturls_text = pycurl.Curl();
3970 geturls_text.setopt(geturls_text.URL, httpurl);
3971 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
3972 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
3973 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
3974 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
3975 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
3976 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
3977 geturls_text.perform();
3978 elif(httpmethod=="POST"):
3979 geturls_text = pycurl.Curl();
3980 geturls_text.setopt(geturls_text.URL, httpurl);
3981 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
3982 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
3983 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
3984 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
3985 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
3986 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
3987 geturls_text.setopt(geturls_text.POST, True);
3988 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
3989 geturls_text.perform();
3990 else:
3991 geturls_text = pycurl.Curl();
3992 geturls_text.setopt(geturls_text.URL, httpurl);
3993 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
3994 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
3995 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
3996 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
3997 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
3998 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
3999 geturls_text.perform();
4000 retrieved_headers.seek(0);
4001 if(sys.version[0]=="2"):
4002 pycurlhead = retrieved_headers.read();
4003 if(sys.version[0]>="3"):
4004 pycurlhead = retrieved_headers.read().decode('UTF-8');
4005 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
4006 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
4007 retrieved_body.seek(0);
4008 except socket.timeout:
4009 log.info("Error With URL "+httpurl);
4010 return False;
4011 except socket.gaierror:
4012 log.info("Error With URL "+httpurl);
4013 return False;
4014 except ValueError:
4015 log.info("Error With URL "+httpurl);
4016 return False;
4017 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
4018 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
4019 httpversionout = pyhttpverinfo[0];
4020 httpmethodout = httpmethod;
4021 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
4022 httpheaderout = pycurlheadersout;
4023 httpheadersentout = httpheaders;
4024 if(isinstance(httpheaderout, list)):
4025 httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
4026 if(sys.version[0]=="2"):
4027 try:
4028 prehttpheaderout = httpheaderout;
4029 httpheaderkeys = httpheaderout.keys();
4030 imax = len(httpheaderkeys);
4031 ic = 0;
4032 httpheaderout = {};
4033 while(ic < imax):
4034 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
4035 ic += 1;
4036 except AttributeError:
4037 pass;
4038 httpheaderout = fix_header_names(httpheaderout);
4039 if(isinstance(httpheadersentout, list)):
4040 httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
4041 httpheadersentout = fix_header_names(httpheadersentout);
4042 log.info("Downloading URL "+httpurl);
4043 downloadsize = httpheaderout.get('Content-Length');
4044 if(downloadsize is not None):
4045 downloadsize = int(downloadsize);
4046 if downloadsize is None: downloadsize = 0;
4047 fulldatasize = 0;
4048 prevdownsize = 0;
4049 log.info("Downloading URL "+httpurl);
4050 with BytesIO() as strbuf:
4051 while True:
4052 databytes = retrieved_body.read(buffersize);
4053 if not databytes: break;
4054 datasize = len(databytes);
4055 fulldatasize = datasize + fulldatasize;
4056 percentage = "";
4057 if(downloadsize>0):
4058 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4059 downloaddiff = fulldatasize - prevdownsize;
4060 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4061 prevdownsize = fulldatasize;
4062 strbuf.write(databytes);
4063 strbuf.seek(0);
4064 returnval_content = strbuf.read();
4065 if(httpheaderout.get("Content-Encoding")=="gzip"):
4066 try:
4067 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
4068 except zlib.error:
4069 pass;
4070 elif(httpheaderout.get("Content-Encoding")=="deflate"):
4071 try:
4072 returnval_content = zlib.decompress(returnval_content);
4073 except zlib.error:
4074 pass;
4075 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
4076 try:
4077 returnval_content = brotli.decompress(returnval_content);
4078 except brotli.error:
4079 pass;
4080 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
4081 try:
4082 returnval_content = zstandard.decompress(returnval_content);
4083 except zstandard.error:
4084 pass;
4085 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
4086 geturls_text.close();
4087 return returnval;
4088
4089 if(not havepycurl):
4090 def download_from_url_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
4091 returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout)
4092 return returnval;
4093
4094 if(havepycurl):
4095 def download_from_url_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
4096 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
4097 exec_time_start = time.time();
4098 myhash = hashlib.new("sha1");
4099 if(sys.version[0]=="2"):
4100 myhash.update(httpurl);
4101 myhash.update(str(buffersize));
4102 myhash.update(str(exec_time_start));
4103 if(sys.version[0]>="3"):
4104 myhash.update(httpurl.encode('utf-8'));
4105 myhash.update(str(buffersize).encode('utf-8'));
4106 myhash.update(str(exec_time_start).encode('utf-8'));
4107 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
4108 if(sleep<0):
4109 sleep = geturls_download_sleep;
4110 if(timeout<=0):
4111 timeout = 10;
4112 pretmpfilename = download_from_url_with_pycurl3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
4113 if(not pretmpfilename):
4114 return False;
4115 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
4116 tmpfilename = f.name;
4117 try:
4118 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
4119 except AttributeError:
4120 try:
4121 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4122 except ValueError:
4123 pass;
4124 except ValueError:
4125 pass;
4126 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
4127 f.write(pretmpfilename['Content']);
4128 f.close();
4129 exec_time_end = time.time();
4130 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
4131 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
4132 return returnval;
4133
4134 if(not havepycurl):
4135 def download_from_url_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
4136 returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize, sleep, timeout)
4137 return returnval;
4138
4139 if(havepycurl):
4140 def download_from_url_to_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
4141 global geturls_download_sleep, havezstd, havebrotli;
4142 if(sleep<0):
4143 sleep = geturls_download_sleep;
4144 if(timeout<=0):
4145 timeout = 10;
4146 if(not outfile=="-"):
4147 outpath = outpath.rstrip(os.path.sep);
4148 filepath = os.path.realpath(outpath+os.path.sep+outfile);
4149 if(not os.path.exists(outpath)):
4150 os.makedirs(outpath);
4151 if(os.path.exists(outpath) and os.path.isfile(outpath)):
4152 return False;
4153 if(os.path.exists(filepath) and os.path.isdir(filepath)):
4154 return False;
4155 pretmpfilename = download_from_url_file_with_pycurl3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
4156 if(not pretmpfilename):
4157 return False;
4158 tmpfilename = pretmpfilename['Filename'];
4159 downloadsize = int(os.path.getsize(tmpfilename));
4160 fulldatasize = 0;
4161 log.info("Moving file "+tmpfilename+" to "+filepath);
4162 exec_time_start = time.time();
4163 shutil.move(tmpfilename, filepath);
4164 try:
4165 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
4166 except AttributeError:
4167 try:
4168 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4169 except ValueError:
4170 pass;
4171 except ValueError:
4172 pass;
4173 exec_time_end = time.time();
4174 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
4175 if(os.path.exists(tmpfilename)):
4176 os.remove(tmpfilename);
4177 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4178 if(outfile=="-"):
4179 pretmpfilename = download_from_url_file_with_pycurl3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
4180 tmpfilename = pretmpfilename['Filename'];
4181 downloadsize = int(os.path.getsize(tmpfilename));
4182 fulldatasize = 0;
4183 prevdownsize = 0;
4184 exec_time_start = time.time();
4185 with open(tmpfilename, 'rb') as ft:
4186 f = BytesIO();
4187 while True:
4188 databytes = ft.read(buffersize[1]);
4189 if not databytes: break;
4190 datasize = len(databytes);
4191 fulldatasize = datasize + fulldatasize;
4192 percentage = "";
4193 if(downloadsize>0):
4194 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4195 downloaddiff = fulldatasize - prevdownsize;
4196 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4197 prevdownsize = fulldatasize;
4198 f.write(databytes);
4199 f.seek(0);
4200 fdata = f.getvalue();
4201 f.close();
4202 ft.close();
4203 os.remove(tmpfilename);
4204 exec_time_end = time.time();
4205 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4206 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4207 return returnval;
4208
4209 if(not havepycurl):
4210 def download_from_url_to_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
4211 returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep, timeout)
4212 return returnval;
4213
4214 def download_file_from_ftp_file(url):
4215 urlparts = urlparse.urlparse(url);
4216 file_name = os.path.basename(urlparts.path);
4217 file_dir = os.path.dirname(urlparts.path);
4218 if(urlparts.username is not None):
4219 ftp_username = urlparts.username;
4220 else:
4221 ftp_username = "anonymous";
4222 if(urlparts.password is not None):
4223 ftp_password = urlparts.password;
4224 elif(urlparts.password is None and urlparts.username=="anonymous"):
4225 ftp_password = "anonymous";
4226 else:
4227 ftp_password = "";
4228 if(urlparts.scheme=="ftp"):
4229 ftp = FTP();
4230 elif(urlparts.scheme=="ftps"):
4231 ftp = FTP_TLS();
4232 else:
4233 return False;
4234 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
4235 return False;
4236 ftp_port = urlparts.port;
4237 if(urlparts.port is None):
4238 ftp_port = 21;
4239 try:
4240 ftp.connect(urlparts.hostname, ftp_port);
4241 except socket.gaierror:
4242 log.info("Error With URL "+httpurl);
4243 return False;
4244 except socket.timeout:
4245 log.info("Error With URL "+httpurl);
4246 return False;
4247 ftp.login(urlparts.username, urlparts.password);
4248 if(urlparts.scheme=="ftps"):
4249 ftp.prot_p();
4250 ftpfile = BytesIO();
4251 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
4252 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
4253 ftp.close();
4254 ftpfile.seek(0, 0);
4255 return ftpfile;
4256
4257 def download_file_from_ftp_string(url):
4258 ftpfile = download_file_from_ftp_file(url);
4259 return ftpfile.read();
4260
4261 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
4262 global geturls_download_sleep, havezstd, havebrotli;
4263 if(sleep<0):
4264 sleep = geturls_download_sleep;
4265 if(timeout<=0):
4266 timeout = 10;
4267 urlparts = urlparse.urlparse(httpurl);
4268 if(isinstance(httpheaders, list)):
4269 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4270 httpheaders = fix_header_names(httpheaders);
4271 if(httpuseragent is not None):
4272 if('User-Agent' in httpheaders):
4273 httpheaders['User-Agent'] = httpuseragent;
4274 else:
4275 httpuseragent.update({'User-Agent': httpuseragent});
4276 if(httpreferer is not None):
4277 if('Referer' in httpheaders):
4278 httpheaders['Referer'] = httpreferer;
4279 else:
4280 httpuseragent.update({'Referer': httpreferer});
4281 if(isinstance(httpheaders, dict)):
4282 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
4283 time.sleep(sleep);
4284 geturls_text = download_file_from_ftp_file(httpurl);
4285 if(not geturls_text):
4286 return False;
4287 downloadsize = None;
4288 if(downloadsize is not None):
4289 downloadsize = int(downloadsize);
4290 if downloadsize is None: downloadsize = 0;
4291 fulldatasize = 0;
4292 prevdownsize = 0;
4293 log.info("Downloading URL "+httpurl);
4294 with BytesIO() as strbuf:
4295 while True:
4296 databytes = geturls_text.read(buffersize);
4297 if not databytes: break;
4298 datasize = len(databytes);
4299 fulldatasize = datasize + fulldatasize;
4300 percentage = "";
4301 if(downloadsize>0):
4302 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4303 downloaddiff = fulldatasize - prevdownsize;
4304 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4305 prevdownsize = fulldatasize;
4306 strbuf.write(databytes);
4307 strbuf.seek(0);
4308 returnval_content = strbuf.read();
4309 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
4310 geturls_text.close();
4311 return returnval;
4312
4313 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
4314 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
4315 exec_time_start = time.time();
4316 myhash = hashlib.new("sha1");
4317 if(sys.version[0]=="2"):
4318 myhash.update(httpurl);
4319 myhash.update(str(buffersize));
4320 myhash.update(str(exec_time_start));
4321 if(sys.version[0]>="3"):
4322 myhash.update(httpurl.encode('utf-8'));
4323 myhash.update(str(buffersize).encode('utf-8'));
4324 myhash.update(str(exec_time_start).encode('utf-8'));
4325 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
4326 if(sleep<0):
4327 sleep = geturls_download_sleep;
4328 if(timeout<=0):
4329 timeout = 10;
4330 pretmpfilename = download_from_url_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
4331 if(not pretmpfilename):
4332 return False;
4333 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
4334 tmpfilename = f.name;
4335 try:
4336 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
4337 except AttributeError:
4338 try:
4339 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4340 except ValueError:
4341 pass;
4342 except ValueError:
4343 pass;
4344 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
4345 f.write(pretmpfilename['Content']);
4346 f.close();
4347 exec_time_end = time.time();
4348 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
4349 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
4350 return returnval;
4351
4352 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
4353 global geturls_download_sleep, havezstd, havebrotli;
4354 if(sleep<0):
4355 sleep = geturls_download_sleep;
4356 if(timeout<=0):
4357 timeout = 10;
4358 if(not outfile=="-"):
4359 outpath = outpath.rstrip(os.path.sep);
4360 filepath = os.path.realpath(outpath+os.path.sep+outfile);
4361 if(not os.path.exists(outpath)):
4362 os.makedirs(outpath);
4363 if(os.path.exists(outpath) and os.path.isfile(outpath)):
4364 return False;
4365 if(os.path.exists(filepath) and os.path.isdir(filepath)):
4366 return False;
4367 pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
4368 if(not pretmpfilename):
4369 return False;
4370 tmpfilename = pretmpfilename['Filename'];
4371 downloadsize = int(os.path.getsize(tmpfilename));
4372 fulldatasize = 0;
4373 log.info("Moving file "+tmpfilename+" to "+filepath);
4374 exec_time_start = time.time();
4375 shutil.move(tmpfilename, filepath);
4376 exec_time_end = time.time();
4377 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
4378 if(os.path.exists(tmpfilename)):
4379 os.remove(tmpfilename);
4380 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4381 if(outfile=="-"):
4382 pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
4383 tmpfilename = pretmpfilename['Filename'];
4384 downloadsize = int(os.path.getsize(tmpfilename));
4385 fulldatasize = 0;
4386 prevdownsize = 0;
4387 exec_time_start = time.time();
4388 with open(tmpfilename, 'rb') as ft:
4389 f = BytesIO();
4390 while True:
4391 databytes = ft.read(buffersize[1]);
4392 if not databytes: break;
4393 datasize = len(databytes);
4394 fulldatasize = datasize + fulldatasize;
4395 percentage = "";
4396 if(downloadsize>0):
4397 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4398 downloaddiff = fulldatasize - prevdownsize;
4399 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4400 prevdownsize = fulldatasize;
4401 f.write(databytes);
4402 f.seek(0);
4403 fdata = f.getvalue();
4404 f.close();
4405 ft.close();
4406 os.remove(tmpfilename);
4407 exec_time_end = time.time();
4408 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4409 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4410 return returnval;
4411
4412 def upload_file_to_ftp_file(ftpfile, url):
4413 urlparts = urlparse.urlparse(url);
4414 file_name = os.path.basename(urlparts.path);
4415 file_dir = os.path.dirname(urlparts.path);
4416 if(urlparts.username is not None):
4417 ftp_username = urlparts.username;
4418 else:
4419 ftp_username = "anonymous";
4420 if(urlparts.password is not None):
4421 ftp_password = urlparts.password;
4422 elif(urlparts.password is None and urlparts.username=="anonymous"):
4423 ftp_password = "anonymous";
4424 else:
4425 ftp_password = "";
4426 if(urlparts.scheme=="ftp"):
4427 ftp = FTP();
4428 elif(urlparts.scheme=="ftps"):
4429 ftp = FTP_TLS();
4430 else:
4431 return False;
4432 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
4433 return False;
4434 ftp_port = urlparts.port;
4435 if(urlparts.port is None):
4436 ftp_port = 21;
4437 try:
4438 ftp.connect(urlparts.hostname, ftp_port);
4439 except socket.gaierror:
4440 log.info("Error With URL "+httpurl);
4441 return False;
4442 except socket.timeout:
4443 log.info("Error With URL "+httpurl);
4444 return False;
4445 ftp.login(urlparts.username, urlparts.password);
4446 if(urlparts.scheme=="ftps"):
4447 ftp.prot_p();
4448 ftp.storbinary("STOR "+urlparts.path, ftpfile);
4449 ftp.close();
4450 ftpfile.seek(0, 0);
4451 return ftpfile;
4452
4453 def upload_file_to_ftp_string(ftpstring, url):
4454 ftpfileo = BytesIO(ftpstring);
4455 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
4456 ftpfileo.close();
4457 return ftpfile;
4458
4459 if(haveparamiko):
4460 def download_file_from_sftp_file(url):
4461 urlparts = urlparse.urlparse(url);
4462 file_name = os.path.basename(urlparts.path);
4463 file_dir = os.path.dirname(urlparts.path);
4464 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
4465 return False;
4466 sftp_port = urlparts.port;
4467 if(urlparts.port is None):
4468 sftp_port = 22;
4469 else:
4470 sftp_port = urlparts.port;
4471 if(urlparts.username is not None):
4472 sftp_username = urlparts.username;
4473 else:
4474 sftp_username = "anonymous";
4475 if(urlparts.password is not None):
4476 sftp_password = urlparts.password;
4477 elif(urlparts.password is None and urlparts.username=="anonymous"):
4478 sftp_password = "anonymous";
4479 else:
4480 sftp_password = "";
4481 if(urlparts.scheme!="sftp"):
4482 return False;
4483 ssh = paramiko.SSHClient();
4484 ssh.load_system_host_keys();
4485 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
4486 try:
4487 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
4488 except paramiko.ssh_exception.SSHException:
4489 return False;
4490 except socket.gaierror:
4491 log.info("Error With URL "+httpurl);
4492 return False;
4493 except socket.timeout:
4494 log.info("Error With URL "+httpurl);
4495 return False;
4496 sftp = ssh.open_sftp();
4497 sftpfile = BytesIO();
4498 sftp.getfo(urlparts.path, sftpfile);
4499 sftp.close();
4500 ssh.close();
4501 sftpfile.seek(0, 0);
4502 return sftpfile;
4503 else:
4504 def download_file_from_sftp_file(url):
4505 return False;
4506
4507 if(haveparamiko):
4508 def download_file_from_sftp_string(url):
4509 sftpfile = download_file_from_sftp_file(url);
4510 return sftpfile.read();
4511 else:
4512 def download_file_from_ftp_string(url):
4513 return False;
4514
4515 if(haveparamiko):
4516 def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
4517 global geturls_download_sleep, havezstd, havebrotli;
4518 if(sleep<0):
4519 sleep = geturls_download_sleep;
4520 if(timeout<=0):
4521 timeout = 10;
4522 urlparts = urlparse.urlparse(httpurl);
4523 if(isinstance(httpheaders, list)):
4524 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4525 httpheaders = fix_header_names(httpheaders);
4526 if(httpuseragent is not None):
4527 if('User-Agent' in httpheaders):
4528 httpheaders['User-Agent'] = httpuseragent;
4529 else:
4530 httpuseragent.update({'User-Agent': httpuseragent});
4531 if(httpreferer is not None):
4532 if('Referer' in httpheaders):
4533 httpheaders['Referer'] = httpreferer;
4534 else:
4535 httpuseragent.update({'Referer': httpreferer});
4536 if(isinstance(httpheaders, dict)):
4537 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
4538 time.sleep(sleep);
4539 geturls_text = download_file_from_sftp_file(httpurl);
4540 if(not geturls_text):
4541 return False;
4542 downloadsize = None;
4543 if(downloadsize is not None):
4544 downloadsize = int(downloadsize);
4545 if downloadsize is None: downloadsize = 0;
4546 fulldatasize = 0;
4547 prevdownsize = 0;
4548 log.info("Downloading URL "+httpurl);
4549 with BytesIO() as strbuf:
4550 while True:
4551 databytes = geturls_text.read(buffersize);
4552 if not databytes: break;
4553 datasize = len(databytes);
4554 fulldatasize = datasize + fulldatasize;
4555 percentage = "";
4556 if(downloadsize>0):
4557 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4558 downloaddiff = fulldatasize - prevdownsize;
4559 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4560 prevdownsize = fulldatasize;
4561 strbuf.write(databytes);
4562 strbuf.seek(0);
4563 returnval_content = strbuf.read();
4564 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
4565 geturls_text.close();
4566 return returnval;
4567
4568 if(not haveparamiko):
4569 def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
4570 return False;
4571
4572 if(haveparamiko):
4573 def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
4574 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
4575 exec_time_start = time.time();
4576 myhash = hashlib.new("sha1");
4577 if(sys.version[0]=="2"):
4578 myhash.update(httpurl);
4579 myhash.update(str(buffersize));
4580 myhash.update(str(exec_time_start));
4581 if(sys.version[0]>="3"):
4582 myhash.update(httpurl.encode('utf-8'));
4583 myhash.update(str(buffersize).encode('utf-8'));
4584 myhash.update(str(exec_time_start).encode('utf-8'));
4585 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
4586 if(sleep<0):
4587 sleep = geturls_download_sleep;
4588 if(timeout<=0):
4589 timeout = 10;
4590 pretmpfilename = download_from_url_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
4591 if(not pretmpfilename):
4592 return False;
4593 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
4594 tmpfilename = f.name;
4595 try:
4596 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
4597 except AttributeError:
4598 try:
4599 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4600 except ValueError:
4601 pass;
4602 except ValueError:
4603 pass;
4604 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
4605 f.write(pretmpfilename['Content']);
4606 f.close();
4607 exec_time_end = time.time();
4608 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
4609 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
4610 return returnval;
4611
4612 if(not haveparamiko):
4613 def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
4614 return False;
4615
4616 if(haveparamiko):
4617 def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
4618 global geturls_download_sleep, havezstd, havebrotli;
4619 if(sleep<0):
4620 sleep = geturls_download_sleep;
4621 if(timeout<=0):
4622 timeout = 10;
4623 if(not outfile=="-"):
4624 outpath = outpath.rstrip(os.path.sep);
4625 filepath = os.path.realpath(outpath+os.path.sep+outfile);
4626 if(not os.path.exists(outpath)):
4627 os.makedirs(outpath);
4628 if(os.path.exists(outpath) and os.path.isfile(outpath)):
4629 return False;
4630 if(os.path.exists(filepath) and os.path.isdir(filepath)):
4631 return False;
4632 pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
4633 if(not pretmpfilename):
4634 return False;
4635 tmpfilename = pretmpfilename['Filename'];
4636 downloadsize = int(os.path.getsize(tmpfilename));
4637 fulldatasize = 0;
4638 log.info("Moving file "+tmpfilename+" to "+filepath);
4639 exec_time_start = time.time();
4640 shutil.move(tmpfilename, filepath);
4641 exec_time_end = time.time();
4642 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
4643 if(os.path.exists(tmpfilename)):
4644 os.remove(tmpfilename);
4645 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4646 if(outfile=="-"):
4647 pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
4648 tmpfilename = pretmpfilename['Filename'];
4649 downloadsize = int(os.path.getsize(tmpfilename));
4650 fulldatasize = 0;
4651 prevdownsize = 0;
4652 exec_time_start = time.time();
4653 with open(tmpfilename, 'rb') as ft:
4654 f = BytesIO();
4655 while True:
4656 databytes = ft.read(buffersize[1]);
4657 if not databytes: break;
4658 datasize = len(databytes);
4659 fulldatasize = datasize + fulldatasize;
4660 percentage = "";
4661 if(downloadsize>0):
4662 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4663 downloaddiff = fulldatasize - prevdownsize;
4664 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4665 prevdownsize = fulldatasize;
4666 f.write(databytes);
4667 f.seek(0);
4668 fdata = f.getvalue();
4669 f.close();
4670 ft.close();
4671 os.remove(tmpfilename);
4672 exec_time_end = time.time();
4673 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4674 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4675 return returnval;
4676
4677 if(not haveparamiko):
4678 def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
4679 return False;
4680
4681 if(haveparamiko):
4682 def upload_file_to_sftp_file(sftpfile, url):
4683 urlparts = urlparse.urlparse(url);
4684 file_name = os.path.basename(urlparts.path);
4685 file_dir = os.path.dirname(urlparts.path);
4686 sftp_port = urlparts.port;
4687 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
4688 return False;
4689 if(urlparts.port is None):
4690 sftp_port = 22;
4691 else:
4692 sftp_port = urlparts.port;
4693 if(urlparts.username is not None):
4694 sftp_username = urlparts.username;
4695 else:
4696 sftp_username = "anonymous";
4697 if(urlparts.password is not None):
4698 sftp_password = urlparts.password;
4699 elif(urlparts.password is None and urlparts.username=="anonymous"):
4700 sftp_password = "anonymous";
4701 else:
4702 sftp_password = "";
4703 if(urlparts.scheme!="sftp"):
4704 return False;
4705 ssh = paramiko.SSHClient();
4706 ssh.load_system_host_keys();
4707 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
4708 try:
4709 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
4710 except paramiko.ssh_exception.SSHException:
4711 return False;
4712 except socket.gaierror:
4713 log.info("Error With URL "+httpurl);
4714 return False;
4715 except socket.timeout:
4716 log.info("Error With URL "+httpurl);
4717 return False;
4718 sftp = ssh.open_sftp();
4719 sftp.putfo(sftpfile, urlparts.path);
4720 sftp.close();
4721 ssh.close();
4722 sftpfile.seek(0, 0);
4723 return sftpfile;
4724 else:
4725 def upload_file_to_sftp_file(sftpfile, url):
4726 return False;
4727
4728 if(haveparamiko):
4729 def upload_file_to_sftp_string(sftpstring, url):
4730 sftpfileo = BytesIO(sftpstring);
4731 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
4732 sftpfileo.close();
4733 return sftpfile;
4734 else:
4735 def upload_file_to_sftp_string(url):
4736 return False;
4737
4738
4739 if(havepysftp):
4740 def download_file_from_pysftp_file(url):
4741 urlparts = urlparse.urlparse(url);
4742 file_name = os.path.basename(urlparts.path);
4743 file_dir = os.path.dirname(urlparts.path);
4744 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
4745 return False;
4746 sftp_port = urlparts.port;
4747 if(urlparts.port is None):
4748 sftp_port = 22;
4749 else:
4750 sftp_port = urlparts.port;
4751 if(urlparts.username is not None):
4752 sftp_username = urlparts.username;
4753 else:
4754 sftp_username = "anonymous";
4755 if(urlparts.password is not None):
4756 sftp_password = urlparts.password;
4757 elif(urlparts.password is None and urlparts.username=="anonymous"):
4758 sftp_password = "anonymous";
4759 else:
4760 sftp_password = "";
4761 if(urlparts.scheme!="sftp"):
4762 return False;
4763 try:
4764 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
4765 except paramiko.ssh_exception.SSHException:
4766 return False;
4767 except socket.gaierror:
4768 log.info("Error With URL "+httpurl);
4769 return False;
4770 except socket.timeout:
4771 log.info("Error With URL "+httpurl);
4772 return False;
4773 sftp = ssh.open_sftp();
4774 sftpfile = BytesIO();
4775 sftp.getfo(urlparts.path, sftpfile);
4776 sftp.close();
4777 ssh.close();
4778 sftpfile.seek(0, 0);
4779 return sftpfile;
4780 else:
4781 def download_file_from_pysftp_file(url):
4782 return False;
4783
4784 if(havepysftp):
4785 def download_file_from_pysftp_string(url):
4786 sftpfile = download_file_from_pysftp_file(url);
4787 return sftpfile.read();
4788 else:
4789 def download_file_from_ftp_string(url):
4790 return False;
4791
4792 if(havepysftp):
4793 def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
4794 global geturls_download_sleep, havezstd, havebrotli;
4795 if(sleep<0):
4796 sleep = geturls_download_sleep;
4797 if(timeout<=0):
4798 timeout = 10;
4799 urlparts = urlparse.urlparse(httpurl);
4800 if(isinstance(httpheaders, list)):
4801 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4802 httpheaders = fix_header_names(httpheaders);
4803 if(isinstance(httpheaders, dict)):
4804 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
4805 time.sleep(sleep);
4806 geturls_text = download_file_from_pysftp_file(httpurl);
4807 if(not geturls_text):
4808 return False;
4809 downloadsize = None;
4810 if(downloadsize is not None):
4811 downloadsize = int(downloadsize);
4812 if downloadsize is None: downloadsize = 0;
4813 fulldatasize = 0;
4814 prevdownsize = 0;
4815 log.info("Downloading URL "+httpurl);
4816 with BytesIO() as strbuf:
4817 while True:
4818 databytes = geturls_text.read(buffersize);
4819 if not databytes: break;
4820 datasize = len(databytes);
4821 fulldatasize = datasize + fulldatasize;
4822 percentage = "";
4823 if(downloadsize>0):
4824 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4825 downloaddiff = fulldatasize - prevdownsize;
4826 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4827 prevdownsize = fulldatasize;
4828 strbuf.write(databytes);
4829 strbuf.seek(0);
4830 returnval_content = strbuf.read();
4831 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
4832 geturls_text.close();
4833 return returnval;
4834
4835 if(not havepysftp):
4836 def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
4837 return False;
4838
4839 if(havepysftp):
4840 def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
4841 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix;
4842 exec_time_start = time.time();
4843 myhash = hashlib.new("sha1");
4844 if(sys.version[0]=="2"):
4845 myhash.update(httpurl);
4846 myhash.update(str(buffersize));
4847 myhash.update(str(exec_time_start));
4848 if(sys.version[0]>="3"):
4849 myhash.update(httpurl.encode('utf-8'));
4850 myhash.update(str(buffersize).encode('utf-8'));
4851 myhash.update(str(exec_time_start).encode('utf-8'));
4852 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
4853 if(sleep<0):
4854 sleep = geturls_download_sleep;
4855 if(timeout<=0):
4856 timeout = 10;
4857 pretmpfilename = download_from_url_with_pysftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep, timeout);
4858 if(not pretmpfilename):
4859 return False;
4860 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
4861 tmpfilename = f.name;
4862 try:
4863 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
4864 except AttributeError:
4865 try:
4866 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4867 except ValueError:
4868 pass;
4869 except ValueError:
4870 pass;
4871 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason')};
4872 f.write(pretmpfilename['Content']);
4873 f.close();
4874 exec_time_end = time.time();
4875 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
4876 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
4877 return returnval;
4878
4879 if(not havepysftp):
4880 def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
4881 return False;
4882
4883 if(havepysftp):
4884 def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
4885 global geturls_download_sleep, havezstd, havebrotli;
4886 if(sleep<0):
4887 sleep = geturls_download_sleep;
4888 if(timeout<=0):
4889 timeout = 10;
4890 if(not outfile=="-"):
4891 outpath = outpath.rstrip(os.path.sep);
4892 filepath = os.path.realpath(outpath+os.path.sep+outfile);
4893 if(not os.path.exists(outpath)):
4894 os.makedirs(outpath);
4895 if(os.path.exists(outpath) and os.path.isfile(outpath)):
4896 return False;
4897 if(os.path.exists(filepath) and os.path.isdir(filepath)):
4898 return False;
4899 pretmpfilename = download_from_url_file_with_pysftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
4900 if(not pretmpfilename):
4901 return False;
4902 tmpfilename = pretmpfilename['Filename'];
4903 downloadsize = int(os.path.getsize(tmpfilename));
4904 fulldatasize = 0;
4905 log.info("Moving file "+tmpfilename+" to "+filepath);
4906 exec_time_start = time.time();
4907 shutil.move(tmpfilename, filepath);
4908 exec_time_end = time.time();
4909 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
4910 if(os.path.exists(tmpfilename)):
4911 os.remove(tmpfilename);
4912 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4913 if(outfile=="-"):
4914 pretmpfilename = download_from_url_file_with_pysftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, ranges, buffersize[0], sleep, timeout);
4915 tmpfilename = pretmpfilename['Filename'];
4916 downloadsize = int(os.path.getsize(tmpfilename));
4917 fulldatasize = 0;
4918 prevdownsize = 0;
4919 exec_time_start = time.time();
4920 with open(tmpfilename, 'rb') as ft:
4921 f = BytesIO();
4922 while True:
4923 databytes = ft.read(buffersize[1]);
4924 if not databytes: break;
4925 datasize = len(databytes);
4926 fulldatasize = datasize + fulldatasize;
4927 percentage = "";
4928 if(downloadsize>0):
4929 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4930 downloaddiff = fulldatasize - prevdownsize;
4931 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4932 prevdownsize = fulldatasize;
4933 f.write(databytes);
4934 f.seek(0);
4935 fdata = f.getvalue();
4936 f.close();
4937 ft.close();
4938 os.remove(tmpfilename);
4939 exec_time_end = time.time();
4940 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4941 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4942 return returnval;
4943
4944 if(not havepysftp):
4945 def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
4946 return False;
4947
4948 if(havepysftp):
4949 def upload_file_to_pysftp_file(sftpfile, url):
4950 urlparts = urlparse.urlparse(url);
4951 file_name = os.path.basename(urlparts.path);
4952 file_dir = os.path.dirname(urlparts.path);
4953 sftp_port = urlparts.port;
4954 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
4955 return False;
4956 if(urlparts.port is None):
4957 sftp_port = 22;
4958 else:
4959 sftp_port = urlparts.port;
4960 if(urlparts.username is not None):
4961 sftp_username = urlparts.username;
4962 else:
4963 sftp_username = "anonymous";
4964 if(urlparts.password is not None):
4965 sftp_password = urlparts.password;
4966 elif(urlparts.password is None and urlparts.username=="anonymous"):
4967 sftp_password = "anonymous";
4968 else:
4969 sftp_password = "";
4970 if(urlparts.scheme!="sftp"):
4971 return False;
4972 try:
4973 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
4974 except paramiko.ssh_exception.SSHException:
4975 return False;
4976 except socket.gaierror:
4977 log.info("Error With URL "+httpurl);
4978 return False;
4979 except socket.timeout:
4980 log.info("Error With URL "+httpurl);
4981 return False;
4982 sftp = ssh.open_sftp();
4983 sftp.putfo(sftpfile, urlparts.path);
4984 sftp.close();
4985 ssh.close();
4986 sftpfile.seek(0, 0);
4987 return sftpfile;
4988 else:
4989 def upload_file_to_pysftp_file(sftpfile, url):
4990 return False;
4991
4992 if(havepysftp):
4993 def upload_file_to_pysftp_string(sftpstring, url):
4994 sftpfileo = BytesIO(sftpstring);
4995 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
4996 sftpfileo.close();
4997 return sftpfile;
4998 else:
4999 def upload_file_to_pysftp_string(url):
5000 return False;
python libary/module to download files.