4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the Revised BSD License.
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 Revised BSD License for more details.
12 Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
13 Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
15 $FileInfo: pywwwget.py - Last Update: 9/15/2023 Ver. 0.9.4 RC 1 - Author: cooldude2k $
18 from __future__
import division
, absolute_import
, print_function
;
19 import re
, os
, sys
, hashlib
, shutil
, platform
, tempfile
, urllib
, gzip
, time
, argparse
, cgi
, subprocess
, socket
;
20 import logging
as log
;
21 from ftplib
import FTP
, FTP_TLS
;
22 from base64
import b64encode
;
29 havemechanize
= False;
34 havemechanize
= False;
49 from httplib2
import HTTPConnectionWithTimeout
, HTTPSConnectionWithTimeout
;
65 if(sys
.version
[0]=="2"):
67 from cStringIO
import StringIO
;
69 from StringIO
import StringIO
;
70 # From http://python-future.org/compatible_idioms.html
71 from urlparse
import urlparse
, urlunparse
, urlsplit
, urlunsplit
, urljoin
;
72 from urllib
import urlencode
;
73 from urllib
import urlopen
as urlopenalt
;
74 from urllib2
import urlopen
, Request
, install_opener
, HTTPError
, URLError
, build_opener
, HTTPCookieProcessor
;
75 import urlparse
, cookielib
;
76 from httplib
import HTTPConnection
, HTTPSConnection
;
77 if(sys
.version
[0]>="3"):
78 from io
import StringIO
, BytesIO
;
79 # From http://python-future.org/compatible_idioms.html
80 from urllib
.parse
import urlparse
, urlunparse
, urlsplit
, urlunsplit
, urljoin
, urlencode
;
81 from urllib
.request
import urlopen
, Request
, install_opener
, build_opener
, HTTPCookieProcessor
;
82 from urllib
.error
import HTTPError
, URLError
;
83 import urllib
.parse
as urlparse
;
84 import http
.cookiejar
as cookielib
;
85 from http
.client
import HTTPConnection
, HTTPSConnection
;
87 __program_name__
= "PyWWW-Get";
88 __program_alt_name__
= "PyWWWGet";
89 __program_small_name__
= "wwwget";
90 __project__
= __program_name__
;
91 __project_url__
= "https://github.com/GameMaker2k/PyWWW-Get";
92 __version_info__
= (0, 9, 4, "RC 1", 1);
93 __version_date_info__
= (2023, 9, 15, "RC 1", 1);
94 __version_date__
= str(__version_date_info__
[0])+"."+str(__version_date_info__
[1]).zfill(2)+"."+str(__version_date_info__
[2]).zfill(2);
95 __revision__
= __version_info__
[3];
96 __revision_id__
= "$Id$";
97 if(__version_info__
[4] is not None):
98 __version_date_plusrc__
= __version_date__
+"-"+str(__version_date_info__
[4]);
99 if(__version_info__
[4] is None):
100 __version_date_plusrc__
= __version_date__
;
101 if(__version_info__
[3] is not None):
102 __version__
= str(__version_info__
[0])+"."+str(__version_info__
[1])+"."+str(__version_info__
[2])+" "+str(__version_info__
[3]);
103 if(__version_info__
[3] is None):
104 __version__
= str(__version_info__
[0])+"."+str(__version_info__
[1])+"."+str(__version_info__
[2]);
106 tmpfileprefix
= "py"+str(sys
.version_info
[0])+__program_small_name__
+str(__version_info__
[0])+"-";
108 pytempdir
= tempfile
.gettempdir();
110 PyBitness
= platform
.architecture();
111 if(PyBitness
=="32bit" or PyBitness
=="32"):
113 elif(PyBitness
=="64bit" or PyBitness
=="64"):
118 compression_supported
= "gzip, deflate";
120 compression_supported
= "gzip, deflate, br";
122 compression_supported
= "gzip, deflate";
124 geturls_cj
= cookielib
.CookieJar();
125 windowsNT4_ua_string
= "Windows NT 4.0";
126 windowsNT4_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
127 windows2k_ua_string
= "Windows NT 5.0";
128 windows2k_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
129 windowsXP_ua_string
= "Windows NT 5.1";
130 windowsXP_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
131 windowsXP64_ua_string
= "Windows NT 5.2; Win64; x64";
132 windowsXP64_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
133 windows7_ua_string
= "Windows NT 6.1; Win64; x64";
134 windows7_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
135 windows8_ua_string
= "Windows NT 6.2; Win64; x64";
136 windows8_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
137 windows81_ua_string
= "Windows NT 6.3; Win64; x64";
138 windows81_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
139 windows10_ua_string
= "Windows NT 10.0; Win64; x64";
140 windows10_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
141 windows11_ua_string
= "Windows NT 11.0; Win64; x64";
142 windows11_ua_addon
= {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
143 geturls_ua_firefox_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:109.0) Gecko/20100101 Firefox/117.0";
144 geturls_ua_seamonkey_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
145 geturls_ua_chrome_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
146 geturls_ua_chromium_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
147 geturls_ua_palemoon_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
148 geturls_ua_opera_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
149 geturls_ua_vivaldi_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
150 geturls_ua_internet_explorer_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+"; Trident/7.0; rv:11.0) like Gecko";
151 geturls_ua_microsoft_edge_windows7
= "Mozilla/5.0 ("+windows7_ua_string
+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
152 geturls_ua_pywwwget_python
= "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname
=__project__
, prover
=__version__
, prourl
=__project_url__
);
153 if(platform
.python_implementation()!=""):
154 py_implementation
= platform
.python_implementation();
155 if(platform
.python_implementation()==""):
156 py_implementation
= "Python";
157 geturls_ua_pywwwget_python_alt
= "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver
=platform
.system()+" "+platform
.release(), archtype
=platform
.machine(), prourl
=__project_url__
, pyimp
=py_implementation
, pyver
=platform
.python_version(), proname
=__project__
, prover
=__version__
);
158 geturls_ua_googlebot_google
= "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
159 geturls_ua_googlebot_google_old
= "Googlebot/2.1 (+http://www.google.com/bot.html)";
160 geturls_ua
= geturls_ua_firefox_windows7
;
161 geturls_headers_firefox_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
162 geturls_headers_seamonkey_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
163 geturls_headers_chrome_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
164 geturls_headers_chrome_windows7
.update(windows7_ua_addon
);
165 geturls_headers_chromium_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
166 geturls_headers_chromium_windows7
.update(windows7_ua_addon
);
167 geturls_headers_palemoon_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
168 geturls_headers_opera_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
169 geturls_headers_opera_windows7
.update(windows7_ua_addon
);
170 geturls_headers_vivaldi_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
171 geturls_headers_vivaldi_windows7
.update(windows7_ua_addon
);
172 geturls_headers_internet_explorer_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
173 geturls_headers_microsoft_edge_windows7
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
174 geturls_headers_microsoft_edge_windows7
.update(windows7_ua_addon
);
175 geturls_headers_pywwwget_python
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
176 geturls_headers_pywwwget_python_alt
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__
+"\";v=\""+str(__version__
)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation
+"\";v=\""+str(platform
.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__
), 'SEC-CH-UA-PLATFORM': ""+py_implementation
+"", 'SEC-CH-UA-ARCH': ""+platform
.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__
), 'SEC-CH-UA-BITNESS': str(PyBitness
)};
177 geturls_headers_googlebot_google
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
178 geturls_headers_googlebot_google_old
= {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old
, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
179 geturls_headers
= geturls_headers_firefox_windows7
;
180 geturls_download_sleep
= 0;
182 def verbose_printout(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
183 if(outtype
=="print" and dbgenable
):
186 elif(outtype
=="log" and dbgenable
):
187 logging
.info(dbgtxt
);
189 elif(outtype
=="warning" and dbgenable
):
190 logging
.warning(dbgtxt
);
192 elif(outtype
=="error" and dbgenable
):
193 logging
.error(dbgtxt
);
195 elif(outtype
=="critical" and dbgenable
):
196 logging
.critical(dbgtxt
);
198 elif(outtype
=="exception" and dbgenable
):
199 logging
.exception(dbgtxt
);
201 elif(outtype
=="logalt" and dbgenable
):
202 logging
.log(dgblevel
, dbgtxt
);
204 elif(outtype
=="debug" and dbgenable
):
205 logging
.debug(dbgtxt
);
213 def verbose_printout_return(dbgtxt
, outtype
="log", dbgenable
=True, dgblevel
=20):
214 dbgout
= verbose_printout(dbgtxt
, outtype
, dbgenable
, dgblevel
);
219 def add_url_param(url
, **params
):
221 parts
= list(urlparse
.urlsplit(url
));
222 d
= dict(cgi
.parse_qsl(parts
[n
])); # use cgi.parse_qs for list values
224 parts
[n
]=urlencode(d
);
225 return urlparse
.urlunsplit(parts
);
227 os
.environ
["PATH"] = os
.environ
["PATH"] + os
.pathsep
+ os
.path
.dirname(os
.path
.realpath(__file__
)) + os
.pathsep
+ os
.getcwd();
228 def which_exec(execfile):
229 for path
in os
.environ
["PATH"].split(":"):
230 if os
.path
.exists(path
+ "/" + execfile):
231 return path
+ "/" + execfile;
233 def listize(varlist
):
241 newlistreg
.update({ilx
: varlist
[il
]});
242 newlistrev
.update({varlist
[il
]: ilx
});
245 newlistfull
= {1: newlistreg
, 2: newlistrev
, 'reg': newlistreg
, 'rev': newlistrev
};
248 def twolistize(varlist
):
258 newlistnamereg
.update({ilx
: varlist
[il
][0].strip()});
259 newlistnamerev
.update({varlist
[il
][0].strip(): ilx
});
260 newlistdescreg
.update({ilx
: varlist
[il
][1].strip()});
261 newlistdescrev
.update({varlist
[il
][1].strip(): ilx
});
264 newlistnametmp
= {1: newlistnamereg
, 2: newlistnamerev
, 'reg': newlistnamereg
, 'rev': newlistnamerev
};
265 newlistdesctmp
= {1: newlistdescreg
, 2: newlistdescrev
, 'reg': newlistdescreg
, 'rev': newlistdescrev
};
266 newlistfull
= {1: newlistnametmp
, 2: newlistdesctmp
, 'name': newlistnametmp
, 'desc': newlistdesctmp
}
269 def arglistize(proexec
, *varlist
):
273 newarglist
= [proexec
];
275 if varlist
[il
][0] is not None:
276 newarglist
.append(varlist
[il
][0]);
277 if varlist
[il
][1] is not None:
278 newarglist
.append(varlist
[il
][1]);
282 # hms_string by ArcGIS Python Recipes
283 # https://arcpy.wordpress.com/2012/04/20/146/
284 def hms_string(sec_elapsed
):
285 h
= int(sec_elapsed
/ (60 * 60));
286 m
= int((sec_elapsed
% (60 * 60)) / 60);
287 s
= sec_elapsed
% 60.0;
288 return "{}:{:>02}:{:>05.2f}".format(h
, m
, s
);
290 # get_readable_size by Lipis
291 # http://stackoverflow.com/posts/14998888/revisions
292 def get_readable_size(bytes
, precision
=1, unit
="IEC"):
294 if(unit
!="IEC" and unit
!="SI"):
297 units
= [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
298 unitswos
= ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
301 units
= [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
302 unitswos
= ["B","kB","MB","GB","TB","PB","EB","ZB"];
307 if abs(bytes
) < unitsize
:
308 strformat
= "%3."+str(precision
)+"f%s";
309 pre_return_val
= (strformat
% (bytes
, unit
));
310 pre_return_val
= re
.sub(r
"([0]+) ([A-Za-z]+)", r
" \2", pre_return_val
);
311 pre_return_val
= re
.sub(r
"\. ([A-Za-z]+)", r
" \1", pre_return_val
);
312 alt_return_val
= pre_return_val
.split();
313 return_val
= {'Bytes': orgbytes
, 'ReadableWithSuffix': pre_return_val
, 'ReadableWithoutSuffix': alt_return_val
[0], 'ReadableSuffix': alt_return_val
[1]}
316 strformat
= "%."+str(precision
)+"f%s";
317 pre_return_val
= (strformat
% (bytes
, "YiB"));
318 pre_return_val
= re
.sub(r
"([0]+) ([A-Za-z]+)", r
" \2", pre_return_val
);
319 pre_return_val
= re
.sub(r
"\. ([A-Za-z]+)", r
" \1", pre_return_val
);
320 alt_return_val
= pre_return_val
.split();
321 return_val
= {'Bytes': orgbytes
, 'ReadableWithSuffix': pre_return_val
, 'ReadableWithoutSuffix': alt_return_val
[0], 'ReadableSuffix': alt_return_val
[1]}
324 def get_readable_size_from_file(infile
, precision
=1, unit
="IEC", usehashes
=False, usehashtypes
="md5,sha1"):
326 usehashtypes
= usehashtypes
.lower();
327 getfilesize
= os
.path
.getsize(infile
);
328 return_val
= get_readable_size(getfilesize
, precision
, unit
);
330 hashtypelist
= usehashtypes
.split(",");
331 openfile
= open(infile
, "rb");
332 filecontents
= openfile
.read();
335 listnumend
= len(hashtypelist
);
336 while(listnumcount
< listnumend
):
337 hashtypelistlow
= hashtypelist
[listnumcount
].strip();
338 hashtypelistup
= hashtypelistlow
.upper();
339 filehash
= hashlib
.new(hashtypelistup
);
340 filehash
.update(filecontents
);
341 filegethash
= filehash
.hexdigest();
342 return_val
.update({hashtypelistup
: filegethash
});
346 def get_readable_size_from_string(instring
, precision
=1, unit
="IEC", usehashes
=False, usehashtypes
="md5,sha1"):
348 usehashtypes
= usehashtypes
.lower();
349 getfilesize
= len(instring
);
350 return_val
= get_readable_size(getfilesize
, precision
, unit
);
352 hashtypelist
= usehashtypes
.split(",");
354 listnumend
= len(hashtypelist
);
355 while(listnumcount
< listnumend
):
356 hashtypelistlow
= hashtypelist
[listnumcount
].strip();
357 hashtypelistup
= hashtypelistlow
.upper();
358 filehash
= hashlib
.new(hashtypelistup
);
359 if(sys
.version
[0]=="2"):
360 filehash
.update(instring
);
361 if(sys
.version
[0]>="3"):
362 filehash
.update(instring
.encode('utf-8'));
363 filegethash
= filehash
.hexdigest();
364 return_val
.update({hashtypelistup
: filegethash
});
368 def make_http_headers_from_dict_to_list(headers
={'Referer': "http://google.com/", 'User-Agent': geturls_ua
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
369 if isinstance(headers
, dict):
371 if(sys
.version
[0]=="2"):
372 for headkey
, headvalue
in headers
.iteritems():
373 returnval
.append((headkey
, headvalue
));
374 if(sys
.version
[0]>="3"):
375 for headkey
, headvalue
in headers
.items():
376 returnval
.append((headkey
, headvalue
));
377 elif isinstance(headers
, list):
383 def make_http_headers_from_dict_to_pycurl(headers
={'Referer': "http://google.com/", 'User-Agent': geturls_ua
, 'Accept-Encoding': compression_supported
, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
384 if isinstance(headers
, dict):
386 if(sys
.version
[0]=="2"):
387 for headkey
, headvalue
in headers
.iteritems():
388 returnval
.append(headkey
+": "+headvalue
);
389 if(sys
.version
[0]>="3"):
390 for headkey
, headvalue
in headers
.items():
391 returnval
.append(headkey
+": "+headvalue
);
392 elif isinstance(headers
, list):
398 def make_http_headers_from_list_to_dict(headers
=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua
), ("Accept-Encoding", compression_supported
), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
399 if isinstance(headers
, list):
404 returnval
.update({headers
[mli
][0]: headers
[mli
][1]});
406 elif isinstance(headers
, dict):
412 def get_httplib_support(checkvalue
=None):
413 global haverequests
, havemechanize
, havehttplib2
, haveurllib3
, havehttpx
;
415 returnval
.append("ftp");
416 returnval
.append("httplib");
418 returnval
.append("httplib2");
419 returnval
.append("urllib");
421 returnval
.append("urllib3");
422 returnval
.append("request3");
423 returnval
.append("request");
425 returnval
.append("requests");
427 returnval
.append("httpx");
428 returnval
.append("httpx2");
430 returnval
.append("mechanize");
432 returnval
.append("sftp");
433 if(not checkvalue
is None):
434 if(checkvalue
=="urllib1" or checkvalue
=="urllib2"):
435 checkvalue
= "urllib";
436 if(checkvalue
=="httplib1"):
437 checkvalue
= "httplib";
438 if(checkvalue
in returnval
):
444 def check_httplib_support(checkvalue
="urllib"):
445 if(checkvalue
=="urllib1" or checkvalue
=="urllib2"):
446 checkvalue
= "urllib";
447 if(checkvalue
=="httplib1"):
448 checkvalue
= "httplib";
449 returnval
= get_httplib_support(checkvalue
);
452 def get_httplib_support_list():
453 returnval
= get_httplib_support(None);
456 def download_from_url(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", sleep
=-1):
457 global geturls_download_sleep
, haverequests
, havemechanize
, havehttplib2
, haveurllib3
, havehttpx
;
459 sleep
= geturls_download_sleep
;
460 if(httplibuse
=="urllib1" or httplibuse
=="urllib2"):
461 httplibuse
= "urllib";
462 if(httplibuse
=="httplib1"):
463 httplibuse
= "httplib";
464 if(not haverequests
and httplibuse
=="requests"):
465 httplibuse
= "urllib";
466 if(not havehttpx
and httplibuse
=="httpx"):
467 httplibuse
= "urllib";
468 if(not havehttpx
and httplibuse
=="httpx2"):
469 httplibuse
= "urllib";
470 if(not havemechanize
and httplibuse
=="mechanize"):
471 httplibuse
= "urllib";
472 if(not havehttplib2
and httplibuse
=="httplib2"):
473 httplibuse
= "httplib";
474 if(not haveparamiko
and httplibuse
=="sftp"):
476 if(httplibuse
=="urllib"):
477 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
478 elif(httplibuse
=="request"):
479 returnval
= download_from_url_with_request(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
480 elif(httplibuse
=="request3"):
481 returnval
= download_from_url_with_request(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
482 elif(httplibuse
=="httplib"):
483 returnval
= download_from_url_with_httplib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
484 elif(httplibuse
=="httplib2"):
485 returnval
= download_from_url_with_httplib2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
486 elif(httplibuse
=="urllib3"):
487 returnval
= download_from_url_with_urllib3(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
488 elif(httplibuse
=="requests"):
489 returnval
= download_from_url_with_requests(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
490 elif(httplibuse
=="httpx"):
491 returnval
= download_from_url_with_httpx(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
492 elif(httplibuse
=="httpx2"):
493 returnval
= download_from_url_with_httpx2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
494 elif(httplibuse
=="mechanize"):
495 returnval
= download_from_url_with_mechanize(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
496 elif(httplibuse
=="ftp"):
497 returnval
= download_from_url_with_ftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
498 elif(httplibuse
=="mechanize"):
499 returnval
= download_from_url_with_sftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
);
504 def download_from_url_file(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", buffersize
=524288, sleep
=-1):
505 global geturls_download_sleep
, haverequests
, havemechanize
, havehttplib2
, haveurllib3
, havehttpx
;
507 sleep
= geturls_download_sleep
;
508 if(httplibuse
=="urllib1" or httplibuse
=="urllib2"):
509 httplibuse
= "urllib";
510 if(httplibuse
=="httplib1"):
511 httplibuse
= "httplib";
512 if(not haverequests
and httplibuse
=="requests"):
513 httplibuse
= "urllib";
514 if(not havehttpx
and httplibuse
=="httpx"):
515 httplibuse
= "urllib";
516 if(not havehttpx
and httplibuse
=="httpx2"):
517 httplibuse
= "urllib";
518 if(not havemechanize
and httplibuse
=="mechanize"):
519 httplibuse
= "urllib";
520 if(not havehttplib2
and httplibuse
=="httplib2"):
521 httplibuse
= "httplib";
522 if(not haveparamiko
and httplibuse
=="sftp"):
524 if(httplibuse
=="urllib"):
525 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
526 elif(httplibuse
=="request"):
527 returnval
= download_from_url_file_with_request(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
528 elif(httplibuse
=="request3"):
529 returnval
= download_from_url_file_with_request(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
530 elif(httplibuse
=="httplib"):
531 returnval
= download_from_url_file_with_httplib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
532 elif(httplibuse
=="httplib2"):
533 returnval
= download_from_url_file_with_httplib2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
534 elif(httplibuse
=="urllib3"):
535 returnval
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
536 elif(httplibuse
=="requests"):
537 returnval
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
538 elif(httplibuse
=="httpx"):
539 returnval
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
540 elif(httplibuse
=="httpx2"):
541 returnval
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
542 elif(httplibuse
=="mechanize"):
543 returnval
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
544 elif(httplibuse
=="ftp"):
545 returnval
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
546 elif(httplibuse
=="sftp"):
547 returnval
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
);
552 def download_from_url_to_file(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, httplibuse
="urllib", outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
553 global geturls_download_sleep
, haverequests
, havemechanize
, havehttplib2
, haveurllib3
, havehttpx
;
555 sleep
= geturls_download_sleep
;
556 if(httplibuse
=="urllib1" or httplibuse
=="urllib2"):
557 httplibuse
= "urllib";
558 if(httplibuse
=="httplib1"):
559 httplibuse
= "httplib";
560 if(not haverequests
and httplibuse
=="requests"):
561 httplibuse
= "urllib";
562 if(not havehttpx
and httplibuse
=="httpx"):
563 httplibuse
= "urllib";
564 if(not havehttpx
and httplibuse
=="httpx2"):
565 httplibuse
= "urllib";
566 if(not havemechanize
and httplibuse
=="mechanize"):
567 httplibuse
= "urllib";
568 if(not havehttplib2
and httplibuse
=="httplib2"):
569 httplibuse
= "httplib";
570 if(not haveparamiko
and httplibuse
=="sftp"):
572 if(httplibuse
=="urllib"):
573 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
574 elif(httplibuse
=="request"):
575 returnval
= download_from_url_to_file_with_request(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
576 elif(httplibuse
=="request3"):
577 returnval
= download_from_url_to_file_with_request(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
578 elif(httplibuse
=="httplib"):
579 returnval
= download_from_url_to_file_with_httplib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
580 elif(httplibuse
=="httplib2"):
581 returnval
= download_from_url_to_file_with_httplib2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
582 elif(httplibuse
=="urllib3"):
583 returnval
= download_from_url_to_file_with_urllib3(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
584 elif(httplibuse
=="requests"):
585 returnval
= download_from_url_to_file_with_requests(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
586 elif(httplibuse
=="httpx"):
587 returnval
= download_from_url_to_file_with_httpx(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
588 elif(httplibuse
=="httpx2"):
589 returnval
= download_from_url_to_file_with_httpx2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
590 elif(httplibuse
=="mechanize"):
591 returnval
= download_from_url_to_file_with_mechanize(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
592 elif(httplibuse
=="ftp"):
593 returnval
= download_from_url_to_file_with_ftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
594 elif(httplibuse
=="sftp"):
595 returnval
= download_from_url_to_file_with_sftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, outfile
, outpath
, buffersize
, sleep
);
600 def download_from_url_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
601 global geturls_download_sleep
, havebrotli
;
603 sleep
= geturls_download_sleep
;
604 urlparts
= urlparse
.urlparse(httpurl
);
605 if(isinstance(httpheaders
, list)):
606 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
607 if(urlparts
.username
is not None or urlparts
.password
is not None):
608 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
609 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
610 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
611 if(isinstance(httpheaders
, dict)):
612 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
613 geturls_opener
.addheaders
= httpheaders
;
615 if(postdata
is not None and not isinstance(postdata
, dict)):
616 postdata
= urlencode(postdata
);
618 if(httpmethod
=="GET"):
619 geturls_text
= geturls_opener
.open(httpurl
);
620 elif(httpmethod
=="POST"):
621 geturls_text
= geturls_opener
.open(httpurl
, data
=postdata
);
623 geturls_text
= geturls_opener
.open(httpurl
);
624 except HTTPError
as geturls_text_error
:
625 geturls_text
= geturls_text_error
;
626 log
.info("Error With URL "+httpurl
);
628 log
.info("Error With URL "+httpurl
);
630 except socket
.timeout
:
631 log
.info("Error With URL "+httpurl
);
633 log
.info("Downloading URL "+httpurl
);
634 if(geturls_text
.info().get("Content-Encoding")=="gzip" or geturls_text
.info().get("Content-Encoding")=="deflate"):
635 if(sys
.version
[0]=="2"):
636 strbuf
= StringIO(geturls_text
.read());
637 if(sys
.version
[0]>="3"):
638 strbuf
= BytesIO(geturls_text
.read());
639 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
640 returnval_content
= gzstrbuf
.read()[:];
641 if(geturls_text
.info().get("Content-Encoding")!="gzip" and geturls_text
.info().get("Content-Encoding")!="deflate"):
642 returnval_content
= geturls_text
.read()[:];
643 if(geturls_text
.info().get("Content-Encoding")=="br" and havebrotli
):
644 returnval_content
= brotli
.decompress(returnval_content
);
645 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.info()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': geturls_text
.geturl(), 'Code': geturls_text
.getcode()};
646 geturls_text
.close();
649 def download_from_url_file_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
650 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
651 exec_time_start
= time
.time();
652 myhash
= hashlib
.new("sha1");
653 if(sys
.version
[0]=="2"):
654 myhash
.update(httpurl
);
655 myhash
.update(str(buffersize
));
656 myhash
.update(str(exec_time_start
));
657 if(sys
.version
[0]>="3"):
658 myhash
.update(httpurl
.encode('utf-8'));
659 myhash
.update(str(buffersize
).encode('utf-8'));
660 myhash
.update(str(exec_time_start
).encode('utf-8'));
661 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
663 sleep
= geturls_download_sleep
;
664 urlparts
= urlparse
.urlparse(httpurl
);
665 if(isinstance(httpheaders
, list)):
666 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
667 if(urlparts
.username
is not None or urlparts
.password
is not None):
668 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
669 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
670 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
671 if(isinstance(httpheaders
, dict)):
672 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
673 geturls_opener
.addheaders
= httpheaders
;
676 if(httpmethod
=="GET"):
677 geturls_text
= geturls_opener
.open(httpurl
);
678 elif(httpmethod
=="POST"):
679 geturls_text
= geturls_opener
.open(httpurl
, data
=postdata
);
681 geturls_text
= geturls_opener
.open(httpurl
);
682 except HTTPError
as geturls_text_error
:
683 geturls_text
= geturls_text_error
;
684 log
.info("Error With URL "+httpurl
);
686 log
.info("Error With URL "+httpurl
);
688 except socket
.timeout
:
689 log
.info("Error With URL "+httpurl
);
691 except socket
.timeout
:
692 log
.info("Error With URL "+httpurl
);
694 downloadsize
= geturls_text
.info().get('Content-Length');
695 if(downloadsize
is not None):
696 downloadsize
= int(downloadsize
);
697 if downloadsize
is None: downloadsize
= 0;
700 log
.info("Downloading URL "+httpurl
);
701 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
702 tmpfilename
= f
.name
;
703 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(geturls_text
.info()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': geturls_text
.geturl(), 'Code': geturls_text
.getcode()};
705 databytes
= geturls_text
.read(buffersize
);
706 if not databytes
: break;
707 datasize
= len(databytes
);
708 fulldatasize
= datasize
+ fulldatasize
;
711 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
712 downloaddiff
= fulldatasize
- prevdownsize
;
713 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
714 prevdownsize
= fulldatasize
;
717 geturls_text
.close();
718 exec_time_end
= time
.time();
719 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
720 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
723 def download_from_url_to_file_with_urllib(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
724 global geturls_download_sleep
;
726 sleep
= geturls_download_sleep
;
727 if(not outfile
=="-"):
728 outpath
= outpath
.rstrip(os
.path
.sep
);
729 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
730 if(not os
.path
.exists(outpath
)):
731 os
.makedirs(outpath
);
732 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
734 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
736 pretmpfilename
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
737 if(not pretmpfilename
):
739 tmpfilename
= pretmpfilename
['Filename'];
740 downloadsize
= os
.path
.getsize(tmpfilename
);
742 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
743 exec_time_start
= time
.time();
744 shutil
.move(tmpfilename
, filepath
);
745 exec_time_end
= time
.time();
746 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
747 if(os
.path
.exists(tmpfilename
)):
748 os
.remove(tmpfilename
);
749 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
750 if(outfile
=="-" and sys
.version
[0]=="2"):
751 pretmpfilename
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
752 if(not pretmpfilename
):
754 tmpfilename
= pretmpfilename
['Filename'];
755 downloadsize
= os
.path
.getsize(tmpfilename
);
758 exec_time_start
= time
.time();
759 with
open(tmpfilename
, 'rb') as ft
:
762 databytes
= ft
.read(buffersize
[1]);
763 if not databytes
: break;
764 datasize
= len(databytes
);
765 fulldatasize
= datasize
+ fulldatasize
;
768 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
769 downloaddiff
= fulldatasize
- prevdownsize
;
770 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
771 prevdownsize
= fulldatasize
;
774 fdata
= f
.getvalue();
777 os
.remove(tmpfilename
);
778 exec_time_end
= time
.time();
779 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
780 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
781 if(outfile
=="-" and sys
.version
[0]>="3"):
782 pretmpfilename
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
783 tmpfilename
= pretmpfilename
['Filename'];
784 downloadsize
= os
.path
.getsize(tmpfilename
);
787 exec_time_start
= time
.time();
788 with
open(tmpfilename
, 'rb') as ft
:
791 databytes
= ft
.read(buffersize
[1]);
792 if not databytes
: break;
793 datasize
= len(databytes
);
794 fulldatasize
= datasize
+ fulldatasize
;
797 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
798 downloaddiff
= fulldatasize
- prevdownsize
;
799 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
800 prevdownsize
= fulldatasize
;
803 fdata
= f
.getvalue();
806 os
.remove(tmpfilename
);
807 exec_time_end
= time
.time();
808 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
809 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
812 def download_from_url_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
813 global geturls_download_sleep
, havebrotli
;
815 sleep
= geturls_download_sleep
;
816 urlparts
= urlparse
.urlparse(httpurl
);
817 if(isinstance(httpheaders
, list)):
818 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
819 if(urlparts
.username
is not None or urlparts
.password
is not None):
820 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
821 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
822 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
823 geturls_opener
.addheaders
= httpheaders
;
825 if(urlparts
[0]=="http"):
826 httpconn
= HTTPConnection(urlparts
[1]);
827 elif(urlparts
[0]=="https"):
828 httpconn
= HTTPSConnection(urlparts
[1]);
831 if(postdata
is not None and not isinstance(postdata
, dict)):
832 postdata
= urlencode(postdata
);
834 if(httpmethod
=="GET"):
835 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
836 elif(httpmethod
=="POST"):
837 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
839 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
840 except socket
.timeout
:
841 log
.info("Error With URL "+httpurl
);
843 geturls_text
= httpconn
.getresponse();
844 log
.info("Downloading URL "+httpurl
);
845 if(dict(geturls_text
.getheaders()).get("Content-Encoding")=="gzip" or dict(geturls_text
.getheaders()).get("Content-Encoding")=="deflate"):
846 if(sys
.version
[0]=="2"):
847 strbuf
= StringIO(geturls_text
.read());
848 if(sys
.version
[0]>="3"):
849 strbuf
= BytesIO(geturls_text
.read());
850 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
851 returnval_content
= gzstrbuf
.read()[:];
852 if(dict(geturls_text
.getheaders()).get("Content-Encoding")!="gzip" and dict(geturls_text
.getheaders()).get("Content-Encoding")!="deflate"):
853 returnval_content
= geturls_text
.read()[:];
854 if(dict(geturls_text
.getheaders()).get("Content-Encoding")=="br" and havebrotli
):
855 returnval_content
= brotli
.decompress(returnval_content
);
856 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.getheaders()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': httpheaders
, 'URL': httpurl
, 'Code': geturls_text
.status
};
857 geturls_text
.close();
860 def download_from_url_file_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
861 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
862 exec_time_start
= time
.time();
863 myhash
= hashlib
.new("sha1");
864 if(sys
.version
[0]=="2"):
865 myhash
.update(httpurl
);
866 myhash
.update(str(buffersize
));
867 myhash
.update(str(exec_time_start
));
868 if(sys
.version
[0]>="3"):
869 myhash
.update(httpurl
.encode('utf-8'));
870 myhash
.update(str(buffersize
).encode('utf-8'));
871 myhash
.update(str(exec_time_start
).encode('utf-8'));
872 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
874 sleep
= geturls_download_sleep
;
875 urlparts
= urlparse
.urlparse(httpurl
);
876 if(isinstance(httpheaders
, list)):
877 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
878 if(urlparts
.username
is not None or urlparts
.password
is not None):
879 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
880 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
881 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
882 geturls_opener
.addheaders
= httpheaders
;
884 if(urlparts
[0]=="http"):
885 httpconn
= HTTPConnection(urlparts
[1]);
886 elif(urlparts
[0]=="https"):
887 httpconn
= HTTPSConnection(urlparts
[1]);
890 if(postdata
is not None and not isinstance(postdata
, dict)):
891 postdata
= urlencode(postdata
);
893 if(httpmethod
=="GET"):
894 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
895 elif(httpmethod
=="POST"):
896 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
898 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
899 except socket
.timeout
:
900 log
.info("Error With URL "+httpurl
);
902 geturls_text
= httpconn
.getresponse();
903 downloadsize
= dict(geturls_text
.getheaders()).get('Content-Length');
904 if(downloadsize
is not None):
905 downloadsize
= int(downloadsize
);
906 if downloadsize
is None: downloadsize
= 0;
909 log
.info("Downloading URL "+httpurl
);
910 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
911 tmpfilename
= f
.name
;
912 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(dict(geturls_text
.getheaders())), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': httpheaders
, 'URL': httpurl
, 'Code': geturls_text
.status
};
914 databytes
= geturls_text
.read(buffersize
);
915 if not databytes
: break;
916 datasize
= len(databytes
);
917 fulldatasize
= datasize
+ fulldatasize
;
920 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
921 downloaddiff
= fulldatasize
- prevdownsize
;
922 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
923 prevdownsize
= fulldatasize
;
926 geturls_text
.close();
927 exec_time_end
= time
.time();
928 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
929 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
932 def download_from_url_to_file_with_httplib(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
933 global geturls_download_sleep
;
935 sleep
= geturls_download_sleep
;
936 if(not outfile
=="-"):
937 outpath
= outpath
.rstrip(os
.path
.sep
);
938 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
939 if(not os
.path
.exists(outpath
)):
940 os
.makedirs(outpath
);
941 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
943 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
945 pretmpfilename
= download_from_url_file_with_httplib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
946 if(not pretmpfilename
):
948 tmpfilename
= pretmpfilename
['Filename'];
949 downloadsize
= os
.path
.getsize(tmpfilename
);
951 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
952 exec_time_start
= time
.time();
953 shutil
.move(tmpfilename
, filepath
);
954 exec_time_end
= time
.time();
955 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
956 if(os
.path
.exists(tmpfilename
)):
957 os
.remove(tmpfilename
);
958 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
959 if(outfile
=="-" and sys
.version
[0]=="2"):
960 pretmpfilename
= download_from_url_file_with_httplib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
961 if(not pretmpfilename
):
963 tmpfilename
= pretmpfilename
['Filename'];
964 downloadsize
= os
.path
.getsize(tmpfilename
);
967 exec_time_start
= time
.time();
968 with
open(tmpfilename
, 'rb') as ft
:
971 databytes
= ft
.read(buffersize
[1]);
972 if not databytes
: break;
973 datasize
= len(databytes
);
974 fulldatasize
= datasize
+ fulldatasize
;
977 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
978 downloaddiff
= fulldatasize
- prevdownsize
;
979 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
980 prevdownsize
= fulldatasize
;
983 fdata
= f
.getvalue();
986 os
.remove(tmpfilename
);
987 exec_time_end
= time
.time();
988 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
989 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
990 if(outfile
=="-" and sys
.version
[0]>="3"):
991 pretmpfilename
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
992 tmpfilename
= pretmpfilename
['Filename'];
993 downloadsize
= os
.path
.getsize(tmpfilename
);
996 exec_time_start
= time
.time();
997 with
open(tmpfilename
, 'rb') as ft
:
1000 databytes
= ft
.read(buffersize
[1]);
1001 if not databytes
: break;
1002 datasize
= len(databytes
);
1003 fulldatasize
= datasize
+ fulldatasize
;
1006 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1007 downloaddiff
= fulldatasize
- prevdownsize
;
1008 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1009 prevdownsize
= fulldatasize
;
1012 fdata
= f
.getvalue();
1015 os
.remove(tmpfilename
);
1016 exec_time_end
= time
.time();
1017 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1018 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1022 def download_from_url_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
1023 global geturls_download_sleep
, havebrotli
;
1025 sleep
= geturls_download_sleep
;
1026 urlparts
= urlparse
.urlparse(httpurl
);
1027 if(isinstance(httpheaders
, list)):
1028 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1029 if(urlparts
.username
is not None or urlparts
.password
is not None):
1030 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1031 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1032 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
1033 geturls_opener
.addheaders
= httpheaders
;
1035 if(urlparts
[0]=="http"):
1036 httpconn
= HTTPConnectionWithTimeout(urlparts
[1]);
1037 elif(urlparts
[0]=="https"):
1038 httpconn
= HTTPSConnectionWithTimeout(urlparts
[1]);
1041 if(postdata
is not None and not isinstance(postdata
, dict)):
1042 postdata
= urlencode(postdata
);
1044 if(httpmethod
=="GET"):
1045 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1046 elif(httpmethod
=="POST"):
1047 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
1049 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1050 except socket
.timeout
:
1051 log
.info("Error With URL "+httpurl
);
1053 geturls_text
= httpconn
.getresponse();
1054 log
.info("Downloading URL "+httpurl
);
1055 if(dict(geturls_text
.getheaders()).get("Content-Encoding")=="gzip" or dict(geturls_text
.getheaders()).get("Content-Encoding")=="deflate"):
1056 if(sys
.version
[0]=="2"):
1057 strbuf
= StringIO(geturls_text
.read());
1058 if(sys
.version
[0]>="3"):
1059 strbuf
= BytesIO(geturls_text
.read());
1060 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
1061 returnval_content
= gzstrbuf
.read()[:];
1062 if(dict(geturls_text
.getheaders()).get("Content-Encoding")!="gzip" and dict(geturls_text
.getheaders()).get("Content-Encoding")!="deflate"):
1063 returnval_content
= geturls_text
.read()[:];
1064 if(dict(geturls_text
.getheaders()).get("Content-Encoding")=="br" and havebrotli
):
1065 returnval_content
= brotli
.decompress(returnval_content
);
1066 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.getheaders()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': httpheaders
, 'URL': httpurl
, 'Code': geturls_text
.status
};
1067 geturls_text
.close();
1070 if(not havehttplib2
):
1071 def download_from_url_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
1072 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
)
1076 def download_from_url_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
1077 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
1078 exec_time_start
= time
.time();
1079 myhash
= hashlib
.new("sha1");
1080 if(sys
.version
[0]=="2"):
1081 myhash
.update(httpurl
);
1082 myhash
.update(str(buffersize
));
1083 myhash
.update(str(exec_time_start
));
1084 if(sys
.version
[0]>="3"):
1085 myhash
.update(httpurl
.encode('utf-8'));
1086 myhash
.update(str(buffersize
).encode('utf-8'));
1087 myhash
.update(str(exec_time_start
).encode('utf-8'));
1088 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1090 sleep
= geturls_download_sleep
;
1091 urlparts
= urlparse
.urlparse(httpurl
);
1092 if(isinstance(httpheaders
, list)):
1093 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1094 if(urlparts
.username
is not None or urlparts
.password
is not None):
1095 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1096 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1097 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
1098 geturls_opener
.addheaders
= httpheaders
;
1100 if(urlparts
[0]=="http"):
1101 httpconn
= HTTPConnectionWithTimeout(urlparts
[1]);
1102 elif(urlparts
[0]=="https"):
1103 httpconn
= HTTPSConnectionWithTimeout(urlparts
[1]);
1106 if(postdata
is not None and not isinstance(postdata
, dict)):
1107 postdata
= urlencode(postdata
);
1109 if(httpmethod
=="GET"):
1110 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1111 elif(httpmethod
=="POST"):
1112 httpconn
.request("GET", urlparts
[2], body
=postdata
, headers
=httpheaders
);
1114 httpconn
.request("GET", urlparts
[2], headers
=httpheaders
);
1115 except socket
.timeout
:
1116 log
.info("Error With URL "+httpurl
);
1118 geturls_text
= httpconn
.getresponse();
1119 downloadsize
= dict(geturls_text
.getheaders()).get('Content-Length');
1120 if(downloadsize
is not None):
1121 downloadsize
= int(downloadsize
);
1122 if downloadsize
is None: downloadsize
= 0;
1125 log
.info("Downloading URL "+httpurl
);
1126 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1127 tmpfilename
= f
.name
;
1128 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(dict(geturls_text
.getheaders())), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': httpheaders
, 'URL': httpurl
, 'Code': geturls_text
.status
};
1130 databytes
= geturls_text
.read(buffersize
);
1131 if not databytes
: break;
1132 datasize
= len(databytes
);
1133 fulldatasize
= datasize
+ fulldatasize
;
1136 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1137 downloaddiff
= fulldatasize
- prevdownsize
;
1138 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1139 prevdownsize
= fulldatasize
;
1142 geturls_text
.close();
1143 exec_time_end
= time
.time();
1144 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1145 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1148 if(not havehttplib2
):
1149 def download_from_url_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
1150 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
)
1154 def download_from_url_to_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
1155 global geturls_download_sleep
;
1157 sleep
= geturls_download_sleep
;
1158 if(not outfile
=="-"):
1159 outpath
= outpath
.rstrip(os
.path
.sep
);
1160 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1161 if(not os
.path
.exists(outpath
)):
1162 os
.makedirs(outpath
);
1163 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1165 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1167 pretmpfilename
= download_from_url_file_with_httplib2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1168 if(not pretmpfilename
):
1170 tmpfilename
= pretmpfilename
['Filename'];
1171 downloadsize
= os
.path
.getsize(tmpfilename
);
1173 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1174 exec_time_start
= time
.time();
1175 shutil
.move(tmpfilename
, filepath
);
1176 exec_time_end
= time
.time();
1177 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1178 if(os
.path
.exists(tmpfilename
)):
1179 os
.remove(tmpfilename
);
1180 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1181 if(outfile
=="-" and sys
.version
[0]=="2"):
1182 pretmpfilename
= download_from_url_file_with_httplib2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1183 if(not pretmpfilename
):
1185 tmpfilename
= pretmpfilename
['Filename'];
1186 downloadsize
= os
.path
.getsize(tmpfilename
);
1189 exec_time_start
= time
.time();
1190 with
open(tmpfilename
, 'rb') as ft
:
1193 databytes
= ft
.read(buffersize
[1]);
1194 if not databytes
: break;
1195 datasize
= len(databytes
);
1196 fulldatasize
= datasize
+ fulldatasize
;
1199 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1200 downloaddiff
= fulldatasize
- prevdownsize
;
1201 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1202 prevdownsize
= fulldatasize
;
1205 fdata
= f
.getvalue();
1208 os
.remove(tmpfilename
);
1209 exec_time_end
= time
.time();
1210 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1211 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1212 if(outfile
=="-" and sys
.version
[0]>="3"):
1213 pretmpfilename
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1214 tmpfilename
= pretmpfilename
['Filename'];
1215 downloadsize
= os
.path
.getsize(tmpfilename
);
1218 exec_time_start
= time
.time();
1219 with
open(tmpfilename
, 'rb') as ft
:
1222 databytes
= ft
.read(buffersize
[1]);
1223 if not databytes
: break;
1224 datasize
= len(databytes
);
1225 fulldatasize
= datasize
+ fulldatasize
;
1228 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1229 downloaddiff
= fulldatasize
- prevdownsize
;
1230 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1231 prevdownsize
= fulldatasize
;
1234 fdata
= f
.getvalue();
1237 os
.remove(tmpfilename
);
1238 exec_time_end
= time
.time();
1239 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1240 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1243 if(not havehttplib2
):
1244 def download_from_url_to_file_with_httplib2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
1245 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
)
1248 def download_from_url_with_request(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
1249 global geturls_download_sleep
, havebrotli
;
1251 sleep
= geturls_download_sleep
;
1252 urlparts
= urlparse
.urlparse(httpurl
);
1253 if(isinstance(httpheaders
, list)):
1254 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1255 if(urlparts
.username
is not None or urlparts
.password
is not None):
1256 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1257 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1258 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
1259 if(isinstance(httpheaders
, dict)):
1260 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
1261 geturls_opener
.addheaders
= httpheaders
;
1262 urllib
.request
.install_opener(geturls_opener
);
1264 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1265 if(postdata
is not None and not isinstance(postdata
, dict)):
1266 postdata
= urlencode(postdata
);
1268 if(httpmethod
=="GET"):
1269 geturls_request
= Request(httpurl
, headers
=httpheaders
);
1270 geturls_text
= urlopen(geturls_request
);
1271 elif(httpmethod
=="POST"):
1272 geturls_request
= Request(httpurl
, headers
=httpheaders
);
1273 geturls_text
= urlopen(geturls_request
, data
=postdata
);
1275 geturls_request
= Request(httpurl
, headers
=httpheaders
);
1276 geturls_text
= urlopen(geturls_request
);
1277 except HTTPError
as geturls_text_error
:
1278 geturls_text
= geturls_text_error
;
1279 log
.info("Error With URL "+httpurl
);
1281 log
.info("Error With URL "+httpurl
);
1283 except socket
.timeout
:
1284 log
.info("Error With URL "+httpurl
);
1286 log
.info("Downloading URL "+httpurl
);
1287 if(geturls_text
.headers
.get("Content-Encoding")=="gzip" or geturls_text
.headers
.get("Content-Encoding")=="deflate"):
1288 if(sys
.version
[0]=="2"):
1289 strbuf
= StringIO(geturls_text
.read());
1290 if(sys
.version
[0]>="3"):
1291 strbuf
= BytesIO(geturls_text
.read());
1292 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
1293 returnval_content
= gzstrbuf
.read()[:];
1294 if(geturls_text
.headers
.get("Content-Encoding")!="gzip" and geturls_text
.headers
.get("Content-Encoding")!="deflate"):
1295 returnval_content
= geturls_text
.read()[:];
1296 if(geturls_text
.headers
.get("Content-Encoding")=="br" and havebrotli
):
1297 returnval_content
= brotli
.decompress(returnval_content
);
1298 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.headers
), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': geturls_text
.geturl(), 'Code': geturls_text
.getcode()};
1299 geturls_text
.close();
1302 def download_from_url_file_with_request(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
1303 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
1304 exec_time_start
= time
.time();
1305 myhash
= hashlib
.new("sha1");
1306 if(sys
.version
[0]=="2"):
1307 myhash
.update(httpurl
);
1308 myhash
.update(str(buffersize
));
1309 myhash
.update(str(exec_time_start
));
1310 if(sys
.version
[0]>="3"):
1311 myhash
.update(httpurl
.encode('utf-8'));
1312 myhash
.update(str(buffersize
).encode('utf-8'));
1313 myhash
.update(str(exec_time_start
).encode('utf-8'));
1314 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1316 sleep
= geturls_download_sleep
;
1317 urlparts
= urlparse
.urlparse(httpurl
);
1318 if(isinstance(httpheaders
, list)):
1319 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1320 if(urlparts
.username
is not None or urlparts
.password
is not None):
1321 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1322 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1323 geturls_opener
= build_opener(HTTPCookieProcessor(httpcookie
));
1324 if(isinstance(httpheaders
, dict)):
1325 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
1326 geturls_opener
.addheaders
= httpheaders
;
1327 urllib
.request
.install_opener(geturls_opener
);
1329 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1330 if(postdata
is not None and not isinstance(postdata
, dict)):
1331 postdata
= urlencode(postdata
);
1333 if(httpmethod
=="GET"):
1334 geturls_request
= Request(httpurl
, headers
=httpheaders
);
1335 geturls_text
= urlopen(geturls_request
);
1336 elif(httpmethod
=="POST"):
1337 geturls_request
= Request(httpurl
, headers
=httpheaders
);
1338 geturls_text
= urlopen(geturls_request
, data
=postdata
);
1340 geturls_request
= Request(httpurl
, headers
=httpheaders
);
1341 geturls_text
= urlopen(geturls_request
);
1342 except HTTPError
as geturls_text_error
:
1343 geturls_text
= geturls_text_error
;
1344 log
.info("Error With URL "+httpurl
);
1346 log
.info("Error With URL "+httpurl
);
1348 except socket
.timeout
:
1349 log
.info("Error With URL "+httpurl
);
1351 downloadsize
= geturls_text
.headers
.get('Content-Length');
1352 if(downloadsize
is not None):
1353 downloadsize
= int(downloadsize
);
1354 if downloadsize
is None: downloadsize
= 0;
1357 log
.info("Downloading URL "+httpurl
);
1358 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1359 tmpfilename
= f
.name
;
1360 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(geturls_text
.headers
), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': geturls_text
.geturl(), 'Code': geturls_text
.getcode()};
1362 databytes
= geturls_text
.read(buffersize
);
1363 if not databytes
: break;
1364 datasize
= len(databytes
);
1365 fulldatasize
= datasize
+ fulldatasize
;
1368 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1369 downloaddiff
= fulldatasize
- prevdownsize
;
1370 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1371 prevdownsize
= fulldatasize
;
1374 geturls_text
.close();
1375 exec_time_end
= time
.time();
1376 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1377 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1380 def download_from_url_to_file_with_request(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
1381 global geturls_download_sleep
;
1383 sleep
= geturls_download_sleep
;
1384 if(not outfile
=="-"):
1385 outpath
= outpath
.rstrip(os
.path
.sep
);
1386 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1387 if(not os
.path
.exists(outpath
)):
1388 os
.makedirs(outpath
);
1389 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1391 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1393 pretmpfilename
= download_from_url_file_with_request(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1394 if(not pretmpfilename
):
1396 tmpfilename
= pretmpfilename
['Filename'];
1397 downloadsize
= os
.path
.getsize(tmpfilename
);
1399 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1400 exec_time_start
= time
.time();
1401 shutil
.move(tmpfilename
, filepath
);
1402 exec_time_end
= time
.time();
1403 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1404 if(os
.path
.exists(tmpfilename
)):
1405 os
.remove(tmpfilename
);
1406 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent':pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1407 if(outfile
=="-" and sys
.version
[0]=="2"):
1408 pretmpfilename
= download_from_url_file_with_request(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1409 if(not pretmpfilename
):
1411 tmpfilename
= pretmpfilename
['Filename'];
1412 downloadsize
= os
.path
.getsize(tmpfilename
);
1415 exec_time_start
= time
.time();
1416 with
open(tmpfilename
, 'rb') as ft
:
1419 databytes
= ft
.read(buffersize
[1]);
1420 if not databytes
: break;
1421 datasize
= len(databytes
);
1422 fulldatasize
= datasize
+ fulldatasize
;
1425 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1426 downloaddiff
= fulldatasize
- prevdownsize
;
1427 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1428 prevdownsize
= fulldatasize
;
1431 fdata
= f
.getvalue();
1434 os
.remove(tmpfilename
);
1435 exec_time_end
= time
.time();
1436 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1437 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1438 if(outfile
=="-" and sys
.version
[0]>="3"):
1439 pretmpfilename
= download_from_url_file_with_request(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1440 tmpfilename
= pretmpfilename
['Filename'];
1441 downloadsize
= os
.path
.getsize(tmpfilename
);
1444 exec_time_start
= time
.time();
1445 with
open(tmpfilename
, 'rb') as ft
:
1448 databytes
= ft
.read(buffersize
[1]);
1449 if not databytes
: break;
1450 datasize
= len(databytes
);
1451 fulldatasize
= datasize
+ fulldatasize
;
1454 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1455 downloaddiff
= fulldatasize
- prevdownsize
;
1456 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1457 prevdownsize
= fulldatasize
;
1460 fdata
= f
.getvalue();
1463 os
.remove(tmpfilename
);
1464 exec_time_end
= time
.time();
1465 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1466 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1470 def download_from_url_with_requests(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
1471 global geturls_download_sleep
, havebrotli
;
1473 sleep
= geturls_download_sleep
;
1474 urlparts
= urlparse
.urlparse(httpurl
);
1475 if(isinstance(httpheaders
, list)):
1476 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1477 if(urlparts
.username
is not None or urlparts
.password
is not None):
1478 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1479 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1481 if(postdata
is not None and not isinstance(postdata
, dict)):
1482 postdata
= urlencode(postdata
);
1484 if(httpmethod
=="GET"):
1485 geturls_text
= requests
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1486 elif(httpmethod
=="POST"):
1487 geturls_text
= requests
.post(httpurl
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
1489 geturls_text
= requests
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1490 except requests
.exceptions
.ConnectTimeout
:
1491 log
.info("Error With URL "+httpurl
);
1493 except socket
.timeout
:
1494 log
.info("Error With URL "+httpurl
);
1496 log
.info("Downloading URL "+httpurl
);
1497 if(geturls_text
.headers
.get('Content-Type')=="gzip" or geturls_text
.headers
.get('Content-Type')=="deflate"):
1498 if(sys
.version
[0]=="2"):
1499 strbuf
= StringIO(geturls_text
.content
);
1500 if(sys
.version
[0]>="3"):
1501 strbuf
= BytesIO(geturls_text
.content
);
1502 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
1503 returnval_content
= gzstrbuf
.content
[:];
1504 if(geturls_text
.headers
.get('Content-Type')!="gzip" and geturls_text
.headers
.get('Content-Type')!="deflate"):
1505 returnval_content
= geturls_text
.content
[:];
1506 if(geturls_text
.headers
.get("Content-Encoding")=="br" and havebrotli
):
1507 returnval_content
= brotli
.decompress(returnval_content
);
1508 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.headers
), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': geturls_text
.url
, 'Code': geturls_text
.status_code
};
1509 geturls_text
.close();
1512 if(not haverequests
):
1513 def download_from_url_with_requests(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
1514 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
)
1518 def download_from_url_file_with_requests(httpurl
, httpheaders
, httpcookie
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
1519 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
1520 exec_time_start
= time
.time();
1521 myhash
= hashlib
.new("sha1");
1522 if(sys
.version
[0]=="2"):
1523 myhash
.update(httpurl
);
1524 myhash
.update(str(buffersize
));
1525 myhash
.update(str(exec_time_start
));
1526 if(sys
.version
[0]>="3"):
1527 myhash
.update(httpurl
.encode('utf-8'));
1528 myhash
.update(str(buffersize
).encode('utf-8'));
1529 myhash
.update(str(exec_time_start
).encode('utf-8'));
1530 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1532 sleep
= geturls_download_sleep
;
1533 urlparts
= urlparse
.urlparse(httpurl
);
1534 if(isinstance(httpheaders
, list)):
1535 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1536 if(urlparts
.username
is not None or urlparts
.password
is not None):
1537 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1538 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1540 if(postdata
is not None and not isinstance(postdata
, dict)):
1541 postdata
= urlencode(postdata
);
1543 if(httpmethod
=="GET"):
1544 geturls_text
= requests
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1545 elif(httpmethod
=="POST"):
1546 geturls_text
= requests
.post(httpurl
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
1548 geturls_text
= requests
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1549 except requests
.exceptions
.ConnectTimeout
:
1550 log
.info("Error With URL "+httpurl
);
1552 except socket
.timeout
:
1553 log
.info("Error With URL "+httpurl
);
1555 downloadsize
= int(geturls_text
.headers
.get('Content-Length'));
1556 if(downloadsize
is not None):
1557 downloadsize
= int(downloadsize
);
1558 if downloadsize
is None: downloadsize
= 0;
1561 log
.info("Downloading URL "+httpurl
);
1562 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1563 tmpfilename
= f
.name
;
1564 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(geturls_text
.headers
), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': geturls_text
.url
, 'Code': geturls_text
.status_code
};
1565 for databytes
in geturls_text
.iter_content(chunk_size
=buffersize
):
1566 datasize
= len(databytes
);
1567 fulldatasize
= datasize
+ fulldatasize
;
1570 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1571 downloaddiff
= fulldatasize
- prevdownsize
;
1572 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1573 prevdownsize
= fulldatasize
;
1576 geturls_text
.close();
1577 exec_time_end
= time
.time();
1578 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1579 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1582 if(not haverequests
):
1583 def download_from_url_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
1584 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
)
1588 def download_from_url_to_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
1589 global geturls_download_sleep
;
1591 sleep
= geturls_download_sleep
;
1592 if(not outfile
=="-"):
1593 outpath
= outpath
.rstrip(os
.path
.sep
);
1594 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1595 if(not os
.path
.exists(outpath
)):
1596 os
.makedirs(outpath
);
1597 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1599 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1601 pretmpfilename
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1602 if(not pretmpfilename
):
1604 tmpfilename
= pretmpfilename
['Filename'];
1605 downloadsize
= os
.path
.getsize(tmpfilename
);
1607 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1608 exec_time_start
= time
.time();
1609 shutil
.move(tmpfilename
, filepath
);
1610 exec_time_end
= time
.time();
1611 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1612 if(os
.path
.exists(tmpfilename
)):
1613 os
.remove(tmpfilename
);
1614 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1615 if(outfile
=="-" and sys
.version
[0]=="2"):
1616 pretmpfilename
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1617 if(not pretmpfilename
):
1619 tmpfilename
= pretmpfilename
['Filename'];
1620 downloadsize
= os
.path
.getsize(tmpfilename
);
1623 exec_time_start
= time
.time();
1624 with
open(tmpfilename
, 'rb') as ft
:
1627 databytes
= ft
.read(buffersize
[1]);
1628 if not databytes
: break;
1629 datasize
= len(databytes
);
1630 fulldatasize
= datasize
+ fulldatasize
;
1633 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1634 downloaddiff
= fulldatasize
- prevdownsize
;
1635 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1636 prevdownsize
= fulldatasize
;
1639 fdata
= f
.getvalue();
1642 os
.remove(tmpfilename
);
1643 exec_time_end
= time
.time();
1644 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1645 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1646 if(outfile
=="-" and sys
.version
[0]>="3"):
1647 pretmpfilename
= download_from_url_file_with_requests(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1648 tmpfilename
= pretmpfilename
['Filename'];
1649 downloadsize
= os
.path
.getsize(tmpfilename
);
1652 exec_time_start
= time
.time();
1653 with
open(tmpfilename
, 'rb') as ft
:
1656 databytes
= ft
.read(buffersize
[1]);
1657 if not databytes
: break;
1658 datasize
= len(databytes
);
1659 fulldatasize
= datasize
+ fulldatasize
;
1662 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1663 downloaddiff
= fulldatasize
- prevdownsize
;
1664 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1665 prevdownsize
= fulldatasize
;
1668 fdata
= f
.getvalue();
1671 os
.remove(tmpfilename
);
1672 exec_time_end
= time
.time();
1673 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1674 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1677 if(not haverequests
):
1678 def download_from_url_to_file_with_requests(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
1679 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
)
1683 def download_from_url_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
1684 global geturls_download_sleep
, havebrotli
;
1686 sleep
= geturls_download_sleep
;
1687 urlparts
= urlparse
.urlparse(httpurl
);
1688 if(isinstance(httpheaders
, list)):
1689 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1690 if(urlparts
.username
is not None or urlparts
.password
is not None):
1691 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1692 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1694 if(postdata
is not None and not isinstance(postdata
, dict)):
1695 postdata
= urlencode(postdata
);
1697 if(httpmethod
=="GET"):
1698 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
1699 geturls_text
= httpx_pool
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1700 elif(httpmethod
=="POST"):
1701 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
1702 geturls_text
= httpx_pool
.post(httpurl
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
1704 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
1705 geturls_text
= httpx_pool
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1706 except httpx
.ConnectTimeout
:
1707 log
.info("Error With URL "+httpurl
);
1709 except socket
.timeout
:
1710 log
.info("Error With URL "+httpurl
);
1712 log
.info("Downloading URL "+httpurl
);
1713 if(geturls_text
.headers
.get('Content-Type')=="gzip" or geturls_text
.headers
.get('Content-Type')=="deflate"):
1714 if(sys
.version
[0]=="2"):
1715 strbuf
= StringIO(geturls_text
.content
);
1716 if(sys
.version
[0]>="3"):
1717 strbuf
= BytesIO(geturls_text
.content
);
1718 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
1719 returnval_content
= gzstrbuf
.content
[:];
1720 if(geturls_text
.headers
.get('Content-Type')!="gzip" and geturls_text
.headers
.get('Content-Type')!="deflate"):
1721 returnval_content
= geturls_text
.content
[:];
1722 if(geturls_text
.headers
.get("Content-Encoding")=="br" and havebrotli
):
1723 returnval_content
= brotli
.decompress(returnval_content
);
1724 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.headers
), 'Version': geturls_text
.http_version
, 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': str(geturls_text
.url
), 'Code': geturls_text
.status_code
};
1725 geturls_text
.close();
1729 def download_from_url_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
1730 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
)
1734 def download_from_url_file_with_httpx(httpurl
, httpheaders
, httpcookie
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
1735 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
1736 exec_time_start
= time
.time();
1737 myhash
= hashlib
.new("sha1");
1738 if(sys
.version
[0]=="2"):
1739 myhash
.update(httpurl
);
1740 myhash
.update(str(buffersize
));
1741 myhash
.update(str(exec_time_start
));
1742 if(sys
.version
[0]>="3"):
1743 myhash
.update(httpurl
.encode('utf-8'));
1744 myhash
.update(str(buffersize
).encode('utf-8'));
1745 myhash
.update(str(exec_time_start
).encode('utf-8'));
1746 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1748 sleep
= geturls_download_sleep
;
1749 urlparts
= urlparse
.urlparse(httpurl
);
1750 if(isinstance(httpheaders
, list)):
1751 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1752 if(urlparts
.username
is not None or urlparts
.password
is not None):
1753 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1754 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1756 if(postdata
is not None and not isinstance(postdata
, dict)):
1757 postdata
= urlencode(postdata
);
1759 if(httpmethod
=="GET"):
1760 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
1761 geturls_text
= httpx_pool
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1762 elif(httpmethod
=="POST"):
1763 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
1764 geturls_text
= httpx_pool
.post(httpurl
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
1766 httpx_pool
= httpx
.Client(http1
=True, http2
=False, trust_env
=True);
1767 geturls_text
= httpx_pool
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1768 except httpx
.ConnectTimeout
:
1769 log
.info("Error With URL "+httpurl
);
1771 except socket
.timeout
:
1772 log
.info("Error With URL "+httpurl
);
1774 downloadsize
= int(geturls_text
.headers
.get('Content-Length'));
1775 if(downloadsize
is not None):
1776 downloadsize
= int(downloadsize
);
1777 if downloadsize
is None: downloadsize
= 0;
1780 log
.info("Downloading URL "+httpurl
);
1781 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
1782 tmpfilename
= f
.name
;
1783 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(geturls_text
.headers
), 'Version': geturls_text
.http_version
, 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': str(geturls_text
.url
), 'Code': geturls_text
.status_code
};
1784 for databytes
in geturls_text
.iter_content(chunk_size
=buffersize
):
1785 datasize
= len(databytes
);
1786 fulldatasize
= datasize
+ fulldatasize
;
1789 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1790 downloaddiff
= fulldatasize
- prevdownsize
;
1791 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1792 prevdownsize
= fulldatasize
;
1795 geturls_text
.close();
1796 exec_time_end
= time
.time();
1797 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
1798 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
1802 def download_from_url_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
1803 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
)
1807 def download_from_url_to_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
1808 global geturls_download_sleep
;
1810 sleep
= geturls_download_sleep
;
1811 if(not outfile
=="-"):
1812 outpath
= outpath
.rstrip(os
.path
.sep
);
1813 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
1814 if(not os
.path
.exists(outpath
)):
1815 os
.makedirs(outpath
);
1816 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
1818 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
1820 pretmpfilename
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1821 if(not pretmpfilename
):
1823 tmpfilename
= pretmpfilename
['Filename'];
1824 downloadsize
= os
.path
.getsize(tmpfilename
);
1826 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
1827 exec_time_start
= time
.time();
1828 shutil
.move(tmpfilename
, filepath
);
1829 exec_time_end
= time
.time();
1830 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
1831 if(os
.path
.exists(tmpfilename
)):
1832 os
.remove(tmpfilename
);
1833 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1834 if(outfile
=="-" and sys
.version
[0]=="2"):
1835 pretmpfilename
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1836 if(not pretmpfilename
):
1838 tmpfilename
= pretmpfilename
['Filename'];
1839 downloadsize
= os
.path
.getsize(tmpfilename
);
1842 exec_time_start
= time
.time();
1843 with
open(tmpfilename
, 'rb') as ft
:
1846 databytes
= ft
.read(buffersize
[1]);
1847 if not databytes
: break;
1848 datasize
= len(databytes
);
1849 fulldatasize
= datasize
+ fulldatasize
;
1852 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1853 downloaddiff
= fulldatasize
- prevdownsize
;
1854 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1855 prevdownsize
= fulldatasize
;
1858 fdata
= f
.getvalue();
1861 os
.remove(tmpfilename
);
1862 exec_time_end
= time
.time();
1863 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1864 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1865 if(outfile
=="-" and sys
.version
[0]>="3"):
1866 pretmpfilename
= download_from_url_file_with_httpx(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
1867 tmpfilename
= pretmpfilename
['Filename'];
1868 downloadsize
= os
.path
.getsize(tmpfilename
);
1871 exec_time_start
= time
.time();
1872 with
open(tmpfilename
, 'rb') as ft
:
1875 databytes
= ft
.read(buffersize
[1]);
1876 if not databytes
: break;
1877 datasize
= len(databytes
);
1878 fulldatasize
= datasize
+ fulldatasize
;
1881 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
1882 downloaddiff
= fulldatasize
- prevdownsize
;
1883 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
1884 prevdownsize
= fulldatasize
;
1887 fdata
= f
.getvalue();
1890 os
.remove(tmpfilename
);
1891 exec_time_end
= time
.time();
1892 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
1893 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
1897 def download_from_url_to_file_with_httpx(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
1898 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
)
1902 def download_from_url_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
1903 global geturls_download_sleep
, havebrotli
;
1905 sleep
= geturls_download_sleep
;
1906 urlparts
= urlparse
.urlparse(httpurl
);
1907 if(isinstance(httpheaders
, list)):
1908 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1909 if(urlparts
.username
is not None or urlparts
.password
is not None):
1910 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1911 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1913 if(postdata
is not None and not isinstance(postdata
, dict)):
1914 postdata
= urlencode(postdata
);
1916 if(httpmethod
=="GET"):
1917 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
1918 geturls_text
= httpx_pool
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1919 elif(httpmethod
=="POST"):
1920 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
1921 geturls_text
= httpx_pool
.post(httpurl
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
1923 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
1924 geturls_text
= httpx_pool
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1925 except httpx
.ConnectTimeout
:
1926 log
.info("Error With URL "+httpurl
);
1928 except socket
.timeout
:
1929 log
.info("Error With URL "+httpurl
);
1931 log
.info("Downloading URL "+httpurl
);
1932 if(geturls_text
.headers
.get('Content-Type')=="gzip" or geturls_text
.headers
.get('Content-Type')=="deflate"):
1933 if(sys
.version
[0]=="2"):
1934 strbuf
= StringIO(geturls_text
.content
);
1935 if(sys
.version
[0]>="3"):
1936 strbuf
= BytesIO(geturls_text
.content
);
1937 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
1938 returnval_content
= gzstrbuf
.content
[:];
1939 if(geturls_text
.headers
.get('Content-Type')!="gzip" and geturls_text
.headers
.get('Content-Type')!="deflate"):
1940 returnval_content
= geturls_text
.content
[:];
1941 if(geturls_text
.headers
.get("Content-Encoding")=="br" and havebrotli
):
1942 returnval_content
= brotli
.decompress(returnval_content
);
1943 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.headers
), 'Version': geturls_text
.http_version
, 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': str(geturls_text
.url
), 'Code': geturls_text
.status_code
};
1944 geturls_text
.close();
1948 def download_from_url_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
1949 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
)
1953 def download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpcookie
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
1954 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
1955 exec_time_start
= time
.time();
1956 myhash
= hashlib
.new("sha1");
1957 if(sys
.version
[0]=="2"):
1958 myhash
.update(httpurl
);
1959 myhash
.update(str(buffersize
));
1960 myhash
.update(str(exec_time_start
));
1961 if(sys
.version
[0]>="3"):
1962 myhash
.update(httpurl
.encode('utf-8'));
1963 myhash
.update(str(buffersize
).encode('utf-8'));
1964 myhash
.update(str(exec_time_start
).encode('utf-8'));
1965 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
1967 sleep
= geturls_download_sleep
;
1968 urlparts
= urlparse
.urlparse(httpurl
);
1969 if(isinstance(httpheaders
, list)):
1970 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
1971 if(urlparts
.username
is not None or urlparts
.password
is not None):
1972 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
1973 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
1975 if(postdata
is not None and not isinstance(postdata
, dict)):
1976 postdata
= urlencode(postdata
);
1978 if(httpmethod
=="GET"):
1979 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
1980 geturls_text
= httpx_pool
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1981 elif(httpmethod
=="POST"):
1982 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
1983 geturls_text
= httpx_pool
.post(httpurl
, data
=postdata
, headers
=httpheaders
, cookies
=httpcookie
);
1985 httpx_pool
= httpx
.Client(http1
=True, http2
=True, trust_env
=True);
1986 geturls_text
= httpx_pool
.get(httpurl
, headers
=httpheaders
, cookies
=httpcookie
);
1987 except httpx
.ConnectTimeout
:
1988 log
.info("Error With URL "+httpurl
);
1990 except socket
.timeout
:
1991 log
.info("Error With URL "+httpurl
);
1993 downloadsize
= int(geturls_text
.headers
.get('Content-Length'));
1994 if(downloadsize
is not None):
1995 downloadsize
= int(downloadsize
);
1996 if downloadsize
is None: downloadsize
= 0;
1999 log
.info("Downloading URL "+httpurl
);
2000 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2001 tmpfilename
= f
.name
;
2002 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(geturls_text
.headers
), 'Version': geturls_text
.http_version
, 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': str(geturls_text
.url
), 'Code': geturls_text
.status_code
};
2003 for databytes
in geturls_text
.iter_content(chunk_size
=buffersize
):
2004 datasize
= len(databytes
);
2005 fulldatasize
= datasize
+ fulldatasize
;
2008 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2009 downloaddiff
= fulldatasize
- prevdownsize
;
2010 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2011 prevdownsize
= fulldatasize
;
2014 geturls_text
.close();
2015 exec_time_end
= time
.time();
2016 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2017 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2021 def download_from_url_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
2022 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
)
2026 def download_from_url_to_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
2027 global geturls_download_sleep
;
2029 sleep
= geturls_download_sleep
;
2030 if(not outfile
=="-"):
2031 outpath
= outpath
.rstrip(os
.path
.sep
);
2032 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2033 if(not os
.path
.exists(outpath
)):
2034 os
.makedirs(outpath
);
2035 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2037 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2039 pretmpfilename
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2040 if(not pretmpfilename
):
2042 tmpfilename
= pretmpfilename
['Filename'];
2043 downloadsize
= os
.path
.getsize(tmpfilename
);
2045 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2046 exec_time_start
= time
.time();
2047 shutil
.move(tmpfilename
, filepath
);
2048 exec_time_end
= time
.time();
2049 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2050 if(os
.path
.exists(tmpfilename
)):
2051 os
.remove(tmpfilename
);
2052 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2053 if(outfile
=="-" and sys
.version
[0]=="2"):
2054 pretmpfilename
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2055 if(not pretmpfilename
):
2057 tmpfilename
= pretmpfilename
['Filename'];
2058 downloadsize
= os
.path
.getsize(tmpfilename
);
2061 exec_time_start
= time
.time();
2062 with
open(tmpfilename
, 'rb') as ft
:
2065 databytes
= ft
.read(buffersize
[1]);
2066 if not databytes
: break;
2067 datasize
= len(databytes
);
2068 fulldatasize
= datasize
+ fulldatasize
;
2071 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2072 downloaddiff
= fulldatasize
- prevdownsize
;
2073 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2074 prevdownsize
= fulldatasize
;
2077 fdata
= f
.getvalue();
2080 os
.remove(tmpfilename
);
2081 exec_time_end
= time
.time();
2082 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2083 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2084 if(outfile
=="-" and sys
.version
[0]>="3"):
2085 pretmpfilename
= download_from_url_file_with_httpx2(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2086 tmpfilename
= pretmpfilename
['Filename'];
2087 downloadsize
= os
.path
.getsize(tmpfilename
);
2090 exec_time_start
= time
.time();
2091 with
open(tmpfilename
, 'rb') as ft
:
2094 databytes
= ft
.read(buffersize
[1]);
2095 if not databytes
: break;
2096 datasize
= len(databytes
);
2097 fulldatasize
= datasize
+ fulldatasize
;
2100 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2101 downloaddiff
= fulldatasize
- prevdownsize
;
2102 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2103 prevdownsize
= fulldatasize
;
2106 fdata
= f
.getvalue();
2109 os
.remove(tmpfilename
);
2110 exec_time_end
= time
.time();
2111 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2112 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2116 def download_from_url_to_file_with_httpx2(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
2117 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
)
2121 def download_from_url_with_request3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
2122 global geturls_download_sleep
, havebrotli
;
2124 sleep
= geturls_download_sleep
;
2125 urlparts
= urlparse
.urlparse(httpurl
);
2126 if(isinstance(httpheaders
, list)):
2127 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2128 if(urlparts
.username
is not None or urlparts
.password
is not None):
2129 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2130 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2132 urllib_pool
= urllib3
.PoolManager(headers
=httpheaders
);
2133 if(postdata
is not None and not isinstance(postdata
, dict)):
2134 postdata
= urlencode(postdata
);
2136 if(httpmethod
=="GET"):
2137 geturls_text
= geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2138 elif(httpmethod
=="POST"):
2139 geturls_text
= geturls_text
= urllib_pool
.request("POST", httpurl
, body
=postdata
, headers
=httpheaders
, preload_content
=False);
2141 geturls_text
= geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2142 except urllib3
.exceptions
.ConnectTimeoutError
:
2143 log
.info("Error With URL "+httpurl
);
2145 except urllib3
.exceptions
.MaxRetryError
:
2146 log
.info("Error With URL "+httpurl
);
2148 except socket
.timeout
:
2149 log
.info("Error With URL "+httpurl
);
2151 log
.info("Downloading URL "+httpurl
);
2152 if(geturls_text
.info().get("Content-Encoding")=="gzip" or geturls_text
.info().get("Content-Encoding")=="deflate"):
2153 if(sys
.version
[0]=="2"):
2154 strbuf
= StringIO(geturls_text
.read());
2155 if(sys
.version
[0]>="3"):
2156 strbuf
= BytesIO(geturls_text
.read());
2157 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
2158 returnval_content
= gzstrbuf
.read()[:];
2159 if(geturls_text
.info().get("Content-Encoding")!="gzip" and geturls_text
.info().get("Content-Encoding")!="deflate"):
2160 returnval_content
= geturls_text
.read()[:];
2161 if(geturls_text
.info().get("Content-Encoding")=="br" and havebrotli
):
2162 returnval_content
= brotli
.decompress(returnval_content
);
2163 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.info()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': httpheaders
, 'URL': geturls_text
.geturl(), 'Code': geturls_text
.status
};
2164 geturls_text
.close();
2167 if(not haveurllib3
):
2168 def download_from_url_with_request3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
2169 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
)
2173 def download_from_url_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
2174 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
2175 exec_time_start
= time
.time();
2176 myhash
= hashlib
.new("sha1");
2177 if(sys
.version
[0]=="2"):
2178 myhash
.update(httpurl
);
2179 myhash
.update(str(buffersize
));
2180 myhash
.update(str(exec_time_start
));
2181 if(sys
.version
[0]>="3"):
2182 myhash
.update(httpurl
.encode('utf-8'));
2183 myhash
.update(str(buffersize
).encode('utf-8'));
2184 myhash
.update(str(exec_time_start
).encode('utf-8'));
2185 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2187 sleep
= geturls_download_sleep
;
2188 urlparts
= urlparse
.urlparse(httpurl
);
2189 if(isinstance(httpheaders
, list)):
2190 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2191 if(urlparts
.username
is not None or urlparts
.password
is not None):
2192 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2193 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2195 urllib_pool
= urllib3
.PoolManager(headers
=httpheaders
);
2196 if(postdata
is not None and not isinstance(postdata
, dict)):
2197 postdata
= urlencode(postdata
);
2199 if(httpmethod
=="GET"):
2200 geturls_text
= geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2201 elif(httpmethod
=="POST"):
2202 geturls_text
= geturls_text
= urllib_pool
.request("POST", httpurl
, body
=postdata
, headers
=httpheaders
, preload_content
=False);
2204 geturls_text
= geturls_text
= urllib_pool
.request("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2205 except urllib3
.exceptions
.ConnectTimeoutError
:
2206 log
.info("Error With URL "+httpurl
);
2208 except urllib3
.exceptions
.MaxRetryError
:
2209 log
.info("Error With URL "+httpurl
);
2211 except socket
.timeout
:
2212 log
.info("Error With URL "+httpurl
);
2214 downloadsize
= int(geturls_text
.headers
.get('Content-Length'));
2215 if(downloadsize
is not None):
2216 downloadsize
= int(downloadsize
);
2217 if downloadsize
is None: downloadsize
= 0;
2220 log
.info("Downloading URL "+httpurl
);
2221 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2222 tmpfilename
= f
.name
;
2223 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(geturls_text
.info()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': httpheaders
, 'URL': geturls_text
.geturl(), 'Code': geturls_text
.status
};
2225 databytes
= geturls_text
.read(buffersize
);
2226 if not databytes
: break;
2227 datasize
= len(databytes
);
2228 fulldatasize
= datasize
+ fulldatasize
;
2231 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2232 downloaddiff
= fulldatasize
- prevdownsize
;
2233 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2234 prevdownsize
= fulldatasize
;
2237 geturls_text
.close();
2238 exec_time_end
= time
.time();
2239 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2240 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2243 if(not haveurllib3
):
2244 def download_from_url_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
2245 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
)
2249 def download_from_url_to_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
2250 global geturls_download_sleep
;
2252 sleep
= geturls_download_sleep
;
2253 if(not outfile
=="-"):
2254 outpath
= outpath
.rstrip(os
.path
.sep
);
2255 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2256 if(not os
.path
.exists(outpath
)):
2257 os
.makedirs(outpath
);
2258 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2260 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2262 pretmpfilename
= download_from_url_file_with_request3(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2263 if(not pretmpfilename
):
2265 tmpfilename
= pretmpfilename
['Filename'];
2266 downloadsize
= os
.path
.getsize(tmpfilename
);
2268 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2269 exec_time_start
= time
.time();
2270 shutil
.move(tmpfilename
, filepath
);
2271 exec_time_end
= time
.time();
2272 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2273 if(os
.path
.exists(tmpfilename
)):
2274 os
.remove(tmpfilename
);
2275 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2276 if(outfile
=="-" and sys
.version
[0]=="2"):
2277 pretmpfilename
= download_from_url_file_with_request3(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2278 if(not pretmpfilename
):
2280 tmpfilename
= pretmpfilename
['Filename'];
2281 downloadsize
= os
.path
.getsize(tmpfilename
);
2284 exec_time_start
= time
.time();
2285 with
open(tmpfilename
, 'rb') as ft
:
2288 databytes
= ft
.read(buffersize
[1]);
2289 if not databytes
: break;
2290 datasize
= len(databytes
);
2291 fulldatasize
= datasize
+ fulldatasize
;
2294 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2295 downloaddiff
= fulldatasize
- prevdownsize
;
2296 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2297 prevdownsize
= fulldatasize
;
2300 fdata
= f
.getvalue();
2303 os
.remove(tmpfilename
);
2304 exec_time_end
= time
.time();
2305 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2306 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2307 if(outfile
=="-" and sys
.version
[0]>="3"):
2308 pretmpfilename
= download_from_url_file_with_request3(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2309 tmpfilename
= pretmpfilename
['Filename'];
2310 downloadsize
= os
.path
.getsize(tmpfilename
);
2313 exec_time_start
= time
.time();
2314 with
open(tmpfilename
, 'rb') as ft
:
2317 databytes
= ft
.read(buffersize
[1]);
2318 if not databytes
: break;
2319 datasize
= len(databytes
);
2320 fulldatasize
= datasize
+ fulldatasize
;
2323 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2324 downloaddiff
= fulldatasize
- prevdownsize
;
2325 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2326 prevdownsize
= fulldatasize
;
2329 fdata
= f
.getvalue();
2332 os
.remove(tmpfilename
);
2333 exec_time_end
= time
.time();
2334 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2335 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2338 if(not haveurllib3
):
2339 def download_from_url_to_file_with_request3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
2340 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
)
2344 def download_from_url_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
2345 global geturls_download_sleep
, havebrotli
;
2347 sleep
= geturls_download_sleep
;
2348 urlparts
= urlparse
.urlparse(httpurl
);
2349 if(isinstance(httpheaders
, list)):
2350 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2351 if(urlparts
.username
is not None or urlparts
.password
is not None):
2352 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2353 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2355 urllib_pool
= urllib3
.PoolManager(headers
=httpheaders
);
2356 if(postdata
is not None and not isinstance(postdata
, dict)):
2357 postdata
= urlencode(postdata
);
2359 if(httpmethod
=="GET"):
2360 geturls_text
= urllib_pool
.urlopen("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2361 elif(httpmethod
=="POST"):
2362 geturls_text
= urllib_pool
.urlopen("GET", httpurl
, body
=postdata
, headers
=httpheaders
, preload_content
=False);
2364 geturls_text
= urllib_pool
.urlopen("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2365 except urllib3
.exceptions
.ConnectTimeoutError
:
2366 log
.info("Error With URL "+httpurl
);
2368 except urllib3
.exceptions
.MaxRetryError
:
2369 log
.info("Error With URL "+httpurl
);
2371 except socket
.timeout
:
2372 log
.info("Error With URL "+httpurl
);
2374 log
.info("Downloading URL "+httpurl
);
2375 if(geturls_text
.info().get("Content-Encoding")=="gzip" or geturls_text
.info().get("Content-Encoding")=="deflate"):
2376 if(sys
.version
[0]=="2"):
2377 strbuf
= StringIO(geturls_text
.read());
2378 if(sys
.version
[0]>="3"):
2379 strbuf
= BytesIO(geturls_text
.read());
2380 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
2381 returnval_content
= gzstrbuf
.read()[:];
2382 if(geturls_text
.info().get("Content-Encoding")!="gzip" and geturls_text
.info().get("Content-Encoding")!="deflate"):
2383 returnval_content
= geturls_text
.read()[:];
2384 if(geturls_text
.info().get("Content-Encoding")=="br" and havebrotli
):
2385 returnval_content
= brotli
.decompress(returnval_content
);
2386 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.info()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': httpheaders
, 'URL': geturls_text
.geturl(), 'Code': geturls_text
.status
};
2387 geturls_text
.close();
2390 if(not haveurllib3
):
2391 def download_from_url_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
2392 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
)
2396 def download_from_url_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
2397 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
2398 exec_time_start
= time
.time();
2399 myhash
= hashlib
.new("sha1");
2400 if(sys
.version
[0]=="2"):
2401 myhash
.update(httpurl
);
2402 myhash
.update(str(buffersize
));
2403 myhash
.update(str(exec_time_start
));
2404 if(sys
.version
[0]>="3"):
2405 myhash
.update(httpurl
.encode('utf-8'));
2406 myhash
.update(str(buffersize
).encode('utf-8'));
2407 myhash
.update(str(exec_time_start
).encode('utf-8'));
2408 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2410 sleep
= geturls_download_sleep
;
2411 urlparts
= urlparse
.urlparse(httpurl
);
2412 if(isinstance(httpheaders
, list)):
2413 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2414 if(urlparts
.username
is not None or urlparts
.password
is not None):
2415 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2416 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2418 urllib_pool
= urllib3
.PoolManager(headers
=httpheaders
);
2419 if(postdata
is not None and not isinstance(postdata
, dict)):
2420 postdata
= urlencode(postdata
);
2422 if(httpmethod
=="GET"):
2423 geturls_text
= urllib_pool
.urlopen("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2424 elif(httpmethod
=="POST"):
2425 geturls_text
= urllib_pool
.urlopen("GET", httpurl
, body
=postdata
, headers
=httpheaders
, preload_content
=False);
2427 geturls_text
= urllib_pool
.urlopen("GET", httpurl
, headers
=httpheaders
, preload_content
=False);
2428 except urllib3
.exceptions
.ConnectTimeoutError
:
2429 log
.info("Error With URL "+httpurl
);
2431 except urllib3
.exceptions
.MaxRetryError
:
2432 log
.info("Error With URL "+httpurl
);
2434 except socket
.timeout
:
2435 log
.info("Error With URL "+httpurl
);
2437 downloadsize
= int(geturls_text
.headers
.get('Content-Length'));
2438 if(downloadsize
is not None):
2439 downloadsize
= int(downloadsize
);
2440 if downloadsize
is None: downloadsize
= 0;
2443 log
.info("Downloading URL "+httpurl
);
2444 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2445 tmpfilename
= f
.name
;
2446 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(geturls_text
.info()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': httpheaders
, 'URL': geturls_text
.geturl(), 'Code': geturls_text
.status
};
2448 databytes
= geturls_text
.read(buffersize
);
2449 if not databytes
: break;
2450 datasize
= len(databytes
);
2451 fulldatasize
= datasize
+ fulldatasize
;
2454 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2455 downloaddiff
= fulldatasize
- prevdownsize
;
2456 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2457 prevdownsize
= fulldatasize
;
2460 geturls_text
.close();
2461 exec_time_end
= time
.time();
2462 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2463 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2466 if(not haveurllib3
):
2467 def download_from_url_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
2468 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
)
2472 def download_from_url_to_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
2473 global geturls_download_sleep
;
2475 sleep
= geturls_download_sleep
;
2476 if(not outfile
=="-"):
2477 outpath
= outpath
.rstrip(os
.path
.sep
);
2478 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2479 if(not os
.path
.exists(outpath
)):
2480 os
.makedirs(outpath
);
2481 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2483 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2485 pretmpfilename
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2486 if(not pretmpfilename
):
2488 tmpfilename
= pretmpfilename
['Filename'];
2489 downloadsize
= os
.path
.getsize(tmpfilename
);
2491 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2492 exec_time_start
= time
.time();
2493 shutil
.move(tmpfilename
, filepath
);
2494 exec_time_end
= time
.time();
2495 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2496 if(os
.path
.exists(tmpfilename
)):
2497 os
.remove(tmpfilename
);
2498 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2499 if(outfile
=="-" and sys
.version
[0]=="2"):
2500 pretmpfilename
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2501 if(not pretmpfilename
):
2503 tmpfilename
= pretmpfilename
['Filename'];
2504 downloadsize
= os
.path
.getsize(tmpfilename
);
2507 exec_time_start
= time
.time();
2508 with
open(tmpfilename
, 'rb') as ft
:
2511 databytes
= ft
.read(buffersize
[1]);
2512 if not databytes
: break;
2513 datasize
= len(databytes
);
2514 fulldatasize
= datasize
+ fulldatasize
;
2517 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2518 downloaddiff
= fulldatasize
- prevdownsize
;
2519 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2520 prevdownsize
= fulldatasize
;
2523 fdata
= f
.getvalue();
2526 os
.remove(tmpfilename
);
2527 exec_time_end
= time
.time();
2528 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2529 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2530 if(outfile
=="-" and sys
.version
[0]>="3"):
2531 pretmpfilename
= download_from_url_file_with_urllib3(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2532 tmpfilename
= pretmpfilename
['Filename'];
2533 downloadsize
= os
.path
.getsize(tmpfilename
);
2536 exec_time_start
= time
.time();
2537 with
open(tmpfilename
, 'rb') as ft
:
2540 databytes
= ft
.read(buffersize
[1]);
2541 if not databytes
: break;
2542 datasize
= len(databytes
);
2543 fulldatasize
= datasize
+ fulldatasize
;
2546 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2547 downloaddiff
= fulldatasize
- prevdownsize
;
2548 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2549 prevdownsize
= fulldatasize
;
2552 fdata
= f
.getvalue();
2555 os
.remove(tmpfilename
);
2556 exec_time_end
= time
.time();
2557 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2558 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2561 if(not haveurllib3
):
2562 def download_from_url_to_file_with_urllib3(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
2563 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
)
2567 def download_from_url_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
2568 global geturls_download_sleep
, havebrotli
;
2570 sleep
= geturls_download_sleep
;
2571 urlparts
= urlparse
.urlparse(httpurl
);
2572 if(isinstance(httpheaders
, list)):
2573 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2574 if(urlparts
.username
is not None or urlparts
.password
is not None):
2575 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2576 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2577 geturls_opener
= mechanize
.Browser();
2578 if(isinstance(httpheaders
, dict)):
2579 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
2581 geturls_opener
.addheaders
= httpheaders
;
2582 geturls_opener
.set_cookiejar(httpcookie
);
2583 geturls_opener
.set_handle_robots(False);
2584 if(postdata
is not None and not isinstance(postdata
, dict)):
2585 postdata
= urlencode(postdata
);
2587 if(httpmethod
=="GET"):
2588 geturls_text
= geturls_opener
.open(httpurl
);
2589 elif(httpmethod
=="POST"):
2590 geturls_text
= geturls_opener
.open(httpurl
, data
=postdata
);
2592 geturls_text
= geturls_opener
.open(httpurl
);
2593 except mechanize
.HTTPError
as geturls_text_error
:
2594 geturls_text
= geturls_text_error
;
2595 log
.info("Error With URL "+httpurl
);
2597 log
.info("Error With URL "+httpurl
);
2599 except socket
.timeout
:
2600 log
.info("Error With URL "+httpurl
);
2602 log
.info("Downloading URL "+httpurl
);
2603 if(geturls_text
.info().get("Content-Encoding")=="gzip" or geturls_text
.info().get("Content-Encoding")=="deflate"):
2604 if(sys
.version
[0]=="2"):
2605 strbuf
= StringIO(geturls_text
.read());
2606 if(sys
.version
[0]>="3"):
2607 strbuf
= BytesIO(geturls_text
.read());
2608 gzstrbuf
= gzip
.GzipFile(fileobj
=strbuf
);
2609 returnval_content
= gzstrbuf
.read()[:];
2610 if(geturls_text
.info().get("Content-Encoding")!="gzip" and geturls_text
.info().get("Content-Encoding")!="deflate"):
2611 returnval_content
= geturls_text
.read()[:];
2612 if(geturls_text
.info().get("Content-Encoding")=="br" and havebrotli
):
2613 returnval_content
= brotli
.decompress(returnval_content
);
2614 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': dict(geturls_text
.info()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': geturls_text
.geturl(), 'Code': geturls_text
.code
};
2615 geturls_text
.close();
2618 if(not havemechanize
):
2619 def download_from_url_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
2620 returnval
= download_from_url_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, sleep
)
2624 def download_from_url_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
2625 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
2626 exec_time_start
= time
.time();
2627 myhash
= hashlib
.new("sha1");
2628 if(sys
.version
[0]=="2"):
2629 myhash
.update(httpurl
);
2630 myhash
.update(str(buffersize
));
2631 myhash
.update(str(exec_time_start
));
2632 if(sys
.version
[0]>="3"):
2633 myhash
.update(httpurl
.encode('utf-8'));
2634 myhash
.update(str(buffersize
).encode('utf-8'));
2635 myhash
.update(str(exec_time_start
).encode('utf-8'));
2636 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2638 sleep
= geturls_download_sleep
;
2639 urlparts
= urlparse
.urlparse(httpurl
);
2640 if(isinstance(httpheaders
, list)):
2641 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2642 if(urlparts
.username
is not None or urlparts
.password
is not None):
2643 inurlencode
= b64encode(str(urlparts
.username
+":"+urlparts
.password
).encode()).decode("UTF-8");
2644 httpheaders
.update( { 'Authorization': "Basic "+inurlencode
} );
2645 geturls_opener
= mechanize
.Browser();
2646 if(isinstance(httpheaders
, dict)):
2647 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
2649 geturls_opener
.addheaders
= httpheaders
;
2650 geturls_opener
.set_cookiejar(httpcookie
);
2651 geturls_opener
.set_handle_robots(False);
2652 if(postdata
is not None and not isinstance(postdata
, dict)):
2653 postdata
= urlencode(postdata
);
2655 if(httpmethod
=="GET"):
2656 geturls_text
= geturls_opener
.open(httpurl
);
2657 elif(httpmethod
=="POST"):
2658 geturls_text
= geturls_opener
.open(httpurl
, data
=postdata
);
2660 geturls_text
= geturls_opener
.open(httpurl
);
2661 except mechanize
.HTTPError
as geturls_text_error
:
2662 geturls_text
= geturls_text_error
;
2663 log
.info("Error With URL "+httpurl
);
2665 log
.info("Error With URL "+httpurl
);
2667 except socket
.timeout
:
2668 log
.info("Error With URL "+httpurl
);
2670 downloadsize
= int(geturls_text
.info().get('Content-Length'));
2671 if(downloadsize
is not None):
2672 downloadsize
= int(downloadsize
);
2673 if downloadsize
is None: downloadsize
= 0;
2676 log
.info("Downloading URL "+httpurl
);
2677 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2678 tmpfilename
= f
.name
;
2679 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': dict(geturls_text
.info()), 'Version': "1.1", 'Method': httpmethod
, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders
), 'URL': geturls_text
.geturl(), 'Code': geturls_text
.code
};
2681 databytes
= geturls_text
.read(buffersize
);
2682 if not databytes
: break;
2683 datasize
= len(databytes
);
2684 fulldatasize
= datasize
+ fulldatasize
;
2687 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2688 downloaddiff
= fulldatasize
- prevdownsize
;
2689 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2690 prevdownsize
= fulldatasize
;
2693 geturls_text
.close();
2694 exec_time_end
= time
.time();
2695 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2696 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2699 if(not havemechanize
):
2700 def download_from_url_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
2701 returnval
= download_from_url_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, sleep
)
2705 def download_from_url_to_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
2706 global geturls_download_sleep
;
2708 sleep
= geturls_download_sleep
;
2709 if(not outfile
=="-"):
2710 outpath
= outpath
.rstrip(os
.path
.sep
);
2711 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2712 if(not os
.path
.exists(outpath
)):
2713 os
.makedirs(outpath
);
2714 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2716 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2718 pretmpfilename
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2719 if(not pretmpfilename
):
2721 tmpfilename
= pretmpfilename
['Filename'];
2722 downloadsize
= os
.path
.getsize(tmpfilename
);
2724 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2725 exec_time_start
= time
.time();
2726 shutil
.move(tmpfilename
, filepath
);
2727 exec_time_end
= time
.time();
2728 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2729 if(os
.path
.exists(tmpfilename
)):
2730 os
.remove(tmpfilename
);
2731 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2732 if(outfile
=="-" and sys
.version
[0]=="2"):
2733 pretmpfilename
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2734 if(not pretmpfilename
):
2736 tmpfilename
= pretmpfilename
['Filename'];
2737 downloadsize
= os
.path
.getsize(tmpfilename
);
2740 exec_time_start
= time
.time();
2741 with
open(tmpfilename
, 'rb') as ft
:
2744 databytes
= ft
.read(buffersize
[1]);
2745 if not databytes
: break;
2746 datasize
= len(databytes
);
2747 fulldatasize
= datasize
+ fulldatasize
;
2750 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2751 downloaddiff
= fulldatasize
- prevdownsize
;
2752 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2753 prevdownsize
= fulldatasize
;
2756 fdata
= f
.getvalue();
2759 os
.remove(tmpfilename
);
2760 exec_time_end
= time
.time();
2761 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2762 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2763 if(outfile
=="-" and sys
.version
[0]>="3"):
2764 pretmpfilename
= download_from_url_file_with_mechanize(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2765 tmpfilename
= pretmpfilename
['Filename'];
2766 downloadsize
= os
.path
.getsize(tmpfilename
);
2769 exec_time_start
= time
.time();
2770 with
open(tmpfilename
, 'rb') as ft
:
2773 databytes
= ft
.read(buffersize
[1]);
2774 if not databytes
: break;
2775 datasize
= len(databytes
);
2776 fulldatasize
= datasize
+ fulldatasize
;
2779 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2780 downloaddiff
= fulldatasize
- prevdownsize
;
2781 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2782 prevdownsize
= fulldatasize
;
2785 fdata
= f
.getvalue();
2788 os
.remove(tmpfilename
);
2789 exec_time_end
= time
.time();
2790 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2791 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2794 if(not havemechanize
):
2795 def download_from_url_to_file_with_mechanize(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
2796 returnval
= download_from_url_to_file_with_urllib(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
, outfile
, outpath
, sleep
)
2799 def download_file_from_ftp_file(url
):
2800 urlparts
= urlparse
.urlparse(url
);
2801 file_name
= os
.path
.basename(urlparts
.path
);
2802 file_dir
= os
.path
.dirname(urlparts
.path
);
2803 if(urlparts
.username
is not None):
2804 ftp_username
= urlparts
.username
;
2806 ftp_username
= "anonymous";
2807 if(urlparts
.password
is not None):
2808 ftp_password
= urlparts
.password
;
2809 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
2810 ftp_password
= "anonymous";
2813 if(urlparts
.scheme
=="ftp"):
2815 elif(urlparts
.scheme
=="ftps"):
2819 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
2821 ftp_port
= urlparts
.port
;
2822 if(urlparts
.port
is None):
2824 ftp
.connect(urlparts
.hostname
, ftp_port
);
2825 ftp
.login(urlparts
.username
, urlparts
.password
);
2826 if(urlparts
.scheme
=="ftps"):
2828 ftpfile
= BytesIO();
2829 ftp
.retrbinary("RETR "+urlparts
.path
, ftpfile
.write
);
2830 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2835 def download_file_from_ftp_string(url
):
2836 ftpfile
= download_file_from_ftp_file(url
);
2837 return ftpfile
.read();
2839 def download_from_url_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
2840 global geturls_download_sleep
, havebrotli
;
2842 sleep
= geturls_download_sleep
;
2843 urlparts
= urlparse
.urlparse(httpurl
);
2844 if(isinstance(httpheaders
, list)):
2845 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2846 if(isinstance(httpheaders
, dict)):
2847 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
2849 geturls_text
= download_file_from_ftp_file(httpurl
);
2850 log
.info("Downloading URL "+httpurl
);
2851 returnval_content
= geturls_text
.read()[:];
2852 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': None, 'Version': None, 'Method': httpmethod
, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
2853 geturls_text
.close();
2856 def download_from_url_file_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
2857 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
2858 exec_time_start
= time
.time();
2859 myhash
= hashlib
.new("sha1");
2860 if(sys
.version
[0]=="2"):
2861 myhash
.update(httpurl
);
2862 myhash
.update(str(buffersize
));
2863 myhash
.update(str(exec_time_start
));
2864 if(sys
.version
[0]>="3"):
2865 myhash
.update(httpurl
.encode('utf-8'));
2866 myhash
.update(str(buffersize
).encode('utf-8'));
2867 myhash
.update(str(exec_time_start
).encode('utf-8'));
2868 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
2870 sleep
= geturls_download_sleep
;
2871 urlparts
= urlparse
.urlparse(httpurl
);
2872 if(isinstance(httpheaders
, list)):
2873 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
2874 if(isinstance(httpheaders
, dict)):
2875 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
2877 geturls_text
= download_file_from_ftp_file(httpurl
);
2878 geturls_text
.seek(0, 2);
2879 downloadsize
= geturls_text
.tell();
2880 geturls_text
.seek(0, 0);
2881 if(downloadsize
is not None):
2882 downloadsize
= int(downloadsize
);
2883 if downloadsize
is None: downloadsize
= 0;
2886 log
.info("Downloading URL "+httpurl
);
2887 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
2888 tmpfilename
= f
.name
;
2889 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
2891 databytes
= geturls_text
.read(buffersize
);
2892 if not databytes
: break;
2893 datasize
= len(databytes
);
2894 fulldatasize
= datasize
+ fulldatasize
;
2897 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2898 downloaddiff
= fulldatasize
- prevdownsize
;
2899 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2900 prevdownsize
= fulldatasize
;
2903 geturls_text
.close();
2904 exec_time_end
= time
.time();
2905 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
2906 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
2909 def download_from_url_to_file_with_ftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
2910 global geturls_download_sleep
;
2912 sleep
= geturls_download_sleep
;
2913 if(not outfile
=="-"):
2914 outpath
= outpath
.rstrip(os
.path
.sep
);
2915 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
2916 if(not os
.path
.exists(outpath
)):
2917 os
.makedirs(outpath
);
2918 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
2920 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
2922 pretmpfilename
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2923 if(not pretmpfilename
):
2925 tmpfilename
= pretmpfilename
['Filename'];
2926 downloadsize
= os
.path
.getsize(tmpfilename
);
2928 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
2929 exec_time_start
= time
.time();
2930 shutil
.move(tmpfilename
, filepath
);
2931 exec_time_end
= time
.time();
2932 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
2933 if(os
.path
.exists(tmpfilename
)):
2934 os
.remove(tmpfilename
);
2935 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2936 if(outfile
=="-" and sys
.version
[0]=="2"):
2937 pretmpfilename
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2938 if(not pretmpfilename
):
2940 tmpfilename
= pretmpfilename
['Filename'];
2941 downloadsize
= os
.path
.getsize(tmpfilename
);
2944 exec_time_start
= time
.time();
2945 with
open(tmpfilename
, 'rb') as ft
:
2948 databytes
= ft
.read(buffersize
[1]);
2949 if not databytes
: break;
2950 datasize
= len(databytes
);
2951 fulldatasize
= datasize
+ fulldatasize
;
2954 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2955 downloaddiff
= fulldatasize
- prevdownsize
;
2956 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2957 prevdownsize
= fulldatasize
;
2960 fdata
= f
.getvalue();
2963 os
.remove(tmpfilename
);
2964 exec_time_end
= time
.time();
2965 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2966 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2967 if(outfile
=="-" and sys
.version
[0]>="3"):
2968 pretmpfilename
= download_from_url_file_with_ftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
2969 tmpfilename
= pretmpfilename
['Filename'];
2970 downloadsize
= os
.path
.getsize(tmpfilename
);
2973 exec_time_start
= time
.time();
2974 with
open(tmpfilename
, 'rb') as ft
:
2977 databytes
= ft
.read(buffersize
[1]);
2978 if not databytes
: break;
2979 datasize
= len(databytes
);
2980 fulldatasize
= datasize
+ fulldatasize
;
2983 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
2984 downloaddiff
= fulldatasize
- prevdownsize
;
2985 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
2986 prevdownsize
= fulldatasize
;
2989 fdata
= f
.getvalue();
2992 os
.remove(tmpfilename
);
2993 exec_time_end
= time
.time();
2994 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
2995 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
2998 def upload_file_to_ftp_file(ftpfile
, url
):
2999 urlparts
= urlparse
.urlparse(url
);
3000 file_name
= os
.path
.basename(urlparts
.path
);
3001 file_dir
= os
.path
.dirname(urlparts
.path
);
3002 if(urlparts
.username
is not None):
3003 ftp_username
= urlparts
.username
;
3005 ftp_username
= "anonymous";
3006 if(urlparts
.password
is not None):
3007 ftp_password
= urlparts
.password
;
3008 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
3009 ftp_password
= "anonymous";
3012 if(urlparts
.scheme
=="ftp"):
3014 elif(urlparts
.scheme
=="ftps"):
3018 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
3020 ftp_port
= urlparts
.port
;
3021 if(urlparts
.port
is None):
3023 ftp
.connect(urlparts
.hostname
, ftp_port
);
3024 ftp
.login(urlparts
.username
, urlparts
.password
);
3025 if(urlparts
.scheme
=="ftps"):
3027 ftp
.storbinary("STOR "+urlparts
.path
, ftpfile
);
3032 def upload_file_to_ftp_string(ftpstring
, url
):
3033 ftpfileo
= BytesIO(ftpstring
);
3034 ftpfile
= upload_file_to_ftp_file(ftpfileo
, url
);
3039 def download_file_from_sftp_file(url
):
3040 urlparts
= urlparse
.urlparse(url
);
3041 file_name
= os
.path
.basename(urlparts
.path
);
3042 file_dir
= os
.path
.dirname(urlparts
.path
);
3043 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
3045 sftp_port
= urlparts
.port
;
3046 if(urlparts
.port
is None):
3049 sftp_port
= urlparts
.port
;
3050 if(urlparts
.username
is not None):
3051 sftp_username
= urlparts
.username
;
3053 sftp_username
= "anonymous";
3054 if(urlparts
.password
is not None):
3055 sftp_password
= urlparts
.password
;
3056 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
3057 sftp_password
= "anonymous";
3060 if(urlparts
.scheme
!="sftp"):
3062 ssh
= paramiko
.SSHClient();
3063 ssh
.load_system_host_keys();
3064 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
3066 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
3067 except paramiko
.ssh_exception
.SSHException
:
3069 sftp
= ssh
.open_sftp();
3070 sftpfile
= BytesIO();
3071 sftp
.getfo(urlparts
.path
, sftpfile
);
3074 sftpfile
.seek(0, 0);
3077 def download_file_from_sftp_file(url
):
3081 def download_file_from_sftp_string(url
):
3082 sftpfile
= download_file_from_sftp_file(url
);
3083 return sftpfile
.read();
3085 def download_file_from_ftp_string(url
):
3089 def download_from_url_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
3090 global geturls_download_sleep
, havebrotli
;
3092 sleep
= geturls_download_sleep
;
3093 urlparts
= urlparse
.urlparse(httpurl
);
3094 if(isinstance(httpheaders
, list)):
3095 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3096 if(isinstance(httpheaders
, dict)):
3097 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
3099 geturls_text
= download_file_from_sftp_file(httpurl
);
3100 log
.info("Downloading URL "+httpurl
);
3101 returnval_content
= geturls_text
.read()[:];
3102 returnval
= {'Type': "Content", 'Content': returnval_content
, 'Headers': None, 'Version': None, 'Method': httpmethod
, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
3103 geturls_text
.close();
3106 if(not haveparamiko
):
3107 def download_from_url_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, sleep
=-1):
3111 def download_from_url_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
3112 global geturls_download_sleep
, tmpfileprefix
, tmpfilesuffix
;
3113 exec_time_start
= time
.time();
3114 myhash
= hashlib
.new("sha1");
3115 if(sys
.version
[0]=="2"):
3116 myhash
.update(httpurl
);
3117 myhash
.update(str(buffersize
));
3118 myhash
.update(str(exec_time_start
));
3119 if(sys
.version
[0]>="3"):
3120 myhash
.update(httpurl
.encode('utf-8'));
3121 myhash
.update(str(buffersize
).encode('utf-8'));
3122 myhash
.update(str(exec_time_start
).encode('utf-8'));
3123 newtmpfilesuffix
= tmpfilesuffix
+ str(myhash
.hexdigest());
3125 sleep
= geturls_download_sleep
;
3126 urlparts
= urlparse
.urlparse(httpurl
);
3127 if(isinstance(httpheaders
, list)):
3128 httpheaders
= make_http_headers_from_list_to_dict(httpheaders
);
3129 if(isinstance(httpheaders
, dict)):
3130 httpheaders
= make_http_headers_from_dict_to_list(httpheaders
);
3132 geturls_text
= download_file_from_sftp_file(httpurl
);
3133 geturls_text
.seek(0, 2);
3134 downloadsize
= geturls_text
.tell();
3135 geturls_text
.seek(0, 0);
3136 if(downloadsize
is not None):
3137 downloadsize
= int(downloadsize
);
3138 if downloadsize
is None: downloadsize
= 0;
3141 log
.info("Downloading URL "+httpurl
);
3142 with tempfile
.NamedTemporaryFile('wb+', prefix
=tmpfileprefix
, suffix
=newtmpfilesuffix
, delete
=False) as f
:
3143 tmpfilename
= f
.name
;
3144 returnval
= {'Type': "File", 'Filename': tmpfilename
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl
, 'Code': None};
3146 databytes
= geturls_text
.read(buffersize
);
3147 if not databytes
: break;
3148 datasize
= len(databytes
);
3149 fulldatasize
= datasize
+ fulldatasize
;
3152 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3153 downloaddiff
= fulldatasize
- prevdownsize
;
3154 log
.info("Downloading "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Downloaded "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3155 prevdownsize
= fulldatasize
;
3158 geturls_text
.close();
3159 exec_time_end
= time
.time();
3160 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to download file.");
3161 returnval
.update({'Filesize': os
.path
.getsize(tmpfilename
), 'DownloadTime': float(exec_time_start
- exec_time_end
), 'DownloadTimeReadable': hms_string(exec_time_start
- exec_time_end
)});
3164 if(not haveparamiko
):
3165 def download_from_url_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, buffersize
=524288, sleep
=-1):
3169 def download_from_url_to_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
3170 global geturls_download_sleep
;
3172 sleep
= geturls_download_sleep
;
3173 if(not outfile
=="-"):
3174 outpath
= outpath
.rstrip(os
.path
.sep
);
3175 filepath
= os
.path
.realpath(outpath
+os
.path
.sep
+outfile
);
3176 if(not os
.path
.exists(outpath
)):
3177 os
.makedirs(outpath
);
3178 if(os
.path
.exists(outpath
) and os
.path
.isfile(outpath
)):
3180 if(os
.path
.exists(filepath
) and os
.path
.isdir(filepath
)):
3182 pretmpfilename
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
3183 if(not pretmpfilename
):
3185 tmpfilename
= pretmpfilename
['Filename'];
3186 downloadsize
= os
.path
.getsize(tmpfilename
);
3188 log
.info("Moving file "+tmpfilename
+" to "+filepath
);
3189 exec_time_start
= time
.time();
3190 shutil
.move(tmpfilename
, filepath
);
3191 exec_time_end
= time
.time();
3192 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to move file.");
3193 if(os
.path
.exists(tmpfilename
)):
3194 os
.remove(tmpfilename
);
3195 returnval
= {'Type': "File", 'Filename': filepath
, 'Filesize': downloadsize
, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
3196 if(outfile
=="-" and sys
.version
[0]=="2"):
3197 pretmpfilename
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
3198 if(not pretmpfilename
):
3200 tmpfilename
= pretmpfilename
['Filename'];
3201 downloadsize
= os
.path
.getsize(tmpfilename
);
3204 exec_time_start
= time
.time();
3205 with
open(tmpfilename
, 'rb') as ft
:
3208 databytes
= ft
.read(buffersize
[1]);
3209 if not databytes
: break;
3210 datasize
= len(databytes
);
3211 fulldatasize
= datasize
+ fulldatasize
;
3214 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3215 downloaddiff
= fulldatasize
- prevdownsize
;
3216 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3217 prevdownsize
= fulldatasize
;
3220 fdata
= f
.getvalue();
3223 os
.remove(tmpfilename
);
3224 exec_time_end
= time
.time();
3225 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3226 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
3227 if(outfile
=="-" and sys
.version
[0]>="3"):
3228 pretmpfilename
= download_from_url_file_with_sftp(httpurl
, httpheaders
, httpcookie
, httpmethod
, postdata
, buffersize
[0], sleep
);
3229 tmpfilename
= pretmpfilename
['Filename'];
3230 downloadsize
= os
.path
.getsize(tmpfilename
);
3233 exec_time_start
= time
.time();
3234 with
open(tmpfilename
, 'rb') as ft
:
3237 databytes
= ft
.read(buffersize
[1]);
3238 if not databytes
: break;
3239 datasize
= len(databytes
);
3240 fulldatasize
= datasize
+ fulldatasize
;
3243 percentage
= str("{0:.2f}".format(float(float(fulldatasize
/ downloadsize
) * 100))).rstrip('0').rstrip('.')+"%";
3244 downloaddiff
= fulldatasize
- prevdownsize
;
3245 log
.info("Copying "+get_readable_size(fulldatasize
, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize
, 2, "SI")['ReadableWithSuffix']+" "+str(percentage
)+" / Copied "+get_readable_size(downloaddiff
, 2, "IEC")['ReadableWithSuffix']);
3246 prevdownsize
= fulldatasize
;
3249 fdata
= f
.getvalue();
3252 os
.remove(tmpfilename
);
3253 exec_time_end
= time
.time();
3254 log
.info("It took "+hms_string(exec_time_start
- exec_time_end
)+" to copy file.");
3255 returnval
= {'Type': "Content", 'Content': fdata
, 'Contentsize': downloadsize
, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize
, 2, "IEC"), 'SI': get_readable_size(downloadsize
, 2, "SI")}, 'DownloadTime': pretmpfilename
['DownloadTime'], 'DownloadTimeReadable': pretmpfilename
['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start
- exec_time_end
), 'MoveFileTimeReadable': hms_string(exec_time_start
- exec_time_end
), 'Headers': pretmpfilename
['Headers'], 'Version': pretmpfilename
['Version'], 'Method': pretmpfilename
['Method'], 'Method': httpmethod
, 'HeadersSent': pretmpfilename
['HeadersSent'], 'URL': pretmpfilename
['URL'], 'Code': pretmpfilename
['Code']};
3258 if(not haveparamiko
):
3259 def download_from_url_to_file_with_sftp(httpurl
, httpheaders
=geturls_headers
, httpcookie
=geturls_cj
, httpmethod
="GET", postdata
=None, outfile
="-", outpath
=os
.getcwd(), buffersize
=[524288, 524288], sleep
=-1):
3263 def upload_file_to_sftp_file(sftpfile
, url
):
3264 urlparts
= urlparse
.urlparse(url
);
3265 file_name
= os
.path
.basename(urlparts
.path
);
3266 file_dir
= os
.path
.dirname(urlparts
.path
);
3267 sftp_port
= urlparts
.port
;
3268 if(urlparts
.scheme
=="http" or urlparts
.scheme
=="https"):
3270 if(urlparts
.port
is None):
3273 sftp_port
= urlparts
.port
;
3274 if(urlparts
.username
is not None):
3275 sftp_username
= urlparts
.username
;
3277 sftp_username
= "anonymous";
3278 if(urlparts
.password
is not None):
3279 sftp_password
= urlparts
.password
;
3280 elif(urlparts
.password
is None and urlparts
.username
=="anonymous"):
3281 sftp_password
= "anonymous";
3284 if(urlparts
.scheme
!="sftp"):
3286 ssh
= paramiko
.SSHClient();
3287 ssh
.load_system_host_keys();
3288 ssh
.set_missing_host_key_policy(paramiko
.AutoAddPolicy());
3290 ssh
.connect(urlparts
.hostname
, port
=sftp_port
, username
=urlparts
.username
, password
=urlparts
.password
);
3291 except paramiko
.ssh_exception
.SSHException
:
3293 sftp
= ssh
.open_sftp();
3294 sftp
.putfo(sftpfile
, urlparts
.path
);
3297 sftpfile
.seek(0, 0);
3300 def upload_file_to_sftp_file(sftpfile
, url
):
3304 def upload_file_to_sftp_string(sftpstring
, url
):
3305 sftpfileo
= BytesIO(sftpstring
);
3306 sftpfile
= upload_file_to_sftp_files(ftpfileo
, url
);
3310 def upload_file_to_sftp_string(url
):