?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Update pywwwgetold.py
[PyWWW-Get.git] / pywwwget.py
blob7e20408b8004e31c9210cb3c44981763fab08489
1 #!/usr/bin/env python
2
3 '''
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the Revised BSD License.
6
7 This program is distributed in the hope that it will be useful,
8 but WITHOUT ANY WARRANTY; without even the implied warranty of
9 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 Revised BSD License for more details.
11
12 Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
13 Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
14
15 $FileInfo: pywwwget.py - Last Update: 10/5/2023 Ver. 2.0.2 RC 1 - Author: cooldude2k $
16 '''
17
18 from __future__ import division, absolute_import, print_function;
19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, zlib, bz2, time, argparse, subprocess, socket, email.utils, datetime, time;
20 import logging as log;
21 from ftplib import FTP, FTP_TLS;
22 from base64 import b64encode;
23 try:
24 from cgi import parse_qsl;
25 except ImportError:
26 from urlparse import parse_qsl;
27 except DeprecationWarning:
28 from urlparse import parse_qsl;
29 haverequests = False;
30 try:
31 import requests;
32 haverequests = True;
33 except ImportError:
34 haverequests = False;
35 havemechanize = False;
36 try:
37 import mechanize;
38 havemechanize = True;
39 except ImportError:
40 havemechanize = False;
41 havepycurl = False;
42 try:
43 import pycurl;
44 havepycurl = True;
45 except ImportError:
46 havepycurl = False;
47 haveparamiko = False;
48 try:
49 import paramiko;
50 haveparamiko = True;
51 except ImportError:
52 haveparamiko = False;
53 havepysftp = False;
54 try:
55 import pysftp;
56 havepysftp = True;
57 except ImportError:
58 havepysftp = False;
59 haveurllib3 = False;
60 try:
61 import urllib3;
62 haveurllib3 = True;
63 except ImportError:
64 haveurllib3 = False;
65 havehttplib2 = False;
66 try:
67 from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
68 havehttplib2 = True;
69 except ImportError:
70 havehttplib2 = False;
71 havehttpx = False;
72 try:
73 import httpx;
74 havehttpx = True;
75 except ImportError:
76 havehttpx = False;
77 havehttpcore = False;
78 try:
79 import httpcore;
80 havehttpcore = True;
81 except ImportError:
82 havehttpcore = False;
83 haveaiohttp = False;
84 try:
85 import aiohttp;
86 haveaiohttp = True;
87 except ImportError:
88 haveaiohttp = False;
89 havebrotli = False;
90 try:
91 import brotli;
92 havebrotli = True;
93 except ImportError:
94 havebrotli = False;
95 havezstd = False;
96 try:
97 import zstandard;
98 havezstd = True;
99 except ImportError:
100 havezstd = False;
101 havelzma = False;
102 try:
103 import lzma;
104 havelzma = True;
105 except ImportError:
106 havelzma = False;
107 if(sys.version[0]=="2"):
108 try:
109 from io import StringIO, BytesIO;
110 except ImportError:
111 try:
112 from cStringIO import StringIO;
113 from cStringIO import StringIO as BytesIO;
114 except ImportError:
115 from StringIO import StringIO;
116 from StringIO import StringIO as BytesIO;
117 # From http://python-future.org/compatible_idioms.html
118 from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
119 from urllib import urlencode;
120 from urllib import urlopen as urlopenalt;
121 from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
122 import urlparse, cookielib;
123 from httplib import HTTPConnection, HTTPSConnection;
124 if(sys.version[0]>="3"):
125 from io import StringIO, BytesIO;
126 # From http://python-future.org/compatible_idioms.html
127 from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
128 from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
129 from urllib.error import HTTPError, URLError;
130 import urllib.parse as urlparse;
131 import http.cookiejar as cookielib;
132 from http.client import HTTPConnection, HTTPSConnection;
133
134 __program_name__ = "PyWWW-Get";
135 __program_alt_name__ = "PyWWWGet";
136 __program_small_name__ = "wwwget";
137 __project__ = __program_name__;
138 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
139 __version_info__ = (2, 0, 2, "RC 1", 1);
140 __version_date_info__ = (2023, 10, 5, "RC 1", 1);
141 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
142 __revision__ = __version_info__[3];
143 __revision_id__ = "$Id$";
144 if(__version_info__[4] is not None):
145 __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
146 if(__version_info__[4] is None):
147 __version_date_plusrc__ = __version_date__;
148 if(__version_info__[3] is not None):
149 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
150 if(__version_info__[3] is None):
151 __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
152
153 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
154 tmpfilesuffix = "-";
155 pytempdir = tempfile.gettempdir();
156
157 PyBitness = platform.architecture();
158 if(PyBitness=="32bit" or PyBitness=="32"):
159 PyBitness = "32";
160 elif(PyBitness=="64bit" or PyBitness=="64"):
161 PyBitness = "64";
162 else:
163 PyBitness = "32";
164
165 compression_supported_list = ['identity', 'gzip', 'deflate', 'bzip2'];
166 if(havebrotli):
167 compression_supported_list.append('br');
168 if(havezstd):
169 compression_supported_list.append('zstd');
170 if(havelzma):
171 compression_supported_list.append('lzma');
172 compression_supported_list.append('xz');
173 compression_supported = ', '.join(compression_supported_list);
174
175 geturls_cj = cookielib.CookieJar();
176 windowsNT4_ua_string = "Windows NT 4.0";
177 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
178 windows2k_ua_string = "Windows NT 5.0";
179 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
180 windowsXP_ua_string = "Windows NT 5.1";
181 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
182 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
183 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
184 windows7_ua_string = "Windows NT 6.1; Win64; x64";
185 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
186 windows8_ua_string = "Windows NT 6.2; Win64; x64";
187 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
188 windows81_ua_string = "Windows NT 6.3; Win64; x64";
189 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
190 windows10_ua_string = "Windows NT 10.0; Win64; x64";
191 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
192 windows11_ua_string = "Windows NT 11.0; Win64; x64";
193 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
194 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
195 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
196 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
197 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
198 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
199 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
200 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
201 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
202 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
203 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
204 if(platform.python_implementation()!=""):
205 py_implementation = platform.python_implementation();
206 if(platform.python_implementation()==""):
207 py_implementation = "Python";
208 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
209 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
210 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
211 geturls_ua = geturls_ua_firefox_windows7;
212 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
214 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
215 geturls_headers_chrome_windows7.update(windows7_ua_addon);
216 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
217 geturls_headers_chromium_windows7.update(windows7_ua_addon);
218 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
219 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
220 geturls_headers_opera_windows7.update(windows7_ua_addon);
221 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
222 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
223 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
224 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
225 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
226 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
227 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
228 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
229 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
230 geturls_headers = geturls_headers_firefox_windows7;
231 geturls_download_sleep = 0;
232
233 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
234 if(outtype=="print" and dbgenable):
235 print(dbgtxt);
236 return True;
237 elif(outtype=="log" and dbgenable):
238 logging.info(dbgtxt);
239 return True;
240 elif(outtype=="warning" and dbgenable):
241 logging.warning(dbgtxt);
242 return True;
243 elif(outtype=="error" and dbgenable):
244 logging.error(dbgtxt);
245 return True;
246 elif(outtype=="critical" and dbgenable):
247 logging.critical(dbgtxt);
248 return True;
249 elif(outtype=="exception" and dbgenable):
250 logging.exception(dbgtxt);
251 return True;
252 elif(outtype=="logalt" and dbgenable):
253 logging.log(dgblevel, dbgtxt);
254 return True;
255 elif(outtype=="debug" and dbgenable):
256 logging.debug(dbgtxt);
257 return True;
258 elif(not dbgenable):
259 return True;
260 else:
261 return False;
262 return False;
263
264 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
265 dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
266 if(not dbgout):
267 return False;
268 return dbgtxt;
269
270 def add_url_param(url, **params):
271 n=3;
272 parts = list(urlparse.urlsplit(url));
273 d = dict(parse_qsl(parts[n])); # use cgi.parse_qs for list values
274 d.update(params);
275 parts[n]=urlencode(d);
276 return urlparse.urlunsplit(parts);
277
278 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
279 def which_exec(execfile):
280 for path in os.environ["PATH"].split(":"):
281 if os.path.exists(path + "/" + execfile):
282 return path + "/" + execfile;
283
284 def listize(varlist):
285 il = 0;
286 ix = len(varlist);
287 ilx = 1;
288 newlistreg = {};
289 newlistrev = {};
290 newlistfull = {};
291 while(il < ix):
292 newlistreg.update({ilx: varlist[il]});
293 newlistrev.update({varlist[il]: ilx});
294 ilx = ilx + 1;
295 il = il + 1;
296 newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
297 return newlistfull;
298
299 def twolistize(varlist):
300 il = 0;
301 ix = len(varlist);
302 ilx = 1;
303 newlistnamereg = {};
304 newlistnamerev = {};
305 newlistdescreg = {};
306 newlistdescrev = {};
307 newlistfull = {};
308 while(il < ix):
309 newlistnamereg.update({ilx: varlist[il][0].strip()});
310 newlistnamerev.update({varlist[il][0].strip(): ilx});
311 newlistdescreg.update({ilx: varlist[il][1].strip()});
312 newlistdescrev.update({varlist[il][1].strip(): ilx});
313 ilx = ilx + 1;
314 il = il + 1;
315 newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
316 newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
317 newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
318 return newlistfull;
319
320 def arglistize(proexec, *varlist):
321 il = 0;
322 ix = len(varlist);
323 ilx = 1;
324 newarglist = [proexec];
325 while(il < ix):
326 if varlist[il][0] is not None:
327 newarglist.append(varlist[il][0]);
328 if varlist[il][1] is not None:
329 newarglist.append(varlist[il][1]);
330 il = il + 1;
331 return newarglist;
332
333 def fix_header_names(header_dict):
334 if(sys.version[0]=="2"):
335 header_dict = {k.title(): v for k, v in header_dict.iteritems()};
336 if(sys.version[0]>="3"):
337 header_dict = {k.title(): v for k, v in header_dict.items()};
338 return header_dict;
339
340 # hms_string by ArcGIS Python Recipes
341 # https://arcpy.wordpress.com/2012/04/20/146/
342 def hms_string(sec_elapsed):
343 h = int(sec_elapsed / (60 * 60));
344 m = int((sec_elapsed % (60 * 60)) / 60);
345 s = sec_elapsed % 60.0;
346 return "{}:{:>02}:{:>05.2f}".format(h, m, s);
347
348 # get_readable_size by Lipis
349 # http://stackoverflow.com/posts/14998888/revisions
350 def get_readable_size(bytes, precision=1, unit="IEC"):
351 unit = unit.upper();
352 if(unit!="IEC" and unit!="SI"):
353 unit = "IEC";
354 if(unit=="IEC"):
355 units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
356 unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
357 unitsize = 1024.0;
358 if(unit=="SI"):
359 units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
360 unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
361 unitsize = 1000.0;
362 return_val = {};
363 orgbytes = bytes;
364 for unit in units:
365 if abs(bytes) < unitsize:
366 strformat = "%3."+str(precision)+"f%s";
367 pre_return_val = (strformat % (bytes, unit));
368 pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
369 pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
370 alt_return_val = pre_return_val.split();
371 return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
372 return return_val;
373 bytes /= unitsize;
374 strformat = "%."+str(precision)+"f%s";
375 pre_return_val = (strformat % (bytes, "YiB"));
376 pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
377 pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
378 alt_return_val = pre_return_val.split();
379 return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
380 return return_val;
381
382 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
383 unit = unit.upper();
384 usehashtypes = usehashtypes.lower();
385 getfilesize = os.path.getsize(infile);
386 return_val = get_readable_size(getfilesize, precision, unit);
387 if(usehashes):
388 hashtypelist = usehashtypes.split(",");
389 openfile = open(infile, "rb");
390 filecontents = openfile.read();
391 openfile.close();
392 listnumcount = 0;
393 listnumend = len(hashtypelist);
394 while(listnumcount < listnumend):
395 hashtypelistlow = hashtypelist[listnumcount].strip();
396 hashtypelistup = hashtypelistlow.upper();
397 filehash = hashlib.new(hashtypelistup);
398 filehash.update(filecontents);
399 filegethash = filehash.hexdigest();
400 return_val.update({hashtypelistup: filegethash});
401 listnumcount += 1;
402 return return_val;
403
404 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
405 unit = unit.upper();
406 usehashtypes = usehashtypes.lower();
407 getfilesize = len(instring);
408 return_val = get_readable_size(getfilesize, precision, unit);
409 if(usehashes):
410 hashtypelist = usehashtypes.split(",");
411 listnumcount = 0;
412 listnumend = len(hashtypelist);
413 while(listnumcount < listnumend):
414 hashtypelistlow = hashtypelist[listnumcount].strip();
415 hashtypelistup = hashtypelistlow.upper();
416 filehash = hashlib.new(hashtypelistup);
417 if(sys.version[0]=="2"):
418 filehash.update(instring);
419 if(sys.version[0]>="3"):
420 filehash.update(instring.encode('utf-8'));
421 filegethash = filehash.hexdigest();
422 return_val.update({hashtypelistup: filegethash});
423 listnumcount += 1;
424 return return_val;
425
426 def http_status_to_reason(code):
427 reasons = {
428 100: 'Continue',
429 101: 'Switching Protocols',
430 102: 'Processing',
431 200: 'OK',
432 201: 'Created',
433 202: 'Accepted',
434 203: 'Non-Authoritative Information',
435 204: 'No Content',
436 205: 'Reset Content',
437 206: 'Partial Content',
438 207: 'Multi-Status',
439 208: 'Already Reported',
440 226: 'IM Used',
441 300: 'Multiple Choices',
442 301: 'Moved Permanently',
443 302: 'Found',
444 303: 'See Other',
445 304: 'Not Modified',
446 305: 'Use Proxy',
447 307: 'Temporary Redirect',
448 308: 'Permanent Redirect',
449 400: 'Bad Request',
450 401: 'Unauthorized',
451 402: 'Payment Required',
452 403: 'Forbidden',
453 404: 'Not Found',
454 405: 'Method Not Allowed',
455 406: 'Not Acceptable',
456 407: 'Proxy Authentication Required',
457 408: 'Request Timeout',
458 409: 'Conflict',
459 410: 'Gone',
460 411: 'Length Required',
461 412: 'Precondition Failed',
462 413: 'Payload Too Large',
463 414: 'URI Too Long',
464 415: 'Unsupported Media Type',
465 416: 'Range Not Satisfiable',
466 417: 'Expectation Failed',
467 421: 'Misdirected Request',
468 422: 'Unprocessable Entity',
469 423: 'Locked',
470 424: 'Failed Dependency',
471 426: 'Upgrade Required',
472 428: 'Precondition Required',
473 429: 'Too Many Requests',
474 431: 'Request Header Fields Too Large',
475 451: 'Unavailable For Legal Reasons',
476 500: 'Internal Server Error',
477 501: 'Not Implemented',
478 502: 'Bad Gateway',
479 503: 'Service Unavailable',
480 504: 'Gateway Timeout',
481 505: 'HTTP Version Not Supported',
482 506: 'Variant Also Negotiates',
483 507: 'Insufficient Storage',
484 508: 'Loop Detected',
485 510: 'Not Extended',
486 511: 'Network Authentication Required'
487 }
488 return reasons.get(code, 'Unknown Status Code');
489
490 def ftp_status_to_reason(code):
491 reasons = {
492 110: 'Restart marker reply',
493 120: 'Service ready in nnn minutes',
494 125: 'Data connection already open; transfer starting',
495 150: 'File status okay; about to open data connection',
496 200: 'Command okay',
497 202: 'Command not implemented, superfluous at this site',
498 211: 'System status, or system help reply',
499 212: 'Directory status',
500 213: 'File status',
501 214: 'Help message',
502 215: 'NAME system type',
503 220: 'Service ready for new user',
504 221: 'Service closing control connection',
505 225: 'Data connection open; no transfer in progress',
506 226: 'Closing data connection',
507 227: 'Entering Passive Mode',
508 230: 'User logged in, proceed',
509 250: 'Requested file action okay, completed',
510 257: '"PATHNAME" created',
511 331: 'User name okay, need password',
512 332: 'Need account for login',
513 350: 'Requested file action pending further information',
514 421: 'Service not available, closing control connection',
515 425: 'Can\'t open data connection',
516 426: 'Connection closed; transfer aborted',
517 450: 'Requested file action not taken',
518 451: 'Requested action aborted. Local error in processing',
519 452: 'Requested action not taken. Insufficient storage space in system',
520 500: 'Syntax error, command unrecognized',
521 501: 'Syntax error in parameters or arguments',
522 502: 'Command not implemented',
523 503: 'Bad sequence of commands',
524 504: 'Command not implemented for that parameter',
525 530: 'Not logged in',
526 532: 'Need account for storing files',
527 550: 'Requested action not taken. File unavailable',
528 551: 'Requested action aborted. Page type unknown',
529 552: 'Requested file action aborted. Exceeded storage allocation',
530 553: 'Requested action not taken. File name not allowed'
531 };
532 return reasons.get(code, 'Unknown Status Code');
533
534 def sftp_status_to_reason(code):
535 reasons = {
536 0: 'SSH_FX_OK',
537 1: 'SSH_FX_EOF',
538 2: 'SSH_FX_NO_SUCH_FILE',
539 3: 'SSH_FX_PERMISSION_DENIED',
540 4: 'SSH_FX_FAILURE',
541 5: 'SSH_FX_BAD_MESSAGE',
542 6: 'SSH_FX_NO_CONNECTION',
543 7: 'SSH_FX_CONNECTION_LOST',
544 8: 'SSH_FX_OP_UNSUPPORTED'
545 };
546 return reasons.get(code, 'Unknown Status Code');
547
548 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
549 if isinstance(headers, dict):
550 returnval = [];
551 if(sys.version[0]=="2"):
552 for headkey, headvalue in headers.iteritems():
553 returnval.append((headkey, headvalue));
554 if(sys.version[0]>="3"):
555 for headkey, headvalue in headers.items():
556 returnval.append((headkey, headvalue));
557 elif isinstance(headers, list):
558 returnval = headers;
559 else:
560 returnval = False;
561 return returnval;
562
563 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
564 if isinstance(headers, dict):
565 returnval = [];
566 if(sys.version[0]=="2"):
567 for headkey, headvalue in headers.iteritems():
568 returnval.append(headkey+": "+headvalue);
569 if(sys.version[0]>="3"):
570 for headkey, headvalue in headers.items():
571 returnval.append(headkey+": "+headvalue);
572 elif isinstance(headers, list):
573 returnval = headers;
574 else:
575 returnval = False;
576 return returnval;
577
578 def make_http_headers_from_pycurl_to_dict(headers):
579 header_dict = {};
580 headers = headers.strip().split('\r\n');
581 for header in headers:
582 parts = header.split(': ', 1)
583 if(len(parts) == 2):
584 key, value = parts;
585 header_dict[key.title()] = value;
586 return header_dict;
587
588 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
589 if isinstance(headers, list):
590 returnval = {};
591 mli = 0;
592 mlil = len(headers);
593 while(mli<mlil):
594 returnval.update({headers[mli][0]: headers[mli][1]});
595 mli = mli + 1;
596 elif isinstance(headers, dict):
597 returnval = headers;
598 else:
599 returnval = False;
600 return returnval;
601
602 def get_httplib_support(checkvalue=None):
603 global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
604 returnval = [];
605 returnval.append("ftp");
606 returnval.append("httplib");
607 if(havehttplib2):
608 returnval.append("httplib2");
609 returnval.append("urllib");
610 if(haveurllib3):
611 returnval.append("urllib3");
612 returnval.append("request3");
613 returnval.append("request");
614 if(haverequests):
615 returnval.append("requests");
616 if(haveaiohttp):
617 returnval.append("aiohttp");
618 if(havehttpx):
619 returnval.append("httpx");
620 returnval.append("httpx2");
621 if(havemechanize):
622 returnval.append("mechanize");
623 if(havepycurl):
624 returnval.append("pycurl");
625 if(hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
626 returnval.append("pycurl2");
627 if(hasattr(pycurl, "CURL_HTTP_VERSION_3_0")):
628 returnval.append("pycurl3");
629 if(haveparamiko):
630 returnval.append("sftp");
631 if(havepysftp):
632 returnval.append("pysftp");
633 if(not checkvalue is None):
634 if(checkvalue=="urllib1" or checkvalue=="urllib2"):
635 checkvalue = "urllib";
636 if(checkvalue=="httplib1"):
637 checkvalue = "httplib";
638 if(checkvalue in returnval):
639 returnval = True;
640 else:
641 returnval = False;
642 return returnval;
643
644 def check_httplib_support(checkvalue="urllib"):
645 if(checkvalue=="urllib1" or checkvalue=="urllib2"):
646 checkvalue = "urllib";
647 if(checkvalue=="httplib1"):
648 checkvalue = "httplib";
649 returnval = get_httplib_support(checkvalue);
650 return returnval;
651
652 def get_httplib_support_list():
653 returnval = get_httplib_support(None);
654 return returnval;
655
656 def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
657 global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
658 if(sleep<0):
659 sleep = geturls_download_sleep;
660 if(timeout<=0):
661 timeout = 10;
662 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
663 httplibuse = "urllib";
664 if(httplibuse=="httplib1"):
665 httplibuse = "httplib";
666 if(not haverequests and httplibuse=="requests"):
667 httplibuse = "urllib";
668 if(not haveaiohttp and httplibuse=="aiohttp"):
669 httplibuse = "urllib";
670 if(not havehttpx and httplibuse=="httpx"):
671 httplibuse = "urllib";
672 if(not havehttpx and httplibuse=="httpx2"):
673 httplibuse = "urllib";
674 if(not havehttpcore and httplibuse=="httpcore"):
675 httplibuse = "urllib";
676 if(not havehttpcore and httplibuse=="httpcore2"):
677 httplibuse = "urllib";
678 if(not havemechanize and httplibuse=="mechanize"):
679 httplibuse = "urllib";
680 if(not havepycurl and httplibuse=="pycurl"):
681 httplibuse = "urllib";
682 if(not havepycurl and httplibuse=="pycurl2"):
683 httplibuse = "urllib";
684 if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
685 httplibuse = "pycurl";
686 if(not havepycurl and httplibuse=="pycurl3"):
687 httplibuse = "urllib";
688 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
689 httplibuse = "pycurl2";
690 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
691 httplibuse = "pycurl";
692 if(not havehttplib2 and httplibuse=="httplib2"):
693 httplibuse = "httplib";
694 if(not haveparamiko and httplibuse=="sftp"):
695 httplibuse = "ftp";
696 if(not havepysftp and httplibuse=="pysftp"):
697 httplibuse = "ftp";
698 urlparts = urlparse.urlparse(httpurl);
699 if(isinstance(httpheaders, list)):
700 httpheaders = make_http_headers_from_list_to_dict(httpheaders);
701 httpheaders = fix_header_names(httpheaders);
702 if(httpuseragent is not None):
703 if('User-Agent' in httpheaders):
704 httpheaders['User-Agent'] = httpuseragent;
705 else:
706 httpuseragent.update({'User-Agent': httpuseragent});
707 if(httpreferer is not None):
708 if('Referer' in httpheaders):
709 httpheaders['Referer'] = httpreferer;
710 else:
711 httpuseragent.update({'Referer': httpreferer});
712 if(urlparts.username is not None or urlparts.password is not None):
713 if(sys.version[0]=="2"):
714 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
715 if(sys.version[0]>="3"):
716 inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
717 httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
718 geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
719 if(httplibuse=="urllib" or httplibuse=="mechanize"):
720 if(isinstance(httpheaders, dict)):
721 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
722 if(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
723 if(isinstance(httpheaders, dict)):
724 httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
725 geturls_opener.addheaders = httpheaders;
726 time.sleep(sleep);
727 if(postdata is not None and not isinstance(postdata, dict)):
728 postdata = urlencode(postdata);
729 if(httplibuse=="urllib" or httplibuse=="request"):
730 geturls_request = Request(httpurl);
731 try:
732 if(httpmethod=="GET"):
733 geturls_text = geturls_opener.open(geturls_request);
734 elif(httpmethod=="POST"):
735 geturls_text = geturls_opener.open(geturls_request, data=postdata);
736 else:
737 geturls_text = geturls_opener.open(geturls_request);
738 except HTTPError as geturls_text_error:
739 geturls_text = geturls_text_error;
740 log.info("Error With URL "+httpurl);
741 except URLError:
742 log.info("Error With URL "+httpurl);
743 return False;
744 except socket.timeout:
745 log.info("Error With URL "+httpurl);
746 return False;
747 httpcodeout = geturls_text.getcode();
748 try:
749 httpcodereason = geturls_text.reason;
750 except AttributeError:
751 httpcodereason = http_status_to_reason(geturls_text.getcode());
752 try:
753 httpversionout = geturls_text.version;
754 except AttributeError:
755 httpversionout = "1.1";
756 httpmethodout = geturls_request.get_method();
757 httpurlout = geturls_text.geturl();
758 httpheaderout = geturls_text.info();
759 httpheadersentout = httpheaders;
760 elif(httplibuse=="httplib"):
761 if(urlparts[0]=="http"):
762 httpconn = HTTPConnection(urlparts[1], timeout=timeout);
763 elif(urlparts[0]=="https"):
764 httpconn = HTTPSConnection(urlparts[1], timeout=timeout);
765 else:
766 return False;
767 if(postdata is not None and not isinstance(postdata, dict)):
768 postdata = urlencode(postdata);
769 try:
770 if(httpmethod=="GET"):
771 httpconn.request("GET", urlparts[2], headers=httpheaders);
772 elif(httpmethod=="POST"):
773 httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
774 else:
775 httpconn.request("GET", urlparts[2], headers=httpheaders);
776 except socket.timeout:
777 log.info("Error With URL "+httpurl);
778 return False;
779 except socket.gaierror:
780 log.info("Error With URL "+httpurl);
781 return False;
782 except BlockingIOError:
783 log.info("Error With URL "+httpurl);
784 return False;
785 geturls_text = httpconn.getresponse();
786 httpcodeout = geturls_text.status;
787 httpcodereason = geturls_text.reason;
788 if(geturls_text.version=="10"):
789 httpversionout = "1.0";
790 else:
791 httpversionout = "1.1";
792 httpmethodout = geturls_text._method;
793 httpurlout = httpurl;
794 httpheaderout = geturls_text.getheaders();
795 httpheadersentout = httpheaders;
796 elif(httplibuse=="httplib2"):
797 if(urlparts[0]=="http"):
798 httpconn = HTTPConnectionWithTimeout(urlparts[1], timeout=timeout);
799 elif(urlparts[0]=="https"):
800 httpconn = HTTPSConnectionWithTimeout(urlparts[1], timeout=timeout);
801 else:
802 return False;
803 if(postdata is not None and not isinstance(postdata, dict)):
804 postdata = urlencode(postdata);
805 try:
806 if(httpmethod=="GET"):
807 httpconn.request("GET", urlparts[2], headers=httpheaders);
808 elif(httpmethod=="POST"):
809 httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
810 else:
811 httpconn.request("GET", urlparts[2], headers=httpheaders);
812 except socket.timeout:
813 log.info("Error With URL "+httpurl);
814 return False;
815 except socket.gaierror:
816 log.info("Error With URL "+httpurl);
817 return False;
818 except BlockingIOError:
819 log.info("Error With URL "+httpurl);
820 return False;
821 geturls_text = httpconn.getresponse();
822 httpcodeout = geturls_text.status;
823 httpcodereason = geturls_text.reason;
824 if(geturls_text.version=="10"):
825 httpversionout = "1.0";
826 else:
827 httpversionout = "1.1";
828 httpmethodout = httpmethod;
829 httpurlout = httpurl;
830 httpheaderout = geturls_text.getheaders();
831 httpheadersentout = httpheaders;
832 elif(httplibuse=="urllib3" or httplibuse=="request3"):
833 timeout = urllib3.util.Timeout(connect=timeout, read=timeout);
834 urllib_pool = urllib3.PoolManager(headers=httpheaders, timeout=timeout);
835 try:
836 if(httpmethod=="GET"):
837 geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
838 elif(httpmethod=="POST"):
839 geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
840 else:
841 geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
842 except urllib3.exceptions.ConnectTimeoutError:
843 log.info("Error With URL "+httpurl);
844 return False;
845 except urllib3.exceptions.ConnectError:
846 log.info("Error With URL "+httpurl);
847 return False;
848 except urllib3.exceptions.MaxRetryError:
849 log.info("Error With URL "+httpurl);
850 return False;
851 except socket.timeout:
852 log.info("Error With URL "+httpurl);
853 return False;
854 except ValueError:
855 log.info("Error With URL "+httpurl);
856 return False;
857 httpcodeout = geturls_text.status;
858 httpcodereason = geturls_text.reason;
859 if(geturls_text.version=="10"):
860 httpversionout = "1.0";
861 else:
862 httpversionout = "1.1";
863 httpmethodout = httpmethod;
864 httpurlout = geturls_text.geturl();
865 httpheaderout = geturls_text.info();
866 httpheadersentout = httpheaders;
867 elif(httplibuse=="requests"):
868 try:
869 reqsession = requests.Session();
870 if(httpmethod=="GET"):
871 geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
872 elif(httpmethod=="POST"):
873 geturls_text = reqsession.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
874 else:
875 geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
876 except requests.exceptions.ConnectTimeout:
877 log.info("Error With URL "+httpurl);
878 return False;
879 except requests.exceptions.ConnectError:
880 log.info("Error With URL "+httpurl);
881 return False;
882 except socket.timeout:
883 log.info("Error With URL "+httpurl);
884 return False;
885 httpcodeout = geturls_text.status_code;
886 httpcodereason = geturls_text.reason;
887 if(geturls_text.raw.version=="10"):
888 httpversionout = "1.0";
889 else:
890 httpversionout = "1.1";
891 httpmethodout = httpmethod;
892 httpurlout = geturls_text.url;
893 httpheaderout = geturls_text.headers;
894 httpheadersentout = geturls_text.request.headers;
895 elif(httplibuse=="aiohttp"):
896 try:
897 reqsession = aiohttp.ClientSession(cookie_jar=httpcookie, headers=httpheaders, timeout=timeout, read_timeout=timeout, conn_timeout=timeout, read_bufsize=buffersize);
898 if(httpmethod=="GET"):
899 geturls_text = reqsession.get(httpurl);
900 elif(httpmethod=="POST"):
901 geturls_text = reqsession.post(httpurl, data=postdata);
902 else:
903 geturls_text = reqsession.get(httpurl);
904 except aiohttp.exceptions.ConnectTimeout:
905 log.info("Error With URL "+httpurl);
906 return False;
907 except aiohttp.exceptions.ConnectError:
908 log.info("Error With URL "+httpurl);
909 return False;
910 except socket.timeout:
911 log.info("Error With URL "+httpurl);
912 return False;
913 httpcodeout = geturls_text.status;
914 httpcodereason = geturls_text.reason;
915 httpversionout = geturls_text.version;
916 httpmethodout = geturls_text.method;
917 httpurlout = geturls_text.url;
918 httpheaderout = geturls_text.headers;
919 httpheadersentout = geturls_text.request_info.headers;
920 elif(httplibuse=="httpx"):
921 try:
922 if(httpmethod=="GET"):
923 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
924 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
925 elif(httpmethod=="POST"):
926 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
927 geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
928 else:
929 httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
930 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
931 except httpx.ConnectTimeout:
932 log.info("Error With URL "+httpurl);
933 return False;
934 except httpx.ConnectError:
935 log.info("Error With URL "+httpurl);
936 return False;
937 except socket.timeout:
938 log.info("Error With URL "+httpurl);
939 return False;
940 httpcodeout = geturls_text.status_code;
941 try:
942 httpcodereason = geturls_text.reason_phrase;
943 except:
944 httpcodereason = http_status_to_reason(geturls_text.status_code);
945 httpversionout = geturls_text.http_version;
946 httpmethodout = httpmethod;
947 httpurlout = str(geturls_text.url);
948 httpheaderout = geturls_text.headers;
949 httpheadersentout = geturls_text.request.headers;
950 elif(httplibuse=="httpx2"):
951 try:
952 if(httpmethod=="GET"):
953 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
954 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
955 elif(httpmethod=="POST"):
956 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
957 geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
958 else:
959 httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
960 geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
961 except httpx.ConnectTimeout:
962 log.info("Error With URL "+httpurl);
963 return False;
964 except httpx.ConnectError:
965 log.info("Error With URL "+httpurl);
966 return False;
967 except socket.timeout:
968 log.info("Error With URL "+httpurl);
969 return False;
970 httpcodeout = geturls_text.status_code;
971 try:
972 httpcodereason = geturls_text.reason_phrase;
973 except:
974 httpcodereason = http_status_to_reason(geturls_text.status_code);
975 httpversionout = geturls_text.http_version;
976 httpmethodout = httpmethod;
977 httpurlout = str(geturls_text.url);
978 httpheaderout = geturls_text.headers;
979 httpheadersentout = geturls_text.request.headers;
980 elif(httplibuse=="httpcore"):
981 try:
982 if(httpmethod=="GET"):
983 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
984 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
985 elif(httpmethod=="POST"):
986 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
987 geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
988 else:
989 httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
990 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
991 except httpcore.ConnectTimeout:
992 log.info("Error With URL "+httpurl);
993 return False;
994 except httpcore.ConnectError:
995 log.info("Error With URL "+httpurl);
996 return False;
997 except socket.timeout:
998 log.info("Error With URL "+httpurl);
999 return False;
1000 httpcodeout = geturls_text.status;
1001 httpcodereason = http_status_to_reason(geturls_text.status);
1002 httpversionout = "1.1";
1003 httpmethodout = httpmethod;
1004 httpurlout = str(httpurl);
1005 httpheaderout = geturls_text.headers;
1006 httpheadersentout = httpheaders;
1007 elif(httplibuse=="httpcore2"):
1008 try:
1009 if(httpmethod=="GET"):
1010 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1011 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1012 elif(httpmethod=="POST"):
1013 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1014 geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
1015 else:
1016 httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1017 geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1018 except httpcore.ConnectTimeout:
1019 log.info("Error With URL "+httpurl);
1020 return False;
1021 except httpcore.ConnectError:
1022 log.info("Error With URL "+httpurl);
1023 return False;
1024 except socket.timeout:
1025 log.info("Error With URL "+httpurl);
1026 return False;
1027 httpcodeout = geturls_text.status;
1028 httpcodereason = http_status_to_reason(geturls_text.status);
1029 httpversionout = "1.1";
1030 httpmethodout = httpmethod;
1031 httpurlout = str(httpurl);
1032 httpheaderout = geturls_text.headers;
1033 httpheadersentout = httpheaders;
1034 elif(httplibuse=="mechanize"):
1035 geturls_opener = mechanize.Browser();
1036 if(isinstance(httpheaders, dict)):
1037 httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1038 time.sleep(sleep);
1039 geturls_opener.addheaders = httpheaders;
1040 geturls_opener.set_cookiejar(httpcookie);
1041 geturls_opener.set_handle_robots(False);
1042 if(postdata is not None and not isinstance(postdata, dict)):
1043 postdata = urlencode(postdata);
1044 try:
1045 if(httpmethod=="GET"):
1046 geturls_text = geturls_opener.open(httpurl);
1047 elif(httpmethod=="POST"):
1048 geturls_text = geturls_opener.open(httpurl, data=postdata);
1049 else:
1050 geturls_text = geturls_opener.open(httpurl);
1051 except mechanize.HTTPError as geturls_text_error:
1052 geturls_text = geturls_text_error;
1053 log.info("Error With URL "+httpurl);
1054 except URLError:
1055 log.info("Error With URL "+httpurl);
1056 return False;
1057 except socket.timeout:
1058 log.info("Error With URL "+httpurl);
1059 return False;
1060 httpcodeout = geturls_text.code;
1061 httpcodereason = geturls_text.msg;
1062 httpversionout = "1.1";
1063 httpmethodout = httpmethod;
1064 httpurlout = geturls_text.geturl();
1065 httpheaderout = geturls_text.info();
1066 reqhead = geturls_opener.request;
1067 httpheadersentout = reqhead.header_items();
1068 elif(httplibuse=="pycurl"):
1069 retrieved_body = BytesIO();
1070 retrieved_headers = BytesIO();
1071 try:
1072 if(httpmethod=="GET"):
1073 geturls_text = pycurl.Curl();
1074 geturls_text.setopt(geturls_text.URL, httpurl);
1075 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1076 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1077 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1078 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1079 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1080 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1081 geturls_text.perform();
1082 elif(httpmethod=="POST"):
1083 geturls_text = pycurl.Curl();
1084 geturls_text.setopt(geturls_text.URL, httpurl);
1085 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1086 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1087 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1088 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1089 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1090 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1091 geturls_text.setopt(geturls_text.POST, True);
1092 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1093 geturls_text.perform();
1094 else:
1095 geturls_text = pycurl.Curl();
1096 geturls_text.setopt(geturls_text.URL, httpurl);
1097 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1098 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1099 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1100 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1101 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1102 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1103 geturls_text.perform();
1104 retrieved_headers.seek(0);
1105 if(sys.version[0]=="2"):
1106 pycurlhead = retrieved_headers.read();
1107 if(sys.version[0]>="3"):
1108 pycurlhead = retrieved_headers.read().decode('UTF-8');
1109 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1110 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1111 retrieved_body.seek(0);
1112 except socket.timeout:
1113 log.info("Error With URL "+httpurl);
1114 return False;
1115 except socket.gaierror:
1116 log.info("Error With URL "+httpurl);
1117 return False;
1118 except ValueError:
1119 log.info("Error With URL "+httpurl);
1120 return False;
1121 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1122 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1123 httpversionout = pyhttpverinfo[0];
1124 httpmethodout = httpmethod;
1125 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1126 httpheaderout = pycurlheadersout;
1127 httpheadersentout = httpheaders;
1128 elif(httplibuse=="pycurl2"):
1129 retrieved_body = BytesIO();
1130 retrieved_headers = BytesIO();
1131 try:
1132 if(httpmethod=="GET"):
1133 geturls_text = pycurl.Curl();
1134 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1135 geturls_text.setopt(geturls_text.URL, httpurl);
1136 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1137 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1138 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1139 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1140 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1141 geturls_text.perform();
1142 elif(httpmethod=="POST"):
1143 geturls_text = pycurl.Curl();
1144 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1145 geturls_text.setopt(geturls_text.URL, httpurl);
1146 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1147 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1148 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1149 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1150 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1151 geturls_text.setopt(geturls_text.POST, True);
1152 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1153 geturls_text.perform();
1154 else:
1155 geturls_text = pycurl.Curl();
1156 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1157 geturls_text.setopt(geturls_text.URL, httpurl);
1158 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1159 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1160 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1161 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1162 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1163 geturls_text.perform();
1164 retrieved_headers.seek(0);
1165 if(sys.version[0]=="2"):
1166 pycurlhead = retrieved_headers.read();
1167 if(sys.version[0]>="3"):
1168 pycurlhead = retrieved_headers.read().decode('UTF-8');
1169 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip())[0];
1170 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1171 retrieved_body.seek(0);
1172 except socket.timeout:
1173 log.info("Error With URL "+httpurl);
1174 return False;
1175 except socket.gaierror:
1176 log.info("Error With URL "+httpurl);
1177 return False;
1178 except ValueError:
1179 log.info("Error With URL "+httpurl);
1180 return False;
1181 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1182 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1183 httpversionout = pyhttpverinfo[0];
1184 httpmethodout = httpmethod;
1185 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1186 httpheaderout = pycurlheadersout;
1187 httpheadersentout = httpheaders;
1188 elif(httplibuse=="pycurl3"):
1189 retrieved_body = BytesIO();
1190 retrieved_headers = BytesIO();
1191 try:
1192 if(httpmethod=="GET"):
1193 geturls_text = pycurl.Curl();
1194 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1195 geturls_text.setopt(geturls_text.URL, httpurl);
1196 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1197 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1198 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1199 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1200 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1201 geturls_text.perform();
1202 elif(httpmethod=="POST"):
1203 geturls_text = pycurl.Curl();
1204 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1205 geturls_text.setopt(geturls_text.URL, httpurl);
1206 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1207 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1208 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1209 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1210 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1211 geturls_text.setopt(geturls_text.POST, True);
1212 geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1213 geturls_text.perform();
1214 else:
1215 geturls_text = pycurl.Curl();
1216 geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1217 geturls_text.setopt(geturls_text.URL, httpurl);
1218 geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1219 geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1220 geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1221 geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1222 geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1223 geturls_text.perform();
1224 retrieved_headers.seek(0);
1225 if(sys.version[0]=="2"):
1226 pycurlhead = retrieved_headers.read();
1227 if(sys.version[0]>="3"):
1228 pycurlhead = retrieved_headers.read().decode('UTF-8');
1229 pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1230 pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1231 retrieved_body.seek(0);
1232 except socket.timeout:
1233 log.info("Error With URL "+httpurl);
1234 return False;
1235 except socket.gaierror:
1236 log.info("Error With URL "+httpurl);
1237 return False;
1238 except ValueError:
1239 log.info("Error With URL "+httpurl);
1240 return False;
1241 httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1242 httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1243 httpversionout = pyhttpverinfo[0];
1244 httpmethodout = httpmethod;
1245 httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1246 httpheaderout = pycurlheadersout;
1247 httpheadersentout = httpheaders;
1248 elif(httplibuse=="ftp"):
1249 geturls_text = download_file_from_ftp_file(httpurl);
1250 if(not geturls_text):
1251 return False;
1252 downloadsize = None;
1253 if(downloadsize is not None):
1254 downloadsize = int(downloadsize);
1255 if downloadsize is None: downloadsize = 0;
1256 fulldatasize = 0;
1257 prevdownsize = 0;
1258 log.info("Downloading URL "+httpurl);
1259 with BytesIO() as strbuf:
1260 while True:
1261 databytes = geturls_text.read(buffersize);
1262 if not databytes: break;
1263 datasize = len(databytes);
1264 fulldatasize = datasize + fulldatasize;
1265 percentage = "";
1266 if(downloadsize>0):
1267 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1268 downloaddiff = fulldatasize - prevdownsize;
1269 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1270 prevdownsize = fulldatasize;
1271 strbuf.write(databytes);
1272 strbuf.seek(0);
1273 returnval_content = strbuf.read();
1274 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1275 geturls_text.close();
1276 elif(httplibuse=="sftp"):
1277 geturls_text = download_file_from_sftp_file(httpurl);
1278 if(not geturls_text):
1279 return False;
1280 downloadsize = None;
1281 if(downloadsize is not None):
1282 downloadsize = int(downloadsize);
1283 if downloadsize is None: downloadsize = 0;
1284 fulldatasize = 0;
1285 prevdownsize = 0;
1286 log.info("Downloading URL "+httpurl);
1287 with BytesIO() as strbuf:
1288 while True:
1289 databytes = geturls_text.read(buffersize);
1290 if not databytes: break;
1291 datasize = len(databytes);
1292 fulldatasize = datasize + fulldatasize;
1293 percentage = "";
1294 if(downloadsize>0):
1295 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1296 downloaddiff = fulldatasize - prevdownsize;
1297 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1298 prevdownsize = fulldatasize;
1299 strbuf.write(databytes);
1300 strbuf.seek(0);
1301 returnval_content = strbuf.read();
1302 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1303 geturls_text.close();
1304 return returnval;
1305 elif(httplibuse=="pysftp"):
1306 geturls_text = download_file_from_pysftp_file(httpurl);
1307 if(not geturls_text):
1308 return False;
1309 downloadsize = None;
1310 if(downloadsize is not None):
1311 downloadsize = int(downloadsize);
1312 if downloadsize is None: downloadsize = 0;
1313 fulldatasize = 0;
1314 prevdownsize = 0;
1315 log.info("Downloading URL "+httpurl);
1316 with BytesIO() as strbuf:
1317 while True:
1318 databytes = geturls_text.read(buffersize);
1319 if not databytes: break;
1320 datasize = len(databytes);
1321 fulldatasize = datasize + fulldatasize;
1322 percentage = "";
1323 if(downloadsize>0):
1324 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1325 downloaddiff = fulldatasize - prevdownsize;
1326 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1327 prevdownsize = fulldatasize;
1328 strbuf.write(databytes);
1329 strbuf.seek(0);
1330 returnval_content = strbuf.read();
1331 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1332 geturls_text.close();
1333 return returnval;
1334 else:
1335 returnval = False;
1336 if(isinstance(httpheaderout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1337 httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1338 if(isinstance(httpheaderout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1339 httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
1340 if(sys.version[0]=="2"):
1341 try:
1342 prehttpheaderout = httpheaderout;
1343 httpheaderkeys = httpheaderout.keys();
1344 imax = len(httpheaderkeys);
1345 ic = 0;
1346 httpheaderout = {};
1347 while(ic < imax):
1348 httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1349 ic += 1;
1350 except AttributeError:
1351 pass;
1352 httpheaderout = fix_header_names(httpheaderout);
1353 if(isinstance(httpheadersentout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1354 httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1355 if(isinstance(httpheadersentout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1356 httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
1357 httpheadersentout = fix_header_names(httpheadersentout);
1358 log.info("Downloading URL "+httpurl);
1359 if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="aiohttp" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize"):
1360 downloadsize = httpheaderout.get('Content-Length');
1361 if(downloadsize is not None):
1362 downloadsize = int(downloadsize);
1363 if downloadsize is None: downloadsize = 0;
1364 fulldatasize = 0;
1365 prevdownsize = 0;
1366 log.info("Downloading URL "+httpurl);
1367 with BytesIO() as strbuf:
1368 while True:
1369 databytes = geturls_text.read(buffersize);
1370 if not databytes: break;
1371 datasize = len(databytes);
1372 fulldatasize = datasize + fulldatasize;
1373 percentage = "";
1374 if(downloadsize>0):
1375 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1376 downloaddiff = fulldatasize - prevdownsize;
1377 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1378 prevdownsize = fulldatasize;
1379 strbuf.write(databytes);
1380 strbuf.seek(0);
1381 returnval_content = strbuf.read();
1382 geturls_text.close();
1383 if(httpheaderout.get("Content-Encoding")=="gzip"):
1384 try:
1385 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1386 except zlib.error:
1387 pass;
1388 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1389 try:
1390 returnval_content = zlib.decompress(returnval_content);
1391 except zlib.error:
1392 pass;
1393 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1394 try:
1395 returnval_content = brotli.decompress(returnval_content);
1396 except brotli.error:
1397 pass;
1398 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1399 try:
1400 returnval_content = zstandard.decompress(returnval_content);
1401 except zstandard.error:
1402 pass;
1403 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1404 try:
1405 returnval_content = lzma.decompress(returnval_content);
1406 except zstandard.error:
1407 pass;
1408 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1409 try:
1410 returnval_content = bz2.decompress(returnval_content);
1411 except zstandard.error:
1412 pass;
1413 elif(httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"):
1414 downloadsize = httpheaderout.get('Content-Length');
1415 if(downloadsize is not None):
1416 downloadsize = int(downloadsize);
1417 if downloadsize is None: downloadsize = 0;
1418 fulldatasize = 0;
1419 prevdownsize = 0;
1420 log.info("Downloading URL "+httpurl);
1421 with BytesIO() as strbuf:
1422 while True:
1423 databytes = geturls_text.read();
1424 if not databytes: break;
1425 datasize = len(databytes);
1426 fulldatasize = datasize + fulldatasize;
1427 percentage = "";
1428 if(downloadsize>0):
1429 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1430 downloaddiff = fulldatasize - prevdownsize;
1431 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1432 prevdownsize = fulldatasize;
1433 strbuf.write(databytes);
1434 break;
1435 strbuf.seek(0);
1436 returnval_content = strbuf.read();
1437 geturls_text.close();
1438 if(httpheaderout.get("Content-Encoding")=="gzip"):
1439 try:
1440 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1441 except zlib.error:
1442 pass;
1443 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1444 try:
1445 returnval_content = zlib.decompress(returnval_content);
1446 except zlib.error:
1447 pass;
1448 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1449 try:
1450 returnval_content = brotli.decompress(returnval_content);
1451 except brotli.error:
1452 pass;
1453 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1454 try:
1455 returnval_content = zstandard.decompress(returnval_content);
1456 except zstandard.error:
1457 pass;
1458 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1459 try:
1460 returnval_content = lzma.decompress(returnval_content);
1461 except zstandard.error:
1462 pass;
1463 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1464 try:
1465 returnval_content = bz2.decompress(returnval_content);
1466 except zstandard.error:
1467 pass;
1468 elif(httplibuse=="requests"):
1469 log.info("Downloading URL "+httpurl);
1470 downloadsize = httpheaderout.get('Content-Length');
1471 if(downloadsize is not None):
1472 downloadsize = int(downloadsize);
1473 if downloadsize is None: downloadsize = 0;
1474 fulldatasize = 0;
1475 prevdownsize = 0;
1476 log.info("Downloading URL "+httpurl);
1477 with BytesIO() as strbuf:
1478 while True:
1479 databytes = geturls_text.raw.read(buffersize);
1480 if not databytes: break;
1481 datasize = len(databytes);
1482 fulldatasize = datasize + fulldatasize;
1483 percentage = "";
1484 if(downloadsize>0):
1485 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1486 downloaddiff = fulldatasize - prevdownsize;
1487 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1488 prevdownsize = fulldatasize;
1489 strbuf.write(databytes);
1490 strbuf.seek(0);
1491 returnval_content = strbuf.read();
1492 geturls_text.close();
1493 if(httpheaderout.get("Content-Encoding")=="gzip"):
1494 try:
1495 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1496 except zlib.error:
1497 pass;
1498 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1499 try:
1500 returnval_content = zlib.decompress(returnval_content);
1501 except zlib.error:
1502 pass;
1503 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1504 try:
1505 returnval_content = brotli.decompress(returnval_content);
1506 except brotli.error:
1507 pass;
1508 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1509 try:
1510 returnval_content = zstandard.decompress(returnval_content);
1511 except zstandard.error:
1512 pass;
1513 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1514 try:
1515 returnval_content = lzma.decompress(returnval_content);
1516 except zstandard.error:
1517 pass;
1518 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1519 try:
1520 returnval_content = bz2.decompress(returnval_content);
1521 except zstandard.error:
1522 pass;
1523 elif(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
1524 log.info("Downloading URL "+httpurl);
1525 downloadsize = httpheaderout.get('Content-Length');
1526 if(downloadsize is not None):
1527 downloadsize = int(downloadsize);
1528 if downloadsize is None: downloadsize = 0;
1529 fulldatasize = 0;
1530 prevdownsize = 0;
1531 log.info("Downloading URL "+httpurl);
1532 with BytesIO() as strbuf:
1533 while True:
1534 databytes = retrieved_body.read(buffersize);
1535 if not databytes: break;
1536 datasize = len(databytes);
1537 fulldatasize = datasize + fulldatasize;
1538 percentage = "";
1539 if(downloadsize>0):
1540 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1541 downloaddiff = fulldatasize - prevdownsize;
1542 log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1543 prevdownsize = fulldatasize;
1544 strbuf.write(databytes);
1545 strbuf.seek(0);
1546 returnval_content = strbuf.read();
1547 geturls_text.close();
1548 if(httpheaderout.get("Content-Encoding")=="gzip"):
1549 try:
1550 returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1551 except zlib.error:
1552 pass;
1553 elif(httpheaderout.get("Content-Encoding")=="deflate"):
1554 try:
1555 returnval_content = zlib.decompress(returnval_content);
1556 except zlib.error:
1557 pass;
1558 elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1559 try:
1560 returnval_content = brotli.decompress(returnval_content);
1561 except brotli.error:
1562 pass;
1563 elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1564 try:
1565 returnval_content = zstandard.decompress(returnval_content);
1566 except zstandard.error:
1567 pass;
1568 elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1569 try:
1570 returnval_content = lzma.decompress(returnval_content);
1571 except zstandard.error:
1572 pass;
1573 elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1574 try:
1575 returnval_content = bz2.decompress(returnval_content);
1576 except zstandard.error:
1577 pass;
1578 elif(httplibuse=="ftp" or httplibuse=="sftp" or httplibuse=="pysftp"):
1579 pass;
1580 else:
1581 returnval = False;
1582 returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason, 'HTTPLib': httplibuse};
1583 return returnval;
1584
1585 def download_from_url_from_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
1586 if(isinstance(httpurl, list)):
1587 pass;
1588 elif(isinstance(httpurl, tuple)):
1589 pass;
1590 elif(isinstance(httpurl, dict)):
1591 httpurl = httpurl.values();
1592 else:
1593 httpurl = [httpurl];
1594 listsize = len(httpurl);
1595 listcount = 0;
1596 returnval = [];
1597 while(listcount<listsize):
1598 ouputval = download_from_url(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout);
1599 returnval.append(ouputval);
1600 listcount += 1;
1601 return returnval;
1602
1603 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1604 global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix, haveaiohttp, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1605 exec_time_start = time.time();
1606 myhash = hashlib.new("sha1");
1607 if(sys.version[0]=="2"):
1608 myhash.update(httpurl);
1609 myhash.update(str(buffersize));
1610 myhash.update(str(exec_time_start));
1611 if(sys.version[0]>="3"):
1612 myhash.update(httpurl.encode('utf-8'));
1613 myhash.update(str(buffersize).encode('utf-8'));
1614 myhash.update(str(exec_time_start).encode('utf-8'));
1615 newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1616 if(sleep<0):
1617 sleep = geturls_download_sleep;
1618 if(timeout<=0):
1619 timeout = 10;
1620 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1621 httplibuse = "urllib";
1622 if(httplibuse=="httplib1"):
1623 httplibuse = "httplib";
1624 if(not haverequests and httplibuse=="requests"):
1625 httplibuse = "urllib";
1626 if(not haveaiohttp and httplibuse=="aiohttp"):
1627 httplibuse = "urllib";
1628 if(not havehttpx and httplibuse=="httpx"):
1629 httplibuse = "urllib";
1630 if(not havehttpx and httplibuse=="httpx2"):
1631 httplibuse = "urllib";
1632 if(not havehttpcore and httplibuse=="httpcore"):
1633 httplibuse = "urllib";
1634 if(not havehttpcore and httplibuse=="httpcore2"):
1635 httplibuse = "urllib";
1636 if(not havemechanize and httplibuse=="mechanize"):
1637 httplibuse = "urllib";
1638 if(not havepycurl and httplibuse=="pycurl"):
1639 httplibuse = "urllib";
1640 if(not havepycurl and httplibuse=="pycurl2"):
1641 httplibuse = "urllib";
1642 if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1643 httplibuse = "pycurl";
1644 if(not havepycurl and httplibuse=="pycurl3"):
1645 httplibuse = "urllib";
1646 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1647 httplibuse = "pycurl2";
1648 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1649 httplibuse = "pycurl";
1650 if(not havehttplib2 and httplibuse=="httplib2"):
1651 httplibuse = "httplib";
1652 if(not haveparamiko and httplibuse=="sftp"):
1653 httplibuse = "ftp";
1654 if(not haveparamiko and httplibuse=="pysftp"):
1655 httplibuse = "ftp";
1656 pretmpfilename = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout);
1657 if(not pretmpfilename):
1658 return False;
1659 with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1660 tmpfilename = f.name;
1661 try:
1662 os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1663 except AttributeError:
1664 try:
1665 os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1666 except ValueError:
1667 pass;
1668 except ValueError:
1669 pass;
1670 returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1671 f.write(pretmpfilename.get('Content'));
1672 f.close();
1673 exec_time_end = time.time();
1674 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1675 returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1676 return returnval;
1677
1678 def download_from_url_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1679 if(isinstance(httpurl, list)):
1680 pass;
1681 elif(isinstance(httpurl, tuple)):
1682 pass;
1683 elif(isinstance(httpurl, dict)):
1684 httpurl = httpurl.values();
1685 else:
1686 httpurl = [httpurl];
1687 listsize = len(httpurl);
1688 listcount = 0;
1689 returnval = [];
1690 while(listcount<listsize):
1691 ouputval = download_from_url_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize, sleep, timeout);
1692 returnval.append(ouputval);
1693 listcount += 1;
1694 return returnval;
1695
1696 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1697 global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1698 if(sleep<0):
1699 sleep = geturls_download_sleep;
1700 if(timeout<=0):
1701 timeout = 10;
1702 if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1703 httplibuse = "urllib";
1704 if(httplibuse=="httplib1"):
1705 httplibuse = "httplib";
1706 if(not haverequests and httplibuse=="requests"):
1707 httplibuse = "urllib";
1708 if(not haveaiohttp and httplibuse=="aiohttp"):
1709 httplibuse = "urllib";
1710 if(not havehttpx and httplibuse=="httpx"):
1711 httplibuse = "urllib";
1712 if(not havehttpx and httplibuse=="httpx2"):
1713 httplibuse = "urllib";
1714 if(not havehttpcore and httplibuse=="httpcore"):
1715 httplibuse = "urllib";
1716 if(not havehttpcore and httplibuse=="httpcore2"):
1717 httplibuse = "urllib";
1718 if(not havemechanize and httplibuse=="mechanize"):
1719 httplibuse = "urllib";
1720 if(not havepycurl and httplibuse=="pycurl"):
1721 httplibuse = "urllib";
1722 if(not havepycurl and httplibuse=="pycurl2"):
1723 httplibuse = "urllib";
1724 if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1725 httplibuse = "pycurl";
1726 if(not havepycurl and httplibuse=="pycurl3"):
1727 httplibuse = "urllib";
1728 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1729 httplibuse = "pycurl2";
1730 if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1731 httplibuse = "pycurl";
1732 if(not havehttplib2 and httplibuse=="httplib2"):
1733 httplibuse = "httplib";
1734 if(not haveparamiko and httplibuse=="sftp"):
1735 httplibuse = "ftp";
1736 if(not havepysftp and httplibuse=="pysftp"):
1737 httplibuse = "ftp";
1738 if(not outfile=="-"):
1739 outpath = outpath.rstrip(os.path.sep);
1740 filepath = os.path.realpath(outpath+os.path.sep+outfile);
1741 if(not os.path.exists(outpath)):
1742 os.makedirs(outpath);
1743 if(os.path.exists(outpath) and os.path.isfile(outpath)):
1744 return False;
1745 if(os.path.exists(filepath) and os.path.isdir(filepath)):
1746 return False;
1747 pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1748 if(not pretmpfilename):
1749 return False;
1750 tmpfilename = pretmpfilename.get('Filename');
1751 downloadsize = int(os.path.getsize(tmpfilename));
1752 fulldatasize = 0;
1753 log.info("Moving file "+tmpfilename+" to "+filepath);
1754 exec_time_start = time.time();
1755 shutil.move(tmpfilename, filepath);
1756 try:
1757 os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1758 except AttributeError:
1759 try:
1760 os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1761 except ValueError:
1762 pass;
1763 except ValueError:
1764 pass;
1765 exec_time_end = time.time();
1766 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1767 if(os.path.exists(tmpfilename)):
1768 os.remove(tmpfilename);
1769 returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1770 if(outfile=="-"):
1771 pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1772 tmpfilename = pretmpfilename.get('Filename');
1773 downloadsize = int(os.path.getsize(tmpfilename));
1774 fulldatasize = 0;
1775 prevdownsize = 0;
1776 exec_time_start = time.time();
1777 with open(tmpfilename, 'rb') as ft:
1778 f = BytesIO();
1779 while True:
1780 databytes = ft.read(buffersize[1]);
1781 if not databytes: break;
1782 datasize = len(databytes);
1783 fulldatasize = datasize + fulldatasize;
1784 percentage = "";
1785 if(downloadsize>0):
1786 percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1787 downloaddiff = fulldatasize - prevdownsize;
1788 log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1789 prevdownsize = fulldatasize;
1790 f.write(databytes);
1791 f.seek(0);
1792 fdata = f.getvalue();
1793 f.close();
1794 ft.close();
1795 os.remove(tmpfilename);
1796 exec_time_end = time.time();
1797 log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1798 returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': httplibuse};
1799 return returnval;
1800
1801 def download_from_url_to_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1802 if(isinstance(httpurl, list)):
1803 pass;
1804 elif(isinstance(httpurl, tuple)):
1805 pass;
1806 elif(isinstance(httpurl, dict)):
1807 httpurl = httpurl.values();
1808 else:
1809 httpurl = [httpurl];
1810 listsize = len(httpurl);
1811 listcount = 0;
1812 returnval = [];
1813 while(listcount<listsize):
1814 ouputval = download_from_url_to_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, outfile, outpath, ranges, buffersize, sleep, timeout);
1815 returnval.append(ouputval);
1816 listcount += 1;
1817 return returnval;
1818
1819 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1820 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1821 return returnval;
1822
1823 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1824 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1825 return returnval;
1826
1827 def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1828 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", buffersize, sleep, timeout);
1829 return returnval;
1830
1831 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1832 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", buffersize, sleep, timeout);
1833 return returnval;
1834
1835 def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1836 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", buffersize, sleep, timeout);
1837 return returnval;
1838
1839 def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1840 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", buffersize, sleep, timeout);
1841 return returnval;
1842
1843 def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1844 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", buffersize, sleep, timeout);
1845 return returnval;
1846
1847 def download_from_url_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1848 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", buffersize, sleep, timeout);
1849 return returnval;
1850
1851 def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1852 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", buffersize, sleep, timeout);
1853 return returnval;
1854
1855 def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1856 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", buffersize, sleep, timeout);
1857 return returnval;
1858
1859 def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1860 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", buffersize, sleep, timeout);
1861 return returnval;
1862
1863 def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1864 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", buffersize, sleep, timeout);
1865 return returnval;
1866
1867 def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1868 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", buffersize, sleep, timeout);
1869 return returnval;
1870
1871 def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1872 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", buffersize, sleep, timeout);
1873 return returnval;
1874
1875 def download_from_url_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1876 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", buffersize, sleep, timeout);
1877 return returnval;
1878
1879 def download_from_url_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1880 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", buffersize, sleep, timeout);
1881 return returnval;
1882
1883 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1884 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", buffersize, sleep, timeout);
1885 return returnval;
1886
1887 def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1888 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", buffersize, sleep, timeout);
1889 return returnval;
1890
1891 def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1892 returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", buffersize, sleep, timeout);
1893 return returnval;
1894
1895 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1896 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1897 return returnval;
1898
1899 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1900 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1901 return returnval;
1902
1903 def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1904 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", ranges, buffersize, sleep, timeout);
1905 return returnval;
1906
1907 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1908 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", ranges, buffersize, sleep, timeout);
1909 return returnval;
1910
1911 def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1912 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", ranges, buffersize, sleep, timeout);
1913 return returnval;
1914
1915 def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1916 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", ranges, buffersize, sleep, timeout);
1917 return returnval;
1918
1919 def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1920 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", ranges, buffersize, sleep, timeout);
1921 return returnval;
1922
1923 def download_from_url_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1924 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", ranges, buffersize, sleep, timeout);
1925 return returnval;
1926
1927 def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1928 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", ranges, buffersize, sleep, timeout);
1929 return returnval;
1930
1931 def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1932 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", ranges, buffersize, sleep, timeout);
1933 return returnval;
1934
1935 def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1936 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", ranges, buffersize, sleep, timeout);
1937 return returnval;
1938
1939 def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1940 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", ranges, buffersize, sleep, timeout);
1941 return returnval;
1942
1943 def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1944 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", ranges, buffersize, sleep, timeout);
1945 return returnval;
1946
1947 def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1948 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", ranges, buffersize, sleep, timeout);
1949 return returnval;
1950
1951 def download_from_url_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1952 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", ranges, buffersize, sleep, timeout);
1953 return returnval;
1954
1955 def download_from_url_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1956 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", ranges, buffersize, sleep, timeout);
1957 return returnval;
1958
1959 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1960 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", ranges, buffersize, sleep, timeout);
1961 return returnval;
1962
1963 def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1964 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", ranges, buffersize, sleep, timeout);
1965 return returnval;
1966
1967 def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1968 returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", ranges, buffersize, sleep, timeout);
1969 return returnval;
1970
1971 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1972 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1973 return returnval;
1974
1975 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1976 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", outfile, outpath, ranges, buffersize, sleep, timeout);
1977 return returnval;
1978
1979 def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1980 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1981 return returnval;
1982
1983 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1984 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", outfile, outpath, ranges, buffersize, sleep, timeout);
1985 return returnval;
1986
1987 def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1988 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", outfile, outpath, ranges, buffersize, sleep, timeout);
1989 return returnval;
1990
1991 def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1992 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", outfile, outpath, ranges, buffersize, sleep, timeout);
1993 return returnval;
1994
1995 def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1996 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", outfile, outpath, ranges, buffersize, sleep, timeout);
1997 return returnval;
1998
1999 def download_from_url_to_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2000 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", outfile, outpath, ranges, buffersize, sleep, timeout);
2001 return returnval;
2002
2003 def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2004 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", outfile, outpath, ranges, buffersize, sleep, timeout);
2005 return returnval;
2006
2007 def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2008 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", outfile, outpath, ranges, buffersize, sleep, timeout);
2009 return returnval;
2010
2011 def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2012 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", outfile, outpath, ranges, buffersize, sleep, timeout);
2013 return returnval;
2014
2015 def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2016 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", outfile, outpath, ranges, buffersize, sleep, timeout);
2017 return returnval;
2018
2019 def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2020 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", outfile, outpath, ranges, buffersize, sleep, timeout);
2021 return returnval;
2022
2023 def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2024 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", outfile, outpath, ranges, buffersize, sleep, timeout);
2025 return returnval;
2026
2027 def download_from_url_to_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2028 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", outfile, outpath, ranges, buffersize, sleep, timeout);
2029 return returnval;
2030
2031 def download_from_url_to_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2032 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", outfile, outpath, ranges, buffersize, sleep, timeout);
2033 return returnval;
2034
2035 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2036 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2037 return returnval;
2038
2039 def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2040 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2041 return returnval;
2042
2043 def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2044 returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2045 return returnval;
2046
2047 def download_file_from_ftp_file(url):
2048 urlparts = urlparse.urlparse(url);
2049 file_name = os.path.basename(urlparts.path);
2050 file_dir = os.path.dirname(urlparts.path);
2051 if(urlparts.username is not None):
2052 ftp_username = urlparts.username;
2053 else:
2054 ftp_username = "anonymous";
2055 if(urlparts.password is not None):
2056 ftp_password = urlparts.password;
2057 elif(urlparts.password is None and urlparts.username=="anonymous"):
2058 ftp_password = "anonymous";
2059 else:
2060 ftp_password = "";
2061 if(urlparts.scheme=="ftp"):
2062 ftp = FTP();
2063 elif(urlparts.scheme=="ftps"):
2064 ftp = FTP_TLS();
2065 else:
2066 return False;
2067 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2068 return False;
2069 ftp_port = urlparts.port;
2070 if(urlparts.port is None):
2071 ftp_port = 21;
2072 try:
2073 ftp.connect(urlparts.hostname, ftp_port);
2074 except socket.gaierror:
2075 log.info("Error With URL "+httpurl);
2076 return False;
2077 except socket.timeout:
2078 log.info("Error With URL "+httpurl);
2079 return False;
2080 ftp.login(urlparts.username, urlparts.password);
2081 if(urlparts.scheme=="ftps"):
2082 ftp.prot_p();
2083 ftpfile = BytesIO();
2084 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
2085 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2086 ftp.close();
2087 ftpfile.seek(0, 0);
2088 return ftpfile;
2089
2090 def download_file_from_ftp_string(url):
2091 ftpfile = download_file_from_ftp_file(url);
2092 return ftpfile.read();
2093
2094 def upload_file_to_ftp_file(ftpfile, url):
2095 urlparts = urlparse.urlparse(url);
2096 file_name = os.path.basename(urlparts.path);
2097 file_dir = os.path.dirname(urlparts.path);
2098 if(urlparts.username is not None):
2099 ftp_username = urlparts.username;
2100 else:
2101 ftp_username = "anonymous";
2102 if(urlparts.password is not None):
2103 ftp_password = urlparts.password;
2104 elif(urlparts.password is None and urlparts.username=="anonymous"):
2105 ftp_password = "anonymous";
2106 else:
2107 ftp_password = "";
2108 if(urlparts.scheme=="ftp"):
2109 ftp = FTP();
2110 elif(urlparts.scheme=="ftps"):
2111 ftp = FTP_TLS();
2112 else:
2113 return False;
2114 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2115 return False;
2116 ftp_port = urlparts.port;
2117 if(urlparts.port is None):
2118 ftp_port = 21;
2119 try:
2120 ftp.connect(urlparts.hostname, ftp_port);
2121 except socket.gaierror:
2122 log.info("Error With URL "+httpurl);
2123 return False;
2124 except socket.timeout:
2125 log.info("Error With URL "+httpurl);
2126 return False;
2127 ftp.login(urlparts.username, urlparts.password);
2128 if(urlparts.scheme=="ftps"):
2129 ftp.prot_p();
2130 ftp.storbinary("STOR "+urlparts.path, ftpfile);
2131 ftp.close();
2132 ftpfile.seek(0, 0);
2133 return ftpfile;
2134
2135 def upload_file_to_ftp_string(ftpstring, url):
2136 ftpfileo = BytesIO(ftpstring);
2137 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
2138 ftpfileo.close();
2139 return ftpfile;
2140
2141 if(haveparamiko):
2142 def download_file_from_sftp_file(url):
2143 urlparts = urlparse.urlparse(url);
2144 file_name = os.path.basename(urlparts.path);
2145 file_dir = os.path.dirname(urlparts.path);
2146 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2147 return False;
2148 sftp_port = urlparts.port;
2149 if(urlparts.port is None):
2150 sftp_port = 22;
2151 else:
2152 sftp_port = urlparts.port;
2153 if(urlparts.username is not None):
2154 sftp_username = urlparts.username;
2155 else:
2156 sftp_username = "anonymous";
2157 if(urlparts.password is not None):
2158 sftp_password = urlparts.password;
2159 elif(urlparts.password is None and urlparts.username=="anonymous"):
2160 sftp_password = "anonymous";
2161 else:
2162 sftp_password = "";
2163 if(urlparts.scheme!="sftp"):
2164 return False;
2165 ssh = paramiko.SSHClient();
2166 ssh.load_system_host_keys();
2167 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2168 try:
2169 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2170 except paramiko.ssh_exception.SSHException:
2171 return False;
2172 except socket.gaierror:
2173 log.info("Error With URL "+httpurl);
2174 return False;
2175 except socket.timeout:
2176 log.info("Error With URL "+httpurl);
2177 return False;
2178 sftp = ssh.open_sftp();
2179 sftpfile = BytesIO();
2180 sftp.getfo(urlparts.path, sftpfile);
2181 sftp.close();
2182 ssh.close();
2183 sftpfile.seek(0, 0);
2184 return sftpfile;
2185 else:
2186 def download_file_from_sftp_file(url):
2187 return False;
2188
2189 if(haveparamiko):
2190 def download_file_from_sftp_string(url):
2191 sftpfile = download_file_from_sftp_file(url);
2192 return sftpfile.read();
2193 else:
2194 def download_file_from_ftp_string(url):
2195 return False;
2196
2197 if(haveparamiko):
2198 def upload_file_to_sftp_file(sftpfile, url):
2199 urlparts = urlparse.urlparse(url);
2200 file_name = os.path.basename(urlparts.path);
2201 file_dir = os.path.dirname(urlparts.path);
2202 sftp_port = urlparts.port;
2203 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2204 return False;
2205 if(urlparts.port is None):
2206 sftp_port = 22;
2207 else:
2208 sftp_port = urlparts.port;
2209 if(urlparts.username is not None):
2210 sftp_username = urlparts.username;
2211 else:
2212 sftp_username = "anonymous";
2213 if(urlparts.password is not None):
2214 sftp_password = urlparts.password;
2215 elif(urlparts.password is None and urlparts.username=="anonymous"):
2216 sftp_password = "anonymous";
2217 else:
2218 sftp_password = "";
2219 if(urlparts.scheme!="sftp"):
2220 return False;
2221 ssh = paramiko.SSHClient();
2222 ssh.load_system_host_keys();
2223 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2224 try:
2225 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2226 except paramiko.ssh_exception.SSHException:
2227 return False;
2228 except socket.gaierror:
2229 log.info("Error With URL "+httpurl);
2230 return False;
2231 except socket.timeout:
2232 log.info("Error With URL "+httpurl);
2233 return False;
2234 sftp = ssh.open_sftp();
2235 sftp.putfo(sftpfile, urlparts.path);
2236 sftp.close();
2237 ssh.close();
2238 sftpfile.seek(0, 0);
2239 return sftpfile;
2240 else:
2241 def upload_file_to_sftp_file(sftpfile, url):
2242 return False;
2243
2244 if(haveparamiko):
2245 def upload_file_to_sftp_string(sftpstring, url):
2246 sftpfileo = BytesIO(sftpstring);
2247 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
2248 sftpfileo.close();
2249 return sftpfile;
2250 else:
2251 def upload_file_to_sftp_string(url):
2252 return False;
2253
2254
2255 if(havepysftp):
2256 def download_file_from_pysftp_file(url):
2257 urlparts = urlparse.urlparse(url);
2258 file_name = os.path.basename(urlparts.path);
2259 file_dir = os.path.dirname(urlparts.path);
2260 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2261 return False;
2262 sftp_port = urlparts.port;
2263 if(urlparts.port is None):
2264 sftp_port = 22;
2265 else:
2266 sftp_port = urlparts.port;
2267 if(urlparts.username is not None):
2268 sftp_username = urlparts.username;
2269 else:
2270 sftp_username = "anonymous";
2271 if(urlparts.password is not None):
2272 sftp_password = urlparts.password;
2273 elif(urlparts.password is None and urlparts.username=="anonymous"):
2274 sftp_password = "anonymous";
2275 else:
2276 sftp_password = "";
2277 if(urlparts.scheme!="sftp"):
2278 return False;
2279 try:
2280 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2281 except paramiko.ssh_exception.SSHException:
2282 return False;
2283 except socket.gaierror:
2284 log.info("Error With URL "+httpurl);
2285 return False;
2286 except socket.timeout:
2287 log.info("Error With URL "+httpurl);
2288 return False;
2289 sftp = ssh.open_sftp();
2290 sftpfile = BytesIO();
2291 sftp.getfo(urlparts.path, sftpfile);
2292 sftp.close();
2293 ssh.close();
2294 sftpfile.seek(0, 0);
2295 return sftpfile;
2296 else:
2297 def download_file_from_pysftp_file(url):
2298 return False;
2299
2300 if(havepysftp):
2301 def download_file_from_pysftp_string(url):
2302 sftpfile = download_file_from_pysftp_file(url);
2303 return sftpfile.read();
2304 else:
2305 def download_file_from_ftp_string(url):
2306 return False;
2307
2308 if(havepysftp):
2309 def upload_file_to_pysftp_file(sftpfile, url):
2310 urlparts = urlparse.urlparse(url);
2311 file_name = os.path.basename(urlparts.path);
2312 file_dir = os.path.dirname(urlparts.path);
2313 sftp_port = urlparts.port;
2314 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2315 return False;
2316 if(urlparts.port is None):
2317 sftp_port = 22;
2318 else:
2319 sftp_port = urlparts.port;
2320 if(urlparts.username is not None):
2321 sftp_username = urlparts.username;
2322 else:
2323 sftp_username = "anonymous";
2324 if(urlparts.password is not None):
2325 sftp_password = urlparts.password;
2326 elif(urlparts.password is None and urlparts.username=="anonymous"):
2327 sftp_password = "anonymous";
2328 else:
2329 sftp_password = "";
2330 if(urlparts.scheme!="sftp"):
2331 return False;
2332 try:
2333 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2334 except paramiko.ssh_exception.SSHException:
2335 return False;
2336 except socket.gaierror:
2337 log.info("Error With URL "+httpurl);
2338 return False;
2339 except socket.timeout:
2340 log.info("Error With URL "+httpurl);
2341 return False;
2342 sftp = ssh.open_sftp();
2343 sftp.putfo(sftpfile, urlparts.path);
2344 sftp.close();
2345 ssh.close();
2346 sftpfile.seek(0, 0);
2347 return sftpfile;
2348 else:
2349 def upload_file_to_pysftp_file(sftpfile, url):
2350 return False;
2351
2352 if(havepysftp):
2353 def upload_file_to_pysftp_string(sftpstring, url):
2354 sftpfileo = BytesIO(sftpstring);
2355 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
2356 sftpfileo.close();
2357 return sftpfile;
2358 else:
2359 def upload_file_to_pysftp_string(url):
2360 return False;
python libary/module to download files.