pywwwget.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
  13     Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  14
  15     $FileInfo: pywwwget.py - Last Update: 10/5/2023 Ver. 2.0.2 RC 1 - Author: cooldude2k $
  16 '''
  17
  18 from __future__ import division, absolute_import, print_function;
  19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, zlib, bz2, time, argparse, subprocess, socket, email.utils, datetime, time;
  20 import logging as log;
  21 from ftplib import FTP, FTP_TLS;
  22 from base64 import b64encode;
  23 try:
  24  from cgi import parse_qsl;
  25 except ImportError:
  26  from urlparse import parse_qsl;
  27 except ModuleNotFoundError:
  28  from urlparse import parse_qsl;
  29 except (DeprecationWarning, TypeError):
  30  from urlparse import parse_qsl;
  31 haverequests = False;
  32 try:
  33  import requests;
  34  haverequests = True;
  35 except ImportError:
  36  haverequests = False;
  37 havemechanize = False;
  38 try:
  39  import mechanize;
  40  havemechanize = True;
  41 except ImportError:
  42  havemechanize = False;
  43 havepycurl = False;
  44 try:
  45  import pycurl;
  46  havepycurl = True;
  47 except ImportError:
  48  havepycurl = False;
  49 haveparamiko = False;
  50 try:
  51  import paramiko;
  52  haveparamiko = True;
  53 except ImportError:
  54  haveparamiko = False;
  55 havepysftp = False;
  56 try:
  57  import pysftp;
  58  havepysftp = True;
  59 except ImportError:
  60  havepysftp = False;
  61 haveurllib3 = False;
  62 try:
  63  import urllib3;
  64  haveurllib3 = True;
  65 except ImportError:
  66  haveurllib3 = False;
  67 havehttplib2 = False;
  68 try:
  69  from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
  70  havehttplib2 = True;
  71 except ImportError:
  72  havehttplib2 = False;
  73 havehttpx = False;
  74 try:
  75  import httpx;
  76  havehttpx = True;
  77 except ImportError:
  78  havehttpx = False;
  79 havehttpcore = False;
  80 try:
  81  import httpcore;
  82  havehttpcore = True;
  83 except ImportError:
  84  havehttpcore = False;
  85 haveaiohttp = False;
  86 try:
  87  import aiohttp;
  88  haveaiohttp = True;
  89 except ImportError:
  90  haveaiohttp = False;
  91 havebrotli = False;
  92 try:
  93  import brotli;
  94  havebrotli = True;
  95 except ImportError:
  96  havebrotli = False;
  97 havezstd = False;
  98 try:
  99  import zstandard;
 100  havezstd = True;
 101 except ImportError:
 102  havezstd = False;
 103 havelzma = False;
 104 try:
 105  import lzma;
 106  havelzma = True;
 107 except ImportError:
 108  havelzma = False;
 109 if(sys.version[0]=="2"):
 110  try:
 111   from io import StringIO, BytesIO;
 112  except ImportError:
 113   try:
 114    from cStringIO import StringIO;
 115    from cStringIO import StringIO as BytesIO;
 116   except ImportError:
 117    from StringIO import StringIO;
 118    from StringIO import StringIO as BytesIO;
 119  # From http://python-future.org/compatible_idioms.html
 120  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
 121  from urllib import urlencode;
 122  from urllib import urlopen as urlopenalt;
 123  from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
 124  import urlparse, cookielib;
 125  from httplib import HTTPConnection, HTTPSConnection;
 126 if(sys.version[0]>="3"):
 127  from io import StringIO, BytesIO;
 128  # From http://python-future.org/compatible_idioms.html
 129  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
 130  from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
 131  from urllib.error import HTTPError, URLError;
 132  import urllib.parse as urlparse;
 133  import http.cookiejar as cookielib;
 134  from http.client import HTTPConnection, HTTPSConnection;
 135
 136 __program_name__ = "PyWWW-Get";
 137 __program_alt_name__ = "PyWWWGet";
 138 __program_small_name__ = "wwwget";
 139 __project__ = __program_name__;
 140 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
 141 __version_info__ = (2, 0, 2, "RC 1", 1);
 142 __version_date_info__ = (2023, 10, 5, "RC 1", 1);
 143 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
 144 __revision__ = __version_info__[3];
 145 __revision_id__ = "$Id$";
 146 if(__version_info__[4] is not None):
 147  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
 148 if(__version_info__[4] is None):
 149  __version_date_plusrc__ = __version_date__;
 150 if(__version_info__[3] is not None):
 151  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
 152 if(__version_info__[3] is None):
 153  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
 154
 155 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
 156 tmpfilesuffix = "-";
 157 pytempdir = tempfile.gettempdir();
 158
 159 PyBitness = platform.architecture();
 160 if(PyBitness=="32bit" or PyBitness=="32"):
 161  PyBitness = "32";
 162 elif(PyBitness=="64bit" or PyBitness=="64"):
 163  PyBitness = "64";
 164 else:
 165  PyBitness = "32";
 166
 167 compression_supported_list = ['identity', 'gzip', 'deflate', 'bzip2'];
 168 if(havebrotli):
 169  compression_supported_list.append('br');
 170 if(havezstd):
 171  compression_supported_list.append('zstd');
 172 if(havelzma):
 173  compression_supported_list.append('lzma');
 174  compression_supported_list.append('xz');
 175 compression_supported = ', '.join(compression_supported_list);
 176
 177 geturls_cj = cookielib.CookieJar();
 178 windowsNT4_ua_string = "Windows NT 4.0";
 179 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
 180 windows2k_ua_string = "Windows NT 5.0";
 181 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
 182 windowsXP_ua_string = "Windows NT 5.1";
 183 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 184 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
 185 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 186 windows7_ua_string = "Windows NT 6.1; Win64; x64";
 187 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
 188 windows8_ua_string = "Windows NT 6.2; Win64; x64";
 189 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
 190 windows81_ua_string = "Windows NT 6.3; Win64; x64";
 191 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
 192 windows10_ua_string = "Windows NT 10.0; Win64; x64";
 193 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
 194 windows11_ua_string = "Windows NT 11.0; Win64; x64";
 195 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
 196 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
 197 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
 198 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
 199 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
 200 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
 201 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
 202 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
 203 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
 204 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
 205 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
 206 if(platform.python_implementation()!=""):
 207  py_implementation = platform.python_implementation();
 208 if(platform.python_implementation()==""):
 209  py_implementation = "Python";
 210 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
 211 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
 212 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
 213 geturls_ua = geturls_ua_firefox_windows7;
 214 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 215 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 216 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 217 geturls_headers_chrome_windows7.update(windows7_ua_addon);
 218 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 219 geturls_headers_chromium_windows7.update(windows7_ua_addon);
 220 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 221 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
 222 geturls_headers_opera_windows7.update(windows7_ua_addon);
 223 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
 224 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
 225 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 226 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
 227 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
 228 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 229 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 230 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 231 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 232 geturls_headers = geturls_headers_firefox_windows7;
 233 geturls_download_sleep = 0;
 234
 235 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 236  if(outtype=="print" and dbgenable):
 237   print(dbgtxt);
 238   return True;
 239  elif(outtype=="log" and dbgenable):
 240   logging.info(dbgtxt);
 241   return True;
 242  elif(outtype=="warning" and dbgenable):
 243   logging.warning(dbgtxt);
 244   return True;
 245  elif(outtype=="error" and dbgenable):
 246   logging.error(dbgtxt);
 247   return True;
 248  elif(outtype=="critical" and dbgenable):
 249   logging.critical(dbgtxt);
 250   return True;
 251  elif(outtype=="exception" and dbgenable):
 252   logging.exception(dbgtxt);
 253   return True;
 254  elif(outtype=="logalt" and dbgenable):
 255   logging.log(dgblevel, dbgtxt);
 256   return True;
 257  elif(outtype=="debug" and dbgenable):
 258   logging.debug(dbgtxt);
 259   return True;
 260  elif(not dbgenable):
 261   return True;
 262  else:
 263   return False;
 264  return False;
 265
 266 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 267  dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
 268  if(not dbgout):
 269   return False;
 270  return dbgtxt;
 271
 272 def add_url_param(url, **params):
 273  n=3;
 274  parts = list(urlparse.urlsplit(url));
 275  d = dict(parse_qsl(parts[n])); # use cgi.parse_qs for list values
 276  d.update(params);
 277  parts[n]=urlencode(d);
 278  return urlparse.urlunsplit(parts);
 279
 280 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 281 def which_exec(execfile):
 282  for path in os.environ["PATH"].split(":"):
 283   if os.path.exists(path + "/" + execfile):
 284    return path + "/" + execfile;
 285
 286 def listize(varlist):
 287  il = 0;
 288  ix = len(varlist);
 289  ilx = 1;
 290  newlistreg = {};
 291  newlistrev = {};
 292  newlistfull = {};
 293  while(il < ix):
 294   newlistreg.update({ilx: varlist[il]});
 295   newlistrev.update({varlist[il]: ilx});
 296   ilx = ilx + 1;
 297   il = il + 1;
 298  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 299  return newlistfull;
 300
 301 def twolistize(varlist):
 302  il = 0;
 303  ix = len(varlist);
 304  ilx = 1;
 305  newlistnamereg = {};
 306  newlistnamerev = {};
 307  newlistdescreg = {};
 308  newlistdescrev = {};
 309  newlistfull = {};
 310  while(il < ix):
 311   newlistnamereg.update({ilx: varlist[il][0].strip()});
 312   newlistnamerev.update({varlist[il][0].strip(): ilx});
 313   newlistdescreg.update({ilx: varlist[il][1].strip()});
 314   newlistdescrev.update({varlist[il][1].strip(): ilx});
 315   ilx = ilx + 1;
 316   il = il + 1;
 317  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 318  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 319  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 320  return newlistfull;
 321
 322 def arglistize(proexec, *varlist):
 323  il = 0;
 324  ix = len(varlist);
 325  ilx = 1;
 326  newarglist = [proexec];
 327  while(il < ix):
 328   if varlist[il][0] is not None:
 329    newarglist.append(varlist[il][0]);
 330   if varlist[il][1] is not None:
 331    newarglist.append(varlist[il][1]);
 332   il = il + 1;
 333  return newarglist;
 334
 335 def fix_header_names(header_dict):
 336  if(sys.version[0]=="2"):
 337   header_dict =  {k.title(): v for k, v in header_dict.iteritems()};
 338  if(sys.version[0]>="3"):
 339   header_dict =  {k.title(): v for k, v in header_dict.items()};
 340  return header_dict;
 341
 342 # hms_string by ArcGIS Python Recipes
 343 # https://arcpy.wordpress.com/2012/04/20/146/
 344 def hms_string(sec_elapsed):
 345  h = int(sec_elapsed / (60 * 60));
 346  m = int((sec_elapsed % (60 * 60)) / 60);
 347  s = sec_elapsed % 60.0;
 348  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 349
 350 # get_readable_size by Lipis
 351 # http://stackoverflow.com/posts/14998888/revisions
 352 def get_readable_size(bytes, precision=1, unit="IEC"):
 353  unit = unit.upper();
 354  if(unit!="IEC" and unit!="SI"):
 355   unit = "IEC";
 356  if(unit=="IEC"):
 357   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 358   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 359   unitsize = 1024.0;
 360  if(unit=="SI"):
 361   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 362   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 363   unitsize = 1000.0;
 364  return_val = {};
 365  orgbytes = bytes;
 366  for unit in units:
 367   if abs(bytes) < unitsize:
 368    strformat = "%3."+str(precision)+"f%s";
 369    pre_return_val = (strformat % (bytes, unit));
 370    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 371    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 372    alt_return_val = pre_return_val.split();
 373    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 374    return return_val;
 375   bytes /= unitsize;
 376  strformat = "%."+str(precision)+"f%s";
 377  pre_return_val = (strformat % (bytes, "YiB"));
 378  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 379  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 380  alt_return_val = pre_return_val.split();
 381  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 382  return return_val;
 383
 384 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 385  unit = unit.upper();
 386  usehashtypes = usehashtypes.lower();
 387  getfilesize = os.path.getsize(infile);
 388  return_val = get_readable_size(getfilesize, precision, unit);
 389  if(usehashes):
 390   hashtypelist = usehashtypes.split(",");
 391   openfile = open(infile, "rb");
 392   filecontents = openfile.read();
 393   openfile.close();
 394   listnumcount = 0;
 395   listnumend = len(hashtypelist);
 396   while(listnumcount < listnumend):
 397    hashtypelistlow = hashtypelist[listnumcount].strip();
 398    hashtypelistup = hashtypelistlow.upper();
 399    filehash = hashlib.new(hashtypelistup);
 400    filehash.update(filecontents);
 401    filegethash = filehash.hexdigest();
 402    return_val.update({hashtypelistup: filegethash});
 403    listnumcount += 1;
 404  return return_val;
 405
 406 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 407  unit = unit.upper();
 408  usehashtypes = usehashtypes.lower();
 409  getfilesize = len(instring);
 410  return_val = get_readable_size(getfilesize, precision, unit);
 411  if(usehashes):
 412   hashtypelist = usehashtypes.split(",");
 413   listnumcount = 0;
 414   listnumend = len(hashtypelist);
 415   while(listnumcount < listnumend):
 416    hashtypelistlow = hashtypelist[listnumcount].strip();
 417    hashtypelistup = hashtypelistlow.upper();
 418    filehash = hashlib.new(hashtypelistup);
 419    if(sys.version[0]=="2"):
 420     filehash.update(instring);
 421    if(sys.version[0]>="3"):
 422     filehash.update(instring.encode('utf-8'));
 423    filegethash = filehash.hexdigest();
 424    return_val.update({hashtypelistup: filegethash});
 425    listnumcount += 1;
 426  return return_val;
 427
 428 def http_status_to_reason(code):
 429     reasons = {
 430         100: 'Continue',
 431         101: 'Switching Protocols',
 432         102: 'Processing',
 433         200: 'OK',
 434         201: 'Created',
 435         202: 'Accepted',
 436         203: 'Non-Authoritative Information',
 437         204: 'No Content',
 438         205: 'Reset Content',
 439         206: 'Partial Content',
 440         207: 'Multi-Status',
 441         208: 'Already Reported',
 442         226: 'IM Used',
 443         300: 'Multiple Choices',
 444         301: 'Moved Permanently',
 445         302: 'Found',
 446         303: 'See Other',
 447         304: 'Not Modified',
 448         305: 'Use Proxy',
 449         307: 'Temporary Redirect',
 450         308: 'Permanent Redirect',
 451         400: 'Bad Request',
 452         401: 'Unauthorized',
 453         402: 'Payment Required',
 454         403: 'Forbidden',
 455         404: 'Not Found',
 456         405: 'Method Not Allowed',
 457         406: 'Not Acceptable',
 458         407: 'Proxy Authentication Required',
 459         408: 'Request Timeout',
 460         409: 'Conflict',
 461         410: 'Gone',
 462         411: 'Length Required',
 463         412: 'Precondition Failed',
 464         413: 'Payload Too Large',
 465         414: 'URI Too Long',
 466         415: 'Unsupported Media Type',
 467         416: 'Range Not Satisfiable',
 468         417: 'Expectation Failed',
 469         421: 'Misdirected Request',
 470         422: 'Unprocessable Entity',
 471         423: 'Locked',
 472         424: 'Failed Dependency',
 473         426: 'Upgrade Required',
 474         428: 'Precondition Required',
 475         429: 'Too Many Requests',
 476         431: 'Request Header Fields Too Large',
 477         451: 'Unavailable For Legal Reasons',
 478         500: 'Internal Server Error',
 479         501: 'Not Implemented',
 480         502: 'Bad Gateway',
 481         503: 'Service Unavailable',
 482         504: 'Gateway Timeout',
 483         505: 'HTTP Version Not Supported',
 484         506: 'Variant Also Negotiates',
 485         507: 'Insufficient Storage',
 486         508: 'Loop Detected',
 487         510: 'Not Extended',
 488         511: 'Network Authentication Required'
 489     }
 490     return reasons.get(code, 'Unknown Status Code');
 491
 492 def ftp_status_to_reason(code):
 493     reasons = {
 494         110: 'Restart marker reply',
 495         120: 'Service ready in nnn minutes',
 496         125: 'Data connection already open; transfer starting',
 497         150: 'File status okay; about to open data connection',
 498         200: 'Command okay',
 499         202: 'Command not implemented, superfluous at this site',
 500         211: 'System status, or system help reply',
 501         212: 'Directory status',
 502         213: 'File status',
 503         214: 'Help message',
 504         215: 'NAME system type',
 505         220: 'Service ready for new user',
 506         221: 'Service closing control connection',
 507         225: 'Data connection open; no transfer in progress',
 508         226: 'Closing data connection',
 509         227: 'Entering Passive Mode',
 510         230: 'User logged in, proceed',
 511         250: 'Requested file action okay, completed',
 512         257: '"PATHNAME" created',
 513         331: 'User name okay, need password',
 514         332: 'Need account for login',
 515         350: 'Requested file action pending further information',
 516         421: 'Service not available, closing control connection',
 517         425: 'Can\'t open data connection',
 518         426: 'Connection closed; transfer aborted',
 519         450: 'Requested file action not taken',
 520         451: 'Requested action aborted. Local error in processing',
 521         452: 'Requested action not taken. Insufficient storage space in system',
 522         500: 'Syntax error, command unrecognized',
 523         501: 'Syntax error in parameters or arguments',
 524         502: 'Command not implemented',
 525         503: 'Bad sequence of commands',
 526         504: 'Command not implemented for that parameter',
 527         530: 'Not logged in',
 528         532: 'Need account for storing files',
 529         550: 'Requested action not taken. File unavailable',
 530         551: 'Requested action aborted. Page type unknown',
 531         552: 'Requested file action aborted. Exceeded storage allocation',
 532         553: 'Requested action not taken. File name not allowed'
 533     };
 534     return reasons.get(code, 'Unknown Status Code');
 535
 536 def sftp_status_to_reason(code):
 537     reasons = {
 538         0: 'SSH_FX_OK',
 539         1: 'SSH_FX_EOF',
 540         2: 'SSH_FX_NO_SUCH_FILE',
 541         3: 'SSH_FX_PERMISSION_DENIED',
 542         4: 'SSH_FX_FAILURE',
 543         5: 'SSH_FX_BAD_MESSAGE',
 544         6: 'SSH_FX_NO_CONNECTION',
 545         7: 'SSH_FX_CONNECTION_LOST',
 546         8: 'SSH_FX_OP_UNSUPPORTED'
 547     };
 548     return reasons.get(code, 'Unknown Status Code');
 549
 550 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 551  if isinstance(headers, dict):
 552   returnval = [];
 553   if(sys.version[0]=="2"):
 554    for headkey, headvalue in headers.iteritems():
 555     returnval.append((headkey, headvalue));
 556   if(sys.version[0]>="3"):
 557    for headkey, headvalue in headers.items():
 558     returnval.append((headkey, headvalue));
 559  elif isinstance(headers, list):
 560   returnval = headers;
 561  else:
 562   returnval = False;
 563  return returnval;
 564
 565 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 566  if isinstance(headers, dict):
 567   returnval = [];
 568   if(sys.version[0]=="2"):
 569    for headkey, headvalue in headers.iteritems():
 570     returnval.append(headkey+": "+headvalue);
 571   if(sys.version[0]>="3"):
 572    for headkey, headvalue in headers.items():
 573     returnval.append(headkey+": "+headvalue);
 574  elif isinstance(headers, list):
 575   returnval = headers;
 576  else:
 577   returnval = False;
 578  return returnval;
 579
 580 def make_http_headers_from_pycurl_to_dict(headers):
 581  header_dict = {};
 582  headers = headers.strip().split('\r\n');
 583  for header in headers:
 584   parts = header.split(': ', 1)
 585   if(len(parts) == 2):
 586    key, value = parts;
 587    header_dict[key.title()] = value;
 588  return header_dict;
 589
 590 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 591  if isinstance(headers, list):
 592   returnval = {};
 593   mli = 0;
 594   mlil = len(headers);
 595   while(mli<mlil):
 596    returnval.update({headers[mli][0]: headers[mli][1]});
 597    mli = mli + 1;
 598  elif isinstance(headers, dict):
 599   returnval = headers;
 600  else:
 601   returnval = False;
 602  return returnval;
 603
 604 def get_httplib_support(checkvalue=None):
 605  global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 606  returnval = [];
 607  returnval.append("ftp");
 608  returnval.append("httplib");
 609  if(havehttplib2):
 610   returnval.append("httplib2");
 611  returnval.append("urllib");
 612  if(haveurllib3):
 613   returnval.append("urllib3");
 614   returnval.append("request3");
 615  returnval.append("request");
 616  if(haverequests):
 617   returnval.append("requests");
 618  if(haveaiohttp):
 619   returnval.append("aiohttp");
 620  if(havehttpx):
 621   returnval.append("httpx");
 622   returnval.append("httpx2");
 623  if(havemechanize):
 624   returnval.append("mechanize");
 625  if(havepycurl):
 626   returnval.append("pycurl");
 627   if(hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
 628    returnval.append("pycurl2");
 629   if(hasattr(pycurl, "CURL_HTTP_VERSION_3_0")):
 630    returnval.append("pycurl3");
 631  if(haveparamiko):
 632   returnval.append("sftp");
 633  if(havepysftp):
 634   returnval.append("pysftp");
 635  if(not checkvalue is None):
 636   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 637    checkvalue = "urllib";
 638   if(checkvalue=="httplib1"):
 639    checkvalue = "httplib";
 640   if(checkvalue in returnval):
 641    returnval = True;
 642   else:
 643    returnval = False;
 644  return returnval;
 645
 646 def check_httplib_support(checkvalue="urllib"):
 647  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 648   checkvalue = "urllib";
 649  if(checkvalue=="httplib1"):
 650   checkvalue = "httplib";
 651  returnval = get_httplib_support(checkvalue);
 652  return returnval;
 653
 654 def get_httplib_support_list():
 655  returnval = get_httplib_support(None);
 656  return returnval;
 657
 658 def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
 659  global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 660  if(sleep<0):
 661   sleep = geturls_download_sleep;
 662  if(timeout<=0):
 663   timeout = 10;
 664  if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
 665   httplibuse = "urllib";
 666  if(httplibuse=="httplib1"):
 667   httplibuse = "httplib";
 668  if(not haverequests and httplibuse=="requests"):
 669   httplibuse = "urllib";
 670  if(not haveaiohttp and httplibuse=="aiohttp"):
 671   httplibuse = "urllib";
 672  if(not havehttpx and httplibuse=="httpx"):
 673   httplibuse = "urllib";
 674  if(not havehttpx and httplibuse=="httpx2"):
 675   httplibuse = "urllib";
 676  if(not havehttpcore and httplibuse=="httpcore"):
 677   httplibuse = "urllib";
 678  if(not havehttpcore and httplibuse=="httpcore2"):
 679   httplibuse = "urllib";
 680  if(not havemechanize and httplibuse=="mechanize"):
 681   httplibuse = "urllib";
 682  if(not havepycurl and httplibuse=="pycurl"):
 683   httplibuse = "urllib";
 684  if(not havepycurl and httplibuse=="pycurl2"):
 685   httplibuse = "urllib";
 686  if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
 687   httplibuse = "pycurl";
 688  if(not havepycurl and httplibuse=="pycurl3"):
 689   httplibuse = "urllib";
 690  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
 691   httplibuse = "pycurl2";
 692  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
 693   httplibuse = "pycurl";
 694  if(not havehttplib2 and httplibuse=="httplib2"):
 695   httplibuse = "httplib";
 696  if(not haveparamiko and httplibuse=="sftp"):
 697   httplibuse = "ftp";
 698  if(not havepysftp and httplibuse=="pysftp"):
 699   httplibuse = "ftp";
 700  urlparts = urlparse.urlparse(httpurl);
 701  if(isinstance(httpheaders, list)):
 702   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 703  httpheaders = fix_header_names(httpheaders);
 704  if(httpuseragent is not None):
 705   if('User-Agent' in httpheaders):
 706    httpheaders['User-Agent'] = httpuseragent;
 707   else:
 708    httpuseragent.update({'User-Agent': httpuseragent});
 709  if(httpreferer is not None):
 710   if('Referer' in httpheaders):
 711    httpheaders['Referer'] = httpreferer;
 712   else:
 713    httpuseragent.update({'Referer': httpreferer});
 714  if(urlparts.username is not None or urlparts.password is not None):
 715   if(sys.version[0]=="2"):
 716    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password));
 717   if(sys.version[0]>="3"):
 718    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 719   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 720  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 721  if(httplibuse=="urllib" or httplibuse=="mechanize"):
 722   if(isinstance(httpheaders, dict)):
 723    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 724  if(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
 725   if(isinstance(httpheaders, dict)):
 726    httpheaders = make_http_headers_from_dict_to_pycurl(httpheaders);
 727  geturls_opener.addheaders = httpheaders;
 728  time.sleep(sleep);
 729  if(postdata is not None and not isinstance(postdata, dict)):
 730   postdata = urlencode(postdata);
 731  if(httplibuse=="urllib" or httplibuse=="request"):
 732   geturls_request = Request(httpurl);
 733   try:
 734    if(httpmethod=="GET"):
 735     geturls_text = geturls_opener.open(geturls_request);
 736    elif(httpmethod=="POST"):
 737     geturls_text = geturls_opener.open(geturls_request, data=postdata);
 738    else:
 739     geturls_text = geturls_opener.open(geturls_request);
 740   except HTTPError as geturls_text_error:
 741    geturls_text = geturls_text_error;
 742    log.info("Error With URL "+httpurl);
 743   except URLError:
 744    log.info("Error With URL "+httpurl);
 745    return False;
 746   except socket.timeout:
 747    log.info("Error With URL "+httpurl);
 748    return False;
 749   httpcodeout = geturls_text.getcode();
 750   try:
 751    httpcodereason = geturls_text.reason;
 752   except AttributeError:
 753    httpcodereason = http_status_to_reason(geturls_text.getcode());
 754   try:
 755    httpversionout = geturls_text.version;
 756   except AttributeError:
 757    httpversionout = "1.1";
 758   httpmethodout = geturls_request.get_method();
 759   httpurlout = geturls_text.geturl();
 760   httpheaderout = geturls_text.info();
 761   httpheadersentout = httpheaders;
 762  elif(httplibuse=="httplib"):
 763   if(urlparts[0]=="http"):
 764    httpconn = HTTPConnection(urlparts[1], timeout=timeout);
 765   elif(urlparts[0]=="https"):
 766    httpconn = HTTPSConnection(urlparts[1], timeout=timeout);
 767   else:
 768    return False;
 769   if(postdata is not None and not isinstance(postdata, dict)):
 770    postdata = urlencode(postdata);
 771   try:
 772    if(httpmethod=="GET"):
 773     httpconn.request("GET", urlparts[2], headers=httpheaders);
 774    elif(httpmethod=="POST"):
 775     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 776    else:
 777     httpconn.request("GET", urlparts[2], headers=httpheaders);
 778   except socket.timeout:
 779    log.info("Error With URL "+httpurl);
 780    return False;
 781   except socket.gaierror:
 782    log.info("Error With URL "+httpurl);
 783    return False;
 784   except BlockingIOError:
 785    log.info("Error With URL "+httpurl);
 786    return False;
 787   geturls_text = httpconn.getresponse();
 788   httpcodeout = geturls_text.status;
 789   httpcodereason = geturls_text.reason;
 790   if(geturls_text.version=="10"):
 791    httpversionout = "1.0";
 792   else:
 793    httpversionout = "1.1";
 794   httpmethodout = geturls_text._method;
 795   httpurlout = httpurl;
 796   httpheaderout = geturls_text.getheaders();
 797   httpheadersentout = httpheaders;
 798  elif(httplibuse=="httplib2"):
 799   if(urlparts[0]=="http"):
 800    httpconn = HTTPConnectionWithTimeout(urlparts[1], timeout=timeout);
 801   elif(urlparts[0]=="https"):
 802    httpconn = HTTPSConnectionWithTimeout(urlparts[1], timeout=timeout);
 803   else:
 804    return False;
 805   if(postdata is not None and not isinstance(postdata, dict)):
 806    postdata = urlencode(postdata);
 807   try:
 808    if(httpmethod=="GET"):
 809     httpconn.request("GET", urlparts[2], headers=httpheaders);
 810    elif(httpmethod=="POST"):
 811     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 812    else:
 813     httpconn.request("GET", urlparts[2], headers=httpheaders);
 814   except socket.timeout:
 815    log.info("Error With URL "+httpurl);
 816    return False;
 817   except socket.gaierror:
 818    log.info("Error With URL "+httpurl);
 819    return False;
 820   except BlockingIOError:
 821    log.info("Error With URL "+httpurl);
 822    return False;
 823   geturls_text = httpconn.getresponse();
 824   httpcodeout = geturls_text.status;
 825   httpcodereason = geturls_text.reason;
 826   if(geturls_text.version=="10"):
 827    httpversionout = "1.0";
 828   else:
 829    httpversionout = "1.1";
 830   httpmethodout = httpmethod;
 831   httpurlout = httpurl;
 832   httpheaderout = geturls_text.getheaders();
 833   httpheadersentout = httpheaders;
 834  elif(httplibuse=="urllib3" or httplibuse=="request3"):
 835   timeout = urllib3.util.Timeout(connect=timeout, read=timeout);
 836   urllib_pool = urllib3.PoolManager(headers=httpheaders, timeout=timeout);
 837   try:
 838    if(httpmethod=="GET"):
 839     geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
 840    elif(httpmethod=="POST"):
 841     geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
 842    else:
 843     geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
 844   except urllib3.exceptions.ConnectTimeoutError:
 845    log.info("Error With URL "+httpurl);
 846    return False;
 847   except urllib3.exceptions.ConnectError:
 848    log.info("Error With URL "+httpurl);
 849    return False;
 850   except urllib3.exceptions.MaxRetryError:
 851    log.info("Error With URL "+httpurl);
 852    return False;
 853   except socket.timeout:
 854    log.info("Error With URL "+httpurl);
 855    return False;
 856   except ValueError:
 857    log.info("Error With URL "+httpurl);
 858    return False;
 859   httpcodeout = geturls_text.status;
 860   httpcodereason = geturls_text.reason;
 861   if(geturls_text.version=="10"):
 862    httpversionout = "1.0";
 863   else:
 864    httpversionout = "1.1";
 865   httpmethodout = httpmethod;
 866   httpurlout = geturls_text.geturl();
 867   httpheaderout = geturls_text.info();
 868   httpheadersentout = httpheaders;
 869  elif(httplibuse=="requests"):
 870   try:
 871    reqsession = requests.Session();
 872    if(httpmethod=="GET"):
 873     geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 874    elif(httpmethod=="POST"):
 875     geturls_text = reqsession.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
 876    else:
 877     geturls_text = reqsession.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 878   except requests.exceptions.ConnectTimeout:
 879    log.info("Error With URL "+httpurl);
 880    return False;
 881   except requests.exceptions.ConnectError:
 882    log.info("Error With URL "+httpurl);
 883    return False;
 884   except socket.timeout:
 885    log.info("Error With URL "+httpurl);
 886    return False;
 887   httpcodeout = geturls_text.status_code;
 888   httpcodereason = geturls_text.reason;
 889   if(geturls_text.raw.version=="10"):
 890    httpversionout = "1.0";
 891   else:
 892    httpversionout = "1.1";
 893   httpmethodout = httpmethod;
 894   httpurlout = geturls_text.url;
 895   httpheaderout = geturls_text.headers;
 896   httpheadersentout = geturls_text.request.headers;
 897  elif(httplibuse=="aiohttp"):
 898   try:
 899    reqsession = aiohttp.ClientSession(cookie_jar=httpcookie, headers=httpheaders, timeout=timeout, read_timeout=timeout, conn_timeout=timeout, read_bufsize=buffersize);
 900    if(httpmethod=="GET"):
 901     geturls_text = reqsession.get(httpurl);
 902    elif(httpmethod=="POST"):
 903     geturls_text = reqsession.post(httpurl, data=postdata);
 904    else:
 905     geturls_text = reqsession.get(httpurl);
 906   except aiohttp.exceptions.ConnectTimeout:
 907    log.info("Error With URL "+httpurl);
 908    return False;
 909   except aiohttp.exceptions.ConnectError:
 910    log.info("Error With URL "+httpurl);
 911    return False;
 912   except socket.timeout:
 913    log.info("Error With URL "+httpurl);
 914    return False;
 915   httpcodeout = geturls_text.status;
 916   httpcodereason = geturls_text.reason;
 917   httpversionout = geturls_text.version;
 918   httpmethodout = geturls_text.method;
 919   httpurlout = geturls_text.url;
 920   httpheaderout = geturls_text.headers;
 921   httpheadersentout = geturls_text.request_info.headers;
 922  elif(httplibuse=="httpx"):
 923   try:
 924    if(httpmethod=="GET"):
 925     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
 926     geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 927    elif(httpmethod=="POST"):
 928     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
 929     geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
 930    else:
 931     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
 932     geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 933   except httpx.ConnectTimeout:
 934    log.info("Error With URL "+httpurl);
 935    return False;
 936   except httpx.ConnectError:
 937    log.info("Error With URL "+httpurl);
 938    return False;
 939   except socket.timeout:
 940    log.info("Error With URL "+httpurl);
 941    return False;
 942   httpcodeout = geturls_text.status_code;
 943   try:
 944    httpcodereason = geturls_text.reason_phrase;
 945   except:
 946    httpcodereason = http_status_to_reason(geturls_text.status_code);
 947   httpversionout = geturls_text.http_version;
 948   httpmethodout = httpmethod;
 949   httpurlout = str(geturls_text.url);
 950   httpheaderout = geturls_text.headers;
 951   httpheadersentout = geturls_text.request.headers;
 952  elif(httplibuse=="httpx2"):
 953   try:
 954    if(httpmethod=="GET"):
 955     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
 956     geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 957    elif(httpmethod=="POST"):
 958     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
 959     geturls_text = httpx_pool.post(httpurl, timeout=timeout, data=postdata, headers=httpheaders, cookies=httpcookie);
 960    else:
 961     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
 962     geturls_text = httpx_pool.get(httpurl, timeout=timeout, headers=httpheaders, cookies=httpcookie);
 963   except httpx.ConnectTimeout:
 964    log.info("Error With URL "+httpurl);
 965    return False;
 966   except httpx.ConnectError:
 967    log.info("Error With URL "+httpurl);
 968    return False;
 969   except socket.timeout:
 970    log.info("Error With URL "+httpurl);
 971    return False;
 972   httpcodeout = geturls_text.status_code;
 973   try:
 974    httpcodereason = geturls_text.reason_phrase;
 975   except:
 976    httpcodereason = http_status_to_reason(geturls_text.status_code);
 977   httpversionout = geturls_text.http_version;
 978   httpmethodout = httpmethod;
 979   httpurlout = str(geturls_text.url);
 980   httpheaderout = geturls_text.headers;
 981   httpheadersentout = geturls_text.request.headers;
 982  elif(httplibuse=="httpcore"):
 983   try:
 984    if(httpmethod=="GET"):
 985     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
 986     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
 987    elif(httpmethod=="POST"):
 988     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
 989     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
 990    else:
 991     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
 992     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
 993   except httpcore.ConnectTimeout:
 994    log.info("Error With URL "+httpurl);
 995    return False;
 996   except httpcore.ConnectError:
 997    log.info("Error With URL "+httpurl);
 998    return False;
 999   except socket.timeout:
1000    log.info("Error With URL "+httpurl);
1001    return False;
1002   httpcodeout = geturls_text.status;
1003   httpcodereason = http_status_to_reason(geturls_text.status);
1004   httpversionout = "1.1";
1005   httpmethodout = httpmethod;
1006   httpurlout = str(httpurl);
1007   httpheaderout = geturls_text.headers;
1008   httpheadersentout = httpheaders;
1009  elif(httplibuse=="httpcore2"):
1010   try:
1011    if(httpmethod=="GET"):
1012     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1013     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1014    elif(httpmethod=="POST"):
1015     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1016     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
1017    else:
1018     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1019     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1020   except httpcore.ConnectTimeout:
1021    log.info("Error With URL "+httpurl);
1022    return False;
1023   except httpcore.ConnectError:
1024    log.info("Error With URL "+httpurl);
1025    return False;
1026   except socket.timeout:
1027    log.info("Error With URL "+httpurl);
1028    return False;
1029   httpcodeout = geturls_text.status;
1030   httpcodereason = http_status_to_reason(geturls_text.status);
1031   httpversionout = "1.1";
1032   httpmethodout = httpmethod;
1033   httpurlout = str(httpurl);
1034   httpheaderout = geturls_text.headers;
1035   httpheadersentout = httpheaders;
1036  elif(httplibuse=="mechanize"):
1037   geturls_opener = mechanize.Browser();
1038   if(isinstance(httpheaders, dict)):
1039    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1040   time.sleep(sleep);
1041   geturls_opener.addheaders = httpheaders;
1042   geturls_opener.set_cookiejar(httpcookie);
1043   geturls_opener.set_handle_robots(False);
1044   if(postdata is not None and not isinstance(postdata, dict)):
1045    postdata = urlencode(postdata);
1046   try:
1047    if(httpmethod=="GET"):
1048     geturls_text = geturls_opener.open(httpurl);
1049    elif(httpmethod=="POST"):
1050     geturls_text = geturls_opener.open(httpurl, data=postdata);
1051    else:
1052     geturls_text = geturls_opener.open(httpurl);
1053   except mechanize.HTTPError as geturls_text_error:
1054    geturls_text = geturls_text_error;
1055    log.info("Error With URL "+httpurl);
1056   except URLError:
1057    log.info("Error With URL "+httpurl);
1058    return False;
1059   except socket.timeout:
1060    log.info("Error With URL "+httpurl);
1061    return False;
1062   httpcodeout = geturls_text.code;
1063   httpcodereason = geturls_text.msg;
1064   httpversionout = "1.1";
1065   httpmethodout = httpmethod;
1066   httpurlout = geturls_text.geturl();
1067   httpheaderout = geturls_text.info();
1068   reqhead = geturls_opener.request;
1069   httpheadersentout = reqhead.header_items();
1070  elif(httplibuse=="pycurl"):
1071   retrieved_body = BytesIO();
1072   retrieved_headers = BytesIO();
1073   try:
1074    if(httpmethod=="GET"):
1075     geturls_text = pycurl.Curl();
1076     geturls_text.setopt(geturls_text.URL, httpurl);
1077     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1078     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1079     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1080     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1081     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1082     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1083     geturls_text.perform();
1084    elif(httpmethod=="POST"):
1085     geturls_text = pycurl.Curl();
1086     geturls_text.setopt(geturls_text.URL, httpurl);
1087     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1088     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1089     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1090     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1091     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1092     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1093     geturls_text.setopt(geturls_text.POST, True);
1094     geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1095     geturls_text.perform();
1096    else:
1097     geturls_text = pycurl.Curl();
1098     geturls_text.setopt(geturls_text.URL, httpurl);
1099     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_1_1);
1100     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1101     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1102     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1103     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1104     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1105     geturls_text.perform();
1106    retrieved_headers.seek(0);
1107    if(sys.version[0]=="2"):
1108     pycurlhead = retrieved_headers.read();
1109    if(sys.version[0]>="3"):
1110     pycurlhead = retrieved_headers.read().decode('UTF-8');
1111    pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1112    pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1113    retrieved_body.seek(0);
1114   except socket.timeout:
1115    log.info("Error With URL "+httpurl);
1116    return False;
1117   except socket.gaierror:
1118    log.info("Error With URL "+httpurl);
1119    return False;
1120   except ValueError:
1121    log.info("Error With URL "+httpurl);
1122    return False;
1123   httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1124   httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1125   httpversionout = pyhttpverinfo[0];
1126   httpmethodout = httpmethod;
1127   httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1128   httpheaderout = pycurlheadersout;
1129   httpheadersentout = httpheaders;
1130  elif(httplibuse=="pycurl2"):
1131   retrieved_body = BytesIO();
1132   retrieved_headers = BytesIO();
1133   try:
1134    if(httpmethod=="GET"):
1135     geturls_text = pycurl.Curl();
1136     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1137     geturls_text.setopt(geturls_text.URL, httpurl);
1138     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1139     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1140     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1141     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1142     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1143     geturls_text.perform();
1144    elif(httpmethod=="POST"):
1145     geturls_text = pycurl.Curl();
1146     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1147     geturls_text.setopt(geturls_text.URL, httpurl);
1148     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1149     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1150     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1151     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1152     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1153     geturls_text.setopt(geturls_text.POST, True);
1154     geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1155     geturls_text.perform();
1156    else:
1157     geturls_text = pycurl.Curl();
1158     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_2_0);
1159     geturls_text.setopt(geturls_text.URL, httpurl);
1160     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1161     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1162     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1163     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1164     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1165     geturls_text.perform();
1166    retrieved_headers.seek(0);
1167    if(sys.version[0]=="2"):
1168     pycurlhead = retrieved_headers.read();
1169    if(sys.version[0]>="3"):
1170     pycurlhead = retrieved_headers.read().decode('UTF-8');
1171    pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip())[0];
1172    pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1173    retrieved_body.seek(0);
1174   except socket.timeout:
1175    log.info("Error With URL "+httpurl);
1176    return False;
1177   except socket.gaierror:
1178    log.info("Error With URL "+httpurl);
1179    return False;
1180   except ValueError:
1181    log.info("Error With URL "+httpurl);
1182    return False;
1183   httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1184   httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1185   httpversionout = pyhttpverinfo[0];
1186   httpmethodout = httpmethod;
1187   httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1188   httpheaderout = pycurlheadersout;
1189   httpheadersentout = httpheaders;
1190  elif(httplibuse=="pycurl3"):
1191   retrieved_body = BytesIO();
1192   retrieved_headers = BytesIO();
1193   try:
1194    if(httpmethod=="GET"):
1195     geturls_text = pycurl.Curl();
1196     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1197     geturls_text.setopt(geturls_text.URL, httpurl);
1198     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1199     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1200     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1201     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1202     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1203     geturls_text.perform();
1204    elif(httpmethod=="POST"):
1205     geturls_text = pycurl.Curl();
1206     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1207     geturls_text.setopt(geturls_text.URL, httpurl);
1208     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1209     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1210     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1211     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1212     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1213     geturls_text.setopt(geturls_text.POST, True);
1214     geturls_text.setopt(geturls_text.POSTFIELDS, postdata);
1215     geturls_text.perform();
1216    else:
1217     geturls_text = pycurl.Curl();
1218     geturls_text.setopt(geturls_text.HTTP_VERSION, geturls_text.CURL_HTTP_VERSION_3_0);
1219     geturls_text.setopt(geturls_text.URL, httpurl);
1220     geturls_text.setopt(geturls_text.WRITEFUNCTION, retrieved_body.write);
1221     geturls_text.setopt(geturls_text.HTTPHEADER, httpheaders);
1222     geturls_text.setopt(geturls_text.HEADERFUNCTION, retrieved_headers.write);
1223     geturls_text.setopt(geturls_text.FOLLOWLOCATION, True);
1224     geturls_text.setopt(geturls_text.TIMEOUT, timeout);
1225     geturls_text.perform();
1226    retrieved_headers.seek(0);
1227    if(sys.version[0]=="2"):
1228     pycurlhead = retrieved_headers.read();
1229    if(sys.version[0]>="3"):
1230     pycurlhead = retrieved_headers.read().decode('UTF-8');
1231    pyhttpverinfo = re.findall(r'^HTTP/([0-9.]+) (\d+)(?: ([A-Za-z\s]+))?$', pycurlhead.splitlines()[0].strip().rstrip('\r\n'))[0];
1232    pycurlheadersout = make_http_headers_from_pycurl_to_dict(pycurlhead);
1233    retrieved_body.seek(0);
1234   except socket.timeout:
1235    log.info("Error With URL "+httpurl);
1236    return False;
1237   except socket.gaierror:
1238    log.info("Error With URL "+httpurl);
1239    return False;
1240   except ValueError:
1241    log.info("Error With URL "+httpurl);
1242    return False;
1243   httpcodeout = geturls_text.getinfo(geturls_text.HTTP_CODE);
1244   httpcodereason = http_status_to_reason(geturls_text.getinfo(geturls_text.HTTP_CODE));
1245   httpversionout = pyhttpverinfo[0];
1246   httpmethodout = httpmethod;
1247   httpurlout = geturls_text.getinfo(geturls_text.EFFECTIVE_URL);
1248   httpheaderout = pycurlheadersout;
1249   httpheadersentout = httpheaders;
1250  elif(httplibuse=="ftp"):
1251   geturls_text = download_file_from_ftp_file(httpurl);
1252   if(not geturls_text):
1253    return False;
1254   downloadsize = None;
1255   if(downloadsize is not None):
1256    downloadsize = int(downloadsize);
1257   if downloadsize is None: downloadsize = 0;
1258   fulldatasize = 0;
1259   prevdownsize = 0;
1260   log.info("Downloading URL "+httpurl);
1261   with BytesIO() as strbuf:
1262    while True:
1263     databytes = geturls_text.read(buffersize);
1264     if not databytes: break;
1265     datasize = len(databytes);
1266     fulldatasize = datasize + fulldatasize;
1267     percentage = "";
1268     if(downloadsize>0):
1269      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1270     downloaddiff = fulldatasize - prevdownsize;
1271     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1272     prevdownsize = fulldatasize;
1273     strbuf.write(databytes);
1274    strbuf.seek(0);
1275    returnval_content = strbuf.read();
1276   returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1277   geturls_text.close();
1278  elif(httplibuse=="sftp"):
1279   geturls_text = download_file_from_sftp_file(httpurl);
1280   if(not geturls_text):
1281    return False;
1282   downloadsize = None;
1283   if(downloadsize is not None):
1284    downloadsize = int(downloadsize);
1285   if downloadsize is None: downloadsize = 0;
1286   fulldatasize = 0;
1287   prevdownsize = 0;
1288   log.info("Downloading URL "+httpurl);
1289   with BytesIO() as strbuf:
1290    while True:
1291     databytes = geturls_text.read(buffersize);
1292     if not databytes: break;
1293     datasize = len(databytes);
1294     fulldatasize = datasize + fulldatasize;
1295     percentage = "";
1296     if(downloadsize>0):
1297      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1298     downloaddiff = fulldatasize - prevdownsize;
1299     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1300     prevdownsize = fulldatasize;
1301     strbuf.write(databytes);
1302    strbuf.seek(0);
1303    returnval_content = strbuf.read();
1304   returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1305   geturls_text.close();
1306   return returnval;
1307  elif(httplibuse=="pysftp"):
1308   geturls_text = download_file_from_pysftp_file(httpurl);
1309   if(not geturls_text):
1310    return False;
1311   downloadsize = None;
1312   if(downloadsize is not None):
1313    downloadsize = int(downloadsize);
1314   if downloadsize is None: downloadsize = 0;
1315   fulldatasize = 0;
1316   prevdownsize = 0;
1317   log.info("Downloading URL "+httpurl);
1318   with BytesIO() as strbuf:
1319    while True:
1320     databytes = geturls_text.read(buffersize);
1321     if not databytes: break;
1322     datasize = len(databytes);
1323     fulldatasize = datasize + fulldatasize;
1324     percentage = "";
1325     if(downloadsize>0):
1326      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1327     downloaddiff = fulldatasize - prevdownsize;
1328     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1329     prevdownsize = fulldatasize;
1330     strbuf.write(databytes);
1331    strbuf.seek(0);
1332    returnval_content = strbuf.read();
1333   returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1334   geturls_text.close();
1335   return returnval;
1336  else:
1337   returnval = False;
1338  if(isinstance(httpheaderout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1339   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1340  if(isinstance(httpheaderout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1341   httpheaderout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheaderout)));
1342  if(sys.version[0]=="2"):
1343   try:
1344    prehttpheaderout = httpheaderout;
1345    httpheaderkeys = httpheaderout.keys();
1346    imax = len(httpheaderkeys);
1347    ic = 0;
1348    httpheaderout = {};
1349    while(ic < imax):
1350     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1351     ic += 1;
1352   except AttributeError:
1353    pass;
1354  httpheaderout = fix_header_names(httpheaderout);
1355  if(isinstance(httpheadersentout, list) and (httplibuse!="pycurl" and httplibuse!="pycurl2" and httplibuse!="pycurl3")):
1356   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1357  if(isinstance(httpheadersentout, list) and (httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3")):
1358   httpheadersentout = dict(make_http_headers_from_pycurl_to_dict("\r\n".join(httpheadersentout)));
1359  httpheadersentout = fix_header_names(httpheadersentout);
1360  log.info("Downloading URL "+httpurl);
1361  if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="aiohttp" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize"):
1362   downloadsize = httpheaderout.get('Content-Length');
1363   if(downloadsize is not None):
1364    downloadsize = int(downloadsize);
1365   if downloadsize is None: downloadsize = 0;
1366   fulldatasize = 0;
1367   prevdownsize = 0;
1368   log.info("Downloading URL "+httpurl);
1369   with BytesIO() as strbuf:
1370    while True:
1371     databytes = geturls_text.read(buffersize);
1372     if not databytes: break;
1373     datasize = len(databytes);
1374     fulldatasize = datasize + fulldatasize;
1375     percentage = "";
1376     if(downloadsize>0):
1377      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1378     downloaddiff = fulldatasize - prevdownsize;
1379     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1380     prevdownsize = fulldatasize;
1381     strbuf.write(databytes);
1382    strbuf.seek(0);
1383    returnval_content = strbuf.read();
1384   geturls_text.close();
1385   if(httpheaderout.get("Content-Encoding")=="gzip"):
1386    try:
1387     returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1388    except zlib.error:
1389     pass;
1390   elif(httpheaderout.get("Content-Encoding")=="deflate"):
1391    try:
1392     returnval_content = zlib.decompress(returnval_content);
1393    except zlib.error:
1394     pass;
1395   elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1396    try:
1397     returnval_content = brotli.decompress(returnval_content);
1398    except brotli.error:
1399     pass;
1400   elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1401    try:
1402     returnval_content = zstandard.decompress(returnval_content);
1403    except zstandard.error:
1404     pass;
1405   elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1406    try:
1407     returnval_content = lzma.decompress(returnval_content);
1408    except zstandard.error:
1409     pass;
1410   elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1411    try:
1412     returnval_content = bz2.decompress(returnval_content);
1413    except zstandard.error:
1414     pass;
1415  elif(httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"):
1416   downloadsize = httpheaderout.get('Content-Length');
1417   if(downloadsize is not None):
1418    downloadsize = int(downloadsize);
1419   if downloadsize is None: downloadsize = 0;
1420   fulldatasize = 0;
1421   prevdownsize = 0;
1422   log.info("Downloading URL "+httpurl);
1423   with BytesIO() as strbuf:
1424    while True:
1425     databytes = geturls_text.read();
1426     if not databytes: break;
1427     datasize = len(databytes);
1428     fulldatasize = datasize + fulldatasize;
1429     percentage = "";
1430     if(downloadsize>0):
1431      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1432     downloaddiff = fulldatasize - prevdownsize;
1433     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1434     prevdownsize = fulldatasize;
1435     strbuf.write(databytes);
1436     break;
1437    strbuf.seek(0);
1438    returnval_content = strbuf.read();
1439   geturls_text.close();
1440   if(httpheaderout.get("Content-Encoding")=="gzip"):
1441    try:
1442     returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1443    except zlib.error:
1444     pass;
1445   elif(httpheaderout.get("Content-Encoding")=="deflate"):
1446    try:
1447     returnval_content = zlib.decompress(returnval_content);
1448    except zlib.error:
1449     pass;
1450   elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1451    try:
1452     returnval_content = brotli.decompress(returnval_content);
1453    except brotli.error:
1454     pass;
1455   elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1456    try:
1457     returnval_content = zstandard.decompress(returnval_content);
1458    except zstandard.error:
1459     pass;
1460   elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1461    try:
1462     returnval_content = lzma.decompress(returnval_content);
1463    except zstandard.error:
1464     pass;
1465   elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1466    try:
1467     returnval_content = bz2.decompress(returnval_content);
1468    except zstandard.error:
1469     pass;
1470  elif(httplibuse=="requests"):
1471   log.info("Downloading URL "+httpurl);
1472   downloadsize = httpheaderout.get('Content-Length');
1473   if(downloadsize is not None):
1474    downloadsize = int(downloadsize);
1475   if downloadsize is None: downloadsize = 0;
1476   fulldatasize = 0;
1477   prevdownsize = 0;
1478   log.info("Downloading URL "+httpurl);
1479   with BytesIO() as strbuf:
1480    while True:
1481     databytes = geturls_text.raw.read(buffersize);
1482     if not databytes: break;
1483     datasize = len(databytes);
1484     fulldatasize = datasize + fulldatasize;
1485     percentage = "";
1486     if(downloadsize>0):
1487      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1488     downloaddiff = fulldatasize - prevdownsize;
1489     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1490     prevdownsize = fulldatasize;
1491     strbuf.write(databytes);
1492    strbuf.seek(0);
1493    returnval_content = strbuf.read();
1494   geturls_text.close();
1495   if(httpheaderout.get("Content-Encoding")=="gzip"):
1496    try:
1497     returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1498    except zlib.error:
1499     pass;
1500   elif(httpheaderout.get("Content-Encoding")=="deflate"):
1501    try:
1502     returnval_content = zlib.decompress(returnval_content);
1503    except zlib.error:
1504     pass;
1505   elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1506    try:
1507     returnval_content = brotli.decompress(returnval_content);
1508    except brotli.error:
1509     pass;
1510   elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1511    try:
1512     returnval_content = zstandard.decompress(returnval_content);
1513    except zstandard.error:
1514     pass;
1515   elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1516    try:
1517     returnval_content = lzma.decompress(returnval_content);
1518    except zstandard.error:
1519     pass;
1520   elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1521    try:
1522     returnval_content = bz2.decompress(returnval_content);
1523    except zstandard.error:
1524     pass;
1525  elif(httplibuse=="pycurl" or httplibuse=="pycurl2" or httplibuse=="pycurl3"):
1526   log.info("Downloading URL "+httpurl);
1527   downloadsize = httpheaderout.get('Content-Length');
1528   if(downloadsize is not None):
1529    downloadsize = int(downloadsize);
1530   if downloadsize is None: downloadsize = 0;
1531   fulldatasize = 0;
1532   prevdownsize = 0;
1533   log.info("Downloading URL "+httpurl);
1534   with BytesIO() as strbuf:
1535    while True:
1536     databytes = retrieved_body.read(buffersize);
1537     if not databytes: break;
1538     datasize = len(databytes);
1539     fulldatasize = datasize + fulldatasize;
1540     percentage = "";
1541     if(downloadsize>0):
1542      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1543     downloaddiff = fulldatasize - prevdownsize;
1544     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1545     prevdownsize = fulldatasize;
1546     strbuf.write(databytes);
1547    strbuf.seek(0);
1548    returnval_content = strbuf.read();
1549   geturls_text.close();
1550   if(httpheaderout.get("Content-Encoding")=="gzip"):
1551    try:
1552     returnval_content = zlib.decompress(returnval_content, 16+zlib.MAX_WBITS);
1553    except zlib.error:
1554     pass;
1555   elif(httpheaderout.get("Content-Encoding")=="deflate"):
1556    try:
1557     returnval_content = zlib.decompress(returnval_content);
1558    except zlib.error:
1559     pass;
1560   elif(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1561    try:
1562     returnval_content = brotli.decompress(returnval_content);
1563    except brotli.error:
1564     pass;
1565   elif(httpheaderout.get("Content-Encoding")=="zstd" and havezstd):
1566    try:
1567     returnval_content = zstandard.decompress(returnval_content);
1568    except zstandard.error:
1569     pass;
1570   elif((httpheaderout.get("Content-Encoding")=="lzma" or httpheaderout.get("Content-Encoding")=="xz") and havelzma):
1571    try:
1572     returnval_content = lzma.decompress(returnval_content);
1573    except zstandard.error:
1574     pass;
1575   elif(httpheaderout.get("Content-Encoding")=="bzip2"):
1576    try:
1577     returnval_content = bz2.decompress(returnval_content);
1578    except zstandard.error:
1579     pass;
1580  elif(httplibuse=="ftp" or httplibuse=="sftp" or httplibuse=="pysftp"):
1581   pass;
1582  else:
1583   returnval = False;
1584  returnval = {'Type': "Content", 'Content': returnval_content, 'Contentsize': fulldatasize, 'ContentsizeAlt': {'IEC': get_readable_size(fulldatasize, 2, "IEC"), 'SI': get_readable_size(fulldatasize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason, 'HTTPLib': httplibuse};
1585  return returnval;
1586
1587 def download_from_url_from_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1, timeout=10):
1588  if(isinstance(httpurl, list)):
1589   pass;
1590  elif(isinstance(httpurl, tuple)):
1591   pass;
1592  elif(isinstance(httpurl, dict)):
1593   httpurl = httpurl.values();
1594  else:
1595   httpurl = [httpurl];
1596  listsize = len(httpurl);
1597  listcount = 0;
1598  returnval = [];
1599  while(listcount<listsize):
1600   ouputval = download_from_url(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout);
1601   returnval.append(ouputval);
1602   listcount += 1;
1603  return returnval;
1604
1605 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1606  global geturls_download_sleep, havezstd, havebrotli, tmpfileprefix, tmpfilesuffix, haveaiohttp, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1607  exec_time_start = time.time();
1608  myhash = hashlib.new("sha1");
1609  if(sys.version[0]=="2"):
1610   myhash.update(httpurl);
1611   myhash.update(str(buffersize));
1612   myhash.update(str(exec_time_start));
1613  if(sys.version[0]>="3"):
1614   myhash.update(httpurl.encode('utf-8'));
1615   myhash.update(str(buffersize).encode('utf-8'));
1616   myhash.update(str(exec_time_start).encode('utf-8'));
1617  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1618  if(sleep<0):
1619   sleep = geturls_download_sleep;
1620  if(timeout<=0):
1621   timeout = 10;
1622  if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1623   httplibuse = "urllib";
1624  if(httplibuse=="httplib1"):
1625   httplibuse = "httplib";
1626  if(not haverequests and httplibuse=="requests"):
1627   httplibuse = "urllib";
1628  if(not haveaiohttp and httplibuse=="aiohttp"):
1629   httplibuse = "urllib";
1630  if(not havehttpx and httplibuse=="httpx"):
1631   httplibuse = "urllib";
1632  if(not havehttpx and httplibuse=="httpx2"):
1633   httplibuse = "urllib";
1634  if(not havehttpcore and httplibuse=="httpcore"):
1635   httplibuse = "urllib";
1636  if(not havehttpcore and httplibuse=="httpcore2"):
1637   httplibuse = "urllib";
1638  if(not havemechanize and httplibuse=="mechanize"):
1639   httplibuse = "urllib";
1640  if(not havepycurl and httplibuse=="pycurl"):
1641   httplibuse = "urllib";
1642  if(not havepycurl and httplibuse=="pycurl2"):
1643   httplibuse = "urllib";
1644  if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1645   httplibuse = "pycurl";
1646  if(not havepycurl and httplibuse=="pycurl3"):
1647   httplibuse = "urllib";
1648  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1649   httplibuse = "pycurl2";
1650  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1651   httplibuse = "pycurl";
1652  if(not havehttplib2 and httplibuse=="httplib2"):
1653   httplibuse = "httplib";
1654  if(not haveparamiko and httplibuse=="sftp"):
1655   httplibuse = "ftp";
1656  if(not haveparamiko and httplibuse=="pysftp"):
1657   httplibuse = "ftp";
1658  pretmpfilename = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize, sleep, timeout);
1659  if(not pretmpfilename):
1660   return False;
1661  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1662   tmpfilename = f.name;
1663   try:
1664    os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1665   except AttributeError:
1666    try:
1667     os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1668    except ValueError:
1669     pass;
1670   except ValueError:
1671    pass;
1672   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': pretmpfilename.get('Contentsize'), 'FilesizeAlt': {'IEC': get_readable_size(pretmpfilename.get('Contentsize'), 2, "IEC"), 'SI': get_readable_size(pretmpfilename.get('Contentsize'), 2, "SI")}, 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1673   f.write(pretmpfilename.get('Content'));
1674   f.close();
1675  exec_time_end = time.time();
1676  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1677  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'FilesizeAlt': {'IEC': get_readable_size(os.path.getsize(tmpfilename), 2, "IEC"), 'SI': get_readable_size(os.path.getsize(tmpfilename), 2, "SI")}, 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1678  return returnval;
1679
1680 def download_from_url_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1681  if(isinstance(httpurl, list)):
1682   pass;
1683  elif(isinstance(httpurl, tuple)):
1684   pass;
1685  elif(isinstance(httpurl, dict)):
1686   httpurl = httpurl.values();
1687  else:
1688   httpurl = [httpurl];
1689  listsize = len(httpurl);
1690  listcount = 0;
1691  returnval = [];
1692  while(listcount<listsize):
1693   ouputval = download_from_url_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize, sleep, timeout);
1694   returnval.append(ouputval);
1695   listcount += 1;
1696  return returnval;
1697
1698 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1699  global geturls_download_sleep, havezstd, havebrotli, haveaiohttp, haverequests, havemechanize, havepycurl, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1700  if(sleep<0):
1701   sleep = geturls_download_sleep;
1702  if(timeout<=0):
1703   timeout = 10;
1704  if(httplibuse=="urllib1" or httplibuse=="urllib2" or httplibuse=="request"):
1705   httplibuse = "urllib";
1706  if(httplibuse=="httplib1"):
1707   httplibuse = "httplib";
1708  if(not haverequests and httplibuse=="requests"):
1709   httplibuse = "urllib";
1710  if(not haveaiohttp and httplibuse=="aiohttp"):
1711   httplibuse = "urllib";
1712  if(not havehttpx and httplibuse=="httpx"):
1713   httplibuse = "urllib";
1714  if(not havehttpx and httplibuse=="httpx2"):
1715   httplibuse = "urllib";
1716  if(not havehttpcore and httplibuse=="httpcore"):
1717   httplibuse = "urllib";
1718  if(not havehttpcore and httplibuse=="httpcore2"):
1719   httplibuse = "urllib";
1720  if(not havemechanize and httplibuse=="mechanize"):
1721   httplibuse = "urllib";
1722  if(not havepycurl and httplibuse=="pycurl"):
1723   httplibuse = "urllib";
1724  if(not havepycurl and httplibuse=="pycurl2"):
1725   httplibuse = "urllib";
1726  if(havepycurl and httplibuse=="pycurl2" and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1727   httplibuse = "pycurl";
1728  if(not havepycurl and httplibuse=="pycurl3"):
1729   httplibuse = "urllib";
1730  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1731   httplibuse = "pycurl2";
1732  if(havepycurl and httplibuse=="pycurl3" and not hasattr(pycurl, "CURL_HTTP_VERSION_3_0") and not hasattr(pycurl, "CURL_HTTP_VERSION_2_0")):
1733   httplibuse = "pycurl";
1734  if(not havehttplib2 and httplibuse=="httplib2"):
1735   httplibuse = "httplib";
1736  if(not haveparamiko and httplibuse=="sftp"):
1737   httplibuse = "ftp";
1738  if(not havepysftp and httplibuse=="pysftp"):
1739   httplibuse = "ftp";
1740  if(not outfile=="-"):
1741   outpath = outpath.rstrip(os.path.sep);
1742   filepath = os.path.realpath(outpath+os.path.sep+outfile);
1743   if(not os.path.exists(outpath)):
1744    os.makedirs(outpath);
1745   if(os.path.exists(outpath) and os.path.isfile(outpath)):
1746    return False;
1747   if(os.path.exists(filepath) and os.path.isdir(filepath)):
1748    return False;
1749   pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1750   if(not pretmpfilename):
1751    return False;
1752   tmpfilename = pretmpfilename.get('Filename');
1753   downloadsize = int(os.path.getsize(tmpfilename));
1754   fulldatasize = 0;
1755   log.info("Moving file "+tmpfilename+" to "+filepath);
1756   exec_time_start = time.time();
1757   shutil.move(tmpfilename, filepath);
1758   try:
1759    os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1760   except AttributeError:
1761    try:
1762     os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1763    except ValueError:
1764     pass;
1765   except ValueError:
1766    pass;
1767   exec_time_end = time.time();
1768   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1769   if(os.path.exists(tmpfilename)):
1770    os.remove(tmpfilename);
1771   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': pretmpfilename.get('HTTPLib')};
1772  if(outfile=="-"):
1773   pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, ranges, buffersize[0], sleep, timeout);
1774   tmpfilename = pretmpfilename.get('Filename');
1775   downloadsize = int(os.path.getsize(tmpfilename));
1776   fulldatasize = 0;
1777   prevdownsize = 0;
1778   exec_time_start = time.time();
1779   with open(tmpfilename, 'rb') as ft:
1780    f = BytesIO();
1781    while True:
1782     databytes = ft.read(buffersize[1]);
1783     if not databytes: break;
1784     datasize = len(databytes);
1785     fulldatasize = datasize + fulldatasize;
1786     percentage = "";
1787     if(downloadsize>0):
1788      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1789     downloaddiff = fulldatasize - prevdownsize;
1790     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1791     prevdownsize = fulldatasize;
1792     f.write(databytes);
1793    f.seek(0);
1794    fdata = f.getvalue();
1795    f.close();
1796    ft.close();
1797    os.remove(tmpfilename);
1798    exec_time_end = time.time();
1799    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1800   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename.get('DownloadTime'), 'DownloadTimeReadable': pretmpfilename.get('DownloadTimeReadable'), 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename.get('Headers'), 'Version': pretmpfilename.get('Version'), 'Method': pretmpfilename.get('Method'), 'Method': httpmethod, 'HeadersSent': pretmpfilename.get('HeadersSent'), 'URL': pretmpfilename.get('URL'), 'Code': pretmpfilename.get('Code'), 'Reason': pretmpfilename.get('Reason'), 'HTTPLib': httplibuse};
1801  return returnval;
1802
1803 def download_from_url_to_file_with_list(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1804  if(isinstance(httpurl, list)):
1805   pass;
1806  elif(isinstance(httpurl, tuple)):
1807   pass;
1808  elif(isinstance(httpurl, dict)):
1809   httpurl = httpurl.values();
1810  else:
1811   httpurl = [httpurl];
1812  listsize = len(httpurl);
1813  listcount = 0;
1814  returnval = [];
1815  while(listcount<listsize):
1816   ouputval = download_from_url_to_file(httpurl[listcount], httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, outfile, outpath, ranges, buffersize, sleep, timeout);
1817   returnval.append(ouputval);
1818   listcount += 1;
1819  return returnval;
1820
1821 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1822   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1823   return returnval;
1824
1825 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1826   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep, timeout);
1827   return returnval;
1828
1829 def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1830   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", buffersize, sleep, timeout);
1831   return returnval;
1832
1833 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1834   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", buffersize, sleep, timeout);
1835   return returnval;
1836
1837 def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1838   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", buffersize, sleep, timeout);
1839   return returnval;
1840
1841 def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1842   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", buffersize, sleep, timeout);
1843   return returnval;
1844
1845 def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1846   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", buffersize, sleep, timeout);
1847   return returnval;
1848
1849 def download_from_url_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1850   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", buffersize, sleep, timeout);
1851   return returnval;
1852
1853 def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1854   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", buffersize, sleep, timeout);
1855   return returnval;
1856
1857 def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1858   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", buffersize, sleep, timeout);
1859   return returnval;
1860
1861 def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1862   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", buffersize, sleep, timeout);
1863   return returnval;
1864
1865 def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1866   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", buffersize, sleep, timeout);
1867   return returnval;
1868
1869 def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1870   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", buffersize, sleep, timeout);
1871   return returnval;
1872
1873 def download_from_url_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1874   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", buffersize, sleep, timeout);
1875   return returnval;
1876
1877 def download_from_url_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1878   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", buffersize, sleep, timeout);
1879   return returnval;
1880
1881 def download_from_url_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1882   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", buffersize, sleep, timeout);
1883   return returnval;
1884
1885 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1886   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", buffersize, sleep, timeout);
1887   return returnval;
1888
1889 def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1890   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", buffersize, sleep, timeout);
1891   return returnval;
1892
1893 def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1, timeout=10):
1894   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", buffersize, sleep, timeout);
1895   return returnval;
1896
1897 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1898   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1899   return returnval;
1900
1901 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1902   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", ranges, buffersize, sleep, timeout);
1903   return returnval;
1904
1905 def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1906   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", ranges, buffersize, sleep, timeout);
1907   return returnval;
1908
1909 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1910   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", ranges, buffersize, sleep, timeout);
1911   return returnval;
1912
1913 def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1914   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", ranges, buffersize, sleep, timeout);
1915   return returnval;
1916
1917 def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1918   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", ranges, buffersize, sleep, timeout);
1919   return returnval;
1920
1921 def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1922   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", ranges, buffersize, sleep, timeout);
1923   return returnval;
1924
1925 def download_from_url_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1926   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", ranges, buffersize, sleep, timeout);
1927   return returnval;
1928
1929 def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1930   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", ranges, buffersize, sleep, timeout);
1931   return returnval;
1932
1933 def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1934   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", ranges, buffersize, sleep, timeout);
1935   return returnval;
1936
1937 def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1938   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", ranges, buffersize, sleep, timeout);
1939   return returnval;
1940
1941 def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1942   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", ranges, buffersize, sleep, timeout);
1943   return returnval;
1944
1945 def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1946   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", ranges, buffersize, sleep, timeout);
1947   return returnval;
1948
1949 def download_from_url_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1950   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", ranges, buffersize, sleep, timeout);
1951   return returnval;
1952
1953 def download_from_url_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1954   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", ranges, buffersize, sleep, timeout);
1955   return returnval;
1956
1957 def download_from_url_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1958   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", ranges, buffersize, sleep, timeout);
1959   return returnval;
1960
1961 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1962   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", ranges, buffersize, sleep, timeout);
1963   return returnval;
1964
1965 def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1966   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", ranges, buffersize, sleep, timeout);
1967   return returnval;
1968
1969 def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, ranges=[None, None], buffersize=524288, sleep=-1, timeout=10):
1970   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", ranges, buffersize, sleep, timeout);
1971   return returnval;
1972
1973 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1974   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1975   return returnval;
1976
1977 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1978   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", outfile, outpath, ranges, buffersize, sleep, timeout);
1979   return returnval;
1980
1981 def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1982   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, ranges, buffersize, sleep, timeout);
1983   return returnval;
1984
1985 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1986   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", outfile, outpath, ranges, buffersize, sleep, timeout);
1987   return returnval;
1988
1989 def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1990   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", outfile, outpath, ranges, buffersize, sleep, timeout);
1991   return returnval;
1992
1993 def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1994   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", outfile, outpath, ranges, buffersize, sleep, timeout);
1995   return returnval;
1996
1997 def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
1998   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", outfile, outpath, ranges, buffersize, sleep, timeout);
1999   return returnval;
2000
2001 def download_from_url_to_file_with_aiohttp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2002   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "aiohttp", outfile, outpath, ranges, buffersize, sleep, timeout);
2003   return returnval;
2004
2005 def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2006   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", outfile, outpath, ranges, buffersize, sleep, timeout);
2007   return returnval;
2008
2009 def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2010   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", outfile, outpath, ranges, buffersize, sleep, timeout);
2011   return returnval;
2012
2013 def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2014   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", outfile, outpath, ranges, buffersize, sleep, timeout);
2015   return returnval;
2016
2017 def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2018   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", outfile, outpath, ranges, buffersize, sleep, timeout);
2019   return returnval;
2020
2021 def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2022   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", outfile, outpath, ranges, buffersize, sleep, timeout);
2023   return returnval;
2024
2025 def download_from_url_to_file_with_pycurl(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2026   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl", outfile, outpath, ranges, buffersize, sleep, timeout);
2027   return returnval;
2028
2029 def download_from_url_to_file_with_pycurl2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2030   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl2", outfile, outpath, ranges, buffersize, sleep, timeout);
2031   return returnval;
2032
2033 def download_from_url_to_file_with_pycurl3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2034   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pycurl3", outfile, outpath, ranges, buffersize, sleep, timeout);
2035   return returnval;
2036
2037 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2038   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2039   return returnval;
2040
2041 def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2042   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2043   return returnval;
2044
2045 def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), ranges=[None, None], buffersize=[524288, 524288], sleep=-1, timeout=10):
2046   returnval = download_from_url_to_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", outfile, outpath, ranges, buffersize, sleep, timeout);
2047   return returnval;
2048
2049 def download_file_from_ftp_file(url):
2050  urlparts = urlparse.urlparse(url);
2051  file_name = os.path.basename(urlparts.path);
2052  file_dir = os.path.dirname(urlparts.path);
2053  if(urlparts.username is not None):
2054   ftp_username = urlparts.username;
2055  else:
2056   ftp_username = "anonymous";
2057  if(urlparts.password is not None):
2058   ftp_password = urlparts.password;
2059  elif(urlparts.password is None and urlparts.username=="anonymous"):
2060   ftp_password = "anonymous";
2061  else:
2062   ftp_password = "";
2063  if(urlparts.scheme=="ftp"):
2064   ftp = FTP();
2065  elif(urlparts.scheme=="ftps"):
2066   ftp = FTP_TLS();
2067  else:
2068   return False;
2069  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2070   return False;
2071  ftp_port = urlparts.port;
2072  if(urlparts.port is None):
2073   ftp_port = 21;
2074  try:
2075   ftp.connect(urlparts.hostname, ftp_port);
2076  except socket.gaierror:
2077   log.info("Error With URL "+httpurl);
2078   return False;
2079  except socket.timeout:
2080   log.info("Error With URL "+httpurl);
2081   return False;
2082  ftp.login(urlparts.username, urlparts.password);
2083  if(urlparts.scheme=="ftps"):
2084   ftp.prot_p();
2085  ftpfile = BytesIO();
2086  ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
2087  #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2088  ftp.close();
2089  ftpfile.seek(0, 0);
2090  return ftpfile;
2091
2092 def download_file_from_ftp_string(url):
2093  ftpfile = download_file_from_ftp_file(url);
2094  return ftpfile.read();
2095
2096 def upload_file_to_ftp_file(ftpfile, url):
2097  urlparts = urlparse.urlparse(url);
2098  file_name = os.path.basename(urlparts.path);
2099  file_dir = os.path.dirname(urlparts.path);
2100  if(urlparts.username is not None):
2101   ftp_username = urlparts.username;
2102  else:
2103   ftp_username = "anonymous";
2104  if(urlparts.password is not None):
2105   ftp_password = urlparts.password;
2106  elif(urlparts.password is None and urlparts.username=="anonymous"):
2107   ftp_password = "anonymous";
2108  else:
2109   ftp_password = "";
2110  if(urlparts.scheme=="ftp"):
2111   ftp = FTP();
2112  elif(urlparts.scheme=="ftps"):
2113   ftp = FTP_TLS();
2114  else:
2115   return False;
2116  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2117   return False;
2118  ftp_port = urlparts.port;
2119  if(urlparts.port is None):
2120   ftp_port = 21;
2121  try:
2122   ftp.connect(urlparts.hostname, ftp_port);
2123  except socket.gaierror:
2124   log.info("Error With URL "+httpurl);
2125   return False;
2126  except socket.timeout:
2127   log.info("Error With URL "+httpurl);
2128   return False;
2129  ftp.login(urlparts.username, urlparts.password);
2130  if(urlparts.scheme=="ftps"):
2131   ftp.prot_p();
2132  ftp.storbinary("STOR "+urlparts.path, ftpfile);
2133  ftp.close();
2134  ftpfile.seek(0, 0);
2135  return ftpfile;
2136
2137 def upload_file_to_ftp_string(ftpstring, url):
2138  ftpfileo = BytesIO(ftpstring);
2139  ftpfile = upload_file_to_ftp_file(ftpfileo, url);
2140  ftpfileo.close();
2141  return ftpfile;
2142
2143 if(haveparamiko):
2144  def download_file_from_sftp_file(url):
2145   urlparts = urlparse.urlparse(url);
2146   file_name = os.path.basename(urlparts.path);
2147   file_dir = os.path.dirname(urlparts.path);
2148   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2149    return False;
2150   sftp_port = urlparts.port;
2151   if(urlparts.port is None):
2152    sftp_port = 22;
2153   else:
2154    sftp_port = urlparts.port;
2155   if(urlparts.username is not None):
2156    sftp_username = urlparts.username;
2157   else:
2158    sftp_username = "anonymous";
2159   if(urlparts.password is not None):
2160    sftp_password = urlparts.password;
2161   elif(urlparts.password is None and urlparts.username=="anonymous"):
2162    sftp_password = "anonymous";
2163   else:
2164    sftp_password = "";
2165   if(urlparts.scheme!="sftp"):
2166    return False;
2167   ssh = paramiko.SSHClient();
2168   ssh.load_system_host_keys();
2169   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2170   try:
2171    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2172   except paramiko.ssh_exception.SSHException:
2173    return False;
2174   except socket.gaierror:
2175    log.info("Error With URL "+httpurl);
2176    return False;
2177   except socket.timeout:
2178    log.info("Error With URL "+httpurl);
2179    return False;
2180   sftp = ssh.open_sftp();
2181   sftpfile = BytesIO();
2182   sftp.getfo(urlparts.path, sftpfile);
2183   sftp.close();
2184   ssh.close();
2185   sftpfile.seek(0, 0);
2186   return sftpfile;
2187 else:
2188  def download_file_from_sftp_file(url):
2189   return False;
2190
2191 if(haveparamiko):
2192  def download_file_from_sftp_string(url):
2193   sftpfile = download_file_from_sftp_file(url);
2194   return sftpfile.read();
2195 else:
2196  def download_file_from_ftp_string(url):
2197   return False;
2198
2199 if(haveparamiko):
2200  def upload_file_to_sftp_file(sftpfile, url):
2201   urlparts = urlparse.urlparse(url);
2202   file_name = os.path.basename(urlparts.path);
2203   file_dir = os.path.dirname(urlparts.path);
2204   sftp_port = urlparts.port;
2205   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2206    return False;
2207   if(urlparts.port is None):
2208    sftp_port = 22;
2209   else:
2210    sftp_port = urlparts.port;
2211   if(urlparts.username is not None):
2212    sftp_username = urlparts.username;
2213   else:
2214    sftp_username = "anonymous";
2215   if(urlparts.password is not None):
2216    sftp_password = urlparts.password;
2217   elif(urlparts.password is None and urlparts.username=="anonymous"):
2218    sftp_password = "anonymous";
2219   else:
2220    sftp_password = "";
2221   if(urlparts.scheme!="sftp"):
2222    return False;
2223   ssh = paramiko.SSHClient();
2224   ssh.load_system_host_keys();
2225   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2226   try:
2227    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2228   except paramiko.ssh_exception.SSHException:
2229    return False;
2230   except socket.gaierror:
2231    log.info("Error With URL "+httpurl);
2232    return False;
2233   except socket.timeout:
2234    log.info("Error With URL "+httpurl);
2235    return False;
2236   sftp = ssh.open_sftp();
2237   sftp.putfo(sftpfile, urlparts.path);
2238   sftp.close();
2239   ssh.close();
2240   sftpfile.seek(0, 0);
2241   return sftpfile;
2242 else:
2243  def upload_file_to_sftp_file(sftpfile, url):
2244   return False;
2245
2246 if(haveparamiko):
2247  def upload_file_to_sftp_string(sftpstring, url):
2248   sftpfileo = BytesIO(sftpstring);
2249   sftpfile = upload_file_to_sftp_files(ftpfileo, url);
2250   sftpfileo.close();
2251   return sftpfile;
2252 else:
2253  def upload_file_to_sftp_string(url):
2254   return False;
2255
2256
2257 if(havepysftp):
2258  def download_file_from_pysftp_file(url):
2259   urlparts = urlparse.urlparse(url);
2260   file_name = os.path.basename(urlparts.path);
2261   file_dir = os.path.dirname(urlparts.path);
2262   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2263    return False;
2264   sftp_port = urlparts.port;
2265   if(urlparts.port is None):
2266    sftp_port = 22;
2267   else:
2268    sftp_port = urlparts.port;
2269   if(urlparts.username is not None):
2270    sftp_username = urlparts.username;
2271   else:
2272    sftp_username = "anonymous";
2273   if(urlparts.password is not None):
2274    sftp_password = urlparts.password;
2275   elif(urlparts.password is None and urlparts.username=="anonymous"):
2276    sftp_password = "anonymous";
2277   else:
2278    sftp_password = "";
2279   if(urlparts.scheme!="sftp"):
2280    return False;
2281   try:
2282    pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2283   except paramiko.ssh_exception.SSHException:
2284    return False;
2285   except socket.gaierror:
2286    log.info("Error With URL "+httpurl);
2287    return False;
2288   except socket.timeout:
2289    log.info("Error With URL "+httpurl);
2290    return False;
2291   sftp = ssh.open_sftp();
2292   sftpfile = BytesIO();
2293   sftp.getfo(urlparts.path, sftpfile);
2294   sftp.close();
2295   ssh.close();
2296   sftpfile.seek(0, 0);
2297   return sftpfile;
2298 else:
2299  def download_file_from_pysftp_file(url):
2300   return False;
2301
2302 if(havepysftp):
2303  def download_file_from_pysftp_string(url):
2304   sftpfile = download_file_from_pysftp_file(url);
2305   return sftpfile.read();
2306 else:
2307  def download_file_from_ftp_string(url):
2308   return False;
2309
2310 if(havepysftp):
2311  def upload_file_to_pysftp_file(sftpfile, url):
2312   urlparts = urlparse.urlparse(url);
2313   file_name = os.path.basename(urlparts.path);
2314   file_dir = os.path.dirname(urlparts.path);
2315   sftp_port = urlparts.port;
2316   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2317    return False;
2318   if(urlparts.port is None):
2319    sftp_port = 22;
2320   else:
2321    sftp_port = urlparts.port;
2322   if(urlparts.username is not None):
2323    sftp_username = urlparts.username;
2324   else:
2325    sftp_username = "anonymous";
2326   if(urlparts.password is not None):
2327    sftp_password = urlparts.password;
2328   elif(urlparts.password is None and urlparts.username=="anonymous"):
2329    sftp_password = "anonymous";
2330   else:
2331    sftp_password = "";
2332   if(urlparts.scheme!="sftp"):
2333    return False;
2334   try:
2335    pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2336   except paramiko.ssh_exception.SSHException:
2337    return False;
2338   except socket.gaierror:
2339    log.info("Error With URL "+httpurl);
2340    return False;
2341   except socket.timeout:
2342    log.info("Error With URL "+httpurl);
2343    return False;
2344   sftp = ssh.open_sftp();
2345   sftp.putfo(sftpfile, urlparts.path);
2346   sftp.close();
2347   ssh.close();
2348   sftpfile.seek(0, 0);
2349   return sftpfile;
2350 else:
2351  def upload_file_to_pysftp_file(sftpfile, url):
2352   return False;
2353
2354 if(havepysftp):
2355  def upload_file_to_pysftp_string(sftpstring, url):
2356   sftpfileo = BytesIO(sftpstring);
2357   sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
2358   sftpfileo.close();
2359   return sftpfile;
2360 else:
2361  def upload_file_to_pysftp_string(url):
2362   return False;