pywwwget.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
  13     Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  14
  15     $FileInfo: pywwwget.py - Last Update: 9/24/2023 Ver. 1.5.0 RC 1 - Author: cooldude2k $
  16 '''
  17
  18 from __future__ import division, absolute_import, print_function;
  19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, argparse, cgi, subprocess, socket, email.utils, datetime, time;
  20 import logging as log;
  21 from ftplib import FTP, FTP_TLS;
  22 from base64 import b64encode;
  23 haverequests = False;
  24 try:
  25  import requests;
  26  haverequests = True;
  27 except ImportError:
  28  haverequests = False;
  29 havemechanize = False;
  30 try:
  31  import mechanize;
  32  havemechanize = True;
  33 except ImportError:
  34  havemechanize = False;
  35 haveparamiko = False;
  36 try:
  37  import paramiko;
  38  haveparamiko = True;
  39 except ImportError:
  40  haveparamiko = False;
  41 havepysftp = False;
  42 try:
  43  import pysftp;
  44  havepysftp = True;
  45 except ImportError:
  46  havepysftp = False;
  47 haveurllib3 = False;
  48 try:
  49  import urllib3;
  50  haveurllib3 = True;
  51 except ImportError:
  52  haveurllib3 = False;
  53 havehttplib2 = False;
  54 try:
  55  from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
  56  havehttplib2 = True;
  57 except ImportError:
  58  havehttplib2 = False;
  59 havehttpx = False;
  60 try:
  61  import httpx;
  62  havehttpx = True;
  63 except ImportError:
  64  havehttpx = False;
  65 havehttpcore = False;
  66 try:
  67  import httpcore;
  68  havehttpcore = True;
  69 except ImportError:
  70  havehttpcore = False;
  71 havebrotli = False;
  72 try:
  73  import brotli;
  74  havebrotli = True;
  75 except ImportError:
  76  havebrotli = False;
  77 havezstd = False;
  78 try:
  79  import zstandard;
  80  havezstd = True;
  81 except ImportError:
  82  havezstd = False;
  83 if(sys.version[0]=="2"):
  84  try:
  85   from cStringIO import StringIO;
  86  except ImportError:
  87   from StringIO import StringIO;
  88  # From http://python-future.org/compatible_idioms.html
  89  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
  90  from urllib import urlencode;
  91  from urllib import urlopen as urlopenalt;
  92  from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
  93  import urlparse, cookielib;
  94  from httplib import HTTPConnection, HTTPSConnection;
  95 if(sys.version[0]>="3"):
  96  from io import StringIO, BytesIO;
  97  # From http://python-future.org/compatible_idioms.html
  98  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
  99  from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
 100  from urllib.error import HTTPError, URLError;
 101  import urllib.parse as urlparse;
 102  import http.cookiejar as cookielib;
 103  from http.client import HTTPConnection, HTTPSConnection;
 104
 105 __program_name__ = "PyWWW-Get";
 106 __program_alt_name__ = "PyWWWGet";
 107 __program_small_name__ = "wwwget";
 108 __project__ = __program_name__;
 109 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
 110 __version_info__ = (1, 5, 0, "RC 1", 1);
 111 __version_date_info__ = (2023, 9, 24, "RC 1", 1);
 112 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
 113 __revision__ = __version_info__[3];
 114 __revision_id__ = "$Id$";
 115 if(__version_info__[4] is not None):
 116  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
 117 if(__version_info__[4] is None):
 118  __version_date_plusrc__ = __version_date__;
 119 if(__version_info__[3] is not None):
 120  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
 121 if(__version_info__[3] is None):
 122  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
 123
 124 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
 125 tmpfilesuffix = "-";
 126 pytempdir = tempfile.gettempdir();
 127
 128 PyBitness = platform.architecture();
 129 if(PyBitness=="32bit" or PyBitness=="32"):
 130  PyBitness = "32";
 131 elif(PyBitness=="64bit" or PyBitness=="64"):
 132  PyBitness = "64";
 133 else:
 134  PyBitness = "32";
 135
 136 compression_supported = "gzip, deflate";
 137 if(havebrotli):
 138  compression_supported = "gzip, deflate, br";
 139 else:
 140  compression_supported = "gzip, deflate";
 141
 142 geturls_cj = cookielib.CookieJar();
 143 windowsNT4_ua_string = "Windows NT 4.0";
 144 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
 145 windows2k_ua_string = "Windows NT 5.0";
 146 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
 147 windowsXP_ua_string = "Windows NT 5.1";
 148 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 149 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
 150 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 151 windows7_ua_string = "Windows NT 6.1; Win64; x64";
 152 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
 153 windows8_ua_string = "Windows NT 6.2; Win64; x64";
 154 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
 155 windows81_ua_string = "Windows NT 6.3; Win64; x64";
 156 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
 157 windows10_ua_string = "Windows NT 10.0; Win64; x64";
 158 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
 159 windows11_ua_string = "Windows NT 11.0; Win64; x64";
 160 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
 161 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
 162 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
 163 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
 164 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
 165 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
 166 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
 167 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
 168 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
 169 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
 170 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
 171 if(platform.python_implementation()!=""):
 172  py_implementation = platform.python_implementation();
 173 if(platform.python_implementation()==""):
 174  py_implementation = "Python";
 175 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
 176 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
 177 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
 178 geturls_ua = geturls_ua_firefox_windows7;
 179 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 180 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 181 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 182 geturls_headers_chrome_windows7.update(windows7_ua_addon);
 183 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 184 geturls_headers_chromium_windows7.update(windows7_ua_addon);
 185 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 186 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
 187 geturls_headers_opera_windows7.update(windows7_ua_addon);
 188 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
 189 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
 190 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 191 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
 192 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
 193 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 194 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 195 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 196 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 197 geturls_headers = geturls_headers_firefox_windows7;
 198 geturls_download_sleep = 0;
 199
 200 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 201  if(outtype=="print" and dbgenable):
 202   print(dbgtxt);
 203   return True;
 204  elif(outtype=="log" and dbgenable):
 205   logging.info(dbgtxt);
 206   return True;
 207  elif(outtype=="warning" and dbgenable):
 208   logging.warning(dbgtxt);
 209   return True;
 210  elif(outtype=="error" and dbgenable):
 211   logging.error(dbgtxt);
 212   return True;
 213  elif(outtype=="critical" and dbgenable):
 214   logging.critical(dbgtxt);
 215   return True;
 216  elif(outtype=="exception" and dbgenable):
 217   logging.exception(dbgtxt);
 218   return True;
 219  elif(outtype=="logalt" and dbgenable):
 220   logging.log(dgblevel, dbgtxt);
 221   return True;
 222  elif(outtype=="debug" and dbgenable):
 223   logging.debug(dbgtxt);
 224   return True;
 225  elif(not dbgenable):
 226   return True;
 227  else:
 228   return False;
 229  return False;
 230
 231 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 232  dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
 233  if(not dbgout):
 234   return False;
 235  return dbgtxt;
 236
 237 def add_url_param(url, **params):
 238  n=3;
 239  parts = list(urlparse.urlsplit(url));
 240  d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 241  d.update(params);
 242  parts[n]=urlencode(d);
 243  return urlparse.urlunsplit(parts);
 244
 245 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 246 def which_exec(execfile):
 247  for path in os.environ["PATH"].split(":"):
 248   if os.path.exists(path + "/" + execfile):
 249    return path + "/" + execfile;
 250
 251 def listize(varlist):
 252  il = 0;
 253  ix = len(varlist);
 254  ilx = 1;
 255  newlistreg = {};
 256  newlistrev = {};
 257  newlistfull = {};
 258  while(il < ix):
 259   newlistreg.update({ilx: varlist[il]});
 260   newlistrev.update({varlist[il]: ilx});
 261   ilx = ilx + 1;
 262   il = il + 1;
 263  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 264  return newlistfull;
 265
 266 def twolistize(varlist):
 267  il = 0;
 268  ix = len(varlist);
 269  ilx = 1;
 270  newlistnamereg = {};
 271  newlistnamerev = {};
 272  newlistdescreg = {};
 273  newlistdescrev = {};
 274  newlistfull = {};
 275  while(il < ix):
 276   newlistnamereg.update({ilx: varlist[il][0].strip()});
 277   newlistnamerev.update({varlist[il][0].strip(): ilx});
 278   newlistdescreg.update({ilx: varlist[il][1].strip()});
 279   newlistdescrev.update({varlist[il][1].strip(): ilx});
 280   ilx = ilx + 1;
 281   il = il + 1;
 282  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 283  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 284  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 285  return newlistfull;
 286
 287 def arglistize(proexec, *varlist):
 288  il = 0;
 289  ix = len(varlist);
 290  ilx = 1;
 291  newarglist = [proexec];
 292  while(il < ix):
 293   if varlist[il][0] is not None:
 294    newarglist.append(varlist[il][0]);
 295   if varlist[il][1] is not None:
 296    newarglist.append(varlist[il][1]);
 297   il = il + 1;
 298  return newarglist;
 299
 300 def fix_header_names(header_dict):
 301  header_dict =  {k.title(): v for k, v in header_dict.items()};
 302  return header_dict;
 303
 304 # hms_string by ArcGIS Python Recipes
 305 # https://arcpy.wordpress.com/2012/04/20/146/
 306 def hms_string(sec_elapsed):
 307  h = int(sec_elapsed / (60 * 60));
 308  m = int((sec_elapsed % (60 * 60)) / 60);
 309  s = sec_elapsed % 60.0;
 310  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 311
 312 # get_readable_size by Lipis
 313 # http://stackoverflow.com/posts/14998888/revisions
 314 def get_readable_size(bytes, precision=1, unit="IEC"):
 315  unit = unit.upper();
 316  if(unit!="IEC" and unit!="SI"):
 317   unit = "IEC";
 318  if(unit=="IEC"):
 319   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 320   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 321   unitsize = 1024.0;
 322  if(unit=="SI"):
 323   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 324   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 325   unitsize = 1000.0;
 326  return_val = {};
 327  orgbytes = bytes;
 328  for unit in units:
 329   if abs(bytes) < unitsize:
 330    strformat = "%3."+str(precision)+"f%s";
 331    pre_return_val = (strformat % (bytes, unit));
 332    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 333    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 334    alt_return_val = pre_return_val.split();
 335    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 336    return return_val;
 337   bytes /= unitsize;
 338  strformat = "%."+str(precision)+"f%s";
 339  pre_return_val = (strformat % (bytes, "YiB"));
 340  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 341  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 342  alt_return_val = pre_return_val.split();
 343  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 344  return return_val;
 345
 346 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 347  unit = unit.upper();
 348  usehashtypes = usehashtypes.lower();
 349  getfilesize = os.path.getsize(infile);
 350  return_val = get_readable_size(getfilesize, precision, unit);
 351  if(usehashes):
 352   hashtypelist = usehashtypes.split(",");
 353   openfile = open(infile, "rb");
 354   filecontents = openfile.read();
 355   openfile.close();
 356   listnumcount = 0;
 357   listnumend = len(hashtypelist);
 358   while(listnumcount < listnumend):
 359    hashtypelistlow = hashtypelist[listnumcount].strip();
 360    hashtypelistup = hashtypelistlow.upper();
 361    filehash = hashlib.new(hashtypelistup);
 362    filehash.update(filecontents);
 363    filegethash = filehash.hexdigest();
 364    return_val.update({hashtypelistup: filegethash});
 365    listnumcount += 1;
 366  return return_val;
 367
 368 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 369  unit = unit.upper();
 370  usehashtypes = usehashtypes.lower();
 371  getfilesize = len(instring);
 372  return_val = get_readable_size(getfilesize, precision, unit);
 373  if(usehashes):
 374   hashtypelist = usehashtypes.split(",");
 375   listnumcount = 0;
 376   listnumend = len(hashtypelist);
 377   while(listnumcount < listnumend):
 378    hashtypelistlow = hashtypelist[listnumcount].strip();
 379    hashtypelistup = hashtypelistlow.upper();
 380    filehash = hashlib.new(hashtypelistup);
 381    if(sys.version[0]=="2"):
 382     filehash.update(instring);
 383    if(sys.version[0]>="3"):
 384     filehash.update(instring.encode('utf-8'));
 385    filegethash = filehash.hexdigest();
 386    return_val.update({hashtypelistup: filegethash});
 387    listnumcount += 1;
 388  return return_val;
 389
 390 def http_status_to_reason(code):
 391     reasons = {
 392         100: 'Continue',
 393         101: 'Switching Protocols',
 394         102: 'Processing',
 395         200: 'OK',
 396         201: 'Created',
 397         202: 'Accepted',
 398         203: 'Non-Authoritative Information',
 399         204: 'No Content',
 400         205: 'Reset Content',
 401         206: 'Partial Content',
 402         207: 'Multi-Status',
 403         208: 'Already Reported',
 404         226: 'IM Used',
 405         300: 'Multiple Choices',
 406         301: 'Moved Permanently',
 407         302: 'Found',
 408         303: 'See Other',
 409         304: 'Not Modified',
 410         305: 'Use Proxy',
 411         307: 'Temporary Redirect',
 412         308: 'Permanent Redirect',
 413         400: 'Bad Request',
 414         401: 'Unauthorized',
 415         402: 'Payment Required',
 416         403: 'Forbidden',
 417         404: 'Not Found',
 418         405: 'Method Not Allowed',
 419         406: 'Not Acceptable',
 420         407: 'Proxy Authentication Required',
 421         408: 'Request Timeout',
 422         409: 'Conflict',
 423         410: 'Gone',
 424         411: 'Length Required',
 425         412: 'Precondition Failed',
 426         413: 'Payload Too Large',
 427         414: 'URI Too Long',
 428         415: 'Unsupported Media Type',
 429         416: 'Range Not Satisfiable',
 430         417: 'Expectation Failed',
 431         421: 'Misdirected Request',
 432         422: 'Unprocessable Entity',
 433         423: 'Locked',
 434         424: 'Failed Dependency',
 435         426: 'Upgrade Required',
 436         428: 'Precondition Required',
 437         429: 'Too Many Requests',
 438         431: 'Request Header Fields Too Large',
 439         451: 'Unavailable For Legal Reasons',
 440         500: 'Internal Server Error',
 441         501: 'Not Implemented',
 442         502: 'Bad Gateway',
 443         503: 'Service Unavailable',
 444         504: 'Gateway Timeout',
 445         505: 'HTTP Version Not Supported',
 446         506: 'Variant Also Negotiates',
 447         507: 'Insufficient Storage',
 448         508: 'Loop Detected',
 449         510: 'Not Extended',
 450         511: 'Network Authentication Required'
 451     }
 452     return reasons.get(code, 'Unknown Status Code');
 453
 454 def ftp_status_to_reason(code):
 455     reasons = {
 456         110: 'Restart marker reply',
 457         120: 'Service ready in nnn minutes',
 458         125: 'Data connection already open; transfer starting',
 459         150: 'File status okay; about to open data connection',
 460         200: 'Command okay',
 461         202: 'Command not implemented, superfluous at this site',
 462         211: 'System status, or system help reply',
 463         212: 'Directory status',
 464         213: 'File status',
 465         214: 'Help message',
 466         215: 'NAME system type',
 467         220: 'Service ready for new user',
 468         221: 'Service closing control connection',
 469         225: 'Data connection open; no transfer in progress',
 470         226: 'Closing data connection',
 471         227: 'Entering Passive Mode',
 472         230: 'User logged in, proceed',
 473         250: 'Requested file action okay, completed',
 474         257: '"PATHNAME" created',
 475         331: 'User name okay, need password',
 476         332: 'Need account for login',
 477         350: 'Requested file action pending further information',
 478         421: 'Service not available, closing control connection',
 479         425: 'Can\'t open data connection',
 480         426: 'Connection closed; transfer aborted',
 481         450: 'Requested file action not taken',
 482         451: 'Requested action aborted. Local error in processing',
 483         452: 'Requested action not taken. Insufficient storage space in system',
 484         500: 'Syntax error, command unrecognized',
 485         501: 'Syntax error in parameters or arguments',
 486         502: 'Command not implemented',
 487         503: 'Bad sequence of commands',
 488         504: 'Command not implemented for that parameter',
 489         530: 'Not logged in',
 490         532: 'Need account for storing files',
 491         550: 'Requested action not taken. File unavailable',
 492         551: 'Requested action aborted. Page type unknown',
 493         552: 'Requested file action aborted. Exceeded storage allocation',
 494         553: 'Requested action not taken. File name not allowed'
 495     };
 496     return reasons.get(code, 'Unknown Status Code');
 497
 498 def sftp_status_to_reason(code):
 499     reasons = {
 500         0: 'SSH_FX_OK',
 501         1: 'SSH_FX_EOF',
 502         2: 'SSH_FX_NO_SUCH_FILE',
 503         3: 'SSH_FX_PERMISSION_DENIED',
 504         4: 'SSH_FX_FAILURE',
 505         5: 'SSH_FX_BAD_MESSAGE',
 506         6: 'SSH_FX_NO_CONNECTION',
 507         7: 'SSH_FX_CONNECTION_LOST',
 508         8: 'SSH_FX_OP_UNSUPPORTED'
 509     };
 510     return reasons.get(code, 'Unknown Status Code');
 511
 512 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 513  if isinstance(headers, dict):
 514   returnval = [];
 515   if(sys.version[0]=="2"):
 516    for headkey, headvalue in headers.iteritems():
 517     returnval.append((headkey, headvalue));
 518   if(sys.version[0]>="3"):
 519    for headkey, headvalue in headers.items():
 520     returnval.append((headkey, headvalue));
 521  elif isinstance(headers, list):
 522   returnval = headers;
 523  else:
 524   returnval = False;
 525  return returnval;
 526
 527 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 528  if isinstance(headers, dict):
 529   returnval = [];
 530   if(sys.version[0]=="2"):
 531    for headkey, headvalue in headers.iteritems():
 532     returnval.append(headkey+": "+headvalue);
 533   if(sys.version[0]>="3"):
 534    for headkey, headvalue in headers.items():
 535     returnval.append(headkey+": "+headvalue);
 536  elif isinstance(headers, list):
 537   returnval = headers;
 538  else:
 539   returnval = False;
 540  return returnval;
 541
 542 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 543  if isinstance(headers, list):
 544   returnval = {};
 545   mli = 0;
 546   mlil = len(headers);
 547   while(mli<mlil):
 548    returnval.update({headers[mli][0]: headers[mli][1]});
 549    mli = mli + 1;
 550  elif isinstance(headers, dict):
 551   returnval = headers;
 552  else:
 553   returnval = False;
 554  return returnval;
 555
 556 def get_httplib_support(checkvalue=None):
 557  global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 558  returnval = [];
 559  returnval.append("ftp");
 560  returnval.append("httplib");
 561  if(havehttplib2):
 562   returnval.append("httplib2");
 563  returnval.append("urllib");
 564  if(haveurllib3):
 565   returnval.append("urllib3");
 566   returnval.append("request3");
 567  returnval.append("request");
 568  if(haverequests):
 569   returnval.append("requests");
 570  if(havehttpx):
 571   returnval.append("httpx");
 572   returnval.append("httpx2");
 573  if(havemechanize):
 574   returnval.append("mechanize");
 575  if(haveparamiko):
 576   returnval.append("sftp");
 577  if(havepysftp):
 578   returnval.append("pysftp");
 579  if(not checkvalue is None):
 580   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 581    checkvalue = "urllib";
 582   if(checkvalue=="httplib1"):
 583    checkvalue = "httplib";
 584   if(checkvalue in returnval):
 585    returnval = True;
 586   else:
 587    returnval = False;
 588  return returnval;
 589
 590 def check_httplib_support(checkvalue="urllib"):
 591  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 592   checkvalue = "urllib";
 593  if(checkvalue=="httplib1"):
 594   checkvalue = "httplib";
 595  returnval = get_httplib_support(checkvalue);
 596  return returnval;
 597
 598 def get_httplib_support_list():
 599  returnval = get_httplib_support(None);
 600  return returnval;
 601
 602 def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", sleep=-1):
 603  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 604  if(sleep<0):
 605   sleep = geturls_download_sleep;
 606  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 607   httplibuse = "urllib";
 608  if(httplibuse=="httplib1"):
 609   httplibuse = "httplib";
 610  if(not haverequests and httplibuse=="requests"):
 611   httplibuse = "urllib";
 612  if(not havehttpx and httplibuse=="httpx"):
 613   httplibuse = "urllib";
 614  if(not havehttpx and httplibuse=="httpx2"):
 615   httplibuse = "urllib";
 616  if(not havehttpcore and httplibuse=="httpcore"):
 617   httplibuse = "urllib";
 618  if(not havehttpcore and httplibuse=="httpcore2"):
 619   httplibuse = "urllib";
 620  if(not havemechanize and httplibuse=="mechanize"):
 621   httplibuse = "urllib";
 622  if(not havehttplib2 and httplibuse=="httplib2"):
 623   httplibuse = "httplib";
 624  if(not haveparamiko and httplibuse=="sftp"):
 625   httplibuse = "ftp";
 626  if(not havepysftp and httplibuse=="pysftp"):
 627   httplibuse = "ftp";
 628  urlparts = urlparse.urlparse(httpurl);
 629  if(isinstance(httpheaders, list)):
 630   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 631  httpheaders = fix_header_names(httpheaders);
 632  if(httpuseragent is not None):
 633   if('User-Agent' in httpheaders):
 634    httpheaders['User-Agent'] = httpuseragent;
 635   else:
 636    httpuseragent.update({'User-Agent': httpuseragent});
 637  if(httpreferer is not None):
 638   if('Referer' in httpheaders):
 639    httpheaders['Referer'] = httpreferer;
 640   else:
 641    httpuseragent.update({'Referer': httpreferer});
 642  if(urlparts.username is not None or urlparts.password is not None):
 643   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 644   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 645  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 646  if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="mechanize"):
 647   if(isinstance(httpheaders, dict)):
 648    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 649  geturls_opener.addheaders = httpheaders;
 650  time.sleep(sleep);
 651  if(postdata is not None and not isinstance(postdata, dict)):
 652   postdata = urlencode(postdata);
 653  if(httplibuse=="urllib"):
 654   try:
 655    if(httpmethod=="GET"):
 656     geturls_text = geturls_opener.open(httpurl);
 657    elif(httpmethod=="POST"):
 658     geturls_text = geturls_opener.open(httpurl, data=postdata);
 659    else:
 660     geturls_text = geturls_opener.open(httpurl);
 661   except HTTPError as geturls_text_error:
 662    geturls_text = geturls_text_error;
 663    log.info("Error With URL "+httpurl);
 664   except URLError:
 665    log.info("Error With URL "+httpurl);
 666    return False;
 667   except socket.timeout:
 668    log.info("Error With URL "+httpurl);
 669    return False;
 670   httpcodeout = geturls_text.getcode();
 671   httpcodereason = geturls_text.reason;
 672   httpversionout = "1.1";
 673   httpmethodout = httpmethod;
 674   httpurlout = geturls_text.geturl();
 675   httpheaderout = geturls_text.info();
 676   httpheadersentout = httpheaders;
 677  elif(httplibuse=="request"):
 678   try:
 679    if(httpmethod=="GET"):
 680     geturls_request = Request(httpurl, headers=httpheaders);
 681     geturls_text = urlopen(geturls_request);
 682    elif(httpmethod=="POST"):
 683     geturls_request = Request(httpurl, headers=httpheaders);
 684     geturls_text = urlopen(geturls_request, data=postdata);
 685    else:
 686     geturls_request = Request(httpurl, headers=httpheaders);
 687     geturls_text = urlopen(geturls_request);
 688   except HTTPError as geturls_text_error:
 689    geturls_text = geturls_text_error;
 690    log.info("Error With URL "+httpurl);
 691   except URLError:
 692    log.info("Error With URL "+httpurl);
 693    return False;
 694   except socket.timeout:
 695    log.info("Error With URL "+httpurl);
 696    return False;
 697   httpcodeout = geturls_text.getcode();
 698   httpcodereason = geturls_text.reason;
 699   httpversionout = "1.1";
 700   httpmethodout = httpmethod;
 701   httpurlout = geturls_text.geturl();
 702   httpheaderout = geturls_text.headers;
 703   httpheadersentout = httpheaders;
 704  elif(httplibuse=="request3"):
 705   urllib_pool = urllib3.PoolManager(headers=httpheaders);
 706   try:
 707    if(httpmethod=="GET"):
 708     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
 709    elif(httpmethod=="POST"):
 710     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
 711    else:
 712     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
 713   except urllib3.exceptions.ConnectTimeoutError:
 714    log.info("Error With URL "+httpurl);
 715    return False;
 716   except urllib3.exceptions.ConnectError:
 717    log.info("Error With URL "+httpurl);
 718    return False;
 719   except urllib3.exceptions.MaxRetryError:
 720    log.info("Error With URL "+httpurl);
 721    return False;
 722   except socket.timeout:
 723    log.info("Error With URL "+httpurl);
 724    return False;
 725   httpcodeout = geturls_text.status;
 726   httpcodereason = geturls_text.reason;
 727   if(geturls_text.version=="10"):
 728    httpversionout = "1.0";
 729   else:
 730    httpversionout = "1.1";
 731   httpmethodout = httpmethod;
 732   httpurlout = geturls_text.geturl();
 733   httpheaderout = geturls_text.info();
 734   httpheadersentout = httpheaders;
 735  elif(httplibuse=="httplib"):
 736   if(urlparts[0]=="http"):
 737    httpconn = HTTPConnection(urlparts[1]);
 738   elif(urlparts[0]=="https"):
 739    httpconn = HTTPSConnection(urlparts[1]);
 740   else:
 741    return False;
 742   if(postdata is not None and not isinstance(postdata, dict)):
 743    postdata = urlencode(postdata);
 744   try:
 745    if(httpmethod=="GET"):
 746     httpconn.request("GET", urlparts[2], headers=httpheaders);
 747    elif(httpmethod=="POST"):
 748     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 749    else:
 750     httpconn.request("GET", urlparts[2], headers=httpheaders);
 751   except socket.timeout:
 752    log.info("Error With URL "+httpurl);
 753    return False;
 754   except socket.gaierror:
 755    log.info("Error With URL "+httpurl);
 756    return False;
 757   geturls_text = httpconn.getresponse();
 758   httpcodeout = geturls_text.status;
 759   httpcodereason = geturls_text.reason;
 760   if(geturls_text.version=="10"):
 761    httpversionout = "1.0";
 762   else:
 763    httpversionout = "1.1";
 764   httpmethodout = httpmethod;
 765   httpurlout = httpurl;
 766   httpheaderout = geturls_text.getheaders();
 767   httpheadersentout = httpheaders;
 768  elif(httplibuse=="httplib2"):
 769   if(urlparts[0]=="http"):
 770    httpconn = HTTPConnectionWithTimeout(urlparts[1]);
 771   elif(urlparts[0]=="https"):
 772    httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
 773   else:
 774    return False;
 775   if(postdata is not None and not isinstance(postdata, dict)):
 776    postdata = urlencode(postdata);
 777   try:
 778    if(httpmethod=="GET"):
 779     httpconn.request("GET", urlparts[2], headers=httpheaders);
 780    elif(httpmethod=="POST"):
 781     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 782    else:
 783     httpconn.request("GET", urlparts[2], headers=httpheaders);
 784   except socket.timeout:
 785    log.info("Error With URL "+httpurl);
 786    return False;
 787   except socket.gaierror:
 788    log.info("Error With URL "+httpurl);
 789    return False;
 790   geturls_text = httpconn.getresponse();
 791   httpcodeout = geturls_text.status;
 792   httpcodereason = geturls_text.reason;
 793   if(geturls_text.version=="10"):
 794    httpversionout = "1.0";
 795   else:
 796    httpversionout = "1.1";
 797   httpmethodout = httpmethod;
 798   httpurlout = httpurl;
 799   httpheaderout = geturls_text.getheaders();
 800   httpheadersentout = httpheaders;
 801  elif(httplibuse=="urllib3"):
 802   urllib_pool = urllib3.PoolManager(headers=httpheaders);
 803   try:
 804    if(httpmethod=="GET"):
 805     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
 806    elif(httpmethod=="POST"):
 807     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
 808    else:
 809     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
 810   except urllib3.exceptions.ConnectTimeoutError:
 811    log.info("Error With URL "+httpurl);
 812    return False;
 813   except urllib3.exceptions.ConnectError:
 814    log.info("Error With URL "+httpurl);
 815    return False;
 816   except urllib3.exceptions.MaxRetryError:
 817    log.info("Error With URL "+httpurl);
 818    return False;
 819   except socket.timeout:
 820    log.info("Error With URL "+httpurl);
 821    return False;
 822   httpcodeout = geturls_text.status;
 823   httpcodereason = geturls_text.reason;
 824   if(geturls_text.version=="10"):
 825    httpversionout = "1.0";
 826   else:
 827    httpversionout = "1.1";
 828   httpmethodout = httpmethod;
 829   httpurlout = geturls_text.geturl();
 830   httpheaderout = geturls_text.info();
 831   httpheadersentout = httpheaders;
 832  elif(httplibuse=="requests"):
 833   try:
 834    reqsession = requests.Session();
 835    if(httpmethod=="GET"):
 836     geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie);
 837    elif(httpmethod=="POST"):
 838     geturls_text = reqsession.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
 839    else:
 840     geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie);
 841   except requests.exceptions.ConnectTimeout:
 842    log.info("Error With URL "+httpurl);
 843    return False;
 844   except requests.exceptions.ConnectError:
 845    log.info("Error With URL "+httpurl);
 846    return False;
 847   except socket.timeout:
 848    log.info("Error With URL "+httpurl);
 849    return False;
 850   httpcodeout = geturls_text.status_code;
 851   httpcodereason = geturls_text.reason;
 852   if(geturls_text.raw.version=="10"):
 853    httpversionout = "1.0";
 854   else:
 855    httpversionout = "1.1";
 856   httpmethodout = httpmethod;
 857   httpurlout = geturls_text.url;
 858   httpheaderout = geturls_text.headers;
 859   httpheadersentout = geturls_text.request.headers;
 860  elif(httplibuse=="httpx"):
 861   try:
 862    if(httpmethod=="GET"):
 863     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
 864     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
 865    elif(httpmethod=="POST"):
 866     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
 867     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
 868    else:
 869     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
 870     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
 871   except httpx.ConnectTimeout:
 872    log.info("Error With URL "+httpurl);
 873    return False;
 874   except httpx.ConnectError:
 875    log.info("Error With URL "+httpurl);
 876    return False;
 877   except socket.timeout:
 878    log.info("Error With URL "+httpurl);
 879    return False;
 880   httpcodeout = geturls_text.status_code;
 881   httpcodereason = geturls_text.reason_phrase;
 882   httpversionout = geturls_text.http_version;
 883   httpmethodout = httpmethod;
 884   httpurlout = str(geturls_text.url);
 885   httpheaderout = geturls_text.headers;
 886   httpheadersentout = geturls_text.request.headers;
 887  elif(httplibuse=="httpx2"):
 888   try:
 889    if(httpmethod=="GET"):
 890     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
 891     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
 892    elif(httpmethod=="POST"):
 893     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
 894     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
 895    else:
 896     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
 897     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
 898   except httpx.ConnectTimeout:
 899    log.info("Error With URL "+httpurl);
 900    return False;
 901   except httpx.ConnectError:
 902    log.info("Error With URL "+httpurl);
 903    return False;
 904   except socket.timeout:
 905    log.info("Error With URL "+httpurl);
 906    return False;
 907   httpcodeout = geturls_text.status_code;
 908   httpcodereason = geturls_text.reason;
 909   httpversionout = geturls_text.http_version;
 910   httpmethodout = httpmethod;
 911   httpurlout = str(geturls_text.url);
 912   httpheaderout = geturls_text.headers;
 913   httpheadersentout = geturls_text.request.headers;
 914  elif(httplibuse=="httpcore"):
 915   try:
 916    if(httpmethod=="GET"):
 917     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
 918     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
 919    elif(httpmethod=="POST"):
 920     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
 921     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
 922    else:
 923     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
 924     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
 925   except httpcore.ConnectTimeout:
 926    log.info("Error With URL "+httpurl);
 927    return False;
 928   except httpcore.ConnectError:
 929    log.info("Error With URL "+httpurl);
 930    return False;
 931   except socket.timeout:
 932    log.info("Error With URL "+httpurl);
 933    return False;
 934   httpcodeout = geturls_text.status;
 935   httpcodereason = http_status_to_reason(geturls_text.status);
 936   httpversionout = "1.1";
 937   httpmethodout = httpmethod;
 938   httpurlout = str(httpurl);
 939   httpheaderout = geturls_text.headers;
 940   httpheadersentout = httpheaders;
 941  elif(httplibuse=="httpcore2"):
 942   try:
 943    if(httpmethod=="GET"):
 944     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
 945     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
 946    elif(httpmethod=="POST"):
 947     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
 948     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
 949    else:
 950     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
 951     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
 952   except httpcore.ConnectTimeout:
 953    log.info("Error With URL "+httpurl);
 954    return False;
 955   except httpcore.ConnectError:
 956    log.info("Error With URL "+httpurl);
 957    return False;
 958   except socket.timeout:
 959    log.info("Error With URL "+httpurl);
 960    return False;
 961   httpcodeout = geturls_text.status;
 962   httpcodereason = geturls_text.reason;
 963   httpversionout = "1.1";
 964   httpmethodout = httpmethod;
 965   httpurlout = str(httpurl);
 966   httpheaderout = geturls_text.headers;
 967   httpheadersentout = httpheaders;
 968  elif(httplibuse=="mechanize"):
 969   geturls_opener = mechanize.Browser();
 970   if(isinstance(httpheaders, dict)):
 971    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 972   time.sleep(sleep);
 973   geturls_opener.addheaders = httpheaders;
 974   geturls_opener.set_cookiejar(httpcookie);
 975   geturls_opener.set_handle_robots(False);
 976   if(postdata is not None and not isinstance(postdata, dict)):
 977    postdata = urlencode(postdata);
 978   try:
 979    if(httpmethod=="GET"):
 980     geturls_text = geturls_opener.open(httpurl);
 981    elif(httpmethod=="POST"):
 982     geturls_text = geturls_opener.open(httpurl, data=postdata);
 983    else:
 984     geturls_text = geturls_opener.open(httpurl);
 985   except mechanize.HTTPError as geturls_text_error:
 986    geturls_text = geturls_text_error;
 987    log.info("Error With URL "+httpurl);
 988   except URLError:
 989    log.info("Error With URL "+httpurl);
 990    return False;
 991   except socket.timeout:
 992    log.info("Error With URL "+httpurl);
 993    return False;
 994   httpcodeout = geturls_text.code;
 995   httpcodereason = http_status_to_reason(geturls_text.code);
 996   httpversionout = "1.1";
 997   httpmethodout = httpmethod;
 998   httpurlout = geturls_text.geturl();
 999   httpheaderout = geturls_text.info();
1000   reqhead = geturls_opener.request;
1001   httpheadersentout = reqhead.header_items();
1002
1003  elif(httplibuse=="ftp"):
1004   geturls_text = download_file_from_ftp_file(httpurl);
1005   if(not geturls_text):
1006    return False;
1007   log.info("Downloading URL "+httpurl);
1008   returnval_content = geturls_text.read()[:];
1009   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1010   geturls_text.close();
1011  elif(httplibuse=="sftp"):
1012   geturls_text = download_file_from_sftp_file(httpurl);
1013   if(not geturls_text):
1014    return False;
1015   log.info("Downloading URL "+httpurl);
1016   returnval_content = geturls_text.read()[:];
1017   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1018   geturls_text.close();
1019   return returnval;
1020  elif(httplibuse=="pysftp"):
1021   geturls_text = download_file_from_pysftp_file(httpurl);
1022   if(not geturls_text):
1023    return False;
1024   log.info("Downloading URL "+httpurl);
1025   returnval_content = geturls_text.read()[:];
1026   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
1027   geturls_text.close();
1028   return returnval;
1029  else:
1030   returnval = False;
1031  if(isinstance(httpheaderout, list)):
1032   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1033  if(sys.version[0]=="2"):
1034   try:
1035    prehttpheaderout = httpheaderout;
1036    httpheaderkeys = httpheaderout.keys();
1037    imax = len(httpheaderkeys);
1038    ic = 0;
1039    httpheaderout = {};
1040    while(ic < imax):
1041     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1042     ic += 1;
1043   except AttributeError:
1044    pass;
1045  httpheaderout = fix_header_names(httpheaderout);
1046  if(isinstance(httpheadersentout, list)):
1047   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1048  httpheadersentout = fix_header_names(httpheadersentout);
1049  log.info("Downloading URL "+httpurl);
1050  if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize" or httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"):
1051   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
1052    if(sys.version[0]=="2"):
1053     strbuf = StringIO(geturls_text.read());
1054    if(sys.version[0]>="3"):
1055     strbuf = BytesIO(geturls_text.read());
1056    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1057    returnval_content = gzstrbuf.read()[:];
1058   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
1059    returnval_content = geturls_text.read()[:];
1060   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1061    returnval_content = geturls_text.read()[:];
1062    returnval_content = brotli.decompress(returnval_content);
1063   geturls_text.close();
1064  elif(httplibuse=="requests"):
1065   log.info("Downloading URL "+httpurl);
1066   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
1067    if(sys.version[0]=="2"):
1068     strbuf = StringIO(geturls_text.raw.read());
1069    if(sys.version[0]>="3"):
1070     strbuf = BytesIO(geturls_text.raw.read());
1071    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1072    returnval_content = gzstrbuf.read()[:];
1073   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
1074    returnval_content = geturls_text.raw.read()[:];
1075   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1076    returnval_content = geturls_text.raw.read()[:];
1077    returnval_content = brotli.decompress(returnval_content);
1078   geturls_text.close();
1079  elif(httplibuse=="ftp" or httplibuse=="sftp" or httplibuse=="pysftp"):
1080   pass;
1081  else:
1082   returnval = False;
1083  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1084  return returnval;
1085
1086 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1):
1087  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1088  exec_time_start = time.time();
1089  myhash = hashlib.new("sha1");
1090  if(sys.version[0]=="2"):
1091   myhash.update(httpurl);
1092   myhash.update(str(buffersize));
1093   myhash.update(str(exec_time_start));
1094  if(sys.version[0]>="3"):
1095   myhash.update(httpurl.encode('utf-8'));
1096   myhash.update(str(buffersize).encode('utf-8'));
1097   myhash.update(str(exec_time_start).encode('utf-8'));
1098  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1099  if(sleep<0):
1100   sleep = geturls_download_sleep;
1101  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
1102   httplibuse = "urllib";
1103  if(httplibuse=="httplib1"):
1104   httplibuse = "httplib";
1105  if(not haverequests and httplibuse=="requests"):
1106   httplibuse = "urllib";
1107  if(not havehttpx and httplibuse=="httpx"):
1108   httplibuse = "urllib";
1109  if(not havehttpx and httplibuse=="httpx2"):
1110   httplibuse = "urllib";
1111  if(not havehttpcore and httplibuse=="httpcore"):
1112   httplibuse = "urllib";
1113  if(not havehttpcore and httplibuse=="httpcore2"):
1114   httplibuse = "urllib";
1115  if(not havemechanize and httplibuse=="mechanize"):
1116   httplibuse = "urllib";
1117  if(not havehttplib2 and httplibuse=="httplib2"):
1118   httplibuse = "httplib";
1119  if(not haveparamiko and httplibuse=="sftp"):
1120   httplibuse = "ftp";
1121  if(not haveparamiko and httplibuse=="pysftp"):
1122   httplibuse = "ftp";
1123  urlparts = urlparse.urlparse(httpurl);
1124  if(isinstance(httpheaders, list)):
1125   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1126  httpheaders = fix_header_names(httpheaders);
1127  if(httpuseragent is not None):
1128   if('User-Agent' in httpheaders):
1129    httpheaders['User-Agent'] = httpuseragent;
1130   else:
1131    httpuseragent.update({'User-Agent': httpuseragent});
1132  if(httpreferer is not None):
1133   if('Referer' in httpheaders):
1134    httpheaders['Referer'] = httpreferer;
1135   else:
1136    httpuseragent.update({'Referer': httpreferer});
1137  if(urlparts.username is not None or urlparts.password is not None):
1138   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1139   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1140  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1141  if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="mechanize"):
1142   if(isinstance(httpheaders, dict)):
1143    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1144  geturls_opener.addheaders = httpheaders;
1145  time.sleep(sleep);
1146  if(httplibuse=="urllib"):
1147   try:
1148    if(httpmethod=="GET"):
1149     geturls_text = geturls_opener.open(httpurl);
1150    elif(httpmethod=="POST"):
1151     geturls_text = geturls_opener.open(httpurl, data=postdata);
1152    else:
1153     geturls_text = geturls_opener.open(httpurl);
1154   except HTTPError as geturls_text_error:
1155    geturls_text = geturls_text_error;
1156    log.info("Error With URL "+httpurl);
1157   except URLError:
1158    log.info("Error With URL "+httpurl);
1159    return False;
1160   except socket.timeout:
1161    log.info("Error With URL "+httpurl);
1162    return False;
1163   except socket.timeout:
1164    log.info("Error With URL "+httpurl);
1165    return False;
1166   httpcodeout = geturls_text.getcode();
1167   httpcodereason = geturls_text.reason;
1168   httpversionout = "1.1";
1169   httpmethodout = httpmethod;
1170   httpurlout = geturls_text.geturl();
1171   httpheaderout = geturls_text.info();
1172   httpheadersentout = httpheaders;
1173  elif(httplibuse=="request"):
1174   try:
1175    if(httpmethod=="GET"):
1176     geturls_request = Request(httpurl, headers=httpheaders);
1177     geturls_text = urlopen(geturls_request);
1178    elif(httpmethod=="POST"):
1179     geturls_request = Request(httpurl, headers=httpheaders);
1180     geturls_text = urlopen(geturls_request, data=postdata);
1181    else:
1182     geturls_request = Request(httpurl, headers=httpheaders);
1183     geturls_text = urlopen(geturls_request);
1184   except HTTPError as geturls_text_error:
1185    geturls_text = geturls_text_error;
1186    log.info("Error With URL "+httpurl);
1187   except URLError:
1188    log.info("Error With URL "+httpurl);
1189    return False;
1190   except socket.timeout:
1191    log.info("Error With URL "+httpurl);
1192    return False;
1193   httpcodeout = geturls_text.getcode();
1194   httpcodereason = geturls_text.reason;
1195   httpversionout = "1.1";
1196   httpmethodout = httpmethod;
1197   httpurlout = geturls_text.geturl();
1198   httpheaderout = geturls_text.headers;
1199   httpheadersentout = httpheaders;
1200  elif(httplibuse=="request3"):
1201   urllib_pool = urllib3.PoolManager(headers=httpheaders);
1202   try:
1203    if(httpmethod=="GET"):
1204     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
1205    elif(httpmethod=="POST"):
1206     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
1207    else:
1208     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
1209   except urllib3.exceptions.ConnectTimeoutError:
1210    log.info("Error With URL "+httpurl);
1211    return False;
1212   except urllib3.exceptions.ConnectError:
1213    log.info("Error With URL "+httpurl);
1214    return False;
1215   except urllib3.exceptions.MaxRetryError:
1216    log.info("Error With URL "+httpurl);
1217    return False;
1218   except socket.timeout:
1219    log.info("Error With URL "+httpurl);
1220    return False;
1221   httpcodeout = geturls_text.status;
1222   httpcodereason = geturls_text.reason;
1223   if(geturls_text.version=="10"):
1224    httpversionout = "1.0";
1225   else:
1226    httpversionout = "1.1";
1227   httpmethodout = httpmethod;
1228   httpurlout = geturls_text.geturl();
1229   httpheaderout = geturls_text.info();
1230   httpheadersentout = httpheaders;
1231  elif(httplibuse=="httplib"):
1232   if(urlparts[0]=="http"):
1233    httpconn = HTTPConnection(urlparts[1]);
1234   elif(urlparts[0]=="https"):
1235    httpconn = HTTPSConnection(urlparts[1]);
1236   else:
1237    return False;
1238   if(postdata is not None and not isinstance(postdata, dict)):
1239    postdata = urlencode(postdata);
1240   try:
1241    if(httpmethod=="GET"):
1242     httpconn.request("GET", urlparts[2], headers=httpheaders);
1243    elif(httpmethod=="POST"):
1244     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1245    else:
1246     httpconn.request("GET", urlparts[2], headers=httpheaders);
1247   except socket.timeout:
1248    log.info("Error With URL "+httpurl);
1249    return False;
1250   except socket.gaierror:
1251    log.info("Error With URL "+httpurl);
1252    return False;
1253   geturls_text = httpconn.getresponse();
1254   httpcodeout = geturls_text.status;
1255   httpcodereason = geturls_text.reason;
1256   if(geturls_text.version=="10"):
1257    httpversionout = "1.0";
1258   else:
1259    httpversionout = "1.1";
1260   httpmethodout = httpmethod;
1261   httpurlout = httpurl;
1262   httpheaderout = geturls_text.getheaders();
1263   httpheadersentout = httpheaders;
1264  elif(httplibuse=="httplib2"):
1265   try:
1266    if(httpmethod=="GET"):
1267     httpconn.request("GET", urlparts[2], headers=httpheaders);
1268    elif(httpmethod=="POST"):
1269     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1270    else:
1271     httpconn.request("GET", urlparts[2], headers=httpheaders);
1272   except socket.timeout:
1273    log.info("Error With URL "+httpurl);
1274    return False;
1275   except socket.gaierror:
1276    log.info("Error With URL "+httpurl);
1277    return False;
1278   geturls_text = httpconn.getresponse();
1279   httpcodeout = geturls_text.status;
1280   httpcodereason = geturls_text.reason;
1281   if(geturls_text.version=="10"):
1282    httpversionout = "1.0";
1283   else:
1284    httpversionout = "1.1";
1285   httpmethodout = httpmethod;
1286   httpurlout = httpurl;
1287   httpheaderout = geturls_text.getheaders();
1288   httpheadersentout = httpheaders;
1289  elif(httplibuse=="urllib3"):
1290   urllib_pool = urllib3.PoolManager(headers=httpheaders);
1291   try:
1292    if(httpmethod=="GET"):
1293     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
1294    elif(httpmethod=="POST"):
1295     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
1296    else:
1297     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
1298   except urllib3.exceptions.ConnectTimeoutError:
1299    log.info("Error With URL "+httpurl);
1300    return False;
1301   except urllib3.exceptions.ConnectError:
1302    log.info("Error With URL "+httpurl);
1303    return False;
1304   except urllib3.exceptions.MaxRetryError:
1305    log.info("Error With URL "+httpurl);
1306    return False;
1307   except socket.timeout:
1308    log.info("Error With URL "+httpurl);
1309    return False;
1310   httpcodeout = geturls_text.status;
1311   httpcodereason = geturls_text.reason;
1312   if(geturls_text.version=="10"):
1313    httpversionout = "1.0";
1314   else:
1315    httpversionout = "1.1";
1316   httpmethodout = httpmethod;
1317   httpurlout = geturls_text.geturl();
1318   httpheaderout = geturls_text.info();
1319   httpheadersentout = httpheaders;
1320  elif(httplibuse=="requests"):
1321   try:
1322    reqsession = requests.Session();
1323    if(httpmethod=="GET"):
1324     geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
1325    elif(httpmethod=="POST"):
1326     geturls_text = reqsession.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie, stream=True);
1327    else:
1328     geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
1329   except requests.exceptions.ConnectTimeout:
1330    log.info("Error With URL "+httpurl);
1331    return False;
1332   except requests.exceptions.ConnectError:
1333    log.info("Error With URL "+httpurl);
1334    return False;
1335   except socket.timeout:
1336    log.info("Error With URL "+httpurl);
1337    return False;
1338   httpcodeout = geturls_text.status_code;
1339   httpcodereason = geturls_text.reason;
1340   if(geturls_text.raw.version=="10"):
1341    httpversionout = "1.0";
1342   else:
1343    httpversionout = "1.1";
1344   httpmethodout = httpmethod;
1345   httpurlout = geturls_text.url;
1346   httpheaderout = geturls_text.headers;
1347   httpheadersentout = geturls_text.request.headers;
1348  elif(httplibuse=="httpx"):
1349   try:
1350    if(httpmethod=="GET"):
1351     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1352     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1353    elif(httpmethod=="POST"):
1354     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1355     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1356    else:
1357     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1358     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1359   except httpx.ConnectTimeout:
1360    log.info("Error With URL "+httpurl);
1361    return False;
1362   except httpx.ConnectError:
1363    log.info("Error With URL "+httpurl);
1364    return False;
1365   except socket.timeout:
1366    log.info("Error With URL "+httpurl);
1367    return False;
1368   httpcodeout = geturls_text.status_code;
1369   httpcodereason = geturls_text.reason_phrase;
1370   httpversionout = geturls_text.http_version;
1371   httpmethodout = httpmethod;
1372   httpurlout = str(geturls_text.url);
1373   httpheaderout = geturls_text.headers;
1374   httpheadersentout = geturls_text.request.headers;
1375  elif(httplibuse=="httpx2"):
1376   try:
1377    if(httpmethod=="GET"):
1378     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1379     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1380    elif(httpmethod=="POST"):
1381     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1382     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1383    else:
1384     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1385     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1386   except httpx.ConnectTimeout:
1387    log.info("Error With URL "+httpurl);
1388    return False;
1389   except httpx.ConnectError:
1390    log.info("Error With URL "+httpurl);
1391    return False;
1392   except socket.timeout:
1393    log.info("Error With URL "+httpurl);
1394    return False;
1395   httpcodeout = geturls_text.status_code;
1396   httpcodereason = geturls_text.reason_phrase;
1397   httpversionout = geturls_text.http_version;
1398   httpmethodout = httpmethod;
1399   httpurlout = str(geturls_text.url);
1400   httpheaderout = geturls_text.headers;
1401   httpheadersentout = geturls_text.request.headers;
1402  elif(httplibuse=="httpcore"):
1403   try:
1404    if(httpmethod=="GET"):
1405     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
1406     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1407    elif(httpmethod=="POST"):
1408     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
1409     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
1410    else:
1411     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
1412     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1413   except httpcore.ConnectTimeout:
1414    log.info("Error With URL "+httpurl);
1415    return False;
1416   except httpcore.ConnectError:
1417    log.info("Error With URL "+httpurl);
1418    return False;
1419   except socket.timeout:
1420    log.info("Error With URL "+httpurl);
1421    return False;
1422   httpcodeout = geturls_text.status;
1423   httpcodereason = http_status_to_reason(geturls_text.status);
1424   httpversionout = "1.1";
1425   httpmethodout = httpmethod;
1426   httpurlout = str(httpurl);
1427   httpheaderout = geturls_text.headers;
1428   httpheadersentout = httpheaders;
1429  elif(httplibuse=="httpcore2"):
1430   try:
1431    if(httpmethod=="GET"):
1432     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1433     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1434    elif(httpmethod=="POST"):
1435     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1436     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
1437    else:
1438     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
1439     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
1440   except httpcore.ConnectTimeout:
1441    log.info("Error With URL "+httpurl);
1442    return False;
1443   except httpcore.ConnectError:
1444    log.info("Error With URL "+httpurl);
1445    return False;
1446   except socket.timeout:
1447    log.info("Error With URL "+httpurl);
1448    return False;
1449   httpcodeout = geturls_text.status;
1450   httpcodereason = geturls_text.reason_phrase;
1451   httpversionout = "1.1";
1452   httpmethodout = httpmethod;
1453   httpurlout = str(httpurl);
1454   httpheaderout = geturls_text.headers;
1455   httpheadersentout = httpheaders;
1456  elif(httplibuse=="mechanize"):
1457   geturls_opener = mechanize.Browser();
1458   if(isinstance(httpheaders, dict)):
1459    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1460   time.sleep(sleep);
1461   geturls_opener.addheaders = httpheaders;
1462   geturls_opener.set_cookiejar(httpcookie);
1463   geturls_opener.set_handle_robots(False);
1464   if(postdata is not None and not isinstance(postdata, dict)):
1465    postdata = urlencode(postdata);
1466   try:
1467    if(httpmethod=="GET"):
1468     geturls_text = geturls_opener.open(httpurl);
1469    elif(httpmethod=="POST"):
1470     geturls_text = geturls_opener.open(httpurl, data=postdata);
1471    else:
1472     geturls_text = geturls_opener.open(httpurl);
1473   except mechanize.HTTPError as geturls_text_error:
1474    geturls_text = geturls_text_error;
1475    log.info("Error With URL "+httpurl);
1476   except URLError:
1477    log.info("Error With URL "+httpurl);
1478    return False;
1479   except socket.timeout:
1480    log.info("Error With URL "+httpurl);
1481    return False;
1482   httpcodeout = geturls_text.code;
1483   httpcodereason = http_status_to_reason(geturls_text.code);
1484   httpversionout = "1.1";
1485   httpmethodout = httpmethod;
1486   httpurlout = geturls_text.geturl();
1487   httpheaderout = geturls_text.info();
1488   reqhead = geturls_opener.request;
1489   httpheadersentout = reqhead.header_items();
1490  elif(httplibuse=="ftp"):
1491   geturls_text = download_file_from_ftp_file(httpurl);
1492   if(not geturls_text):
1493    return False;
1494   geturls_text.seek(0, 2);
1495   downloadsize = geturls_text.tell();
1496   geturls_text.seek(0, 0);
1497  elif(httplibuse=="sftp"):
1498   geturls_text = download_file_from_sftp_file(httpurl);
1499   if(not geturls_text):
1500    return False;
1501   geturls_text.seek(0, 2);
1502   downloadsize = geturls_text.tell();
1503   geturls_text.seek(0, 0);
1504   if(downloadsize is not None):
1505    downloadsize = int(downloadsize);
1506   if downloadsize is None: downloadsize = 0;
1507   fulldatasize = 0;
1508   prevdownsize = 0;
1509  elif(httplibuse=="pysftp"):
1510   geturls_text = download_file_from_pysftp_file(httpurl);
1511   if(not geturls_text):
1512    return False;
1513   geturls_text.seek(0, 2);
1514   downloadsize = geturls_text.tell();
1515   geturls_text.seek(0, 0);
1516  else:
1517   returnval = False;
1518  if(isinstance(httpheaderout, list)):
1519   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1520  if(sys.version[0]=="2"):
1521   try:
1522    prehttpheaderout = httpheaderout;
1523    httpheaderkeys = httpheaderout.keys();
1524    imax = len(httpheaderkeys);
1525    ic = 0;
1526    httpheaderout = {};
1527    while(ic < imax):
1528     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1529     ic += 1;
1530   except AttributeError:
1531    pass;
1532  httpheaderout = fix_header_names(httpheaderout);
1533  if(isinstance(httpheadersentout, list)):
1534   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1535  httpheadersentout = fix_header_names(httpheadersentout);
1536  if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="requests" or httplibuse=="mechanize" or httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2"):
1537   downloadsize = httpheaderout.get('Content-Length');
1538  if(downloadsize is not None):
1539   downloadsize = int(downloadsize);
1540  if downloadsize is None: downloadsize = 0;
1541  fulldatasize = 0;
1542  prevdownsize = 0;
1543  log.info("Downloading URL "+httpurl);
1544  if(httplibuse=="urllib" or httplibuse=="request" or httplibuse=="request3" or httplibuse=="httplib" or httplibuse=="httplib2" or httplibuse=="urllib3" or httplibuse=="mechanize" or httplibuse=="httpx" or httplibuse=="httpx2" or httplibuse=="httpcore" or httplibuse=="httpcore2" or httplibuse=="ftp" or httplibuse=="sftp" or httplibuse=="pysftp"):
1545   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1546    tmpfilename = f.name;
1547    try:
1548     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
1549    except AttributeError:
1550     try:
1551      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1552     except ValueError:
1553      pass;
1554    except ValueError:
1555     pass;
1556    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1557    while True:
1558     databytes = geturls_text.read(buffersize);
1559     if not databytes: break;
1560     datasize = len(databytes);
1561     fulldatasize = datasize + fulldatasize;
1562     percentage = "";
1563     if(downloadsize>0):
1564      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1565     downloaddiff = fulldatasize - prevdownsize;
1566     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1567     prevdownsize = fulldatasize;
1568     f.write(databytes);
1569    f.close();
1570  elif(httplibuse=="requests"):
1571   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1572    tmpfilename = f.name;
1573    try:
1574     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
1575    except AttributeError:
1576     try:
1577      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1578     except ValueError:
1579      pass;
1580    except ValueError:
1581     pass;
1582    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1583    for databytes in geturls_text.iter_content(chunk_size=buffersize):
1584     datasize = len(databytes);
1585     fulldatasize = datasize + fulldatasize;
1586     percentage = "";
1587     if(downloadsize>0):
1588      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1589     downloaddiff = fulldatasize - prevdownsize;
1590     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1591     prevdownsize = fulldatasize;
1592     f.write(databytes);
1593    f.close();
1594  else:
1595   pass;
1596  geturls_text.close();
1597  exec_time_end = time.time();
1598  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1599  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1600  return returnval;
1601
1602 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1603  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
1604  if(sleep<0):
1605   sleep = geturls_download_sleep;
1606  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
1607   httplibuse = "urllib";
1608  if(httplibuse=="httplib1"):
1609   httplibuse = "httplib";
1610  if(not haverequests and httplibuse=="requests"):
1611   httplibuse = "urllib";
1612  if(not havehttpx and httplibuse=="httpx"):
1613   httplibuse = "urllib";
1614  if(not havehttpx and httplibuse=="httpx2"):
1615   httplibuse = "urllib";
1616  if(not havehttpcore and httplibuse=="httpcore"):
1617   httplibuse = "urllib";
1618  if(not havehttpcore and httplibuse=="httpcore2"):
1619   httplibuse = "urllib";
1620  if(not havemechanize and httplibuse=="mechanize"):
1621   httplibuse = "urllib";
1622  if(not havehttplib2 and httplibuse=="httplib2"):
1623   httplibuse = "httplib";
1624  if(not haveparamiko and httplibuse=="sftp"):
1625   httplibuse = "ftp";
1626  if(not havepysftp and httplibuse=="pysftp"):
1627   httplibuse = "ftp";
1628  if(not outfile=="-"):
1629   outpath = outpath.rstrip(os.path.sep);
1630   filepath = os.path.realpath(outpath+os.path.sep+outfile);
1631   if(not os.path.exists(outpath)):
1632    os.makedirs(outpath);
1633   if(os.path.exists(outpath) and os.path.isfile(outpath)):
1634    return False;
1635   if(os.path.exists(filepath) and os.path.isdir(filepath)):
1636    return False;
1637   pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize[0], sleep);
1638   if(not pretmpfilename):
1639    return False;
1640   tmpfilename = pretmpfilename['Filename'];
1641   downloadsize = os.path.getsize(tmpfilename);
1642   fulldatasize = 0;
1643   log.info("Moving file "+tmpfilename+" to "+filepath);
1644   exec_time_start = time.time();
1645   shutil.move(tmpfilename, filepath);
1646   try:
1647    os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1648   except AttributeError:
1649    try:
1650     os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1651    except ValueError:
1652     pass;
1653   except ValueError:
1654    pass;
1655   exec_time_end = time.time();
1656   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1657   if(os.path.exists(tmpfilename)):
1658    os.remove(tmpfilename);
1659   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1660  if(outfile=="-" and sys.version[0]=="2"):
1661   pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize[0], sleep);
1662   if(not pretmpfilename):
1663    return False;
1664   tmpfilename = pretmpfilename['Filename'];
1665   downloadsize = os.path.getsize(tmpfilename);
1666   fulldatasize = 0;
1667   prevdownsize = 0;
1668   exec_time_start = time.time();
1669   with open(tmpfilename, 'rb') as ft:
1670    f = StringIO();
1671    while True:
1672     databytes = ft.read(buffersize[1]);
1673     if not databytes: break;
1674     datasize = len(databytes);
1675     fulldatasize = datasize + fulldatasize;
1676     percentage = "";
1677     if(downloadsize>0):
1678      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1679     downloaddiff = fulldatasize - prevdownsize;
1680     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1681     prevdownsize = fulldatasize;
1682     f.write(databytes);
1683    f.seek(0);
1684    fdata = f.getvalue();
1685    f.close();
1686    ft.close();
1687    os.remove(tmpfilename);
1688    exec_time_end = time.time();
1689    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1690   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1691  if(outfile=="-" and sys.version[0]>="3"):
1692   pretmpfilename = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, httplibuse, buffersize[0], sleep);
1693   tmpfilename = pretmpfilename['Filename'];
1694   downloadsize = os.path.getsize(tmpfilename);
1695   fulldatasize = 0;
1696   prevdownsize = 0;
1697   exec_time_start = time.time();
1698   with open(tmpfilename, 'rb') as ft:
1699    f = BytesIO();
1700    while True:
1701     databytes = ft.read(buffersize[1]);
1702     if not databytes: break;
1703     datasize = len(databytes);
1704     fulldatasize = datasize + fulldatasize;
1705     percentage = "";
1706     if(downloadsize>0):
1707      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1708     downloaddiff = fulldatasize - prevdownsize;
1709     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1710     prevdownsize = fulldatasize;
1711     f.write(databytes);
1712    f.seek(0);
1713    fdata = f.getvalue();
1714    f.close();
1715    ft.close();
1716    os.remove(tmpfilename);
1717    exec_time_end = time.time();
1718    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1719   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1720  return returnval;
1721
1722 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1723   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", sleep);
1724   return returnval;
1725
1726 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1727   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", sleep);
1728   return returnval;
1729
1730 def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1731   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", sleep);
1732   return returnval;
1733
1734 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1735   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", sleep);
1736   return returnval;
1737
1738 def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1739   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", sleep);
1740   return returnval;
1741
1742 def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1743   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", sleep);
1744   return returnval;
1745
1746 def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1747   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", sleep);
1748   return returnval;
1749
1750 def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1751   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", sleep);
1752   return returnval;
1753
1754 def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1755   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", sleep);
1756   return returnval;
1757
1758 def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1759   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", sleep);
1760   return returnval;
1761
1762 def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1763   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", sleep);
1764   return returnval;
1765
1766 def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1767   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", sleep);
1768   return returnval;
1769
1770 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1771   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", sleep);
1772   return returnval;
1773
1774 def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1775   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", sleep);
1776   return returnval;
1777
1778 def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1779   returnval = download_from_url(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", sleep);
1780   return returnval;
1781
1782 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1783   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", buffersize, sleep);
1784   return returnval;
1785
1786 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1787   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", buffersize, sleep);
1788   return returnval;
1789
1790 def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1791   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", buffersize, sleep);
1792   return returnval;
1793
1794 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1795   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", buffersize, sleep);
1796   return returnval;
1797
1798 def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1799   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", buffersize, sleep);
1800   return returnval;
1801
1802 def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1803   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", buffersize, sleep);
1804   return returnval;
1805
1806 def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1807   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", buffersize, sleep);
1808   return returnval;
1809
1810 def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1811   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", buffersize, sleep);
1812   return returnval;
1813
1814 def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1815   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", buffersize, sleep);
1816   return returnval;
1817
1818 def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1819   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", buffersize, sleep);
1820   return returnval;
1821
1822 def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1823   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", buffersize, sleep);
1824   return returnval;
1825
1826 def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1827   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", buffersize, sleep);
1828   return returnval;
1829
1830 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1831   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", buffersize, sleep);
1832   return returnval;
1833
1834 def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1835   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", buffersize, sleep);
1836   return returnval;
1837
1838 def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1839   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", buffersize, sleep);
1840   return returnval;
1841
1842 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1843   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib", outfile, outpath, buffersize, sleep);
1844   return returnval;
1845
1846 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1847   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request", outfile, outpath, buffersize, sleep);
1848   return returnval;
1849
1850 def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1851   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "request3", outfile, outpath, buffersize, sleep);
1852   return returnval;
1853
1854 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1855   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib", outfile, outpath, buffersize, sleep);
1856   return returnval;
1857
1858 def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1859   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httplib2", outfile, outpath, buffersize, sleep);
1860   return returnval;
1861
1862 def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1863   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "urllib3", outfile, outpath, buffersize, sleep);
1864   return returnval;
1865
1866 def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1867   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "requests", outfile, outpath, buffersize, sleep);
1868   return returnval;
1869
1870 def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1871   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx", outfile, outpath, buffersize, sleep);
1872   return returnval;
1873
1874 def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1875   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpx2", outfile, outpath, buffersize, sleep);
1876   return returnval;
1877
1878 def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1879   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore", outfile, outpath, buffersize, sleep);
1880   return returnval;
1881
1882 def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1883   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "httpcore2", outfile, outpath, buffersize, sleep);
1884   return returnval;
1885
1886 def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1887   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "mechanize", outfile, outpath, buffersize, sleep);
1888   return returnval;
1889
1890 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1891   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "ftp", outfile, outpath, buffersize, sleep);
1892   return returnval;
1893
1894 def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1895   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "sftp", outfile, outpath, buffersize, sleep);
1896   return returnval;
1897
1898 def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1899   returnval = download_from_url_file(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, "pysftp", outfile, outpath, buffersize, sleep);
1900   return returnval;
1901
1902 def download_file_from_ftp_file(url):
1903  urlparts = urlparse.urlparse(url);
1904  file_name = os.path.basename(urlparts.path);
1905  file_dir = os.path.dirname(urlparts.path);
1906  if(urlparts.username is not None):
1907   ftp_username = urlparts.username;
1908  else:
1909   ftp_username = "anonymous";
1910  if(urlparts.password is not None):
1911   ftp_password = urlparts.password;
1912  elif(urlparts.password is None and urlparts.username=="anonymous"):
1913   ftp_password = "anonymous";
1914  else:
1915   ftp_password = "";
1916  if(urlparts.scheme=="ftp"):
1917   ftp = FTP();
1918  elif(urlparts.scheme=="ftps"):
1919   ftp = FTP_TLS();
1920  else:
1921   return False;
1922  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
1923   return False;
1924  ftp_port = urlparts.port;
1925  if(urlparts.port is None):
1926   ftp_port = 21;
1927  try:
1928   ftp.connect(urlparts.hostname, ftp_port);
1929  except socket.gaierror:
1930   log.info("Error With URL "+httpurl);
1931   return False;
1932  except socket.timeout:
1933   log.info("Error With URL "+httpurl);
1934   return False;
1935  ftp.login(urlparts.username, urlparts.password);
1936  if(urlparts.scheme=="ftps"):
1937   ftp.prot_p();
1938  ftpfile = BytesIO();
1939  ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
1940  #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
1941  ftp.close();
1942  ftpfile.seek(0, 0);
1943  return ftpfile;
1944
1945 def download_file_from_ftp_string(url):
1946  ftpfile = download_file_from_ftp_file(url);
1947  return ftpfile.read();
1948
1949 def upload_file_to_ftp_file(ftpfile, url):
1950  urlparts = urlparse.urlparse(url);
1951  file_name = os.path.basename(urlparts.path);
1952  file_dir = os.path.dirname(urlparts.path);
1953  if(urlparts.username is not None):
1954   ftp_username = urlparts.username;
1955  else:
1956   ftp_username = "anonymous";
1957  if(urlparts.password is not None):
1958   ftp_password = urlparts.password;
1959  elif(urlparts.password is None and urlparts.username=="anonymous"):
1960   ftp_password = "anonymous";
1961  else:
1962   ftp_password = "";
1963  if(urlparts.scheme=="ftp"):
1964   ftp = FTP();
1965  elif(urlparts.scheme=="ftps"):
1966   ftp = FTP_TLS();
1967  else:
1968   return False;
1969  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
1970   return False;
1971  ftp_port = urlparts.port;
1972  if(urlparts.port is None):
1973   ftp_port = 21;
1974  try:
1975   ftp.connect(urlparts.hostname, ftp_port);
1976  except socket.gaierror:
1977   log.info("Error With URL "+httpurl);
1978   return False;
1979  except socket.timeout:
1980   log.info("Error With URL "+httpurl);
1981   return False;
1982  ftp.login(urlparts.username, urlparts.password);
1983  if(urlparts.scheme=="ftps"):
1984   ftp.prot_p();
1985  ftp.storbinary("STOR "+urlparts.path, ftpfile);
1986  ftp.close();
1987  ftpfile.seek(0, 0);
1988  return ftpfile;
1989
1990 def upload_file_to_ftp_string(ftpstring, url):
1991  ftpfileo = BytesIO(ftpstring);
1992  ftpfile = upload_file_to_ftp_file(ftpfileo, url);
1993  ftpfileo.close();
1994  return ftpfile;
1995
1996 if(haveparamiko):
1997  def download_file_from_sftp_file(url):
1998   urlparts = urlparse.urlparse(url);
1999   file_name = os.path.basename(urlparts.path);
2000   file_dir = os.path.dirname(urlparts.path);
2001   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2002    return False;
2003   sftp_port = urlparts.port;
2004   if(urlparts.port is None):
2005    sftp_port = 22;
2006   else:
2007    sftp_port = urlparts.port;
2008   if(urlparts.username is not None):
2009    sftp_username = urlparts.username;
2010   else:
2011    sftp_username = "anonymous";
2012   if(urlparts.password is not None):
2013    sftp_password = urlparts.password;
2014   elif(urlparts.password is None and urlparts.username=="anonymous"):
2015    sftp_password = "anonymous";
2016   else:
2017    sftp_password = "";
2018   if(urlparts.scheme!="sftp"):
2019    return False;
2020   ssh = paramiko.SSHClient();
2021   ssh.load_system_host_keys();
2022   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2023   try:
2024    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2025   except paramiko.ssh_exception.SSHException:
2026    return False;
2027   except socket.gaierror:
2028    log.info("Error With URL "+httpurl);
2029    return False;
2030   except socket.timeout:
2031    log.info("Error With URL "+httpurl);
2032    return False;
2033   sftp = ssh.open_sftp();
2034   sftpfile = BytesIO();
2035   sftp.getfo(urlparts.path, sftpfile);
2036   sftp.close();
2037   ssh.close();
2038   sftpfile.seek(0, 0);
2039   return sftpfile;
2040 else:
2041  def download_file_from_sftp_file(url):
2042   return False;
2043
2044 if(haveparamiko):
2045  def download_file_from_sftp_string(url):
2046   sftpfile = download_file_from_sftp_file(url);
2047   return sftpfile.read();
2048 else:
2049  def download_file_from_ftp_string(url):
2050   return False;
2051
2052 if(haveparamiko):
2053  def upload_file_to_sftp_file(sftpfile, url):
2054   urlparts = urlparse.urlparse(url);
2055   file_name = os.path.basename(urlparts.path);
2056   file_dir = os.path.dirname(urlparts.path);
2057   sftp_port = urlparts.port;
2058   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2059    return False;
2060   if(urlparts.port is None):
2061    sftp_port = 22;
2062   else:
2063    sftp_port = urlparts.port;
2064   if(urlparts.username is not None):
2065    sftp_username = urlparts.username;
2066   else:
2067    sftp_username = "anonymous";
2068   if(urlparts.password is not None):
2069    sftp_password = urlparts.password;
2070   elif(urlparts.password is None and urlparts.username=="anonymous"):
2071    sftp_password = "anonymous";
2072   else:
2073    sftp_password = "";
2074   if(urlparts.scheme!="sftp"):
2075    return False;
2076   ssh = paramiko.SSHClient();
2077   ssh.load_system_host_keys();
2078   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
2079   try:
2080    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2081   except paramiko.ssh_exception.SSHException:
2082    return False;
2083   except socket.gaierror:
2084    log.info("Error With URL "+httpurl);
2085    return False;
2086   except socket.timeout:
2087    log.info("Error With URL "+httpurl);
2088    return False;
2089   sftp = ssh.open_sftp();
2090   sftp.putfo(sftpfile, urlparts.path);
2091   sftp.close();
2092   ssh.close();
2093   sftpfile.seek(0, 0);
2094   return sftpfile;
2095 else:
2096  def upload_file_to_sftp_file(sftpfile, url):
2097   return False;
2098
2099 if(haveparamiko):
2100  def upload_file_to_sftp_string(sftpstring, url):
2101   sftpfileo = BytesIO(sftpstring);
2102   sftpfile = upload_file_to_sftp_files(ftpfileo, url);
2103   sftpfileo.close();
2104   return sftpfile;
2105 else:
2106  def upload_file_to_sftp_string(url):
2107   return False;
2108
2109
2110 if(havepysftp):
2111  def download_file_from_pysftp_file(url):
2112   urlparts = urlparse.urlparse(url);
2113   file_name = os.path.basename(urlparts.path);
2114   file_dir = os.path.dirname(urlparts.path);
2115   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2116    return False;
2117   sftp_port = urlparts.port;
2118   if(urlparts.port is None):
2119    sftp_port = 22;
2120   else:
2121    sftp_port = urlparts.port;
2122   if(urlparts.username is not None):
2123    sftp_username = urlparts.username;
2124   else:
2125    sftp_username = "anonymous";
2126   if(urlparts.password is not None):
2127    sftp_password = urlparts.password;
2128   elif(urlparts.password is None and urlparts.username=="anonymous"):
2129    sftp_password = "anonymous";
2130   else:
2131    sftp_password = "";
2132   if(urlparts.scheme!="sftp"):
2133    return False;
2134   try:
2135    pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2136   except paramiko.ssh_exception.SSHException:
2137    return False;
2138   except socket.gaierror:
2139    log.info("Error With URL "+httpurl);
2140    return False;
2141   except socket.timeout:
2142    log.info("Error With URL "+httpurl);
2143    return False;
2144   sftp = ssh.open_sftp();
2145   sftpfile = BytesIO();
2146   sftp.getfo(urlparts.path, sftpfile);
2147   sftp.close();
2148   ssh.close();
2149   sftpfile.seek(0, 0);
2150   return sftpfile;
2151 else:
2152  def download_file_from_pysftp_file(url):
2153   return False;
2154
2155 if(havepysftp):
2156  def download_file_from_pysftp_string(url):
2157   sftpfile = download_file_from_pysftp_file(url);
2158   return sftpfile.read();
2159 else:
2160  def download_file_from_ftp_string(url):
2161   return False;
2162
2163 if(havepysftp):
2164  def upload_file_to_pysftp_file(sftpfile, url):
2165   urlparts = urlparse.urlparse(url);
2166   file_name = os.path.basename(urlparts.path);
2167   file_dir = os.path.dirname(urlparts.path);
2168   sftp_port = urlparts.port;
2169   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2170    return False;
2171   if(urlparts.port is None):
2172    sftp_port = 22;
2173   else:
2174    sftp_port = urlparts.port;
2175   if(urlparts.username is not None):
2176    sftp_username = urlparts.username;
2177   else:
2178    sftp_username = "anonymous";
2179   if(urlparts.password is not None):
2180    sftp_password = urlparts.password;
2181   elif(urlparts.password is None and urlparts.username=="anonymous"):
2182    sftp_password = "anonymous";
2183   else:
2184    sftp_password = "";
2185   if(urlparts.scheme!="sftp"):
2186    return False;
2187   try:
2188    pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
2189   except paramiko.ssh_exception.SSHException:
2190    return False;
2191   except socket.gaierror:
2192    log.info("Error With URL "+httpurl);
2193    return False;
2194   except socket.timeout:
2195    log.info("Error With URL "+httpurl);
2196    return False;
2197   sftp = ssh.open_sftp();
2198   sftp.putfo(sftpfile, urlparts.path);
2199   sftp.close();
2200   ssh.close();
2201   sftpfile.seek(0, 0);
2202   return sftpfile;
2203 else:
2204  def upload_file_to_pysftp_file(sftpfile, url):
2205   return False;
2206
2207 if(havepysftp):
2208  def upload_file_to_pysftp_string(sftpstring, url):
2209   sftpfileo = BytesIO(sftpstring);
2210   sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
2211   sftpfileo.close();
2212   return sftpfile;
2213 else:
2214  def upload_file_to_pysftp_string(url):
2215   return False;