pywwwget.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
  13     Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  14
  15     $FileInfo: pywwwget.py - Last Update: 9/16/2023 Ver. 1.0.0 RC 1 - Author: cooldude2k $
  16 '''
  17
  18 from __future__ import division, absolute_import, print_function;
  19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, argparse, cgi, subprocess, socket;
  20 import logging as log;
  21 from ftplib import FTP, FTP_TLS;
  22 from base64 import b64encode;
  23 haverequests = False;
  24 try:
  25  import requests;
  26  haverequests = True;
  27 except ImportError:
  28  haverequests = False;
  29 havemechanize = False;
  30 try:
  31  import mechanize;
  32  havemechanize = True;
  33 except ImportError:
  34  havemechanize = False;
  35 haveparamiko = False;
  36 try:
  37  import paramiko;
  38  haveparamiko = True;
  39 except ImportError:
  40  haveparamiko = False;
  41 haveurllib3 = False;
  42 try:
  43  import urllib3;
  44  haveurllib3 = True;
  45 except ImportError:
  46  haveurllib3 = False;
  47 havehttplib2 = False;
  48 try:
  49  from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
  50  havehttplib2 = True;
  51 except ImportError:
  52  havehttplib2 = False;
  53 havehttpx = False;
  54 try:
  55  import httpx;
  56  havehttpx = True;
  57 except ImportError:
  58  havehttpx = False;
  59 havebrotli = False;
  60 try:
  61  import brotli;
  62  havebrotli = True;
  63 except ImportError:
  64  havebrotli = False;
  65 if(sys.version[0]=="2"):
  66  try:
  67   from cStringIO import StringIO;
  68  except ImportError:
  69   from StringIO import StringIO;
  70  # From http://python-future.org/compatible_idioms.html
  71  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
  72  from urllib import urlencode;
  73  from urllib import urlopen as urlopenalt;
  74  from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
  75  import urlparse, cookielib;
  76  from httplib import HTTPConnection, HTTPSConnection;
  77 if(sys.version[0]>="3"):
  78  from io import StringIO, BytesIO;
  79  # From http://python-future.org/compatible_idioms.html
  80  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
  81  from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
  82  from urllib.error import HTTPError, URLError;
  83  import urllib.parse as urlparse;
  84  import http.cookiejar as cookielib;
  85  from http.client import HTTPConnection, HTTPSConnection;
  86
  87 __program_name__ = "PyWWW-Get";
  88 __program_alt_name__ = "PyWWWGet";
  89 __program_small_name__ = "wwwget";
  90 __project__ = __program_name__;
  91 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
  92 __version_info__ = (1, 0, 0, "RC 1", 1);
  93 __version_date_info__ = (2023, 9, 16, "RC 1", 1);
  94 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
  95 __revision__ = __version_info__[3];
  96 __revision_id__ = "$Id$";
  97 if(__version_info__[4] is not None):
  98  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
  99 if(__version_info__[4] is None):
 100  __version_date_plusrc__ = __version_date__;
 101 if(__version_info__[3] is not None):
 102  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
 103 if(__version_info__[3] is None):
 104  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
 105
 106 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
 107 tmpfilesuffix = "-";
 108 pytempdir = tempfile.gettempdir();
 109
 110 PyBitness = platform.architecture();
 111 if(PyBitness=="32bit" or PyBitness=="32"):
 112  PyBitness = "32";
 113 elif(PyBitness=="64bit" or PyBitness=="64"):
 114  PyBitness = "64";
 115 else:
 116  PyBitness = "32";
 117
 118 compression_supported = "gzip, deflate";
 119 if(havebrotli):
 120  compression_supported = "gzip, deflate, br";
 121 else:
 122  compression_supported = "gzip, deflate";
 123
 124 geturls_cj = cookielib.CookieJar();
 125 windowsNT4_ua_string = "Windows NT 4.0";
 126 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
 127 windows2k_ua_string = "Windows NT 5.0";
 128 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
 129 windowsXP_ua_string = "Windows NT 5.1";
 130 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 131 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
 132 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 133 windows7_ua_string = "Windows NT 6.1; Win64; x64";
 134 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
 135 windows8_ua_string = "Windows NT 6.2; Win64; x64";
 136 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
 137 windows81_ua_string = "Windows NT 6.3; Win64; x64";
 138 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
 139 windows10_ua_string = "Windows NT 10.0; Win64; x64";
 140 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
 141 windows11_ua_string = "Windows NT 11.0; Win64; x64";
 142 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
 143 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
 144 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
 145 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
 146 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
 147 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
 148 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
 149 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
 150 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
 151 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
 152 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
 153 if(platform.python_implementation()!=""):
 154  py_implementation = platform.python_implementation();
 155 if(platform.python_implementation()==""):
 156  py_implementation = "Python";
 157 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
 158 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
 159 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
 160 geturls_ua = geturls_ua_firefox_windows7;
 161 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 162 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 163 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 164 geturls_headers_chrome_windows7.update(windows7_ua_addon);
 165 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 166 geturls_headers_chromium_windows7.update(windows7_ua_addon);
 167 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 168 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
 169 geturls_headers_opera_windows7.update(windows7_ua_addon);
 170 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
 171 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
 172 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 173 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
 174 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
 175 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 176 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 177 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 178 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 179 geturls_headers = geturls_headers_firefox_windows7;
 180 geturls_download_sleep = 0;
 181
 182 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 183  if(outtype=="print" and dbgenable):
 184   print(dbgtxt);
 185   return True;
 186  elif(outtype=="log" and dbgenable):
 187   logging.info(dbgtxt);
 188   return True;
 189  elif(outtype=="warning" and dbgenable):
 190   logging.warning(dbgtxt);
 191   return True;
 192  elif(outtype=="error" and dbgenable):
 193   logging.error(dbgtxt);
 194   return True;
 195  elif(outtype=="critical" and dbgenable):
 196   logging.critical(dbgtxt);
 197   return True;
 198  elif(outtype=="exception" and dbgenable):
 199   logging.exception(dbgtxt);
 200   return True;
 201  elif(outtype=="logalt" and dbgenable):
 202   logging.log(dgblevel, dbgtxt);
 203   return True;
 204  elif(outtype=="debug" and dbgenable):
 205   logging.debug(dbgtxt);
 206   return True;
 207  elif(not dbgenable):
 208   return True;
 209  else:
 210   return False;
 211  return False;
 212
 213 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 214  dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
 215  if(not dbgout):
 216   return False;
 217  return dbgtxt;
 218
 219 def add_url_param(url, **params):
 220  n=3;
 221  parts = list(urlparse.urlsplit(url));
 222  d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 223  d.update(params);
 224  parts[n]=urlencode(d);
 225  return urlparse.urlunsplit(parts);
 226
 227 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 228 def which_exec(execfile):
 229  for path in os.environ["PATH"].split(":"):
 230   if os.path.exists(path + "/" + execfile):
 231    return path + "/" + execfile;
 232
 233 def listize(varlist):
 234  il = 0;
 235  ix = len(varlist);
 236  ilx = 1;
 237  newlistreg = {};
 238  newlistrev = {};
 239  newlistfull = {};
 240  while(il < ix):
 241   newlistreg.update({ilx: varlist[il]});
 242   newlistrev.update({varlist[il]: ilx});
 243   ilx = ilx + 1;
 244   il = il + 1;
 245  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 246  return newlistfull;
 247
 248 def twolistize(varlist):
 249  il = 0;
 250  ix = len(varlist);
 251  ilx = 1;
 252  newlistnamereg = {};
 253  newlistnamerev = {};
 254  newlistdescreg = {};
 255  newlistdescrev = {};
 256  newlistfull = {};
 257  while(il < ix):
 258   newlistnamereg.update({ilx: varlist[il][0].strip()});
 259   newlistnamerev.update({varlist[il][0].strip(): ilx});
 260   newlistdescreg.update({ilx: varlist[il][1].strip()});
 261   newlistdescrev.update({varlist[il][1].strip(): ilx});
 262   ilx = ilx + 1;
 263   il = il + 1;
 264  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 265  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 266  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 267  return newlistfull;
 268
 269 def arglistize(proexec, *varlist):
 270  il = 0;
 271  ix = len(varlist);
 272  ilx = 1;
 273  newarglist = [proexec];
 274  while(il < ix):
 275   if varlist[il][0] is not None:
 276    newarglist.append(varlist[il][0]);
 277   if varlist[il][1] is not None:
 278    newarglist.append(varlist[il][1]);
 279   il = il + 1;
 280  return newarglist;
 281
 282 # hms_string by ArcGIS Python Recipes
 283 # https://arcpy.wordpress.com/2012/04/20/146/
 284 def hms_string(sec_elapsed):
 285  h = int(sec_elapsed / (60 * 60));
 286  m = int((sec_elapsed % (60 * 60)) / 60);
 287  s = sec_elapsed % 60.0;
 288  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 289
 290 # get_readable_size by Lipis
 291 # http://stackoverflow.com/posts/14998888/revisions
 292 def get_readable_size(bytes, precision=1, unit="IEC"):
 293  unit = unit.upper();
 294  if(unit!="IEC" and unit!="SI"):
 295   unit = "IEC";
 296  if(unit=="IEC"):
 297   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 298   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 299   unitsize = 1024.0;
 300  if(unit=="SI"):
 301   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 302   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 303   unitsize = 1000.0;
 304  return_val = {};
 305  orgbytes = bytes;
 306  for unit in units:
 307   if abs(bytes) < unitsize:
 308    strformat = "%3."+str(precision)+"f%s";
 309    pre_return_val = (strformat % (bytes, unit));
 310    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 311    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 312    alt_return_val = pre_return_val.split();
 313    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 314    return return_val;
 315   bytes /= unitsize;
 316  strformat = "%."+str(precision)+"f%s";
 317  pre_return_val = (strformat % (bytes, "YiB"));
 318  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 319  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 320  alt_return_val = pre_return_val.split();
 321  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 322  return return_val;
 323
 324 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 325  unit = unit.upper();
 326  usehashtypes = usehashtypes.lower();
 327  getfilesize = os.path.getsize(infile);
 328  return_val = get_readable_size(getfilesize, precision, unit);
 329  if(usehashes):
 330   hashtypelist = usehashtypes.split(",");
 331   openfile = open(infile, "rb");
 332   filecontents = openfile.read();
 333   openfile.close();
 334   listnumcount = 0;
 335   listnumend = len(hashtypelist);
 336   while(listnumcount < listnumend):
 337    hashtypelistlow = hashtypelist[listnumcount].strip();
 338    hashtypelistup = hashtypelistlow.upper();
 339    filehash = hashlib.new(hashtypelistup);
 340    filehash.update(filecontents);
 341    filegethash = filehash.hexdigest();
 342    return_val.update({hashtypelistup: filegethash});
 343    listnumcount += 1;
 344  return return_val;
 345
 346 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 347  unit = unit.upper();
 348  usehashtypes = usehashtypes.lower();
 349  getfilesize = len(instring);
 350  return_val = get_readable_size(getfilesize, precision, unit);
 351  if(usehashes):
 352   hashtypelist = usehashtypes.split(",");
 353   listnumcount = 0;
 354   listnumend = len(hashtypelist);
 355   while(listnumcount < listnumend):
 356    hashtypelistlow = hashtypelist[listnumcount].strip();
 357    hashtypelistup = hashtypelistlow.upper();
 358    filehash = hashlib.new(hashtypelistup);
 359    if(sys.version[0]=="2"):
 360     filehash.update(instring);
 361    if(sys.version[0]>="3"):
 362     filehash.update(instring.encode('utf-8'));
 363    filegethash = filehash.hexdigest();
 364    return_val.update({hashtypelistup: filegethash});
 365    listnumcount += 1;
 366  return return_val;
 367
 368 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 369  if isinstance(headers, dict):
 370   returnval = [];
 371   if(sys.version[0]=="2"):
 372    for headkey, headvalue in headers.iteritems():
 373     returnval.append((headkey, headvalue));
 374   if(sys.version[0]>="3"):
 375    for headkey, headvalue in headers.items():
 376     returnval.append((headkey, headvalue));
 377  elif isinstance(headers, list):
 378   returnval = headers;
 379  else:
 380   returnval = False;
 381  return returnval;
 382
 383 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 384  if isinstance(headers, dict):
 385   returnval = [];
 386   if(sys.version[0]=="2"):
 387    for headkey, headvalue in headers.iteritems():
 388     returnval.append(headkey+": "+headvalue);
 389   if(sys.version[0]>="3"):
 390    for headkey, headvalue in headers.items():
 391     returnval.append(headkey+": "+headvalue);
 392  elif isinstance(headers, list):
 393   returnval = headers;
 394  else:
 395   returnval = False;
 396  return returnval;
 397
 398 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 399  if isinstance(headers, list):
 400   returnval = {};
 401   mli = 0;
 402   mlil = len(headers);
 403   while(mli<mlil):
 404    returnval.update({headers[mli][0]: headers[mli][1]});
 405    mli = mli + 1;
 406  elif isinstance(headers, dict):
 407   returnval = headers;
 408  else:
 409   returnval = False;
 410  return returnval;
 411
 412 def get_httplib_support(checkvalue=None):
 413  global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, haveparamiko;
 414  returnval = [];
 415  returnval.append("ftp");
 416  returnval.append("httplib");
 417  if(havehttplib2):
 418   returnval.append("httplib2");
 419  returnval.append("urllib");
 420  if(haveurllib3):
 421   returnval.append("urllib3");
 422   returnval.append("request3");
 423  returnval.append("request");
 424  if(haverequests):
 425   returnval.append("requests");
 426  if(havehttpx):
 427   returnval.append("httpx");
 428   returnval.append("httpx2");
 429  if(havemechanize):
 430   returnval.append("mechanize");
 431  if(haveparamiko):
 432   returnval.append("sftp");
 433  if(not checkvalue is None):
 434   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 435    checkvalue = "urllib";
 436   if(checkvalue=="httplib1"):
 437    checkvalue = "httplib";
 438   if(checkvalue in returnval):
 439    returnval = True;
 440   else:
 441    returnval = False;
 442  return returnval;
 443
 444 def check_httplib_support(checkvalue="urllib"):
 445  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 446   checkvalue = "urllib";
 447  if(checkvalue=="httplib1"):
 448   checkvalue = "httplib";
 449  returnval = get_httplib_support(checkvalue);
 450  return returnval;
 451
 452 def get_httplib_support_list():
 453  returnval = get_httplib_support(None);
 454  return returnval;
 455
 456 def download_from_url(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", sleep=-1):
 457  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, haveparamiko;
 458  if(sleep<0):
 459   sleep = geturls_download_sleep;
 460  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 461   httplibuse = "urllib";
 462  if(httplibuse=="httplib1"):
 463   httplibuse = "httplib";
 464  if(not haverequests and httplibuse=="requests"):
 465   httplibuse = "urllib";
 466  if(not havehttpx and httplibuse=="httpx"):
 467   httplibuse = "urllib";
 468  if(not havehttpx and httplibuse=="httpx2"):
 469   httplibuse = "urllib";
 470  if(not havemechanize and httplibuse=="mechanize"):
 471   httplibuse = "urllib";
 472  if(not havehttplib2 and httplibuse=="httplib2"):
 473   httplibuse = "httplib";
 474  if(not haveparamiko and httplibuse=="sftp"):
 475   httplibuse = "ftp";
 476  if(httplibuse=="urllib"):
 477   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 478  elif(httplibuse=="request"):
 479   returnval = download_from_url_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 480  elif(httplibuse=="request3"):
 481   returnval = download_from_url_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 482  elif(httplibuse=="httplib"):
 483   returnval = download_from_url_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 484  elif(httplibuse=="httplib2"):
 485   returnval = download_from_url_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 486  elif(httplibuse=="urllib3"):
 487   returnval = download_from_url_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 488  elif(httplibuse=="requests"):
 489   returnval = download_from_url_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 490  elif(httplibuse=="httpx"):
 491   returnval = download_from_url_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 492  elif(httplibuse=="httpx2"):
 493   returnval = download_from_url_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 494  elif(httplibuse=="mechanize"):
 495   returnval = download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 496  elif(httplibuse=="ftp"):
 497   returnval = download_from_url_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 498  elif(httplibuse=="sftp"):
 499   returnval = download_from_url_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep);
 500  else:
 501   returnval = False;
 502  return returnval;
 503
 504 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1):
 505  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, haveparamiko;
 506  if(sleep<0):
 507   sleep = geturls_download_sleep;
 508  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 509   httplibuse = "urllib";
 510  if(httplibuse=="httplib1"):
 511   httplibuse = "httplib";
 512  if(not haverequests and httplibuse=="requests"):
 513   httplibuse = "urllib";
 514  if(not havehttpx and httplibuse=="httpx"):
 515   httplibuse = "urllib";
 516  if(not havehttpx and httplibuse=="httpx2"):
 517   httplibuse = "urllib";
 518  if(not havemechanize and httplibuse=="mechanize"):
 519   httplibuse = "urllib";
 520  if(not havehttplib2 and httplibuse=="httplib2"):
 521   httplibuse = "httplib";
 522  if(not haveparamiko and httplibuse=="sftp"):
 523   httplibuse = "ftp";
 524  if(httplibuse=="urllib"):
 525   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 526  elif(httplibuse=="request"):
 527   returnval = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 528  elif(httplibuse=="request3"):
 529   returnval = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 530  elif(httplibuse=="httplib"):
 531   returnval = download_from_url_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 532  elif(httplibuse=="httplib2"):
 533   returnval = download_from_url_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 534  elif(httplibuse=="urllib3"):
 535   returnval = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 536  elif(httplibuse=="requests"):
 537   returnval = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 538  elif(httplibuse=="httpx"):
 539   returnval = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 540  elif(httplibuse=="httpx2"):
 541   returnval = download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 542  elif(httplibuse=="mechanize"):
 543   returnval = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 544  elif(httplibuse=="ftp"):
 545   returnval = download_from_url_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 546  elif(httplibuse=="sftp"):
 547   returnval = download_from_url_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep);
 548  else:
 549   returnval = False;
 550  return returnval;
 551
 552 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 553  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, haveparamiko;
 554  if(sleep<0):
 555   sleep = geturls_download_sleep;
 556  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 557   httplibuse = "urllib";
 558  if(httplibuse=="httplib1"):
 559   httplibuse = "httplib";
 560  if(not haverequests and httplibuse=="requests"):
 561   httplibuse = "urllib";
 562  if(not havehttpx and httplibuse=="httpx"):
 563   httplibuse = "urllib";
 564  if(not havehttpx and httplibuse=="httpx2"):
 565   httplibuse = "urllib";
 566  if(not havemechanize and httplibuse=="mechanize"):
 567   httplibuse = "urllib";
 568  if(not havehttplib2 and httplibuse=="httplib2"):
 569   httplibuse = "httplib";
 570  if(not haveparamiko and httplibuse=="sftp"):
 571   httplibuse = "ftp";
 572  if(httplibuse=="urllib"):
 573   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 574  elif(httplibuse=="request"):
 575   returnval = download_from_url_to_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 576  elif(httplibuse=="request3"):
 577   returnval = download_from_url_to_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 578  elif(httplibuse=="httplib"):
 579   returnval = download_from_url_to_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 580  elif(httplibuse=="httplib2"):
 581   returnval = download_from_url_to_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 582  elif(httplibuse=="urllib3"):
 583   returnval = download_from_url_to_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 584  elif(httplibuse=="requests"):
 585   returnval = download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 586  elif(httplibuse=="httpx"):
 587   returnval = download_from_url_to_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 588  elif(httplibuse=="httpx2"):
 589   returnval = download_from_url_to_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 590  elif(httplibuse=="mechanize"):
 591   returnval = download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 592  elif(httplibuse=="ftp"):
 593   returnval = download_from_url_to_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 594  elif(httplibuse=="sftp"):
 595   returnval = download_from_url_to_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 596  else:
 597   returnval = False;
 598  return returnval;
 599
 600 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
 601  global geturls_download_sleep, havebrotli;
 602  if(sleep<0):
 603   sleep = geturls_download_sleep;
 604  urlparts = urlparse.urlparse(httpurl);
 605  if(isinstance(httpheaders, list)):
 606    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 607  if(urlparts.username is not None or urlparts.password is not None):
 608   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 609   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 610  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 611  if(isinstance(httpheaders, dict)):
 612   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 613  geturls_opener.addheaders = httpheaders;
 614  time.sleep(sleep);
 615  if(postdata is not None and not isinstance(postdata, dict)):
 616   postdata = urlencode(postdata);
 617  try:
 618   if(httpmethod=="GET"):
 619    geturls_text = geturls_opener.open(httpurl);
 620   elif(httpmethod=="POST"):
 621    geturls_text = geturls_opener.open(httpurl, data=postdata);
 622   else:
 623    geturls_text = geturls_opener.open(httpurl);
 624  except HTTPError as geturls_text_error:
 625   geturls_text = geturls_text_error;
 626   log.info("Error With URL "+httpurl);
 627  except URLError:
 628   log.info("Error With URL "+httpurl);
 629   return False;
 630  except socket.timeout:
 631   log.info("Error With URL "+httpurl);
 632   return False;
 633  log.info("Downloading URL "+httpurl);
 634  if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 635   if(sys.version[0]=="2"):
 636    strbuf = StringIO(geturls_text.read());
 637   if(sys.version[0]>="3"):
 638    strbuf = BytesIO(geturls_text.read());
 639   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 640   returnval_content = gzstrbuf.read()[:];
 641  if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 642   returnval_content = geturls_text.read()[:];
 643  if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
 644   returnval_content = brotli.decompress(returnval_content);
 645  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 646  geturls_text.close();
 647  return returnval;
 648
 649 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
 650  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 651  exec_time_start = time.time();
 652  myhash = hashlib.new("sha1");
 653  if(sys.version[0]=="2"):
 654   myhash.update(httpurl);
 655   myhash.update(str(buffersize));
 656   myhash.update(str(exec_time_start));
 657  if(sys.version[0]>="3"):
 658   myhash.update(httpurl.encode('utf-8'));
 659   myhash.update(str(buffersize).encode('utf-8'));
 660   myhash.update(str(exec_time_start).encode('utf-8'));
 661  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 662  if(sleep<0):
 663   sleep = geturls_download_sleep;
 664  urlparts = urlparse.urlparse(httpurl);
 665  if(isinstance(httpheaders, list)):
 666    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 667  if(urlparts.username is not None or urlparts.password is not None):
 668   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 669   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 670  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 671  if(isinstance(httpheaders, dict)):
 672   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 673  geturls_opener.addheaders = httpheaders;
 674  time.sleep(sleep);
 675  try:
 676   if(httpmethod=="GET"):
 677    geturls_text = geturls_opener.open(httpurl);
 678   elif(httpmethod=="POST"):
 679    geturls_text = geturls_opener.open(httpurl, data=postdata);
 680   else:
 681    geturls_text = geturls_opener.open(httpurl);
 682  except HTTPError as geturls_text_error:
 683   geturls_text = geturls_text_error;
 684   log.info("Error With URL "+httpurl);
 685  except URLError:
 686   log.info("Error With URL "+httpurl);
 687   return False;
 688  except socket.timeout:
 689   log.info("Error With URL "+httpurl);
 690   return False;
 691  except socket.timeout:
 692   log.info("Error With URL "+httpurl);
 693   return False;
 694  downloadsize = geturls_text.info().get('Content-Length');
 695  if(downloadsize is not None):
 696   downloadsize = int(downloadsize);
 697  if downloadsize is None: downloadsize = 0;
 698  fulldatasize = 0;
 699  prevdownsize = 0;
 700  log.info("Downloading URL "+httpurl);
 701  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 702   tmpfilename = f.name;
 703   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 704   while True:
 705    databytes = geturls_text.read(buffersize);
 706    if not databytes: break;
 707    datasize = len(databytes);
 708    fulldatasize = datasize + fulldatasize;
 709    percentage = "";
 710    if(downloadsize>0):
 711     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 712    downloaddiff = fulldatasize - prevdownsize;
 713    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 714    prevdownsize = fulldatasize;
 715    f.write(databytes);
 716   f.close();
 717  geturls_text.close();
 718  exec_time_end = time.time();
 719  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 720  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 721  return returnval;
 722
 723 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 724  global geturls_download_sleep;
 725  if(sleep<0):
 726   sleep = geturls_download_sleep;
 727  if(not outfile=="-"):
 728   outpath = outpath.rstrip(os.path.sep);
 729   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 730   if(not os.path.exists(outpath)):
 731    os.makedirs(outpath);
 732   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 733    return False;
 734   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 735    return False;
 736   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 737   if(not pretmpfilename):
 738    return False;
 739   tmpfilename = pretmpfilename['Filename'];
 740   downloadsize = os.path.getsize(tmpfilename);
 741   fulldatasize = 0;
 742   log.info("Moving file "+tmpfilename+" to "+filepath);
 743   exec_time_start = time.time();
 744   shutil.move(tmpfilename, filepath);
 745   exec_time_end = time.time();
 746   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 747   if(os.path.exists(tmpfilename)):
 748    os.remove(tmpfilename);
 749   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 750  if(outfile=="-" and sys.version[0]=="2"):
 751   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 752   if(not pretmpfilename):
 753    return False;
 754   tmpfilename = pretmpfilename['Filename'];
 755   downloadsize = os.path.getsize(tmpfilename);
 756   fulldatasize = 0;
 757   prevdownsize = 0;
 758   exec_time_start = time.time();
 759   with open(tmpfilename, 'rb') as ft:
 760    f = StringIO();
 761    while True:
 762     databytes = ft.read(buffersize[1]);
 763     if not databytes: break;
 764     datasize = len(databytes);
 765     fulldatasize = datasize + fulldatasize;
 766     percentage = "";
 767     if(downloadsize>0):
 768      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 769     downloaddiff = fulldatasize - prevdownsize;
 770     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 771     prevdownsize = fulldatasize;
 772     f.write(databytes);
 773    f.seek(0);
 774    fdata = f.getvalue();
 775    f.close();
 776    ft.close();
 777    os.remove(tmpfilename);
 778    exec_time_end = time.time();
 779    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 780   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 781  if(outfile=="-" and sys.version[0]>="3"):
 782   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 783   tmpfilename = pretmpfilename['Filename'];
 784   downloadsize = os.path.getsize(tmpfilename);
 785   fulldatasize = 0;
 786   prevdownsize = 0;
 787   exec_time_start = time.time();
 788   with open(tmpfilename, 'rb') as ft:
 789    f = BytesIO();
 790    while True:
 791     databytes = ft.read(buffersize[1]);
 792     if not databytes: break;
 793     datasize = len(databytes);
 794     fulldatasize = datasize + fulldatasize;
 795     percentage = "";
 796     if(downloadsize>0):
 797      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 798     downloaddiff = fulldatasize - prevdownsize;
 799     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 800     prevdownsize = fulldatasize;
 801     f.write(databytes);
 802    f.seek(0);
 803    fdata = f.getvalue();
 804    f.close();
 805    ft.close();
 806    os.remove(tmpfilename);
 807    exec_time_end = time.time();
 808    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 809   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 810  return returnval;
 811
 812 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
 813  global geturls_download_sleep, havebrotli;
 814  if(sleep<0):
 815   sleep = geturls_download_sleep;
 816  urlparts = urlparse.urlparse(httpurl);
 817  if(isinstance(httpheaders, list)):
 818    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 819  if(urlparts.username is not None or urlparts.password is not None):
 820   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 821   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 822  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 823  geturls_opener.addheaders = httpheaders;
 824  time.sleep(sleep);
 825  if(urlparts[0]=="http"):
 826   httpconn = HTTPConnection(urlparts[1]);
 827  elif(urlparts[0]=="https"):
 828   httpconn = HTTPSConnection(urlparts[1]);
 829  else:
 830   return False;
 831  if(postdata is not None and not isinstance(postdata, dict)):
 832   postdata = urlencode(postdata);
 833  try:
 834   if(httpmethod=="GET"):
 835    httpconn.request("GET", urlparts[2], headers=httpheaders);
 836   elif(httpmethod=="POST"):
 837    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 838   else:
 839    httpconn.request("GET", urlparts[2], headers=httpheaders);
 840  except socket.timeout:
 841   log.info("Error With URL "+httpurl);
 842   return False;
 843  except socket.gaierror:
 844   log.info("Error With URL "+httpurl);
 845   return False;
 846  geturls_text = httpconn.getresponse();
 847  log.info("Downloading URL "+httpurl);
 848  if(dict(geturls_text.getheaders()).get("Content-Encoding")=="gzip" or dict(geturls_text.getheaders()).get("Content-Encoding")=="deflate"):
 849   if(sys.version[0]=="2"):
 850    strbuf = StringIO(geturls_text.read());
 851   if(sys.version[0]>="3"):
 852    strbuf = BytesIO(geturls_text.read());
 853   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 854   returnval_content = gzstrbuf.read()[:];
 855  if(dict(geturls_text.getheaders()).get("Content-Encoding")!="gzip" and dict(geturls_text.getheaders()).get("Content-Encoding")!="deflate"):
 856   returnval_content = geturls_text.read()[:];
 857  if(dict(geturls_text.getheaders()).get("Content-Encoding")=="br" and havebrotli):
 858   returnval_content = brotli.decompress(returnval_content);
 859  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.getheaders()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
 860  geturls_text.close();
 861  return returnval;
 862
 863 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
 864  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 865  exec_time_start = time.time();
 866  myhash = hashlib.new("sha1");
 867  if(sys.version[0]=="2"):
 868   myhash.update(httpurl);
 869   myhash.update(str(buffersize));
 870   myhash.update(str(exec_time_start));
 871  if(sys.version[0]>="3"):
 872   myhash.update(httpurl.encode('utf-8'));
 873   myhash.update(str(buffersize).encode('utf-8'));
 874   myhash.update(str(exec_time_start).encode('utf-8'));
 875  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 876  if(sleep<0):
 877   sleep = geturls_download_sleep;
 878  urlparts = urlparse.urlparse(httpurl);
 879  if(isinstance(httpheaders, list)):
 880    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 881  if(urlparts.username is not None or urlparts.password is not None):
 882   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 883   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 884  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 885  geturls_opener.addheaders = httpheaders;
 886  time.sleep(sleep);
 887  if(urlparts[0]=="http"):
 888   httpconn = HTTPConnection(urlparts[1]);
 889  elif(urlparts[0]=="https"):
 890   httpconn = HTTPSConnection(urlparts[1]);
 891  else:
 892   return False;
 893  if(postdata is not None and not isinstance(postdata, dict)):
 894   postdata = urlencode(postdata);
 895  try:
 896   if(httpmethod=="GET"):
 897    httpconn.request("GET", urlparts[2], headers=httpheaders);
 898   elif(httpmethod=="POST"):
 899    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
 900   else:
 901    httpconn.request("GET", urlparts[2], headers=httpheaders);
 902  except socket.timeout:
 903   log.info("Error With URL "+httpurl);
 904   return False;
 905  except socket.gaierror:
 906   log.info("Error With URL "+httpurl);
 907   return False;
 908  geturls_text = httpconn.getresponse();
 909  downloadsize = dict(geturls_text.getheaders()).get('Content-Length');
 910  if(downloadsize is not None):
 911   downloadsize = int(downloadsize);
 912  if downloadsize is None: downloadsize = 0;
 913  fulldatasize = 0;
 914  prevdownsize = 0;
 915  log.info("Downloading URL "+httpurl);
 916  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 917   tmpfilename = f.name;
 918   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(dict(geturls_text.getheaders())), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
 919   while True:
 920    databytes = geturls_text.read(buffersize);
 921    if not databytes: break;
 922    datasize = len(databytes);
 923    fulldatasize = datasize + fulldatasize;
 924    percentage = "";
 925    if(downloadsize>0):
 926     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 927    downloaddiff = fulldatasize - prevdownsize;
 928    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 929    prevdownsize = fulldatasize;
 930    f.write(databytes);
 931   f.close();
 932  geturls_text.close();
 933  exec_time_end = time.time();
 934  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 935  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 936  return returnval;
 937
 938 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 939  global geturls_download_sleep;
 940  if(sleep<0):
 941   sleep = geturls_download_sleep;
 942  if(not outfile=="-"):
 943   outpath = outpath.rstrip(os.path.sep);
 944   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 945   if(not os.path.exists(outpath)):
 946    os.makedirs(outpath);
 947   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 948    return False;
 949   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 950    return False;
 951   pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 952   if(not pretmpfilename):
 953    return False;
 954   tmpfilename = pretmpfilename['Filename'];
 955   downloadsize = os.path.getsize(tmpfilename);
 956   fulldatasize = 0;
 957   log.info("Moving file "+tmpfilename+" to "+filepath);
 958   exec_time_start = time.time();
 959   shutil.move(tmpfilename, filepath);
 960   exec_time_end = time.time();
 961   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 962   if(os.path.exists(tmpfilename)):
 963    os.remove(tmpfilename);
 964   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 965  if(outfile=="-" and sys.version[0]=="2"):
 966   pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 967   if(not pretmpfilename):
 968    return False;
 969   tmpfilename = pretmpfilename['Filename'];
 970   downloadsize = os.path.getsize(tmpfilename);
 971   fulldatasize = 0;
 972   prevdownsize = 0;
 973   exec_time_start = time.time();
 974   with open(tmpfilename, 'rb') as ft:
 975    f = StringIO();
 976    while True:
 977     databytes = ft.read(buffersize[1]);
 978     if not databytes: break;
 979     datasize = len(databytes);
 980     fulldatasize = datasize + fulldatasize;
 981     percentage = "";
 982     if(downloadsize>0):
 983      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 984     downloaddiff = fulldatasize - prevdownsize;
 985     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 986     prevdownsize = fulldatasize;
 987     f.write(databytes);
 988    f.seek(0);
 989    fdata = f.getvalue();
 990    f.close();
 991    ft.close();
 992    os.remove(tmpfilename);
 993    exec_time_end = time.time();
 994    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 995   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 996  if(outfile=="-" and sys.version[0]>="3"):
 997   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
 998   tmpfilename = pretmpfilename['Filename'];
 999   downloadsize = os.path.getsize(tmpfilename);
1000   fulldatasize = 0;
1001   prevdownsize = 0;
1002   exec_time_start = time.time();
1003   with open(tmpfilename, 'rb') as ft:
1004    f = BytesIO();
1005    while True:
1006     databytes = ft.read(buffersize[1]);
1007     if not databytes: break;
1008     datasize = len(databytes);
1009     fulldatasize = datasize + fulldatasize;
1010     percentage = "";
1011     if(downloadsize>0):
1012      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1013     downloaddiff = fulldatasize - prevdownsize;
1014     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1015     prevdownsize = fulldatasize;
1016     f.write(databytes);
1017    f.seek(0);
1018    fdata = f.getvalue();
1019    f.close();
1020    ft.close();
1021    os.remove(tmpfilename);
1022    exec_time_end = time.time();
1023    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1024   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1025  return returnval;
1026
1027 if(havehttplib2):
1028  def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1029   global geturls_download_sleep, havebrotli;
1030   if(sleep<0):
1031    sleep = geturls_download_sleep;
1032   urlparts = urlparse.urlparse(httpurl);
1033   if(isinstance(httpheaders, list)):
1034    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1035   if(urlparts.username is not None or urlparts.password is not None):
1036    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1037    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1038   geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1039   geturls_opener.addheaders = httpheaders;
1040   time.sleep(sleep);
1041   if(urlparts[0]=="http"):
1042    httpconn = HTTPConnectionWithTimeout(urlparts[1]);
1043   elif(urlparts[0]=="https"):
1044    httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
1045   else:
1046    return False;
1047   if(postdata is not None and not isinstance(postdata, dict)):
1048    postdata = urlencode(postdata);
1049   try:
1050    if(httpmethod=="GET"):
1051     httpconn.request("GET", urlparts[2], headers=httpheaders);
1052    elif(httpmethod=="POST"):
1053     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1054    else:
1055     httpconn.request("GET", urlparts[2], headers=httpheaders);
1056   except socket.timeout:
1057    log.info("Error With URL "+httpurl);
1058    return False;
1059   except socket.gaierror:
1060    log.info("Error With URL "+httpurl);
1061    return False;
1062   geturls_text = httpconn.getresponse();
1063   log.info("Downloading URL "+httpurl);
1064   if(dict(geturls_text.getheaders()).get("Content-Encoding")=="gzip" or dict(geturls_text.getheaders()).get("Content-Encoding")=="deflate"):
1065    if(sys.version[0]=="2"):
1066     strbuf = StringIO(geturls_text.read());
1067    if(sys.version[0]>="3"):
1068     strbuf = BytesIO(geturls_text.read());
1069    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1070    returnval_content = gzstrbuf.read()[:];
1071   if(dict(geturls_text.getheaders()).get("Content-Encoding")!="gzip" and dict(geturls_text.getheaders()).get("Content-Encoding")!="deflate"):
1072    returnval_content = geturls_text.read()[:];
1073   if(dict(geturls_text.getheaders()).get("Content-Encoding")=="br" and havebrotli):
1074    returnval_content = brotli.decompress(returnval_content);
1075   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.getheaders()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
1076   geturls_text.close();
1077   return returnval;
1078
1079 if(not havehttplib2):
1080  def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1081   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1082   return returnval;
1083
1084 if(havehttplib2):
1085  def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1086   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1087   exec_time_start = time.time();
1088   myhash = hashlib.new("sha1");
1089   if(sys.version[0]=="2"):
1090    myhash.update(httpurl);
1091    myhash.update(str(buffersize));
1092    myhash.update(str(exec_time_start));
1093   if(sys.version[0]>="3"):
1094    myhash.update(httpurl.encode('utf-8'));
1095    myhash.update(str(buffersize).encode('utf-8'));
1096    myhash.update(str(exec_time_start).encode('utf-8'));
1097   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1098   if(sleep<0):
1099    sleep = geturls_download_sleep;
1100   urlparts = urlparse.urlparse(httpurl);
1101   if(isinstance(httpheaders, list)):
1102    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1103   if(urlparts.username is not None or urlparts.password is not None):
1104    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1105    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1106   geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1107   geturls_opener.addheaders = httpheaders;
1108   time.sleep(sleep);
1109   if(urlparts[0]=="http"):
1110    httpconn = HTTPConnectionWithTimeout(urlparts[1]);
1111   elif(urlparts[0]=="https"):
1112    httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
1113   else:
1114    return False;
1115   if(postdata is not None and not isinstance(postdata, dict)):
1116    postdata = urlencode(postdata);
1117   try:
1118    if(httpmethod=="GET"):
1119     httpconn.request("GET", urlparts[2], headers=httpheaders);
1120    elif(httpmethod=="POST"):
1121     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1122    else:
1123     httpconn.request("GET", urlparts[2], headers=httpheaders);
1124   except socket.timeout:
1125    log.info("Error With URL "+httpurl);
1126    return False;
1127   except socket.gaierror:
1128    log.info("Error With URL "+httpurl);
1129    return False;
1130   geturls_text = httpconn.getresponse();
1131   downloadsize = dict(geturls_text.getheaders()).get('Content-Length');
1132   if(downloadsize is not None):
1133    downloadsize = int(downloadsize);
1134   if downloadsize is None: downloadsize = 0;
1135   fulldatasize = 0;
1136   prevdownsize = 0;
1137   log.info("Downloading URL "+httpurl);
1138   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1139    tmpfilename = f.name;
1140    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(dict(geturls_text.getheaders())), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': httpurl, 'Code': geturls_text.status};
1141    while True:
1142     databytes = geturls_text.read(buffersize);
1143     if not databytes: break;
1144     datasize = len(databytes);
1145     fulldatasize = datasize + fulldatasize;
1146     percentage = "";
1147     if(downloadsize>0):
1148      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1149     downloaddiff = fulldatasize - prevdownsize;
1150     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1151     prevdownsize = fulldatasize;
1152     f.write(databytes);
1153    f.close();
1154   geturls_text.close();
1155   exec_time_end = time.time();
1156   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1157   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1158   return returnval;
1159
1160 if(not havehttplib2):
1161  def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1162   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1163   return returnval;
1164
1165 if(havehttplib2):
1166  def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1167   global geturls_download_sleep;
1168   if(sleep<0):
1169    sleep = geturls_download_sleep;
1170   if(not outfile=="-"):
1171    outpath = outpath.rstrip(os.path.sep);
1172    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1173    if(not os.path.exists(outpath)):
1174     os.makedirs(outpath);
1175    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1176     return False;
1177    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1178     return False;
1179    pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1180    if(not pretmpfilename):
1181     return False;
1182    tmpfilename = pretmpfilename['Filename'];
1183    downloadsize = os.path.getsize(tmpfilename);
1184    fulldatasize = 0;
1185    log.info("Moving file "+tmpfilename+" to "+filepath);
1186    exec_time_start = time.time();
1187    shutil.move(tmpfilename, filepath);
1188    exec_time_end = time.time();
1189    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1190    if(os.path.exists(tmpfilename)):
1191     os.remove(tmpfilename);
1192    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1193   if(outfile=="-" and sys.version[0]=="2"):
1194    pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1195    if(not pretmpfilename):
1196     return False;
1197    tmpfilename = pretmpfilename['Filename'];
1198    downloadsize = os.path.getsize(tmpfilename);
1199    fulldatasize = 0;
1200    prevdownsize = 0;
1201    exec_time_start = time.time();
1202    with open(tmpfilename, 'rb') as ft:
1203     f = StringIO();
1204     while True:
1205      databytes = ft.read(buffersize[1]);
1206      if not databytes: break;
1207      datasize = len(databytes);
1208      fulldatasize = datasize + fulldatasize;
1209      percentage = "";
1210      if(downloadsize>0):
1211       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1212      downloaddiff = fulldatasize - prevdownsize;
1213      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1214      prevdownsize = fulldatasize;
1215      f.write(databytes);
1216     f.seek(0);
1217     fdata = f.getvalue();
1218     f.close();
1219     ft.close();
1220     os.remove(tmpfilename);
1221     exec_time_end = time.time();
1222     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1223    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1224   if(outfile=="-" and sys.version[0]>="3"):
1225    pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1226    tmpfilename = pretmpfilename['Filename'];
1227    downloadsize = os.path.getsize(tmpfilename);
1228    fulldatasize = 0;
1229    prevdownsize = 0;
1230    exec_time_start = time.time();
1231    with open(tmpfilename, 'rb') as ft:
1232     f = BytesIO();
1233     while True:
1234      databytes = ft.read(buffersize[1]);
1235      if not databytes: break;
1236      datasize = len(databytes);
1237      fulldatasize = datasize + fulldatasize;
1238      percentage = "";
1239      if(downloadsize>0):
1240       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1241      downloaddiff = fulldatasize - prevdownsize;
1242      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1243      prevdownsize = fulldatasize;
1244      f.write(databytes);
1245     f.seek(0);
1246     fdata = f.getvalue();
1247     f.close();
1248     ft.close();
1249     os.remove(tmpfilename);
1250     exec_time_end = time.time();
1251     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1252    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1253   return returnval;
1254
1255 if(not havehttplib2):
1256  def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1257   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1258   return returnval;
1259
1260 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1261  global geturls_download_sleep, havebrotli;
1262  if(sleep<0):
1263   sleep = geturls_download_sleep;
1264  urlparts = urlparse.urlparse(httpurl);
1265  if(isinstance(httpheaders, list)):
1266    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1267  if(urlparts.username is not None or urlparts.password is not None):
1268   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1269   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1270  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1271  if(isinstance(httpheaders, dict)):
1272   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1273  geturls_opener.addheaders = httpheaders;
1274  urllib.request.install_opener(geturls_opener);
1275  time.sleep(sleep);
1276  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1277  if(postdata is not None and not isinstance(postdata, dict)):
1278   postdata = urlencode(postdata);
1279  try:
1280   if(httpmethod=="GET"):
1281    geturls_request = Request(httpurl, headers=httpheaders);
1282    geturls_text = urlopen(geturls_request);
1283   elif(httpmethod=="POST"):
1284    geturls_request = Request(httpurl, headers=httpheaders);
1285    geturls_text = urlopen(geturls_request, data=postdata);
1286   else:
1287    geturls_request = Request(httpurl, headers=httpheaders);
1288    geturls_text = urlopen(geturls_request);
1289  except HTTPError as geturls_text_error:
1290   geturls_text = geturls_text_error;
1291   log.info("Error With URL "+httpurl);
1292  except URLError:
1293   log.info("Error With URL "+httpurl);
1294   return False;
1295  except socket.timeout:
1296   log.info("Error With URL "+httpurl);
1297   return False;
1298  log.info("Downloading URL "+httpurl);
1299  if(geturls_text.headers.get("Content-Encoding")=="gzip" or geturls_text.headers.get("Content-Encoding")=="deflate"):
1300   if(sys.version[0]=="2"):
1301    strbuf = StringIO(geturls_text.read());
1302   if(sys.version[0]>="3"):
1303    strbuf = BytesIO(geturls_text.read());
1304   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1305   returnval_content = gzstrbuf.read()[:];
1306  if(geturls_text.headers.get("Content-Encoding")!="gzip" and geturls_text.headers.get("Content-Encoding")!="deflate"):
1307   returnval_content = geturls_text.read()[:];
1308  if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1309   returnval_content = brotli.decompress(returnval_content);
1310  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
1311  geturls_text.close();
1312  return returnval;
1313
1314 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1315  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1316  exec_time_start = time.time();
1317  myhash = hashlib.new("sha1");
1318  if(sys.version[0]=="2"):
1319   myhash.update(httpurl);
1320   myhash.update(str(buffersize));
1321   myhash.update(str(exec_time_start));
1322  if(sys.version[0]>="3"):
1323   myhash.update(httpurl.encode('utf-8'));
1324   myhash.update(str(buffersize).encode('utf-8'));
1325   myhash.update(str(exec_time_start).encode('utf-8'));
1326  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1327  if(sleep<0):
1328   sleep = geturls_download_sleep;
1329  urlparts = urlparse.urlparse(httpurl);
1330  if(isinstance(httpheaders, list)):
1331    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1332  if(urlparts.username is not None or urlparts.password is not None):
1333   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1334   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1335  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1336  if(isinstance(httpheaders, dict)):
1337   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1338  geturls_opener.addheaders = httpheaders;
1339  urllib.request.install_opener(geturls_opener);
1340  time.sleep(sleep);
1341  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1342  if(postdata is not None and not isinstance(postdata, dict)):
1343   postdata = urlencode(postdata);
1344  try:
1345   if(httpmethod=="GET"):
1346    geturls_request = Request(httpurl, headers=httpheaders);
1347    geturls_text = urlopen(geturls_request);
1348   elif(httpmethod=="POST"):
1349    geturls_request = Request(httpurl, headers=httpheaders);
1350    geturls_text = urlopen(geturls_request, data=postdata);
1351   else:
1352    geturls_request = Request(httpurl, headers=httpheaders);
1353    geturls_text = urlopen(geturls_request);
1354  except HTTPError as geturls_text_error:
1355   geturls_text = geturls_text_error;
1356   log.info("Error With URL "+httpurl);
1357  except URLError:
1358   log.info("Error With URL "+httpurl);
1359   return False;
1360  except socket.timeout:
1361   log.info("Error With URL "+httpurl);
1362   return False;
1363  downloadsize = geturls_text.headers.get('Content-Length');
1364  if(downloadsize is not None):
1365   downloadsize = int(downloadsize);
1366  if downloadsize is None: downloadsize = 0;
1367  fulldatasize = 0;
1368  prevdownsize = 0;
1369  log.info("Downloading URL "+httpurl);
1370  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1371   tmpfilename = f.name;
1372   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
1373   while True:
1374    databytes = geturls_text.read(buffersize);
1375    if not databytes: break;
1376    datasize = len(databytes);
1377    fulldatasize = datasize + fulldatasize;
1378    percentage = "";
1379    if(downloadsize>0):
1380     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1381    downloaddiff = fulldatasize - prevdownsize;
1382    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1383    prevdownsize = fulldatasize;
1384    f.write(databytes);
1385   f.close();
1386  geturls_text.close();
1387  exec_time_end = time.time();
1388  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1389  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1390  return returnval;
1391
1392 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1393  global geturls_download_sleep;
1394  if(sleep<0):
1395   sleep = geturls_download_sleep;
1396  if(not outfile=="-"):
1397   outpath = outpath.rstrip(os.path.sep);
1398   filepath = os.path.realpath(outpath+os.path.sep+outfile);
1399   if(not os.path.exists(outpath)):
1400    os.makedirs(outpath);
1401   if(os.path.exists(outpath) and os.path.isfile(outpath)):
1402    return False;
1403   if(os.path.exists(filepath) and os.path.isdir(filepath)):
1404    return False;
1405   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1406   if(not pretmpfilename):
1407    return False;
1408   tmpfilename = pretmpfilename['Filename'];
1409   downloadsize = os.path.getsize(tmpfilename);
1410   fulldatasize = 0;
1411   log.info("Moving file "+tmpfilename+" to "+filepath);
1412   exec_time_start = time.time();
1413   shutil.move(tmpfilename, filepath);
1414   exec_time_end = time.time();
1415   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1416   if(os.path.exists(tmpfilename)):
1417    os.remove(tmpfilename);
1418   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent':pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1419  if(outfile=="-" and sys.version[0]=="2"):
1420   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1421   if(not pretmpfilename):
1422    return False;
1423   tmpfilename = pretmpfilename['Filename'];
1424   downloadsize = os.path.getsize(tmpfilename);
1425   fulldatasize = 0;
1426   prevdownsize = 0;
1427   exec_time_start = time.time();
1428   with open(tmpfilename, 'rb') as ft:
1429    f = StringIO();
1430    while True:
1431     databytes = ft.read(buffersize[1]);
1432     if not databytes: break;
1433     datasize = len(databytes);
1434     fulldatasize = datasize + fulldatasize;
1435     percentage = "";
1436     if(downloadsize>0):
1437      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1438     downloaddiff = fulldatasize - prevdownsize;
1439     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1440     prevdownsize = fulldatasize;
1441     f.write(databytes);
1442    f.seek(0);
1443    fdata = f.getvalue();
1444    f.close();
1445    ft.close();
1446    os.remove(tmpfilename);
1447    exec_time_end = time.time();
1448    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1449   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1450  if(outfile=="-" and sys.version[0]>="3"):
1451   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1452   tmpfilename = pretmpfilename['Filename'];
1453   downloadsize = os.path.getsize(tmpfilename);
1454   fulldatasize = 0;
1455   prevdownsize = 0;
1456   exec_time_start = time.time();
1457   with open(tmpfilename, 'rb') as ft:
1458    f = BytesIO();
1459    while True:
1460     databytes = ft.read(buffersize[1]);
1461     if not databytes: break;
1462     datasize = len(databytes);
1463     fulldatasize = datasize + fulldatasize;
1464     percentage = "";
1465     if(downloadsize>0):
1466      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1467     downloaddiff = fulldatasize - prevdownsize;
1468     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1469     prevdownsize = fulldatasize;
1470     f.write(databytes);
1471    f.seek(0);
1472    fdata = f.getvalue();
1473    f.close();
1474    ft.close();
1475    os.remove(tmpfilename);
1476    exec_time_end = time.time();
1477    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1478   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1479  return returnval;
1480
1481 if(haverequests):
1482  def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1483   global geturls_download_sleep, havebrotli;
1484   if(sleep<0):
1485    sleep = geturls_download_sleep;
1486   urlparts = urlparse.urlparse(httpurl);
1487   if(isinstance(httpheaders, list)):
1488    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1489   if(urlparts.username is not None or urlparts.password is not None):
1490    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1491    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1492   time.sleep(sleep);
1493   if(postdata is not None and not isinstance(postdata, dict)):
1494    postdata = urlencode(postdata);
1495   try:
1496    if(httpmethod=="GET"):
1497     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1498    elif(httpmethod=="POST"):
1499     geturls_text = requests.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1500    else:
1501     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1502   except requests.exceptions.ConnectTimeout:
1503    log.info("Error With URL "+httpurl);
1504    return False;
1505   except requests.exceptions.ConnectError:
1506    log.info("Error With URL "+httpurl);
1507    return False;
1508   except socket.timeout:
1509    log.info("Error With URL "+httpurl);
1510    return False;
1511   log.info("Downloading URL "+httpurl);
1512   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
1513    if(sys.version[0]=="2"):
1514     strbuf = StringIO(geturls_text.content);
1515    if(sys.version[0]>="3"):
1516     strbuf = BytesIO(geturls_text.content);
1517    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1518    returnval_content = gzstrbuf.content[:];
1519   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
1520    returnval_content = geturls_text.content[:];
1521   if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1522    returnval_content = brotli.decompress(returnval_content);
1523   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
1524   geturls_text.close();
1525   return returnval;
1526
1527 if(not haverequests):
1528  def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1529   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1530   return returnval;
1531
1532 if(haverequests):
1533  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1534   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1535   exec_time_start = time.time();
1536   myhash = hashlib.new("sha1");
1537   if(sys.version[0]=="2"):
1538    myhash.update(httpurl);
1539    myhash.update(str(buffersize));
1540    myhash.update(str(exec_time_start));
1541   if(sys.version[0]>="3"):
1542    myhash.update(httpurl.encode('utf-8'));
1543    myhash.update(str(buffersize).encode('utf-8'));
1544    myhash.update(str(exec_time_start).encode('utf-8'));
1545   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1546   if(sleep<0):
1547    sleep = geturls_download_sleep;
1548   urlparts = urlparse.urlparse(httpurl);
1549   if(isinstance(httpheaders, list)):
1550    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1551   if(urlparts.username is not None or urlparts.password is not None):
1552    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1553    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1554   time.sleep(sleep);
1555   if(postdata is not None and not isinstance(postdata, dict)):
1556    postdata = urlencode(postdata);
1557   try:
1558    if(httpmethod=="GET"):
1559     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1560    elif(httpmethod=="POST"):
1561     geturls_text = requests.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1562    else:
1563     geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
1564   except requests.exceptions.ConnectTimeout:
1565    log.info("Error With URL "+httpurl);
1566    return False;
1567   except requests.exceptions.ConnectError:
1568    log.info("Error With URL "+httpurl);
1569    return False;
1570   except socket.timeout:
1571    log.info("Error With URL "+httpurl);
1572    return False;
1573   downloadsize = int(geturls_text.headers.get('Content-Length'));
1574   if(downloadsize is not None):
1575    downloadsize = int(downloadsize);
1576   if downloadsize is None: downloadsize = 0;
1577   fulldatasize = 0;
1578   prevdownsize = 0;
1579   log.info("Downloading URL "+httpurl);
1580   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1581    tmpfilename = f.name;
1582    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
1583    for databytes in geturls_text.iter_content(chunk_size=buffersize):
1584     datasize = len(databytes);
1585     fulldatasize = datasize + fulldatasize;
1586     percentage = "";
1587     if(downloadsize>0):
1588      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1589     downloaddiff = fulldatasize - prevdownsize;
1590     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1591     prevdownsize = fulldatasize;
1592     f.write(databytes);
1593    f.close();
1594   geturls_text.close();
1595   exec_time_end = time.time();
1596   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1597   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1598   return returnval;
1599
1600 if(not haverequests):
1601  def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1602   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1603   return returnval;
1604
1605 if(haverequests):
1606  def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1607   global geturls_download_sleep;
1608   if(sleep<0):
1609    sleep = geturls_download_sleep;
1610   if(not outfile=="-"):
1611    outpath = outpath.rstrip(os.path.sep);
1612    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1613    if(not os.path.exists(outpath)):
1614     os.makedirs(outpath);
1615    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1616     return False;
1617    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1618     return False;
1619    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1620    if(not pretmpfilename):
1621     return False;
1622    tmpfilename = pretmpfilename['Filename'];
1623    downloadsize = os.path.getsize(tmpfilename);
1624    fulldatasize = 0;
1625    log.info("Moving file "+tmpfilename+" to "+filepath);
1626    exec_time_start = time.time();
1627    shutil.move(tmpfilename, filepath);
1628    exec_time_end = time.time();
1629    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1630    if(os.path.exists(tmpfilename)):
1631     os.remove(tmpfilename);
1632    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1633   if(outfile=="-" and sys.version[0]=="2"):
1634    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1635    if(not pretmpfilename):
1636     return False;
1637    tmpfilename = pretmpfilename['Filename'];
1638    downloadsize = os.path.getsize(tmpfilename);
1639    fulldatasize = 0;
1640    prevdownsize = 0;
1641    exec_time_start = time.time();
1642    with open(tmpfilename, 'rb') as ft:
1643     f = StringIO();
1644     while True:
1645      databytes = ft.read(buffersize[1]);
1646      if not databytes: break;
1647      datasize = len(databytes);
1648      fulldatasize = datasize + fulldatasize;
1649      percentage = "";
1650      if(downloadsize>0):
1651       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1652      downloaddiff = fulldatasize - prevdownsize;
1653      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1654      prevdownsize = fulldatasize;
1655      f.write(databytes);
1656     f.seek(0);
1657     fdata = f.getvalue();
1658     f.close();
1659     ft.close();
1660     os.remove(tmpfilename);
1661     exec_time_end = time.time();
1662     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1663    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1664   if(outfile=="-" and sys.version[0]>="3"):
1665    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1666    tmpfilename = pretmpfilename['Filename'];
1667    downloadsize = os.path.getsize(tmpfilename);
1668    fulldatasize = 0;
1669    prevdownsize = 0;
1670    exec_time_start = time.time();
1671    with open(tmpfilename, 'rb') as ft:
1672     f = BytesIO();
1673     while True:
1674      databytes = ft.read(buffersize[1]);
1675      if not databytes: break;
1676      datasize = len(databytes);
1677      fulldatasize = datasize + fulldatasize;
1678      percentage = "";
1679      if(downloadsize>0):
1680       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1681      downloaddiff = fulldatasize - prevdownsize;
1682      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1683      prevdownsize = fulldatasize;
1684      f.write(databytes);
1685     f.seek(0);
1686     fdata = f.getvalue();
1687     f.close();
1688     ft.close();
1689     os.remove(tmpfilename);
1690     exec_time_end = time.time();
1691     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1692    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1693   return returnval;
1694
1695 if(not haverequests):
1696  def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1697   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1698   return returnval;
1699
1700 if(havehttpx):
1701  def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1702   global geturls_download_sleep, havebrotli;
1703   if(sleep<0):
1704    sleep = geturls_download_sleep;
1705   urlparts = urlparse.urlparse(httpurl);
1706   if(isinstance(httpheaders, list)):
1707    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1708   if(urlparts.username is not None or urlparts.password is not None):
1709    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1710    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1711   time.sleep(sleep);
1712   if(postdata is not None and not isinstance(postdata, dict)):
1713    postdata = urlencode(postdata);
1714   try:
1715    if(httpmethod=="GET"):
1716     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1717     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1718    elif(httpmethod=="POST"):
1719     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1720     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1721    else:
1722     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1723     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1724   except httpx.ConnectTimeout:
1725    log.info("Error With URL "+httpurl);
1726    return False;
1727   except httpx.ConnectError:
1728    log.info("Error With URL "+httpurl);
1729    return False;
1730   except socket.timeout:
1731    log.info("Error With URL "+httpurl);
1732    return False;
1733   log.info("Downloading URL "+httpurl);
1734   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
1735    if(sys.version[0]=="2"):
1736     strbuf = StringIO(geturls_text.content);
1737    if(sys.version[0]>="3"):
1738     strbuf = BytesIO(geturls_text.content);
1739    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1740    returnval_content = gzstrbuf.content[:];
1741   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
1742    returnval_content = geturls_text.content[:];
1743   if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1744    returnval_content = brotli.decompress(returnval_content);
1745   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'Version': geturls_text.http_version, 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': str(geturls_text.url), 'Code': geturls_text.status_code};
1746   geturls_text.close();
1747   return returnval;
1748
1749 if(not havehttpx):
1750  def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1751   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1752   return returnval;
1753
1754 if(havehttpx):
1755  def download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1756   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1757   exec_time_start = time.time();
1758   myhash = hashlib.new("sha1");
1759   if(sys.version[0]=="2"):
1760    myhash.update(httpurl);
1761    myhash.update(str(buffersize));
1762    myhash.update(str(exec_time_start));
1763   if(sys.version[0]>="3"):
1764    myhash.update(httpurl.encode('utf-8'));
1765    myhash.update(str(buffersize).encode('utf-8'));
1766    myhash.update(str(exec_time_start).encode('utf-8'));
1767   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1768   if(sleep<0):
1769    sleep = geturls_download_sleep;
1770   urlparts = urlparse.urlparse(httpurl);
1771   if(isinstance(httpheaders, list)):
1772    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1773   if(urlparts.username is not None or urlparts.password is not None):
1774    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1775    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1776   time.sleep(sleep);
1777   if(postdata is not None and not isinstance(postdata, dict)):
1778    postdata = urlencode(postdata);
1779   try:
1780    if(httpmethod=="GET"):
1781     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1782     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1783    elif(httpmethod=="POST"):
1784     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1785     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1786    else:
1787     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
1788     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1789   except httpx.ConnectTimeout:
1790    log.info("Error With URL "+httpurl);
1791    return False;
1792   except httpx.ConnectError:
1793    log.info("Error With URL "+httpurl);
1794    return False;
1795   except socket.timeout:
1796    log.info("Error With URL "+httpurl);
1797    return False;
1798   downloadsize = int(geturls_text.headers.get('Content-Length'));
1799   if(downloadsize is not None):
1800    downloadsize = int(downloadsize);
1801   if downloadsize is None: downloadsize = 0;
1802   fulldatasize = 0;
1803   prevdownsize = 0;
1804   log.info("Downloading URL "+httpurl);
1805   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1806    tmpfilename = f.name;
1807    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'Version': geturls_text.http_version, 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': str(geturls_text.url), 'Code': geturls_text.status_code};
1808    for databytes in geturls_text.iter_content(chunk_size=buffersize):
1809     datasize = len(databytes);
1810     fulldatasize = datasize + fulldatasize;
1811     percentage = "";
1812     if(downloadsize>0):
1813      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1814     downloaddiff = fulldatasize - prevdownsize;
1815     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1816     prevdownsize = fulldatasize;
1817     f.write(databytes);
1818    f.close();
1819   geturls_text.close();
1820   exec_time_end = time.time();
1821   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1822   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1823   return returnval;
1824
1825 if(not havehttpx):
1826  def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1827   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
1828   return returnval;
1829
1830 if(havehttpx):
1831  def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1832   global geturls_download_sleep;
1833   if(sleep<0):
1834    sleep = geturls_download_sleep;
1835   if(not outfile=="-"):
1836    outpath = outpath.rstrip(os.path.sep);
1837    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1838    if(not os.path.exists(outpath)):
1839     os.makedirs(outpath);
1840    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1841     return False;
1842    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1843     return False;
1844    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1845    if(not pretmpfilename):
1846     return False;
1847    tmpfilename = pretmpfilename['Filename'];
1848    downloadsize = os.path.getsize(tmpfilename);
1849    fulldatasize = 0;
1850    log.info("Moving file "+tmpfilename+" to "+filepath);
1851    exec_time_start = time.time();
1852    shutil.move(tmpfilename, filepath);
1853    exec_time_end = time.time();
1854    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1855    if(os.path.exists(tmpfilename)):
1856     os.remove(tmpfilename);
1857    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1858   if(outfile=="-" and sys.version[0]=="2"):
1859    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1860    if(not pretmpfilename):
1861     return False;
1862    tmpfilename = pretmpfilename['Filename'];
1863    downloadsize = os.path.getsize(tmpfilename);
1864    fulldatasize = 0;
1865    prevdownsize = 0;
1866    exec_time_start = time.time();
1867    with open(tmpfilename, 'rb') as ft:
1868     f = StringIO();
1869     while True:
1870      databytes = ft.read(buffersize[1]);
1871      if not databytes: break;
1872      datasize = len(databytes);
1873      fulldatasize = datasize + fulldatasize;
1874      percentage = "";
1875      if(downloadsize>0):
1876       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1877      downloaddiff = fulldatasize - prevdownsize;
1878      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1879      prevdownsize = fulldatasize;
1880      f.write(databytes);
1881     f.seek(0);
1882     fdata = f.getvalue();
1883     f.close();
1884     ft.close();
1885     os.remove(tmpfilename);
1886     exec_time_end = time.time();
1887     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1888    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1889   if(outfile=="-" and sys.version[0]>="3"):
1890    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1891    tmpfilename = pretmpfilename['Filename'];
1892    downloadsize = os.path.getsize(tmpfilename);
1893    fulldatasize = 0;
1894    prevdownsize = 0;
1895    exec_time_start = time.time();
1896    with open(tmpfilename, 'rb') as ft:
1897     f = BytesIO();
1898     while True:
1899      databytes = ft.read(buffersize[1]);
1900      if not databytes: break;
1901      datasize = len(databytes);
1902      fulldatasize = datasize + fulldatasize;
1903      percentage = "";
1904      if(downloadsize>0):
1905       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1906      downloaddiff = fulldatasize - prevdownsize;
1907      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1908      prevdownsize = fulldatasize;
1909      f.write(databytes);
1910     f.seek(0);
1911     fdata = f.getvalue();
1912     f.close();
1913     ft.close();
1914     os.remove(tmpfilename);
1915     exec_time_end = time.time();
1916     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1917    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
1918   return returnval;
1919
1920 if(not havehttpx):
1921  def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1922   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1923   return returnval;
1924
1925 if(havehttpx):
1926  def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1927   global geturls_download_sleep, havebrotli;
1928   if(sleep<0):
1929    sleep = geturls_download_sleep;
1930   urlparts = urlparse.urlparse(httpurl);
1931   if(isinstance(httpheaders, list)):
1932    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1933   if(urlparts.username is not None or urlparts.password is not None):
1934    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1935    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1936   time.sleep(sleep);
1937   if(postdata is not None and not isinstance(postdata, dict)):
1938    postdata = urlencode(postdata);
1939   try:
1940    if(httpmethod=="GET"):
1941     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1942     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1943    elif(httpmethod=="POST"):
1944     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1945     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
1946    else:
1947     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
1948     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
1949   except httpx.ConnectTimeout:
1950    log.info("Error With URL "+httpurl);
1951    return False;
1952   except httpx.ConnectError:
1953    log.info("Error With URL "+httpurl);
1954    return False;
1955   except socket.timeout:
1956    log.info("Error With URL "+httpurl);
1957    return False;
1958   log.info("Downloading URL "+httpurl);
1959   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
1960    if(sys.version[0]=="2"):
1961     strbuf = StringIO(geturls_text.content);
1962    if(sys.version[0]>="3"):
1963     strbuf = BytesIO(geturls_text.content);
1964    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1965    returnval_content = gzstrbuf.content[:];
1966   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
1967    returnval_content = geturls_text.content[:];
1968   if(geturls_text.headers.get("Content-Encoding")=="br" and havebrotli):
1969    returnval_content = brotli.decompress(returnval_content);
1970   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'Version': geturls_text.http_version, 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': str(geturls_text.url), 'Code': geturls_text.status_code};
1971   geturls_text.close();
1972   return returnval;
1973
1974 if(not havehttpx):
1975  def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1976   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
1977   return returnval;
1978
1979 if(havehttpx):
1980  def download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1981   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1982   exec_time_start = time.time();
1983   myhash = hashlib.new("sha1");
1984   if(sys.version[0]=="2"):
1985    myhash.update(httpurl);
1986    myhash.update(str(buffersize));
1987    myhash.update(str(exec_time_start));
1988   if(sys.version[0]>="3"):
1989    myhash.update(httpurl.encode('utf-8'));
1990    myhash.update(str(buffersize).encode('utf-8'));
1991    myhash.update(str(exec_time_start).encode('utf-8'));
1992   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1993   if(sleep<0):
1994    sleep = geturls_download_sleep;
1995   urlparts = urlparse.urlparse(httpurl);
1996   if(isinstance(httpheaders, list)):
1997    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1998   if(urlparts.username is not None or urlparts.password is not None):
1999    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2000    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2001   time.sleep(sleep);
2002   if(postdata is not None and not isinstance(postdata, dict)):
2003    postdata = urlencode(postdata);
2004   try:
2005    if(httpmethod=="GET"):
2006     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2007     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2008    elif(httpmethod=="POST"):
2009     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2010     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
2011    else:
2012     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2013     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2014   except httpx.ConnectTimeout:
2015    log.info("Error With URL "+httpurl);
2016    return False;
2017   except httpx.ConnectError:
2018    log.info("Error With URL "+httpurl);
2019    return False;
2020   except socket.timeout:
2021    log.info("Error With URL "+httpurl);
2022    return False;
2023   downloadsize = int(geturls_text.headers.get('Content-Length'));
2024   if(downloadsize is not None):
2025    downloadsize = int(downloadsize);
2026   if downloadsize is None: downloadsize = 0;
2027   fulldatasize = 0;
2028   prevdownsize = 0;
2029   log.info("Downloading URL "+httpurl);
2030   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2031    tmpfilename = f.name;
2032    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'Version': geturls_text.http_version, 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': str(geturls_text.url), 'Code': geturls_text.status_code};
2033    for databytes in geturls_text.iter_content(chunk_size=buffersize):
2034     datasize = len(databytes);
2035     fulldatasize = datasize + fulldatasize;
2036     percentage = "";
2037     if(downloadsize>0):
2038      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2039     downloaddiff = fulldatasize - prevdownsize;
2040     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2041     prevdownsize = fulldatasize;
2042     f.write(databytes);
2043    f.close();
2044   geturls_text.close();
2045   exec_time_end = time.time();
2046   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2047   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2048   return returnval;
2049
2050 if(not havehttpx):
2051  def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2052   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2053   return returnval;
2054
2055 if(havehttpx):
2056  def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2057   global geturls_download_sleep;
2058   if(sleep<0):
2059    sleep = geturls_download_sleep;
2060   if(not outfile=="-"):
2061    outpath = outpath.rstrip(os.path.sep);
2062    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2063    if(not os.path.exists(outpath)):
2064     os.makedirs(outpath);
2065    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2066     return False;
2067    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2068     return False;
2069    pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2070    if(not pretmpfilename):
2071     return False;
2072    tmpfilename = pretmpfilename['Filename'];
2073    downloadsize = os.path.getsize(tmpfilename);
2074    fulldatasize = 0;
2075    log.info("Moving file "+tmpfilename+" to "+filepath);
2076    exec_time_start = time.time();
2077    shutil.move(tmpfilename, filepath);
2078    exec_time_end = time.time();
2079    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2080    if(os.path.exists(tmpfilename)):
2081     os.remove(tmpfilename);
2082    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2083   if(outfile=="-" and sys.version[0]=="2"):
2084    pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2085    if(not pretmpfilename):
2086     return False;
2087    tmpfilename = pretmpfilename['Filename'];
2088    downloadsize = os.path.getsize(tmpfilename);
2089    fulldatasize = 0;
2090    prevdownsize = 0;
2091    exec_time_start = time.time();
2092    with open(tmpfilename, 'rb') as ft:
2093     f = StringIO();
2094     while True:
2095      databytes = ft.read(buffersize[1]);
2096      if not databytes: break;
2097      datasize = len(databytes);
2098      fulldatasize = datasize + fulldatasize;
2099      percentage = "";
2100      if(downloadsize>0):
2101       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2102      downloaddiff = fulldatasize - prevdownsize;
2103      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2104      prevdownsize = fulldatasize;
2105      f.write(databytes);
2106     f.seek(0);
2107     fdata = f.getvalue();
2108     f.close();
2109     ft.close();
2110     os.remove(tmpfilename);
2111     exec_time_end = time.time();
2112     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2113    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2114   if(outfile=="-" and sys.version[0]>="3"):
2115    pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2116    tmpfilename = pretmpfilename['Filename'];
2117    downloadsize = os.path.getsize(tmpfilename);
2118    fulldatasize = 0;
2119    prevdownsize = 0;
2120    exec_time_start = time.time();
2121    with open(tmpfilename, 'rb') as ft:
2122     f = BytesIO();
2123     while True:
2124      databytes = ft.read(buffersize[1]);
2125      if not databytes: break;
2126      datasize = len(databytes);
2127      fulldatasize = datasize + fulldatasize;
2128      percentage = "";
2129      if(downloadsize>0):
2130       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2131      downloaddiff = fulldatasize - prevdownsize;
2132      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2133      prevdownsize = fulldatasize;
2134      f.write(databytes);
2135     f.seek(0);
2136     fdata = f.getvalue();
2137     f.close();
2138     ft.close();
2139     os.remove(tmpfilename);
2140     exec_time_end = time.time();
2141     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2142    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2143   return returnval;
2144
2145 if(not havehttpx):
2146  def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2147   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2148   return returnval;
2149
2150 if(haveurllib3):
2151  def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2152   global geturls_download_sleep, havebrotli;
2153   if(sleep<0):
2154    sleep = geturls_download_sleep;
2155   urlparts = urlparse.urlparse(httpurl);
2156   if(isinstance(httpheaders, list)):
2157    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2158   if(urlparts.username is not None or urlparts.password is not None):
2159    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2160    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2161   time.sleep(sleep);
2162   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2163   if(postdata is not None and not isinstance(postdata, dict)):
2164    postdata = urlencode(postdata);
2165   try:
2166    if(httpmethod=="GET"):
2167     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2168    elif(httpmethod=="POST"):
2169     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2170    else:
2171     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2172   except urllib3.exceptions.ConnectTimeoutError:
2173    log.info("Error With URL "+httpurl);
2174    return False;
2175   except urllib3.exceptions.ConnectError:
2176    log.info("Error With URL "+httpurl);
2177    return False;
2178   except urllib3.exceptions.MaxRetryError:
2179    log.info("Error With URL "+httpurl);
2180    return False;
2181   except socket.timeout:
2182    log.info("Error With URL "+httpurl);
2183    return False;
2184   log.info("Downloading URL "+httpurl);
2185   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
2186    if(sys.version[0]=="2"):
2187     strbuf = StringIO(geturls_text.read());
2188    if(sys.version[0]>="3"):
2189     strbuf = BytesIO(geturls_text.read());
2190    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2191    returnval_content = gzstrbuf.read()[:];
2192   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
2193    returnval_content = geturls_text.read()[:];
2194   if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
2195    returnval_content = brotli.decompress(returnval_content);
2196   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2197   geturls_text.close();
2198   return returnval;
2199
2200 if(not haveurllib3):
2201  def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2202   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
2203   return returnval;
2204
2205 if(haveurllib3):
2206  def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2207   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2208   exec_time_start = time.time();
2209   myhash = hashlib.new("sha1");
2210   if(sys.version[0]=="2"):
2211    myhash.update(httpurl);
2212    myhash.update(str(buffersize));
2213    myhash.update(str(exec_time_start));
2214   if(sys.version[0]>="3"):
2215    myhash.update(httpurl.encode('utf-8'));
2216    myhash.update(str(buffersize).encode('utf-8'));
2217    myhash.update(str(exec_time_start).encode('utf-8'));
2218   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2219   if(sleep<0):
2220    sleep = geturls_download_sleep;
2221   urlparts = urlparse.urlparse(httpurl);
2222   if(isinstance(httpheaders, list)):
2223    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2224   if(urlparts.username is not None or urlparts.password is not None):
2225    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2226    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2227   time.sleep(sleep);
2228   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2229   if(postdata is not None and not isinstance(postdata, dict)):
2230    postdata = urlencode(postdata);
2231   try:
2232    if(httpmethod=="GET"):
2233     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2234    elif(httpmethod=="POST"):
2235     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2236    else:
2237     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
2238   except urllib3.exceptions.ConnectTimeoutError:
2239    log.info("Error With URL "+httpurl);
2240    return False;
2241   except urllib3.exceptions.ConnectError:
2242    log.info("Error With URL "+httpurl);
2243    return False;
2244   except urllib3.exceptions.MaxRetryError:
2245    log.info("Error With URL "+httpurl);
2246    return False;
2247   except socket.timeout:
2248    log.info("Error With URL "+httpurl);
2249    return False;
2250   downloadsize = int(geturls_text.headers.get('Content-Length'));
2251   if(downloadsize is not None):
2252    downloadsize = int(downloadsize);
2253   if downloadsize is None: downloadsize = 0;
2254   fulldatasize = 0;
2255   prevdownsize = 0;
2256   log.info("Downloading URL "+httpurl);
2257   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2258    tmpfilename = f.name;
2259    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2260    while True:
2261     databytes = geturls_text.read(buffersize);
2262     if not databytes: break;
2263     datasize = len(databytes);
2264     fulldatasize = datasize + fulldatasize;
2265     percentage = "";
2266     if(downloadsize>0):
2267      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2268     downloaddiff = fulldatasize - prevdownsize;
2269     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2270     prevdownsize = fulldatasize;
2271     f.write(databytes);
2272    f.close();
2273   geturls_text.close();
2274   exec_time_end = time.time();
2275   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2276   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2277   return returnval;
2278
2279 if(not haveurllib3):
2280  def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2281   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2282   return returnval;
2283
2284 if(haveurllib3):
2285  def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2286   global geturls_download_sleep;
2287   if(sleep<0):
2288    sleep = geturls_download_sleep;
2289   if(not outfile=="-"):
2290    outpath = outpath.rstrip(os.path.sep);
2291    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2292    if(not os.path.exists(outpath)):
2293     os.makedirs(outpath);
2294    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2295     return False;
2296    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2297     return False;
2298    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2299    if(not pretmpfilename):
2300     return False;
2301    tmpfilename = pretmpfilename['Filename'];
2302    downloadsize = os.path.getsize(tmpfilename);
2303    fulldatasize = 0;
2304    log.info("Moving file "+tmpfilename+" to "+filepath);
2305    exec_time_start = time.time();
2306    shutil.move(tmpfilename, filepath);
2307    exec_time_end = time.time();
2308    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2309    if(os.path.exists(tmpfilename)):
2310     os.remove(tmpfilename);
2311    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2312   if(outfile=="-" and sys.version[0]=="2"):
2313    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2314    if(not pretmpfilename):
2315     return False;
2316    tmpfilename = pretmpfilename['Filename'];
2317    downloadsize = os.path.getsize(tmpfilename);
2318    fulldatasize = 0;
2319    prevdownsize = 0;
2320    exec_time_start = time.time();
2321    with open(tmpfilename, 'rb') as ft:
2322     f = StringIO();
2323     while True:
2324      databytes = ft.read(buffersize[1]);
2325      if not databytes: break;
2326      datasize = len(databytes);
2327      fulldatasize = datasize + fulldatasize;
2328      percentage = "";
2329      if(downloadsize>0):
2330       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2331      downloaddiff = fulldatasize - prevdownsize;
2332      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2333      prevdownsize = fulldatasize;
2334      f.write(databytes);
2335     f.seek(0);
2336     fdata = f.getvalue();
2337     f.close();
2338     ft.close();
2339     os.remove(tmpfilename);
2340     exec_time_end = time.time();
2341     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2342    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2343   if(outfile=="-" and sys.version[0]>="3"):
2344    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2345    tmpfilename = pretmpfilename['Filename'];
2346    downloadsize = os.path.getsize(tmpfilename);
2347    fulldatasize = 0;
2348    prevdownsize = 0;
2349    exec_time_start = time.time();
2350    with open(tmpfilename, 'rb') as ft:
2351     f = BytesIO();
2352     while True:
2353      databytes = ft.read(buffersize[1]);
2354      if not databytes: break;
2355      datasize = len(databytes);
2356      fulldatasize = datasize + fulldatasize;
2357      percentage = "";
2358      if(downloadsize>0):
2359       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2360      downloaddiff = fulldatasize - prevdownsize;
2361      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2362      prevdownsize = fulldatasize;
2363      f.write(databytes);
2364     f.seek(0);
2365     fdata = f.getvalue();
2366     f.close();
2367     ft.close();
2368     os.remove(tmpfilename);
2369     exec_time_end = time.time();
2370     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2371    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2372   return returnval;
2373
2374 if(not haveurllib3):
2375  def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2376   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2377   return returnval;
2378
2379 if(haveurllib3):
2380  def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2381   global geturls_download_sleep, havebrotli;
2382   if(sleep<0):
2383    sleep = geturls_download_sleep;
2384   urlparts = urlparse.urlparse(httpurl);
2385   if(isinstance(httpheaders, list)):
2386    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2387   if(urlparts.username is not None or urlparts.password is not None):
2388    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2389    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2390   time.sleep(sleep);
2391   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2392   if(postdata is not None and not isinstance(postdata, dict)):
2393    postdata = urlencode(postdata);
2394   try:
2395    if(httpmethod=="GET"):
2396     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2397    elif(httpmethod=="POST"):
2398     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2399    else:
2400     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2401   except urllib3.exceptions.ConnectTimeoutError:
2402    log.info("Error With URL "+httpurl);
2403    return False;
2404   except urllib3.exceptions.ConnectError:
2405    log.info("Error With URL "+httpurl);
2406    return False;
2407   except urllib3.exceptions.MaxRetryError:
2408    log.info("Error With URL "+httpurl);
2409    return False;
2410   except socket.timeout:
2411    log.info("Error With URL "+httpurl);
2412    return False;
2413   log.info("Downloading URL "+httpurl);
2414   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
2415    if(sys.version[0]=="2"):
2416     strbuf = StringIO(geturls_text.read());
2417    if(sys.version[0]>="3"):
2418     strbuf = BytesIO(geturls_text.read());
2419    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2420    returnval_content = gzstrbuf.read()[:];
2421   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
2422    returnval_content = geturls_text.read()[:];
2423   if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
2424    returnval_content = brotli.decompress(returnval_content);
2425   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2426   geturls_text.close();
2427   return returnval;
2428
2429 if(not haveurllib3):
2430  def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2431   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
2432   return returnval;
2433
2434 if(haveurllib3):
2435  def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2436   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2437   exec_time_start = time.time();
2438   myhash = hashlib.new("sha1");
2439   if(sys.version[0]=="2"):
2440    myhash.update(httpurl);
2441    myhash.update(str(buffersize));
2442    myhash.update(str(exec_time_start));
2443   if(sys.version[0]>="3"):
2444    myhash.update(httpurl.encode('utf-8'));
2445    myhash.update(str(buffersize).encode('utf-8'));
2446    myhash.update(str(exec_time_start).encode('utf-8'));
2447   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2448   if(sleep<0):
2449    sleep = geturls_download_sleep;
2450   urlparts = urlparse.urlparse(httpurl);
2451   if(isinstance(httpheaders, list)):
2452    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2453   if(urlparts.username is not None or urlparts.password is not None):
2454    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2455    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2456   time.sleep(sleep);
2457   urllib_pool = urllib3.PoolManager(headers=httpheaders);
2458   if(postdata is not None and not isinstance(postdata, dict)):
2459    postdata = urlencode(postdata);
2460   try:
2461    if(httpmethod=="GET"):
2462     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2463    elif(httpmethod=="POST"):
2464     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
2465    else:
2466     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
2467   except urllib3.exceptions.ConnectTimeoutError:
2468    log.info("Error With URL "+httpurl);
2469    return False;
2470   except urllib3.exceptions.ConnectError:
2471    log.info("Error With URL "+httpurl);
2472    return False;
2473   except urllib3.exceptions.MaxRetryError:
2474    log.info("Error With URL "+httpurl);
2475    return False;
2476   except socket.timeout:
2477    log.info("Error With URL "+httpurl);
2478    return False;
2479   downloadsize = int(geturls_text.headers.get('Content-Length'));
2480   if(downloadsize is not None):
2481    downloadsize = int(downloadsize);
2482   if downloadsize is None: downloadsize = 0;
2483   fulldatasize = 0;
2484   prevdownsize = 0;
2485   log.info("Downloading URL "+httpurl);
2486   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2487    tmpfilename = f.name;
2488    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': httpheaders, 'URL': geturls_text.geturl(), 'Code': geturls_text.status};
2489    while True:
2490     databytes = geturls_text.read(buffersize);
2491     if not databytes: break;
2492     datasize = len(databytes);
2493     fulldatasize = datasize + fulldatasize;
2494     percentage = "";
2495     if(downloadsize>0):
2496      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2497     downloaddiff = fulldatasize - prevdownsize;
2498     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2499     prevdownsize = fulldatasize;
2500     f.write(databytes);
2501    f.close();
2502   geturls_text.close();
2503   exec_time_end = time.time();
2504   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2505   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2506   return returnval;
2507
2508 if(not haveurllib3):
2509  def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2510   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2511   return returnval;
2512
2513 if(haveurllib3):
2514  def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2515   global geturls_download_sleep;
2516   if(sleep<0):
2517    sleep = geturls_download_sleep;
2518   if(not outfile=="-"):
2519    outpath = outpath.rstrip(os.path.sep);
2520    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2521    if(not os.path.exists(outpath)):
2522     os.makedirs(outpath);
2523    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2524     return False;
2525    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2526     return False;
2527    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2528    if(not pretmpfilename):
2529     return False;
2530    tmpfilename = pretmpfilename['Filename'];
2531    downloadsize = os.path.getsize(tmpfilename);
2532    fulldatasize = 0;
2533    log.info("Moving file "+tmpfilename+" to "+filepath);
2534    exec_time_start = time.time();
2535    shutil.move(tmpfilename, filepath);
2536    exec_time_end = time.time();
2537    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2538    if(os.path.exists(tmpfilename)):
2539     os.remove(tmpfilename);
2540    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2541   if(outfile=="-" and sys.version[0]=="2"):
2542    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2543    if(not pretmpfilename):
2544     return False;
2545    tmpfilename = pretmpfilename['Filename'];
2546    downloadsize = os.path.getsize(tmpfilename);
2547    fulldatasize = 0;
2548    prevdownsize = 0;
2549    exec_time_start = time.time();
2550    with open(tmpfilename, 'rb') as ft:
2551     f = StringIO();
2552     while True:
2553      databytes = ft.read(buffersize[1]);
2554      if not databytes: break;
2555      datasize = len(databytes);
2556      fulldatasize = datasize + fulldatasize;
2557      percentage = "";
2558      if(downloadsize>0):
2559       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2560      downloaddiff = fulldatasize - prevdownsize;
2561      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2562      prevdownsize = fulldatasize;
2563      f.write(databytes);
2564     f.seek(0);
2565     fdata = f.getvalue();
2566     f.close();
2567     ft.close();
2568     os.remove(tmpfilename);
2569     exec_time_end = time.time();
2570     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2571    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2572   if(outfile=="-" and sys.version[0]>="3"):
2573    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2574    tmpfilename = pretmpfilename['Filename'];
2575    downloadsize = os.path.getsize(tmpfilename);
2576    fulldatasize = 0;
2577    prevdownsize = 0;
2578    exec_time_start = time.time();
2579    with open(tmpfilename, 'rb') as ft:
2580     f = BytesIO();
2581     while True:
2582      databytes = ft.read(buffersize[1]);
2583      if not databytes: break;
2584      datasize = len(databytes);
2585      fulldatasize = datasize + fulldatasize;
2586      percentage = "";
2587      if(downloadsize>0):
2588       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2589      downloaddiff = fulldatasize - prevdownsize;
2590      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2591      prevdownsize = fulldatasize;
2592      f.write(databytes);
2593     f.seek(0);
2594     fdata = f.getvalue();
2595     f.close();
2596     ft.close();
2597     os.remove(tmpfilename);
2598     exec_time_end = time.time();
2599     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2600    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2601   return returnval;
2602
2603 if(not haveurllib3):
2604  def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2605   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2606   return returnval;
2607
2608 if(havemechanize):
2609  def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2610   global geturls_download_sleep, havebrotli;
2611   if(sleep<0):
2612    sleep = geturls_download_sleep;
2613   urlparts = urlparse.urlparse(httpurl);
2614   if(isinstance(httpheaders, list)):
2615    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2616   if(urlparts.username is not None or urlparts.password is not None):
2617    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2618    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2619   geturls_opener = mechanize.Browser();
2620   if(isinstance(httpheaders, dict)):
2621    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2622   time.sleep(sleep);
2623   geturls_opener.addheaders = httpheaders;
2624   geturls_opener.set_cookiejar(httpcookie);
2625   geturls_opener.set_handle_robots(False);
2626   if(postdata is not None and not isinstance(postdata, dict)):
2627    postdata = urlencode(postdata);
2628   try:
2629    if(httpmethod=="GET"):
2630     geturls_text = geturls_opener.open(httpurl);
2631    elif(httpmethod=="POST"):
2632     geturls_text = geturls_opener.open(httpurl, data=postdata);
2633    else:
2634     geturls_text = geturls_opener.open(httpurl);
2635   except mechanize.HTTPError as geturls_text_error:
2636    geturls_text = geturls_text_error;
2637    log.info("Error With URL "+httpurl);
2638   except URLError:
2639    log.info("Error With URL "+httpurl);
2640    return False;
2641   except socket.timeout:
2642    log.info("Error With URL "+httpurl);
2643    return False;
2644   log.info("Downloading URL "+httpurl);
2645   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
2646    if(sys.version[0]=="2"):
2647     strbuf = StringIO(geturls_text.read());
2648    if(sys.version[0]>="3"):
2649     strbuf = BytesIO(geturls_text.read());
2650    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2651    returnval_content = gzstrbuf.read()[:];
2652   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
2653    returnval_content = geturls_text.read()[:];
2654   if(geturls_text.info().get("Content-Encoding")=="br" and havebrotli):
2655    returnval_content = brotli.decompress(returnval_content);
2656   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
2657   geturls_text.close();
2658   return returnval;
2659
2660 if(not havemechanize):
2661  def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2662   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, sleep)
2663   return returnval;
2664
2665 if(havemechanize):
2666  def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2667   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2668   exec_time_start = time.time();
2669   myhash = hashlib.new("sha1");
2670   if(sys.version[0]=="2"):
2671    myhash.update(httpurl);
2672    myhash.update(str(buffersize));
2673    myhash.update(str(exec_time_start));
2674   if(sys.version[0]>="3"):
2675    myhash.update(httpurl.encode('utf-8'));
2676    myhash.update(str(buffersize).encode('utf-8'));
2677    myhash.update(str(exec_time_start).encode('utf-8'));
2678   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2679   if(sleep<0):
2680    sleep = geturls_download_sleep;
2681   urlparts = urlparse.urlparse(httpurl);
2682   if(isinstance(httpheaders, list)):
2683    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2684   if(urlparts.username is not None or urlparts.password is not None):
2685    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2686    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2687   geturls_opener = mechanize.Browser();
2688   if(isinstance(httpheaders, dict)):
2689    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2690   time.sleep(sleep);
2691   geturls_opener.addheaders = httpheaders;
2692   geturls_opener.set_cookiejar(httpcookie);
2693   geturls_opener.set_handle_robots(False);
2694   if(postdata is not None and not isinstance(postdata, dict)):
2695    postdata = urlencode(postdata);
2696   try:
2697    if(httpmethod=="GET"):
2698     geturls_text = geturls_opener.open(httpurl);
2699    elif(httpmethod=="POST"):
2700     geturls_text = geturls_opener.open(httpurl, data=postdata);
2701    else:
2702     geturls_text = geturls_opener.open(httpurl);
2703   except mechanize.HTTPError as geturls_text_error:
2704    geturls_text = geturls_text_error;
2705    log.info("Error With URL "+httpurl);
2706   except URLError:
2707    log.info("Error With URL "+httpurl);
2708    return False;
2709   except socket.timeout:
2710    log.info("Error With URL "+httpurl);
2711    return False;
2712   downloadsize = int(geturls_text.info().get('Content-Length'));
2713   if(downloadsize is not None):
2714    downloadsize = int(downloadsize);
2715   if downloadsize is None: downloadsize = 0;
2716   fulldatasize = 0;
2717   prevdownsize = 0;
2718   log.info("Downloading URL "+httpurl);
2719   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2720    tmpfilename = f.name;
2721    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'Version': "1.1", 'Method': httpmethod, 'HeadersSent': make_http_headers_from_list_to_dict(httpheaders), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
2722    while True:
2723     databytes = geturls_text.read(buffersize);
2724     if not databytes: break;
2725     datasize = len(databytes);
2726     fulldatasize = datasize + fulldatasize;
2727     percentage = "";
2728     if(downloadsize>0):
2729      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2730     downloaddiff = fulldatasize - prevdownsize;
2731     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2732     prevdownsize = fulldatasize;
2733     f.write(databytes);
2734    f.close();
2735   geturls_text.close();
2736   exec_time_end = time.time();
2737   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2738   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2739   return returnval;
2740
2741 if(not havemechanize):
2742  def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2743   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, sleep)
2744   return returnval;
2745
2746 if(havemechanize):
2747  def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2748   global geturls_download_sleep;
2749   if(sleep<0):
2750    sleep = geturls_download_sleep;
2751   if(not outfile=="-"):
2752    outpath = outpath.rstrip(os.path.sep);
2753    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2754    if(not os.path.exists(outpath)):
2755     os.makedirs(outpath);
2756    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2757     return False;
2758    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2759     return False;
2760    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2761    if(not pretmpfilename):
2762     return False;
2763    tmpfilename = pretmpfilename['Filename'];
2764    downloadsize = os.path.getsize(tmpfilename);
2765    fulldatasize = 0;
2766    log.info("Moving file "+tmpfilename+" to "+filepath);
2767    exec_time_start = time.time();
2768    shutil.move(tmpfilename, filepath);
2769    exec_time_end = time.time();
2770    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2771    if(os.path.exists(tmpfilename)):
2772     os.remove(tmpfilename);
2773    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2774   if(outfile=="-" and sys.version[0]=="2"):
2775    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2776    if(not pretmpfilename):
2777     return False;
2778    tmpfilename = pretmpfilename['Filename'];
2779    downloadsize = os.path.getsize(tmpfilename);
2780    fulldatasize = 0;
2781    prevdownsize = 0;
2782    exec_time_start = time.time();
2783    with open(tmpfilename, 'rb') as ft:
2784     f = StringIO();
2785     while True:
2786      databytes = ft.read(buffersize[1]);
2787      if not databytes: break;
2788      datasize = len(databytes);
2789      fulldatasize = datasize + fulldatasize;
2790      percentage = "";
2791      if(downloadsize>0):
2792       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2793      downloaddiff = fulldatasize - prevdownsize;
2794      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2795      prevdownsize = fulldatasize;
2796      f.write(databytes);
2797     f.seek(0);
2798     fdata = f.getvalue();
2799     f.close();
2800     ft.close();
2801     os.remove(tmpfilename);
2802     exec_time_end = time.time();
2803     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2804    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2805   if(outfile=="-" and sys.version[0]>="3"):
2806    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2807    tmpfilename = pretmpfilename['Filename'];
2808    downloadsize = os.path.getsize(tmpfilename);
2809    fulldatasize = 0;
2810    prevdownsize = 0;
2811    exec_time_start = time.time();
2812    with open(tmpfilename, 'rb') as ft:
2813     f = BytesIO();
2814     while True:
2815      databytes = ft.read(buffersize[1]);
2816      if not databytes: break;
2817      datasize = len(databytes);
2818      fulldatasize = datasize + fulldatasize;
2819      percentage = "";
2820      if(downloadsize>0):
2821       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2822      downloaddiff = fulldatasize - prevdownsize;
2823      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2824      prevdownsize = fulldatasize;
2825      f.write(databytes);
2826     f.seek(0);
2827     fdata = f.getvalue();
2828     f.close();
2829     ft.close();
2830     os.remove(tmpfilename);
2831     exec_time_end = time.time();
2832     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2833    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2834   return returnval;
2835
2836 if(not havemechanize):
2837  def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2838   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2839   return returnval;
2840
2841 def download_file_from_ftp_file(url):
2842  urlparts = urlparse.urlparse(url);
2843  file_name = os.path.basename(urlparts.path);
2844  file_dir = os.path.dirname(urlparts.path);
2845  if(urlparts.username is not None):
2846   ftp_username = urlparts.username;
2847  else:
2848   ftp_username = "anonymous";
2849  if(urlparts.password is not None):
2850   ftp_password = urlparts.password;
2851  elif(urlparts.password is None and urlparts.username=="anonymous"):
2852   ftp_password = "anonymous";
2853  else:
2854   ftp_password = "";
2855  if(urlparts.scheme=="ftp"):
2856   ftp = FTP();
2857  elif(urlparts.scheme=="ftps"):
2858   ftp = FTP_TLS();
2859  else:
2860   return False;
2861  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
2862   return False;
2863  ftp_port = urlparts.port;
2864  if(urlparts.port is None):
2865   ftp_port = 21;
2866  try:
2867   ftp.connect(urlparts.hostname, ftp_port);
2868  except socket.gaierror:
2869   log.info("Error With URL "+httpurl);
2870   return False;
2871  except socket.timeout:
2872   log.info("Error With URL "+httpurl);
2873   return False;
2874  ftp.login(urlparts.username, urlparts.password);
2875  if(urlparts.scheme=="ftps"):
2876   ftp.prot_p();
2877  ftpfile = BytesIO();
2878  ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
2879  #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
2880  ftp.close();
2881  ftpfile.seek(0, 0);
2882  return ftpfile;
2883
2884 def download_file_from_ftp_string(url):
2885  ftpfile = download_file_from_ftp_file(url);
2886  return ftpfile.read();
2887
2888 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2889  global geturls_download_sleep, havebrotli;
2890  if(sleep<0):
2891   sleep = geturls_download_sleep;
2892  urlparts = urlparse.urlparse(httpurl);
2893  if(isinstance(httpheaders, list)):
2894    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2895  if(isinstance(httpheaders, dict)):
2896   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2897  time.sleep(sleep);
2898  geturls_text = download_file_from_ftp_file(httpurl);
2899  log.info("Downloading URL "+httpurl);
2900  returnval_content = geturls_text.read()[:];
2901  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
2902  geturls_text.close();
2903  return returnval;
2904
2905 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2906  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2907  exec_time_start = time.time();
2908  myhash = hashlib.new("sha1");
2909  if(sys.version[0]=="2"):
2910   myhash.update(httpurl);
2911   myhash.update(str(buffersize));
2912   myhash.update(str(exec_time_start));
2913  if(sys.version[0]>="3"):
2914   myhash.update(httpurl.encode('utf-8'));
2915   myhash.update(str(buffersize).encode('utf-8'));
2916   myhash.update(str(exec_time_start).encode('utf-8'));
2917  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2918  if(sleep<0):
2919   sleep = geturls_download_sleep;
2920  urlparts = urlparse.urlparse(httpurl);
2921  if(isinstance(httpheaders, list)):
2922    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2923  if(isinstance(httpheaders, dict)):
2924   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
2925  time.sleep(sleep);
2926  geturls_text = download_file_from_ftp_file(httpurl);
2927  geturls_text.seek(0, 2);
2928  downloadsize = geturls_text.tell();
2929  geturls_text.seek(0, 0);
2930  if(downloadsize is not None):
2931   downloadsize = int(downloadsize);
2932  if downloadsize is None: downloadsize = 0;
2933  fulldatasize = 0;
2934  prevdownsize = 0;
2935  log.info("Downloading URL "+httpurl);
2936  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2937   tmpfilename = f.name;
2938   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
2939   while True:
2940    databytes = geturls_text.read(buffersize);
2941    if not databytes: break;
2942    datasize = len(databytes);
2943    fulldatasize = datasize + fulldatasize;
2944    percentage = "";
2945    if(downloadsize>0):
2946     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2947    downloaddiff = fulldatasize - prevdownsize;
2948    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2949    prevdownsize = fulldatasize;
2950    f.write(databytes);
2951   f.close();
2952  geturls_text.close();
2953  exec_time_end = time.time();
2954  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2955  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2956  return returnval;
2957
2958 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2959  global geturls_download_sleep;
2960  if(sleep<0):
2961   sleep = geturls_download_sleep;
2962  if(not outfile=="-"):
2963   outpath = outpath.rstrip(os.path.sep);
2964   filepath = os.path.realpath(outpath+os.path.sep+outfile);
2965   if(not os.path.exists(outpath)):
2966    os.makedirs(outpath);
2967   if(os.path.exists(outpath) and os.path.isfile(outpath)):
2968    return False;
2969   if(os.path.exists(filepath) and os.path.isdir(filepath)):
2970    return False;
2971   pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2972   if(not pretmpfilename):
2973    return False;
2974   tmpfilename = pretmpfilename['Filename'];
2975   downloadsize = os.path.getsize(tmpfilename);
2976   fulldatasize = 0;
2977   log.info("Moving file "+tmpfilename+" to "+filepath);
2978   exec_time_start = time.time();
2979   shutil.move(tmpfilename, filepath);
2980   exec_time_end = time.time();
2981   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2982   if(os.path.exists(tmpfilename)):
2983    os.remove(tmpfilename);
2984   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
2985  if(outfile=="-" and sys.version[0]=="2"):
2986   pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2987   if(not pretmpfilename):
2988    return False;
2989   tmpfilename = pretmpfilename['Filename'];
2990   downloadsize = os.path.getsize(tmpfilename);
2991   fulldatasize = 0;
2992   prevdownsize = 0;
2993   exec_time_start = time.time();
2994   with open(tmpfilename, 'rb') as ft:
2995    f = StringIO();
2996    while True:
2997     databytes = ft.read(buffersize[1]);
2998     if not databytes: break;
2999     datasize = len(databytes);
3000     fulldatasize = datasize + fulldatasize;
3001     percentage = "";
3002     if(downloadsize>0):
3003      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3004     downloaddiff = fulldatasize - prevdownsize;
3005     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3006     prevdownsize = fulldatasize;
3007     f.write(databytes);
3008    f.seek(0);
3009    fdata = f.getvalue();
3010    f.close();
3011    ft.close();
3012    os.remove(tmpfilename);
3013    exec_time_end = time.time();
3014    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3015   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
3016  if(outfile=="-" and sys.version[0]>="3"):
3017   pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3018   tmpfilename = pretmpfilename['Filename'];
3019   downloadsize = os.path.getsize(tmpfilename);
3020   fulldatasize = 0;
3021   prevdownsize = 0;
3022   exec_time_start = time.time();
3023   with open(tmpfilename, 'rb') as ft:
3024    f = BytesIO();
3025    while True:
3026     databytes = ft.read(buffersize[1]);
3027     if not databytes: break;
3028     datasize = len(databytes);
3029     fulldatasize = datasize + fulldatasize;
3030     percentage = "";
3031     if(downloadsize>0):
3032      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3033     downloaddiff = fulldatasize - prevdownsize;
3034     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3035     prevdownsize = fulldatasize;
3036     f.write(databytes);
3037    f.seek(0);
3038    fdata = f.getvalue();
3039    f.close();
3040    ft.close();
3041    os.remove(tmpfilename);
3042    exec_time_end = time.time();
3043    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3044   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
3045  return returnval;
3046
3047 def upload_file_to_ftp_file(ftpfile, url):
3048  urlparts = urlparse.urlparse(url);
3049  file_name = os.path.basename(urlparts.path);
3050  file_dir = os.path.dirname(urlparts.path);
3051  if(urlparts.username is not None):
3052   ftp_username = urlparts.username;
3053  else:
3054   ftp_username = "anonymous";
3055  if(urlparts.password is not None):
3056   ftp_password = urlparts.password;
3057  elif(urlparts.password is None and urlparts.username=="anonymous"):
3058   ftp_password = "anonymous";
3059  else:
3060   ftp_password = "";
3061  if(urlparts.scheme=="ftp"):
3062   ftp = FTP();
3063  elif(urlparts.scheme=="ftps"):
3064   ftp = FTP_TLS();
3065  else:
3066   return False;
3067  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
3068   return False;
3069  ftp_port = urlparts.port;
3070  if(urlparts.port is None):
3071   ftp_port = 21;
3072  try:
3073   ftp.connect(urlparts.hostname, ftp_port);
3074  except socket.gaierror:
3075   log.info("Error With URL "+httpurl);
3076   return False;
3077  except socket.timeout:
3078   log.info("Error With URL "+httpurl);
3079   return False;
3080  ftp.login(urlparts.username, urlparts.password);
3081  if(urlparts.scheme=="ftps"):
3082   ftp.prot_p();
3083  ftp.storbinary("STOR "+urlparts.path, ftpfile);
3084  ftp.close();
3085  ftpfile.seek(0, 0);
3086  return ftpfile;
3087
3088 def upload_file_to_ftp_string(ftpstring, url):
3089  ftpfileo = BytesIO(ftpstring);
3090  ftpfile = upload_file_to_ftp_file(ftpfileo, url);
3091  ftpfileo.close();
3092  return ftpfile;
3093
3094 if(haveparamiko):
3095  def download_file_from_sftp_file(url):
3096   urlparts = urlparse.urlparse(url);
3097   file_name = os.path.basename(urlparts.path);
3098   file_dir = os.path.dirname(urlparts.path);
3099   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
3100    return False;
3101   sftp_port = urlparts.port;
3102   if(urlparts.port is None):
3103    sftp_port = 22;
3104   else:
3105    sftp_port = urlparts.port;
3106   if(urlparts.username is not None):
3107    sftp_username = urlparts.username;
3108   else:
3109    sftp_username = "anonymous";
3110   if(urlparts.password is not None):
3111    sftp_password = urlparts.password;
3112   elif(urlparts.password is None and urlparts.username=="anonymous"):
3113    sftp_password = "anonymous";
3114   else:
3115    sftp_password = "";
3116   if(urlparts.scheme!="sftp"):
3117    return False;
3118   ssh = paramiko.SSHClient();
3119   ssh.load_system_host_keys();
3120   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
3121   try:
3122    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
3123   except paramiko.ssh_exception.SSHException:
3124    return False;
3125   except socket.gaierror:
3126    log.info("Error With URL "+httpurl);
3127    return False;
3128   except socket.timeout:
3129    log.info("Error With URL "+httpurl);
3130    return False;
3131   sftp = ssh.open_sftp();
3132   sftpfile = BytesIO();
3133   sftp.getfo(urlparts.path, sftpfile);
3134   sftp.close();
3135   ssh.close();
3136   sftpfile.seek(0, 0);
3137   return sftpfile;
3138 else:
3139  def download_file_from_sftp_file(url):
3140   return False;
3141
3142 if(haveparamiko):
3143  def download_file_from_sftp_string(url):
3144   sftpfile = download_file_from_sftp_file(url);
3145   return sftpfile.read();
3146 else:
3147  def download_file_from_ftp_string(url):
3148   return False;
3149
3150 if(haveparamiko):
3151  def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3152   global geturls_download_sleep, havebrotli;
3153   if(sleep<0):
3154    sleep = geturls_download_sleep;
3155   urlparts = urlparse.urlparse(httpurl);
3156   if(isinstance(httpheaders, list)):
3157     httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3158   if(isinstance(httpheaders, dict)):
3159    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
3160   time.sleep(sleep);
3161   geturls_text = download_file_from_sftp_file(httpurl);
3162   log.info("Downloading URL "+httpurl);
3163   returnval_content = geturls_text.read()[:];
3164   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
3165   geturls_text.close();
3166   return returnval;
3167
3168 if(not haveparamiko):
3169  def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3170   return False;
3171
3172 if(haveparamiko):
3173  def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3174   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
3175   exec_time_start = time.time();
3176   myhash = hashlib.new("sha1");
3177   if(sys.version[0]=="2"):
3178    myhash.update(httpurl);
3179    myhash.update(str(buffersize));
3180    myhash.update(str(exec_time_start));
3181   if(sys.version[0]>="3"):
3182    myhash.update(httpurl.encode('utf-8'));
3183    myhash.update(str(buffersize).encode('utf-8'));
3184    myhash.update(str(exec_time_start).encode('utf-8'));
3185   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
3186   if(sleep<0):
3187    sleep = geturls_download_sleep;
3188   urlparts = urlparse.urlparse(httpurl);
3189   if(isinstance(httpheaders, list)):
3190     httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3191   if(isinstance(httpheaders, dict)):
3192    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
3193   time.sleep(sleep);
3194   geturls_text = download_file_from_sftp_file(httpurl);
3195   geturls_text.seek(0, 2);
3196   downloadsize = geturls_text.tell();
3197   geturls_text.seek(0, 0);
3198   if(downloadsize is not None):
3199    downloadsize = int(downloadsize);
3200   if downloadsize is None: downloadsize = 0;
3201   fulldatasize = 0;
3202   prevdownsize = 0;
3203   log.info("Downloading URL "+httpurl);
3204   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
3205    tmpfilename = f.name;
3206    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
3207    while True:
3208     databytes = geturls_text.read(buffersize);
3209     if not databytes: break;
3210     datasize = len(databytes);
3211     fulldatasize = datasize + fulldatasize;
3212     percentage = "";
3213     if(downloadsize>0):
3214      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3215     downloaddiff = fulldatasize - prevdownsize;
3216     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3217     prevdownsize = fulldatasize;
3218     f.write(databytes);
3219    f.close();
3220   geturls_text.close();
3221   exec_time_end = time.time();
3222   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
3223   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
3224   return returnval;
3225
3226 if(not haveparamiko):
3227  def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3228   return False;
3229
3230 if(haveparamiko):
3231  def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3232   global geturls_download_sleep;
3233   if(sleep<0):
3234    sleep = geturls_download_sleep;
3235   if(not outfile=="-"):
3236    outpath = outpath.rstrip(os.path.sep);
3237    filepath = os.path.realpath(outpath+os.path.sep+outfile);
3238    if(not os.path.exists(outpath)):
3239     os.makedirs(outpath);
3240    if(os.path.exists(outpath) and os.path.isfile(outpath)):
3241     return False;
3242    if(os.path.exists(filepath) and os.path.isdir(filepath)):
3243     return False;
3244    pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3245    if(not pretmpfilename):
3246     return False;
3247    tmpfilename = pretmpfilename['Filename'];
3248    downloadsize = os.path.getsize(tmpfilename);
3249    fulldatasize = 0;
3250    log.info("Moving file "+tmpfilename+" to "+filepath);
3251    exec_time_start = time.time();
3252    shutil.move(tmpfilename, filepath);
3253    exec_time_end = time.time();
3254    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
3255    if(os.path.exists(tmpfilename)):
3256     os.remove(tmpfilename);
3257    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
3258   if(outfile=="-" and sys.version[0]=="2"):
3259    pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3260    if(not pretmpfilename):
3261     return False;
3262    tmpfilename = pretmpfilename['Filename'];
3263    downloadsize = os.path.getsize(tmpfilename);
3264    fulldatasize = 0;
3265    prevdownsize = 0;
3266    exec_time_start = time.time();
3267    with open(tmpfilename, 'rb') as ft:
3268     f = StringIO();
3269     while True:
3270      databytes = ft.read(buffersize[1]);
3271      if not databytes: break;
3272      datasize = len(databytes);
3273      fulldatasize = datasize + fulldatasize;
3274      percentage = "";
3275      if(downloadsize>0):
3276       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3277      downloaddiff = fulldatasize - prevdownsize;
3278      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3279      prevdownsize = fulldatasize;
3280      f.write(databytes);
3281     f.seek(0);
3282     fdata = f.getvalue();
3283     f.close();
3284     ft.close();
3285     os.remove(tmpfilename);
3286     exec_time_end = time.time();
3287     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3288    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
3289   if(outfile=="-" and sys.version[0]>="3"):
3290    pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3291    tmpfilename = pretmpfilename['Filename'];
3292    downloadsize = os.path.getsize(tmpfilename);
3293    fulldatasize = 0;
3294    prevdownsize = 0;
3295    exec_time_start = time.time();
3296    with open(tmpfilename, 'rb') as ft:
3297     f = BytesIO();
3298     while True:
3299      databytes = ft.read(buffersize[1]);
3300      if not databytes: break;
3301      datasize = len(databytes);
3302      fulldatasize = datasize + fulldatasize;
3303      percentage = "";
3304      if(downloadsize>0):
3305       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3306      downloaddiff = fulldatasize - prevdownsize;
3307      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3308      prevdownsize = fulldatasize;
3309      f.write(databytes);
3310     f.seek(0);
3311     fdata = f.getvalue();
3312     f.close();
3313     ft.close();
3314     os.remove(tmpfilename);
3315     exec_time_end = time.time();
3316     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3317    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
3318   return returnval;
3319
3320 if(not haveparamiko):
3321  def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3322   return False;
3323
3324 if(haveparamiko):
3325  def upload_file_to_sftp_file(sftpfile, url):
3326   urlparts = urlparse.urlparse(url);
3327   file_name = os.path.basename(urlparts.path);
3328   file_dir = os.path.dirname(urlparts.path);
3329   sftp_port = urlparts.port;
3330   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
3331    return False;
3332   if(urlparts.port is None):
3333    sftp_port = 22;
3334   else:
3335    sftp_port = urlparts.port;
3336   if(urlparts.username is not None):
3337    sftp_username = urlparts.username;
3338   else:
3339    sftp_username = "anonymous";
3340   if(urlparts.password is not None):
3341    sftp_password = urlparts.password;
3342   elif(urlparts.password is None and urlparts.username=="anonymous"):
3343    sftp_password = "anonymous";
3344   else:
3345    sftp_password = "";
3346   if(urlparts.scheme!="sftp"):
3347    return False;
3348   ssh = paramiko.SSHClient();
3349   ssh.load_system_host_keys();
3350   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
3351   try:
3352    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
3353   except paramiko.ssh_exception.SSHException:
3354    return False;
3355   except socket.gaierror:
3356    log.info("Error With URL "+httpurl);
3357    return False;
3358   except socket.timeout:
3359    log.info("Error With URL "+httpurl);
3360    return False;
3361   sftp = ssh.open_sftp();
3362   sftp.putfo(sftpfile, urlparts.path);
3363   sftp.close();
3364   ssh.close();
3365   sftpfile.seek(0, 0);
3366   return sftpfile;
3367 else:
3368  def upload_file_to_sftp_file(sftpfile, url):
3369   return False;
3370
3371 if(haveparamiko):
3372  def upload_file_to_sftp_string(sftpstring, url):
3373   sftpfileo = BytesIO(sftpstring);
3374   sftpfile = upload_file_to_sftp_files(ftpfileo, url);
3375   sftpfileo.close();
3376   return sftpfile;
3377 else:
3378  def upload_file_to_sftp_string(url):
3379   return False;