pywwwgetold.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016-2023 Game Maker 2k - https://github.com/GameMaker2k
  13     Copyright 2016-2023 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  14
  15     $FileInfo: pywwwgetold.py - Last Update: 9/24/2023 Ver. 1.5.0 RC 1 - Author: cooldude2k $
  16 '''
  17
  18 from __future__ import division, absolute_import, print_function;
  19 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, argparse, cgi, subprocess, socket, email.utils, datetime, time;
  20 import logging as log;
  21 from ftplib import FTP, FTP_TLS;
  22 from base64 import b64encode;
  23 haverequests = False;
  24 try:
  25  import requests;
  26  haverequests = True;
  27 except ImportError:
  28  haverequests = False;
  29 havemechanize = False;
  30 try:
  31  import mechanize;
  32  havemechanize = True;
  33 except ImportError:
  34  havemechanize = False;
  35 haveparamiko = False;
  36 try:
  37  import paramiko;
  38  haveparamiko = True;
  39 except ImportError:
  40  haveparamiko = False;
  41 havepysftp = False;
  42 try:
  43  import pysftp;
  44  havepysftp = True;
  45 except ImportError:
  46  havepysftp = False;
  47 haveurllib3 = False;
  48 try:
  49  import urllib3;
  50  haveurllib3 = True;
  51 except ImportError:
  52  haveurllib3 = False;
  53 havehttplib2 = False;
  54 try:
  55  from httplib2 import HTTPConnectionWithTimeout, HTTPSConnectionWithTimeout;
  56  havehttplib2 = True;
  57 except ImportError:
  58  havehttplib2 = False;
  59 havehttpx = False;
  60 try:
  61  import httpx;
  62  havehttpx = True;
  63 except ImportError:
  64  havehttpx = False;
  65 havehttpcore = False;
  66 try:
  67  import httpcore;
  68  havehttpcore = True;
  69 except ImportError:
  70  havehttpcore = False;
  71 havebrotli = False;
  72 try:
  73  import brotli;
  74  havebrotli = True;
  75 except ImportError:
  76  havebrotli = False;
  77 havezstd = False;
  78 try:
  79  import zstandard;
  80  havezstd = True;
  81 except ImportError:
  82  havezstd = False;
  83 if(sys.version[0]=="2"):
  84  try:
  85   from cStringIO import StringIO;
  86  except ImportError:
  87   from StringIO import StringIO;
  88  # From http://python-future.org/compatible_idioms.html
  89  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
  90  from urllib import urlencode;
  91  from urllib import urlopen as urlopenalt;
  92  from urllib2 import urlopen, Request, install_opener, HTTPError, URLError, build_opener, HTTPCookieProcessor;
  93  import urlparse, cookielib;
  94  from httplib import HTTPConnection, HTTPSConnection;
  95 if(sys.version[0]>="3"):
  96  from io import StringIO, BytesIO;
  97  # From http://python-future.org/compatible_idioms.html
  98  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
  99  from urllib.request import urlopen, Request, install_opener, build_opener, HTTPCookieProcessor;
 100  from urllib.error import HTTPError, URLError;
 101  import urllib.parse as urlparse;
 102  import http.cookiejar as cookielib;
 103  from http.client import HTTPConnection, HTTPSConnection;
 104
 105 __program_name__ = "PyWWW-Get";
 106 __program_alt_name__ = "PyWWWGet";
 107 __program_small_name__ = "wwwget";
 108 __project__ = __program_name__;
 109 __project_url__ = "https://github.com/GameMaker2k/PyWWW-Get";
 110 __version_info__ = (1, 5, 0, "RC 1", 1);
 111 __version_date_info__ = (2023, 9, 24, "RC 1", 1);
 112 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
 113 __revision__ = __version_info__[3];
 114 __revision_id__ = "$Id$";
 115 if(__version_info__[4] is not None):
 116  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
 117 if(__version_info__[4] is None):
 118  __version_date_plusrc__ = __version_date__;
 119 if(__version_info__[3] is not None):
 120  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
 121 if(__version_info__[3] is None):
 122  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
 123
 124 tmpfileprefix = "py"+str(sys.version_info[0])+__program_small_name__+str(__version_info__[0])+"-";
 125 tmpfilesuffix = "-";
 126 pytempdir = tempfile.gettempdir();
 127
 128 PyBitness = platform.architecture();
 129 if(PyBitness=="32bit" or PyBitness=="32"):
 130  PyBitness = "32";
 131 elif(PyBitness=="64bit" or PyBitness=="64"):
 132  PyBitness = "64";
 133 else:
 134  PyBitness = "32";
 135
 136 compression_supported = "gzip, deflate";
 137 if(havebrotli):
 138  compression_supported = "gzip, deflate, br";
 139 else:
 140  compression_supported = "gzip, deflate";
 141
 142 geturls_cj = cookielib.CookieJar();
 143 windowsNT4_ua_string = "Windows NT 4.0";
 144 windowsNT4_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "4.0.0"};
 145 windows2k_ua_string = "Windows NT 5.0";
 146 windows2k_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.0.0"};
 147 windowsXP_ua_string = "Windows NT 5.1";
 148 windowsXP_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "32", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 149 windowsXP64_ua_string = "Windows NT 5.2; Win64; x64";
 150 windowsXP64_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "5.1.0"};
 151 windows7_ua_string = "Windows NT 6.1; Win64; x64";
 152 windows7_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.1.0"};
 153 windows8_ua_string = "Windows NT 6.2; Win64; x64";
 154 windows8_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.2.0"};
 155 windows81_ua_string = "Windows NT 6.3; Win64; x64";
 156 windows81_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "6.3.0"};
 157 windows10_ua_string = "Windows NT 10.0; Win64; x64";
 158 windows10_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "10.0.0"};
 159 windows11_ua_string = "Windows NT 11.0; Win64; x64";
 160 windows11_ua_addon = {'SEC-CH-UA-PLATFORM': "Windows", 'SEC-CH-UA-ARCH': "x86", 'SEC-CH-UA-BITNESS': "64", 'SEC-CH-UA-PLATFORM': "11.0.0"};
 161 geturls_ua_firefox_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:109.0) Gecko/20100101 Firefox/117.0";
 162 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:91.0) Gecko/20100101 Firefox/91.0 SeaMonkey/2.53.17";
 163 geturls_ua_chrome_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36";
 164 geturls_ua_chromium_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chromium/117.0.0.0 Chrome/117.0.0.0 Safari/537.36";
 165 geturls_ua_palemoon_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; rv:102.0) Gecko/20100101 Goanna/6.3 Firefox/102.0 PaleMoon/32.4.0.1";
 166 geturls_ua_opera_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 OPR/102.0.0.0";
 167 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Vivaldi/6.2.3105.48";
 168 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 ("+windows7_ua_string+"; Trident/7.0; rv:11.0) like Gecko";
 169 geturls_ua_microsoft_edge_windows7 = "Mozilla/5.0 ("+windows7_ua_string+") AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 Edg/117.0.2045.31";
 170 geturls_ua_pywwwget_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
 171 if(platform.python_implementation()!=""):
 172  py_implementation = platform.python_implementation();
 173 if(platform.python_implementation()==""):
 174  py_implementation = "Python";
 175 geturls_ua_pywwwget_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
 176 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
 177 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
 178 geturls_ua = geturls_ua_firefox_windows7;
 179 geturls_headers_firefox_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 180 geturls_headers_seamonkey_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 181 geturls_headers_chrome_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 182 geturls_headers_chrome_windows7.update(windows7_ua_addon);
 183 geturls_headers_chromium_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"24\"", 'SEC-CH-UA-FULL-VERSION': "117.0.5938.63"};
 184 geturls_headers_chromium_windows7.update(windows7_ua_addon);
 185 geturls_headers_palemoon_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 186 geturls_headers_opera_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Chromium\";v=\"116\", \"Not;A=Brand\";v=\"8\", \"Opera\";v=\"102\"", 'SEC-CH-UA-FULL-VERSION': "102.0.4880.56"};
 187 geturls_headers_opera_windows7.update(windows7_ua_addon);
 188 geturls_headers_vivaldi_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Google Chrome\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Vivaldi\";v=\"6.2\"", 'SEC-CH-UA-FULL-VERSION': "6.2.3105.48"};
 189 geturls_headers_vivaldi_windows7.update(windows7_ua_addon);
 190 geturls_headers_internet_explorer_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 191 geturls_headers_microsoft_edge_windows7 = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_microsoft_edge_windows7, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\"Microsoft Edge\";v=\"117\", \"Not;A=Brand\";v=\"8\", \"Chromium\";v=\"117\"", 'SEC-CH-UA-FULL-VERSION': "117.0.2045.31"}
 192 geturls_headers_microsoft_edge_windows7.update(windows7_ua_addon);
 193 geturls_headers_pywwwget_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 194 geturls_headers_pywwwget_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pywwwget_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
 195 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 196 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 197 geturls_headers = geturls_headers_firefox_windows7;
 198 geturls_download_sleep = 0;
 199
 200 def verbose_printout(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 201  if(outtype=="print" and dbgenable):
 202   print(dbgtxt);
 203   return True;
 204  elif(outtype=="log" and dbgenable):
 205   logging.info(dbgtxt);
 206   return True;
 207  elif(outtype=="warning" and dbgenable):
 208   logging.warning(dbgtxt);
 209   return True;
 210  elif(outtype=="error" and dbgenable):
 211   logging.error(dbgtxt);
 212   return True;
 213  elif(outtype=="critical" and dbgenable):
 214   logging.critical(dbgtxt);
 215   return True;
 216  elif(outtype=="exception" and dbgenable):
 217   logging.exception(dbgtxt);
 218   return True;
 219  elif(outtype=="logalt" and dbgenable):
 220   logging.log(dgblevel, dbgtxt);
 221   return True;
 222  elif(outtype=="debug" and dbgenable):
 223   logging.debug(dbgtxt);
 224   return True;
 225  elif(not dbgenable):
 226   return True;
 227  else:
 228   return False;
 229  return False;
 230
 231 def verbose_printout_return(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
 232  dbgout = verbose_printout(dbgtxt, outtype, dbgenable, dgblevel);
 233  if(not dbgout):
 234   return False;
 235  return dbgtxt;
 236
 237 def add_url_param(url, **params):
 238  n=3;
 239  parts = list(urlparse.urlsplit(url));
 240  d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 241  d.update(params);
 242  parts[n]=urlencode(d);
 243  return urlparse.urlunsplit(parts);
 244
 245 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 246 def which_exec(execfile):
 247  for path in os.environ["PATH"].split(":"):
 248   if os.path.exists(path + "/" + execfile):
 249    return path + "/" + execfile;
 250
 251 def listize(varlist):
 252  il = 0;
 253  ix = len(varlist);
 254  ilx = 1;
 255  newlistreg = {};
 256  newlistrev = {};
 257  newlistfull = {};
 258  while(il < ix):
 259   newlistreg.update({ilx: varlist[il]});
 260   newlistrev.update({varlist[il]: ilx});
 261   ilx = ilx + 1;
 262   il = il + 1;
 263  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 264  return newlistfull;
 265
 266 def twolistize(varlist):
 267  il = 0;
 268  ix = len(varlist);
 269  ilx = 1;
 270  newlistnamereg = {};
 271  newlistnamerev = {};
 272  newlistdescreg = {};
 273  newlistdescrev = {};
 274  newlistfull = {};
 275  while(il < ix):
 276   newlistnamereg.update({ilx: varlist[il][0].strip()});
 277   newlistnamerev.update({varlist[il][0].strip(): ilx});
 278   newlistdescreg.update({ilx: varlist[il][1].strip()});
 279   newlistdescrev.update({varlist[il][1].strip(): ilx});
 280   ilx = ilx + 1;
 281   il = il + 1;
 282  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 283  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 284  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 285  return newlistfull;
 286
 287 def arglistize(proexec, *varlist):
 288  il = 0;
 289  ix = len(varlist);
 290  ilx = 1;
 291  newarglist = [proexec];
 292  while(il < ix):
 293   if varlist[il][0] is not None:
 294    newarglist.append(varlist[il][0]);
 295   if varlist[il][1] is not None:
 296    newarglist.append(varlist[il][1]);
 297   il = il + 1;
 298  return newarglist;
 299
 300 def fix_header_names(header_dict):
 301  header_dict =  {k.title(): v for k, v in header_dict.items()};
 302  return header_dict;
 303
 304 # hms_string by ArcGIS Python Recipes
 305 # https://arcpy.wordpress.com/2012/04/20/146/
 306 def hms_string(sec_elapsed):
 307  h = int(sec_elapsed / (60 * 60));
 308  m = int((sec_elapsed % (60 * 60)) / 60);
 309  s = sec_elapsed % 60.0;
 310  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 311
 312 # get_readable_size by Lipis
 313 # http://stackoverflow.com/posts/14998888/revisions
 314 def get_readable_size(bytes, precision=1, unit="IEC"):
 315  unit = unit.upper();
 316  if(unit!="IEC" and unit!="SI"):
 317   unit = "IEC";
 318  if(unit=="IEC"):
 319   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 320   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 321   unitsize = 1024.0;
 322  if(unit=="SI"):
 323   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 324   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 325   unitsize = 1000.0;
 326  return_val = {};
 327  orgbytes = bytes;
 328  for unit in units:
 329   if abs(bytes) < unitsize:
 330    strformat = "%3."+str(precision)+"f%s";
 331    pre_return_val = (strformat % (bytes, unit));
 332    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 333    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 334    alt_return_val = pre_return_val.split();
 335    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 336    return return_val;
 337   bytes /= unitsize;
 338  strformat = "%."+str(precision)+"f%s";
 339  pre_return_val = (strformat % (bytes, "YiB"));
 340  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 341  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 342  alt_return_val = pre_return_val.split();
 343  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 344  return return_val;
 345
 346 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 347  unit = unit.upper();
 348  usehashtypes = usehashtypes.lower();
 349  getfilesize = os.path.getsize(infile);
 350  return_val = get_readable_size(getfilesize, precision, unit);
 351  if(usehashes):
 352   hashtypelist = usehashtypes.split(",");
 353   openfile = open(infile, "rb");
 354   filecontents = openfile.read();
 355   openfile.close();
 356   listnumcount = 0;
 357   listnumend = len(hashtypelist);
 358   while(listnumcount < listnumend):
 359    hashtypelistlow = hashtypelist[listnumcount].strip();
 360    hashtypelistup = hashtypelistlow.upper();
 361    filehash = hashlib.new(hashtypelistup);
 362    filehash.update(filecontents);
 363    filegethash = filehash.hexdigest();
 364    return_val.update({hashtypelistup: filegethash});
 365    listnumcount += 1;
 366  return return_val;
 367
 368 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 369  unit = unit.upper();
 370  usehashtypes = usehashtypes.lower();
 371  getfilesize = len(instring);
 372  return_val = get_readable_size(getfilesize, precision, unit);
 373  if(usehashes):
 374   hashtypelist = usehashtypes.split(",");
 375   listnumcount = 0;
 376   listnumend = len(hashtypelist);
 377   while(listnumcount < listnumend):
 378    hashtypelistlow = hashtypelist[listnumcount].strip();
 379    hashtypelistup = hashtypelistlow.upper();
 380    filehash = hashlib.new(hashtypelistup);
 381    if(sys.version[0]=="2"):
 382     filehash.update(instring);
 383    if(sys.version[0]>="3"):
 384     filehash.update(instring.encode('utf-8'));
 385    filegethash = filehash.hexdigest();
 386    return_val.update({hashtypelistup: filegethash});
 387    listnumcount += 1;
 388  return return_val;
 389
 390 def http_status_to_reason(code):
 391     reasons = {
 392         100: 'Continue',
 393         101: 'Switching Protocols',
 394         102: 'Processing',
 395         200: 'OK',
 396         201: 'Created',
 397         202: 'Accepted',
 398         203: 'Non-Authoritative Information',
 399         204: 'No Content',
 400         205: 'Reset Content',
 401         206: 'Partial Content',
 402         207: 'Multi-Status',
 403         208: 'Already Reported',
 404         226: 'IM Used',
 405         300: 'Multiple Choices',
 406         301: 'Moved Permanently',
 407         302: 'Found',
 408         303: 'See Other',
 409         304: 'Not Modified',
 410         305: 'Use Proxy',
 411         307: 'Temporary Redirect',
 412         308: 'Permanent Redirect',
 413         400: 'Bad Request',
 414         401: 'Unauthorized',
 415         402: 'Payment Required',
 416         403: 'Forbidden',
 417         404: 'Not Found',
 418         405: 'Method Not Allowed',
 419         406: 'Not Acceptable',
 420         407: 'Proxy Authentication Required',
 421         408: 'Request Timeout',
 422         409: 'Conflict',
 423         410: 'Gone',
 424         411: 'Length Required',
 425         412: 'Precondition Failed',
 426         413: 'Payload Too Large',
 427         414: 'URI Too Long',
 428         415: 'Unsupported Media Type',
 429         416: 'Range Not Satisfiable',
 430         417: 'Expectation Failed',
 431         421: 'Misdirected Request',
 432         422: 'Unprocessable Entity',
 433         423: 'Locked',
 434         424: 'Failed Dependency',
 435         426: 'Upgrade Required',
 436         428: 'Precondition Required',
 437         429: 'Too Many Requests',
 438         431: 'Request Header Fields Too Large',
 439         451: 'Unavailable For Legal Reasons',
 440         500: 'Internal Server Error',
 441         501: 'Not Implemented',
 442         502: 'Bad Gateway',
 443         503: 'Service Unavailable',
 444         504: 'Gateway Timeout',
 445         505: 'HTTP Version Not Supported',
 446         506: 'Variant Also Negotiates',
 447         507: 'Insufficient Storage',
 448         508: 'Loop Detected',
 449         510: 'Not Extended',
 450         511: 'Network Authentication Required'
 451     };
 452
 453     return reasons.get(code, 'Unknown Status Code');
 454
 455 def ftp_status_to_reason(code):
 456     reasons = {
 457         110: 'Restart marker reply',
 458         120: 'Service ready in nnn minutes',
 459         125: 'Data connection already open; transfer starting',
 460         150: 'File status okay; about to open data connection',
 461         200: 'Command okay',
 462         202: 'Command not implemented, superfluous at this site',
 463         211: 'System status, or system help reply',
 464         212: 'Directory status',
 465         213: 'File status',
 466         214: 'Help message',
 467         215: 'NAME system type',
 468         220: 'Service ready for new user',
 469         221: 'Service closing control connection',
 470         225: 'Data connection open; no transfer in progress',
 471         226: 'Closing data connection',
 472         227: 'Entering Passive Mode',
 473         230: 'User logged in, proceed',
 474         250: 'Requested file action okay, completed',
 475         257: '"PATHNAME" created',
 476         331: 'User name okay, need password',
 477         332: 'Need account for login',
 478         350: 'Requested file action pending further information',
 479         421: 'Service not available, closing control connection',
 480         425: 'Can\'t open data connection',
 481         426: 'Connection closed; transfer aborted',
 482         450: 'Requested file action not taken',
 483         451: 'Requested action aborted. Local error in processing',
 484         452: 'Requested action not taken. Insufficient storage space in system',
 485         500: 'Syntax error, command unrecognized',
 486         501: 'Syntax error in parameters or arguments',
 487         502: 'Command not implemented',
 488         503: 'Bad sequence of commands',
 489         504: 'Command not implemented for that parameter',
 490         530: 'Not logged in',
 491         532: 'Need account for storing files',
 492         550: 'Requested action not taken. File unavailable',
 493         551: 'Requested action aborted. Page type unknown',
 494         552: 'Requested file action aborted. Exceeded storage allocation',
 495         553: 'Requested action not taken. File name not allowed'
 496     };
 497     return reasons.get(code, 'Unknown Status Code');
 498
 499 def sftp_status_to_reason(code):
 500     reasons = {
 501         0: 'SSH_FX_OK',
 502         1: 'SSH_FX_EOF',
 503         2: 'SSH_FX_NO_SUCH_FILE',
 504         3: 'SSH_FX_PERMISSION_DENIED',
 505         4: 'SSH_FX_FAILURE',
 506         5: 'SSH_FX_BAD_MESSAGE',
 507         6: 'SSH_FX_NO_CONNECTION',
 508         7: 'SSH_FX_CONNECTION_LOST',
 509         8: 'SSH_FX_OP_UNSUPPORTED'
 510     };
 511     return reasons.get(code, 'Unknown Status Code
 512
 513 def make_http_headers_from_dict_to_list(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 514  if isinstance(headers, dict):
 515   returnval = [];
 516   if(sys.version[0]=="2"):
 517    for headkey, headvalue in headers.iteritems():
 518     returnval.append((headkey, headvalue));
 519   if(sys.version[0]>="3"):
 520    for headkey, headvalue in headers.items():
 521     returnval.append((headkey, headvalue));
 522  elif isinstance(headers, list):
 523   returnval = headers;
 524  else:
 525   returnval = False;
 526  return returnval;
 527
 528 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://google.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': compression_supported, 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 529  if isinstance(headers, dict):
 530   returnval = [];
 531   if(sys.version[0]=="2"):
 532    for headkey, headvalue in headers.iteritems():
 533     returnval.append(headkey+": "+headvalue);
 534   if(sys.version[0]>="3"):
 535    for headkey, headvalue in headers.items():
 536     returnval.append(headkey+": "+headvalue);
 537  elif isinstance(headers, list):
 538   returnval = headers;
 539  else:
 540   returnval = False;
 541  return returnval;
 542
 543 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://google.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", compression_supported), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 544  if isinstance(headers, list):
 545   returnval = {};
 546   mli = 0;
 547   mlil = len(headers);
 548   while(mli<mlil):
 549    returnval.update({headers[mli][0]: headers[mli][1]});
 550    mli = mli + 1;
 551  elif isinstance(headers, dict):
 552   returnval = headers;
 553  else:
 554   returnval = False;
 555  return returnval;
 556
 557 def get_httplib_support(checkvalue=None):
 558  global haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 559  returnval = [];
 560  returnval.append("ftp");
 561  returnval.append("httplib");
 562  if(havehttplib2):
 563   returnval.append("httplib2");
 564  returnval.append("urllib");
 565  if(haveurllib3):
 566   returnval.append("urllib3");
 567   returnval.append("request3");
 568  returnval.append("request");
 569  if(haverequests):
 570   returnval.append("requests");
 571  if(havehttpx):
 572   returnval.append("httpx");
 573   returnval.append("httpx2");
 574  if(havemechanize):
 575   returnval.append("mechanize");
 576  if(haveparamiko):
 577   returnval.append("sftp");
 578  if(havepysftp):
 579   returnval.append("pysftp");
 580  if(not checkvalue is None):
 581   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 582    checkvalue = "urllib";
 583   if(checkvalue=="httplib1"):
 584    checkvalue = "httplib";
 585   if(checkvalue in returnval):
 586    returnval = True;
 587   else:
 588    returnval = False;
 589  return returnval;
 590
 591 def check_httplib_support(checkvalue="urllib"):
 592  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 593   checkvalue = "urllib";
 594  if(checkvalue=="httplib1"):
 595   checkvalue = "httplib";
 596  returnval = get_httplib_support(checkvalue);
 597  return returnval;
 598
 599 def get_httplib_support_list():
 600  returnval = get_httplib_support(None);
 601  return returnval;
 602
 603 def download_from_url(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", sleep=-1):
 604  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 605  if(sleep<0):
 606   sleep = geturls_download_sleep;
 607  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 608   httplibuse = "urllib";
 609  if(httplibuse=="httplib1"):
 610   httplibuse = "httplib";
 611  if(not haverequests and httplibuse=="requests"):
 612   httplibuse = "urllib";
 613  if(not havehttpx and httplibuse=="httpx"):
 614   httplibuse = "urllib";
 615  if(not havehttpx and httplibuse=="httpx2"):
 616   httplibuse = "urllib";
 617  if(not havehttpcore and httplibuse=="httpcore"):
 618   httplibuse = "urllib";
 619  if(not havehttpcore and httplibuse=="httpcore2"):
 620   httplibuse = "urllib";
 621  if(not havemechanize and httplibuse=="mechanize"):
 622   httplibuse = "urllib";
 623  if(not havehttplib2 and httplibuse=="httplib2"):
 624   httplibuse = "httplib";
 625  if(not haveparamiko and httplibuse=="sftp"):
 626   httplibuse = "ftp";
 627  if(not havepysftp and httplibuse=="pysftp"):
 628   httplibuse = "ftp";
 629  if(httplibuse=="urllib"):
 630   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 631  elif(httplibuse=="request"):
 632   returnval = download_from_url_with_request(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 633  elif(httplibuse=="request3"):
 634   returnval = download_from_url_with_request3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 635  elif(httplibuse=="httplib"):
 636   returnval = download_from_url_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 637  elif(httplibuse=="httplib2"):
 638   returnval = download_from_url_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 639  elif(httplibuse=="urllib3"):
 640   returnval = download_from_url_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 641  elif(httplibuse=="requests"):
 642   returnval = download_from_url_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 643  elif(httplibuse=="httpx"):
 644   returnval = download_from_url_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 645  elif(httplibuse=="httpx2"):
 646   returnval = download_from_url_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 647  elif(httplibuse=="httpcore"):
 648   returnval = download_from_url_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 649  elif(httplibuse=="httpcore2"):
 650   returnval = download_from_url_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 651  elif(httplibuse=="mechanize"):
 652   returnval = download_from_url_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 653  elif(httplibuse=="ftp"):
 654   returnval = download_from_url_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 655  elif(httplibuse=="sftp"):
 656   returnval = download_from_url_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 657  elif(httplibuse=="pysftp"):
 658   returnval = download_from_url_with_pysftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep);
 659  else:
 660   returnval = False;
 661  return returnval;
 662
 663 def download_from_url_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", buffersize=524288, sleep=-1):
 664  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 665  if(sleep<0):
 666   sleep = geturls_download_sleep;
 667  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 668   httplibuse = "urllib";
 669  if(httplibuse=="httplib1"):
 670   httplibuse = "httplib";
 671  if(not haverequests and httplibuse=="requests"):
 672   httplibuse = "urllib";
 673  if(not havehttpx and httplibuse=="httpx"):
 674   httplibuse = "urllib";
 675  if(not havehttpx and httplibuse=="httpx2"):
 676   httplibuse = "urllib";
 677  if(not havehttpcore and httplibuse=="httpcore"):
 678   httplibuse = "urllib";
 679  if(not havehttpcore and httplibuse=="httpcore2"):
 680   httplibuse = "urllib";
 681  if(not havemechanize and httplibuse=="mechanize"):
 682   httplibuse = "urllib";
 683  if(not havehttplib2 and httplibuse=="httplib2"):
 684   httplibuse = "httplib";
 685  if(not haveparamiko and httplibuse=="sftp"):
 686   httplibuse = "ftp";
 687  if(not haveparamiko and httplibuse=="pysftp"):
 688   httplibuse = "ftp";
 689  if(httplibuse=="urllib"):
 690   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 691  elif(httplibuse=="request"):
 692   returnval = download_from_url_file_with_request(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 693  elif(httplibuse=="request3"):
 694   returnval = download_from_url_file_with_request3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 695  elif(httplibuse=="httplib"):
 696   returnval = download_from_url_file_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 697  elif(httplibuse=="httplib2"):
 698   returnval = download_from_url_file_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 699  elif(httplibuse=="urllib3"):
 700   returnval = download_from_url_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 701  elif(httplibuse=="requests"):
 702   returnval = download_from_url_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 703  elif(httplibuse=="httpx"):
 704   returnval = download_from_url_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 705  elif(httplibuse=="httpx2"):
 706   returnval = download_from_url_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 707  elif(httplibuse=="httpcore"):
 708   returnval = download_from_url_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 709  elif(httplibuse=="httpcore2"):
 710   returnval = download_from_url_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 711  elif(httplibuse=="mechanize"):
 712   returnval = download_from_url_file_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 713  elif(httplibuse=="ftp"):
 714   returnval = download_from_url_file_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 715  elif(httplibuse=="sftp"):
 716   returnval = download_from_url_file_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 717  elif(httplibuse=="pysftp"):
 718   returnval = download_from_url_file_with_pysftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 719  else:
 720   returnval = False;
 721  return returnval;
 722
 723 def download_from_url_to_file(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 724  global geturls_download_sleep, haverequests, havemechanize, havehttplib2, haveurllib3, havehttpx, havehttpcore, haveparamiko, havepysftp;
 725  if(sleep<0):
 726   sleep = geturls_download_sleep;
 727  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 728   httplibuse = "urllib";
 729  if(httplibuse=="httplib1"):
 730   httplibuse = "httplib";
 731  if(not haverequests and httplibuse=="requests"):
 732   httplibuse = "urllib";
 733  if(not havehttpx and httplibuse=="httpx"):
 734   httplibuse = "urllib";
 735  if(not havehttpx and httplibuse=="httpx2"):
 736   httplibuse = "urllib";
 737  if(not havehttpcore and httplibuse=="httpcore"):
 738   httplibuse = "urllib";
 739  if(not havehttpcore and httplibuse=="httpcore2"):
 740   httplibuse = "urllib";
 741  if(not havemechanize and httplibuse=="mechanize"):
 742   httplibuse = "urllib";
 743  if(not havehttplib2 and httplibuse=="httplib2"):
 744   httplibuse = "httplib";
 745  if(not haveparamiko and httplibuse=="sftp"):
 746   httplibuse = "ftp";
 747  if(not havepysftp and httplibuse=="pysftp"):
 748   httplibuse = "ftp";
 749  if(httplibuse=="urllib"):
 750   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 751  elif(httplibuse=="request"):
 752   returnval = download_from_url_to_file_with_request(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 753  elif(httplibuse=="request3"):
 754   returnval = download_from_url_to_file_with_request3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 755  elif(httplibuse=="httplib"):
 756   returnval = download_from_url_to_file_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 757  elif(httplibuse=="httplib2"):
 758   returnval = download_from_url_to_file_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 759  elif(httplibuse=="urllib3"):
 760   returnval = download_from_url_to_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 761  elif(httplibuse=="requests"):
 762   returnval = download_from_url_to_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 763  elif(httplibuse=="httpx"):
 764   returnval = download_from_url_to_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 765  elif(httplibuse=="httpx2"):
 766   returnval = download_from_url_to_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 767  elif(httplibuse=="httpcore"):
 768   returnval = download_from_url_to_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 769  elif(httplibuse=="httpcore2"):
 770   returnval = download_from_url_to_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep);
 771  elif(httplibuse=="mechanize"):
 772   returnval = download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 773  elif(httplibuse=="ftp"):
 774   returnval = download_from_url_to_file_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 775  elif(httplibuse=="sftp"):
 776   returnval = download_from_url_to_file_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 777  elif(httplibuse=="pysftp"):
 778   returnval = download_from_url_to_file_with_pysftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, outfile, outpath, buffersize, sleep);
 779  else:
 780   returnval = False;
 781  return returnval;
 782
 783 def download_from_url_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
 784  global geturls_download_sleep, havebrotli;
 785  if(sleep<0):
 786   sleep = geturls_download_sleep;
 787  urlparts = urlparse.urlparse(httpurl);
 788  if(isinstance(httpheaders, list)):
 789   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 790  httpheaders = fix_header_names(httpheaders);
 791  if(httpuseragent is not None):
 792   if('User-Agent' in httpheaders):
 793    httpheaders['User-Agent'] = httpuseragent;
 794   else:
 795    httpuseragent.update({'User-Agent': httpuseragent});
 796  if(httpreferer is not None):
 797   if('Referer' in httpheaders):
 798    httpheaders['Referer'] = httpreferer;
 799   else:
 800    httpuseragent.update({'Referer': httpreferer});
 801  if(urlparts.username is not None or urlparts.password is not None):
 802   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 803   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 804  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 805  if(isinstance(httpheaders, dict)):
 806   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 807  geturls_opener.addheaders = httpheaders;
 808  time.sleep(sleep);
 809  if(postdata is not None and not isinstance(postdata, dict)):
 810   postdata = urlencode(postdata);
 811  try:
 812   if(httpmethod=="GET"):
 813    geturls_text = geturls_opener.open(httpurl);
 814   elif(httpmethod=="POST"):
 815    geturls_text = geturls_opener.open(httpurl, data=postdata);
 816   else:
 817    geturls_text = geturls_opener.open(httpurl);
 818  except HTTPError as geturls_text_error:
 819   geturls_text = geturls_text_error;
 820   log.info("Error With URL "+httpurl);
 821  except URLError:
 822   log.info("Error With URL "+httpurl);
 823   return False;
 824  except socket.timeout:
 825   log.info("Error With URL "+httpurl);
 826   return False;
 827  httpcodeout = geturls_text.getcode();
 828  httpcodereason = geturls_text.reason;
 829  httpversionout = "1.1";
 830  httpmethodout = httpmethod;
 831  httpurlout = geturls_text.geturl();
 832  httpheaderout = geturls_text.info();
 833  httpheadersentout = httpheaders;
 834  if(isinstance(httpheaderout, list)):
 835   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
 836  httpheaderout = fix_header_names(httpheaderout);
 837  if(sys.version[0]=="2"):
 838   try:
 839    prehttpheaderout = httpheaderout;
 840    httpheaderkeys = httpheaderout.keys();
 841    imax = len(httpheaderkeys);
 842    ic = 0;
 843    httpheaderout = {};
 844    while(ic < imax):
 845     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
 846     ic += 1;
 847   except AttributeError:
 848    pass;
 849  if(isinstance(httpheadersentout, list)):
 850   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
 851  httpheadersentout = fix_header_names(httpheadersentout);
 852  log.info("Downloading URL "+httpurl);
 853  if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
 854   if(sys.version[0]=="2"):
 855    strbuf = StringIO(geturls_text.read());
 856   if(sys.version[0]>="3"):
 857    strbuf = BytesIO(geturls_text.read());
 858   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 859   returnval_content = gzstrbuf.read()[:];
 860  if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
 861   returnval_content = geturls_text.read()[:];
 862  if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
 863   returnval_content = geturls_text.read()[:];
 864   returnval_content = brotli.decompress(returnval_content);
 865  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
 866  geturls_text.close();
 867  return returnval;
 868
 869 def download_from_url_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
 870  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 871  exec_time_start = time.time();
 872  myhash = hashlib.new("sha1");
 873  if(sys.version[0]=="2"):
 874   myhash.update(httpurl);
 875   myhash.update(str(buffersize));
 876   myhash.update(str(exec_time_start));
 877  if(sys.version[0]>="3"):
 878   myhash.update(httpurl.encode('utf-8'));
 879   myhash.update(str(buffersize).encode('utf-8'));
 880   myhash.update(str(exec_time_start).encode('utf-8'));
 881  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 882  if(sleep<0):
 883   sleep = geturls_download_sleep;
 884  urlparts = urlparse.urlparse(httpurl);
 885  if(isinstance(httpheaders, list)):
 886   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 887  httpheaders = fix_header_names(httpheaders);
 888  if(httpuseragent is not None):
 889   if('User-Agent' in httpheaders):
 890    httpheaders['User-Agent'] = httpuseragent;
 891   else:
 892    httpuseragent.update({'User-Agent': httpuseragent});
 893  if(httpreferer is not None):
 894   if('Referer' in httpheaders):
 895    httpheaders['Referer'] = httpreferer;
 896   else:
 897    httpuseragent.update({'Referer': httpreferer});
 898  if(urlparts.username is not None or urlparts.password is not None):
 899   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
 900   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
 901  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
 902  if(isinstance(httpheaders, dict)):
 903   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 904  geturls_opener.addheaders = httpheaders;
 905  time.sleep(sleep);
 906  try:
 907   if(httpmethod=="GET"):
 908    geturls_text = geturls_opener.open(httpurl);
 909   elif(httpmethod=="POST"):
 910    geturls_text = geturls_opener.open(httpurl, data=postdata);
 911   else:
 912    geturls_text = geturls_opener.open(httpurl);
 913  except HTTPError as geturls_text_error:
 914   geturls_text = geturls_text_error;
 915   log.info("Error With URL "+httpurl);
 916  except URLError:
 917   log.info("Error With URL "+httpurl);
 918   return False;
 919  except socket.timeout:
 920   log.info("Error With URL "+httpurl);
 921   return False;
 922  except socket.timeout:
 923   log.info("Error With URL "+httpurl);
 924   return False;
 925  httpcodeout = geturls_text.getcode();
 926  httpcodereason = geturls_text.reason;
 927  httpversionout = "1.1";
 928  httpmethodout = httpmethod;
 929  httpurlout = geturls_text.geturl();
 930  httpheaderout = geturls_text.info();
 931  httpheadersentout = httpheaders;
 932  if(isinstance(httpheaderout, list)):
 933   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
 934  if(sys.version[0]=="2"):
 935   try:
 936    prehttpheaderout = httpheaderout;
 937    httpheaderkeys = httpheaderout.keys();
 938    imax = len(httpheaderkeys);
 939    ic = 0;
 940    httpheaderout = {};
 941    while(ic < imax):
 942     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
 943     ic += 1;
 944   except AttributeError:
 945    pass;
 946  httpheaderout = fix_header_names(httpheaderout);
 947  if(isinstance(httpheadersentout, list)):
 948   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
 949  httpheadersentout = fix_header_names(httpheadersentout);
 950  downloadsize = httpheaderout.get('Content-Length');
 951  if(downloadsize is not None):
 952   downloadsize = int(downloadsize);
 953  if downloadsize is None: downloadsize = 0;
 954  fulldatasize = 0;
 955  prevdownsize = 0;
 956  log.info("Downloading URL "+httpurl);
 957  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
 958   tmpfilename = f.name;
 959   try:
 960    os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
 961   except AttributeError:
 962    try:
 963     os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
 964    except ValueError:
 965     pass;
 966   except ValueError:
 967    pass;
 968   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
 969   while True:
 970    databytes = geturls_text.read(buffersize);
 971    if not databytes: break;
 972    datasize = len(databytes);
 973    fulldatasize = datasize + fulldatasize;
 974    percentage = "";
 975    if(downloadsize>0):
 976     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 977    downloaddiff = fulldatasize - prevdownsize;
 978    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 979    prevdownsize = fulldatasize;
 980    f.write(databytes);
 981   f.close();
 982  geturls_text.close();
 983  exec_time_end = time.time();
 984  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 985  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 986  return returnval;
 987
 988 def download_from_url_to_file_with_urllib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
 989  global geturls_download_sleep;
 990  if(sleep<0):
 991   sleep = geturls_download_sleep;
 992  if(not outfile=="-"):
 993   outpath = outpath.rstrip(os.path.sep);
 994   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 995   if(not os.path.exists(outpath)):
 996    os.makedirs(outpath);
 997   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 998    return False;
 999   if(os.path.exists(filepath) and os.path.isdir(filepath)):
1000    return False;
1001   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1002   if(not pretmpfilename):
1003    return False;
1004   tmpfilename = pretmpfilename['Filename'];
1005   downloadsize = os.path.getsize(tmpfilename);
1006   fulldatasize = 0;
1007   log.info("Moving file "+tmpfilename+" to "+filepath);
1008   exec_time_start = time.time();
1009   shutil.move(tmpfilename, filepath);
1010   try:
1011    os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1012   except AttributeError:
1013    try:
1014     os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1015    except ValueError:
1016     pass;
1017   except ValueError:
1018    pass;
1019   exec_time_end = time.time();
1020   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1021   if(os.path.exists(tmpfilename)):
1022    os.remove(tmpfilename);
1023   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1024  if(outfile=="-" and sys.version[0]=="2"):
1025   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1026   if(not pretmpfilename):
1027    return False;
1028   tmpfilename = pretmpfilename['Filename'];
1029   downloadsize = os.path.getsize(tmpfilename);
1030   fulldatasize = 0;
1031   prevdownsize = 0;
1032   exec_time_start = time.time();
1033   with open(tmpfilename, 'rb') as ft:
1034    f = StringIO();
1035    while True:
1036     databytes = ft.read(buffersize[1]);
1037     if not databytes: break;
1038     datasize = len(databytes);
1039     fulldatasize = datasize + fulldatasize;
1040     percentage = "";
1041     if(downloadsize>0):
1042      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1043     downloaddiff = fulldatasize - prevdownsize;
1044     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1045     prevdownsize = fulldatasize;
1046     f.write(databytes);
1047    f.seek(0);
1048    fdata = f.getvalue();
1049    f.close();
1050    ft.close();
1051    os.remove(tmpfilename);
1052    exec_time_end = time.time();
1053    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1054   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1055  if(outfile=="-" and sys.version[0]>="3"):
1056   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1057   tmpfilename = pretmpfilename['Filename'];
1058   downloadsize = os.path.getsize(tmpfilename);
1059   fulldatasize = 0;
1060   prevdownsize = 0;
1061   exec_time_start = time.time();
1062   with open(tmpfilename, 'rb') as ft:
1063    f = BytesIO();
1064    while True:
1065     databytes = ft.read(buffersize[1]);
1066     if not databytes: break;
1067     datasize = len(databytes);
1068     fulldatasize = datasize + fulldatasize;
1069     percentage = "";
1070     if(downloadsize>0):
1071      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1072     downloaddiff = fulldatasize - prevdownsize;
1073     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1074     prevdownsize = fulldatasize;
1075     f.write(databytes);
1076    f.seek(0);
1077    fdata = f.getvalue();
1078    f.close();
1079    ft.close();
1080    os.remove(tmpfilename);
1081    exec_time_end = time.time();
1082    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1083   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1084  return returnval;
1085
1086 def download_from_url_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1087  global geturls_download_sleep, havebrotli;
1088  if(sleep<0):
1089   sleep = geturls_download_sleep;
1090  urlparts = urlparse.urlparse(httpurl);
1091  if(isinstance(httpheaders, list)):
1092   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1093  httpheaders = fix_header_names(httpheaders);
1094  if(httpuseragent is not None):
1095   if('User-Agent' in httpheaders):
1096    httpheaders['User-Agent'] = httpuseragent;
1097   else:
1098    httpuseragent.update({'User-Agent': httpuseragent});
1099  if(httpreferer is not None):
1100   if('Referer' in httpheaders):
1101    httpheaders['Referer'] = httpreferer;
1102   else:
1103    httpuseragent.update({'Referer': httpreferer});
1104  if(urlparts.username is not None or urlparts.password is not None):
1105   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1106   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1107  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1108  geturls_opener.addheaders = httpheaders;
1109  time.sleep(sleep);
1110  if(urlparts[0]=="http"):
1111   httpconn = HTTPConnection(urlparts[1]);
1112  elif(urlparts[0]=="https"):
1113   httpconn = HTTPSConnection(urlparts[1]);
1114  else:
1115   return False;
1116  if(postdata is not None and not isinstance(postdata, dict)):
1117   postdata = urlencode(postdata);
1118  try:
1119   if(httpmethod=="GET"):
1120    httpconn.request("GET", urlparts[2], headers=httpheaders);
1121   elif(httpmethod=="POST"):
1122    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1123   else:
1124    httpconn.request("GET", urlparts[2], headers=httpheaders);
1125  except socket.timeout:
1126   log.info("Error With URL "+httpurl);
1127   return False;
1128  except socket.gaierror:
1129   log.info("Error With URL "+httpurl);
1130   return False;
1131  geturls_text = httpconn.getresponse();
1132  httpcodeout = geturls_text.status;
1133  httpcodereason = geturls_text.reason;
1134  if(geturls_text.version=="10"):
1135   httpversionout = "1.0";
1136  else:
1137   httpversionout = "1.1";
1138  httpmethodout = httpmethod;
1139  httpurlout = httpurl;
1140  httpheaderout = geturls_text.getheaders();
1141  httpheadersentout = httpheaders;
1142  if(isinstance(httpheaderout, list)):
1143   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1144  if(sys.version[0]=="2"):
1145   try:
1146    prehttpheaderout = httpheaderout;
1147    httpheaderkeys = httpheaderout.keys();
1148    imax = len(httpheaderkeys);
1149    ic = 0;
1150    httpheaderout = {};
1151    while(ic < imax):
1152     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1153     ic += 1;
1154   except AttributeError:
1155    pass;
1156  httpheaderout = fix_header_names(httpheaderout);
1157  if(isinstance(httpheadersentout, list)):
1158   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1159  httpheadersentout = fix_header_names(httpheadersentout);
1160  log.info("Downloading URL "+httpurl);
1161  if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
1162   if(sys.version[0]=="2"):
1163    strbuf = StringIO(geturls_text.read());
1164   if(sys.version[0]>="3"):
1165    strbuf = BytesIO(geturls_text.read());
1166   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1167   returnval_content = gzstrbuf.read()[:];
1168  if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
1169   returnval_content = geturls_text.read()[:];
1170  if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1171   returnval_content = geturls_text.read()[:];
1172   returnval_content = brotli.decompress(returnval_content);
1173  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1174  geturls_text.close();
1175  return returnval;
1176
1177 def download_from_url_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1178  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1179  exec_time_start = time.time();
1180  myhash = hashlib.new("sha1");
1181  if(sys.version[0]=="2"):
1182   myhash.update(httpurl);
1183   myhash.update(str(buffersize));
1184   myhash.update(str(exec_time_start));
1185  if(sys.version[0]>="3"):
1186   myhash.update(httpurl.encode('utf-8'));
1187   myhash.update(str(buffersize).encode('utf-8'));
1188   myhash.update(str(exec_time_start).encode('utf-8'));
1189  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1190  if(sleep<0):
1191   sleep = geturls_download_sleep;
1192  urlparts = urlparse.urlparse(httpurl);
1193  if(isinstance(httpheaders, list)):
1194   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1195  httpheaders = fix_header_names(httpheaders);
1196  if(httpuseragent is not None):
1197   if('User-Agent' in httpheaders):
1198    httpheaders['User-Agent'] = httpuseragent;
1199   else:
1200    httpuseragent.update({'User-Agent': httpuseragent});
1201  if(httpreferer is not None):
1202   if('Referer' in httpheaders):
1203    httpheaders['Referer'] = httpreferer;
1204   else:
1205    httpuseragent.update({'Referer': httpreferer});
1206  if(urlparts.username is not None or urlparts.password is not None):
1207   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1208   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1209  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1210  geturls_opener.addheaders = httpheaders;
1211  time.sleep(sleep);
1212  if(urlparts[0]=="http"):
1213   httpconn = HTTPConnection(urlparts[1]);
1214  elif(urlparts[0]=="https"):
1215   httpconn = HTTPSConnection(urlparts[1]);
1216  else:
1217   return False;
1218  if(postdata is not None and not isinstance(postdata, dict)):
1219   postdata = urlencode(postdata);
1220  try:
1221   if(httpmethod=="GET"):
1222    httpconn.request("GET", urlparts[2], headers=httpheaders);
1223   elif(httpmethod=="POST"):
1224    httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1225   else:
1226    httpconn.request("GET", urlparts[2], headers=httpheaders);
1227  except socket.timeout:
1228   log.info("Error With URL "+httpurl);
1229   return False;
1230  except socket.gaierror:
1231   log.info("Error With URL "+httpurl);
1232   return False;
1233  geturls_text = httpconn.getresponse();
1234  httpcodeout = geturls_text.status;
1235  httpcodereason = geturls_text.reason;
1236  if(geturls_text.version=="10"):
1237   httpversionout = "1.0";
1238  else:
1239   httpversionout = "1.1";
1240  httpmethodout = httpmethod;
1241  httpurlout = httpurl;
1242  httpheaderout = geturls_text.getheaders();
1243  httpheadersentout = httpheaders;
1244  if(isinstance(httpheaderout, list)):
1245   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1246  if(sys.version[0]=="2"):
1247   try:
1248    prehttpheaderout = httpheaderout;
1249    httpheaderkeys = httpheaderout.keys();
1250    imax = len(httpheaderkeys);
1251    ic = 0;
1252    httpheaderout = {};
1253    while(ic < imax):
1254     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1255     ic += 1;
1256   except AttributeError:
1257    pass;
1258  httpheaderout = fix_header_names(httpheaderout);
1259  if(isinstance(httpheadersentout, list)):
1260   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1261  httpheadersentout = fix_header_names(httpheadersentout);
1262  downloadsize = httpheaderout.get('Content-Length');
1263  if(downloadsize is not None):
1264   downloadsize = int(downloadsize);
1265  if downloadsize is None: downloadsize = 0;
1266  fulldatasize = 0;
1267  prevdownsize = 0;
1268  log.info("Downloading URL "+httpurl);
1269  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1270   tmpfilename = f.name;
1271   try:
1272    os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
1273   except AttributeError:
1274    try:
1275     os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1276    except ValueError:
1277     pass;
1278   except ValueError:
1279    pass;
1280   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1281   while True:
1282    databytes = geturls_text.read(buffersize);
1283    if not databytes: break;
1284    datasize = len(databytes);
1285    fulldatasize = datasize + fulldatasize;
1286    percentage = "";
1287    if(downloadsize>0):
1288     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1289    downloaddiff = fulldatasize - prevdownsize;
1290    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1291    prevdownsize = fulldatasize;
1292    f.write(databytes);
1293   f.close();
1294  geturls_text.close();
1295  exec_time_end = time.time();
1296  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1297  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1298  return returnval;
1299
1300 def download_from_url_to_file_with_httplib(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1301  global geturls_download_sleep;
1302  if(sleep<0):
1303   sleep = geturls_download_sleep;
1304  if(not outfile=="-"):
1305   outpath = outpath.rstrip(os.path.sep);
1306   filepath = os.path.realpath(outpath+os.path.sep+outfile);
1307   if(not os.path.exists(outpath)):
1308    os.makedirs(outpath);
1309   if(os.path.exists(outpath) and os.path.isfile(outpath)):
1310    return False;
1311   if(os.path.exists(filepath) and os.path.isdir(filepath)):
1312    return False;
1313   pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1314   if(not pretmpfilename):
1315    return False;
1316   tmpfilename = pretmpfilename['Filename'];
1317   downloadsize = os.path.getsize(tmpfilename);
1318   fulldatasize = 0;
1319   log.info("Moving file "+tmpfilename+" to "+filepath);
1320   exec_time_start = time.time();
1321   shutil.move(tmpfilename, filepath);
1322   try:
1323    os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1324   except AttributeError:
1325    try:
1326     os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1327    except ValueError:
1328     pass;
1329   except ValueError:
1330    pass;
1331   exec_time_end = time.time();
1332   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1333   if(os.path.exists(tmpfilename)):
1334    os.remove(tmpfilename);
1335   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1336  if(outfile=="-" and sys.version[0]=="2"):
1337   pretmpfilename = download_from_url_file_with_httplib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1338   if(not pretmpfilename):
1339    return False;
1340   tmpfilename = pretmpfilename['Filename'];
1341   downloadsize = os.path.getsize(tmpfilename);
1342   fulldatasize = 0;
1343   prevdownsize = 0;
1344   exec_time_start = time.time();
1345   with open(tmpfilename, 'rb') as ft:
1346    f = StringIO();
1347    while True:
1348     databytes = ft.read(buffersize[1]);
1349     if not databytes: break;
1350     datasize = len(databytes);
1351     fulldatasize = datasize + fulldatasize;
1352     percentage = "";
1353     if(downloadsize>0):
1354      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1355     downloaddiff = fulldatasize - prevdownsize;
1356     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1357     prevdownsize = fulldatasize;
1358     f.write(databytes);
1359    f.seek(0);
1360    fdata = f.getvalue();
1361    f.close();
1362    ft.close();
1363    os.remove(tmpfilename);
1364    exec_time_end = time.time();
1365    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1366   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1367  if(outfile=="-" and sys.version[0]>="3"):
1368   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1369   tmpfilename = pretmpfilename['Filename'];
1370   downloadsize = os.path.getsize(tmpfilename);
1371   fulldatasize = 0;
1372   prevdownsize = 0;
1373   exec_time_start = time.time();
1374   with open(tmpfilename, 'rb') as ft:
1375    f = BytesIO();
1376    while True:
1377     databytes = ft.read(buffersize[1]);
1378     if not databytes: break;
1379     datasize = len(databytes);
1380     fulldatasize = datasize + fulldatasize;
1381     percentage = "";
1382     if(downloadsize>0):
1383      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1384     downloaddiff = fulldatasize - prevdownsize;
1385     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1386     prevdownsize = fulldatasize;
1387     f.write(databytes);
1388    f.seek(0);
1389    fdata = f.getvalue();
1390    f.close();
1391    ft.close();
1392    os.remove(tmpfilename);
1393    exec_time_end = time.time();
1394    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1395   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1396  return returnval;
1397
1398 if(havehttplib2):
1399  def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1400   global geturls_download_sleep, havebrotli;
1401   if(sleep<0):
1402    sleep = geturls_download_sleep;
1403   urlparts = urlparse.urlparse(httpurl);
1404   if(isinstance(httpheaders, list)):
1405    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1406   httpheaders = fix_header_names(httpheaders);
1407   if(httpuseragent is not None):
1408    if('User-Agent' in httpheaders):
1409     httpheaders['User-Agent'] = httpuseragent;
1410    else:
1411     httpuseragent.update({'User-Agent': httpuseragent});
1412   if(httpreferer is not None):
1413    if('Referer' in httpheaders):
1414     httpheaders['Referer'] = httpreferer;
1415    else:
1416     httpuseragent.update({'Referer': httpreferer});
1417   if(urlparts.username is not None or urlparts.password is not None):
1418    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1419    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1420   geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1421   geturls_opener.addheaders = httpheaders;
1422   time.sleep(sleep);
1423   if(urlparts[0]=="http"):
1424    httpconn = HTTPConnectionWithTimeout(urlparts[1]);
1425   elif(urlparts[0]=="https"):
1426    httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
1427   else:
1428    return False;
1429   if(postdata is not None and not isinstance(postdata, dict)):
1430    postdata = urlencode(postdata);
1431   try:
1432    if(httpmethod=="GET"):
1433     httpconn.request("GET", urlparts[2], headers=httpheaders);
1434    elif(httpmethod=="POST"):
1435     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1436    else:
1437     httpconn.request("GET", urlparts[2], headers=httpheaders);
1438   except socket.timeout:
1439    log.info("Error With URL "+httpurl);
1440    return False;
1441   except socket.gaierror:
1442    log.info("Error With URL "+httpurl);
1443    return False;
1444   geturls_text = httpconn.getresponse();
1445   httpcodeout = geturls_text.status;
1446   httpcodereason = geturls_text.reason;
1447   if(geturls_text.version=="10"):
1448    httpversionout = "1.0";
1449   else:
1450    httpversionout = "1.1";
1451   httpmethodout = httpmethod;
1452   httpurlout = httpurl;
1453   httpheaderout = geturls_text.getheaders();
1454   httpheadersentout = httpheaders;
1455   if(isinstance(httpheaderout, list)):
1456    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1457   if(sys.version[0]=="2"):
1458    try:
1459     prehttpheaderout = httpheaderout;
1460     httpheaderkeys = httpheaderout.keys();
1461     imax = len(httpheaderkeys);
1462     ic = 0;
1463     httpheaderout = {};
1464     while(ic < imax):
1465      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1466      ic += 1;
1467    except AttributeError:
1468     pass;
1469   httpheaderout = fix_header_names(httpheaderout);
1470   if(isinstance(httpheadersentout, list)):
1471    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1472   httpheadersentout = fix_header_names(httpheadersentout);
1473   log.info("Downloading URL "+httpurl);
1474   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
1475    if(sys.version[0]=="2"):
1476     strbuf = StringIO(geturls_text.read());
1477    if(sys.version[0]>="3"):
1478     strbuf = BytesIO(geturls_text.read());
1479    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1480    returnval_content = gzstrbuf.read()[:];
1481   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
1482    returnval_content = geturls_text.read()[:];
1483   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1484    returnval_content = geturls_text.read()[:];
1485    returnval_content = brotli.decompress(returnval_content);
1486   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1487   geturls_text.close();
1488   return returnval;
1489
1490 if(not havehttplib2):
1491  def download_from_url_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1492   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep)
1493   return returnval;
1494
1495 if(havehttplib2):
1496  def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1497   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1498   exec_time_start = time.time();
1499   myhash = hashlib.new("sha1");
1500   if(sys.version[0]=="2"):
1501    myhash.update(httpurl);
1502    myhash.update(str(buffersize));
1503    myhash.update(str(exec_time_start));
1504   if(sys.version[0]>="3"):
1505    myhash.update(httpurl.encode('utf-8'));
1506    myhash.update(str(buffersize).encode('utf-8'));
1507    myhash.update(str(exec_time_start).encode('utf-8'));
1508   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1509   if(sleep<0):
1510    sleep = geturls_download_sleep;
1511   urlparts = urlparse.urlparse(httpurl);
1512   if(isinstance(httpheaders, list)):
1513    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1514   httpheaders = fix_header_names(httpheaders);
1515   if(httpuseragent is not None):
1516    if('User-Agent' in httpheaders):
1517     httpheaders['User-Agent'] = httpuseragent;
1518    else:
1519     httpuseragent.update({'User-Agent': httpuseragent});
1520   if(httpreferer is not None):
1521    if('Referer' in httpheaders):
1522     httpheaders['Referer'] = httpreferer;
1523    else:
1524     httpuseragent.update({'Referer': httpreferer});
1525   if(urlparts.username is not None or urlparts.password is not None):
1526    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1527    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1528   geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1529   geturls_opener.addheaders = httpheaders;
1530   time.sleep(sleep);
1531   if(urlparts[0]=="http"):
1532    httpconn = HTTPConnectionWithTimeout(urlparts[1]);
1533   elif(urlparts[0]=="https"):
1534    httpconn = HTTPSConnectionWithTimeout(urlparts[1]);
1535   else:
1536    return False;
1537   if(postdata is not None and not isinstance(postdata, dict)):
1538    postdata = urlencode(postdata);
1539   try:
1540    if(httpmethod=="GET"):
1541     httpconn.request("GET", urlparts[2], headers=httpheaders);
1542    elif(httpmethod=="POST"):
1543     httpconn.request("GET", urlparts[2], body=postdata, headers=httpheaders);
1544    else:
1545     httpconn.request("GET", urlparts[2], headers=httpheaders);
1546   except socket.timeout:
1547    log.info("Error With URL "+httpurl);
1548    return False;
1549   except socket.gaierror:
1550    log.info("Error With URL "+httpurl);
1551    return False;
1552   geturls_text = httpconn.getresponse();
1553   httpcodeout = geturls_text.status;
1554   httpcodereason = geturls_text.reason;
1555   if(geturls_text.version=="10"):
1556    httpversionout = "1.0";
1557   else:
1558    httpversionout = "1.1";
1559   httpmethodout = httpmethod;
1560   httpurlout = httpurl;
1561   httpheaderout = geturls_text.getheaders();
1562   httpheadersentout = httpheaders;
1563   if(isinstance(httpheaderout, list)):
1564    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1565   if(sys.version[0]=="2"):
1566    try:
1567     prehttpheaderout = httpheaderout;
1568     httpheaderkeys = httpheaderout.keys();
1569     imax = len(httpheaderkeys);
1570     ic = 0;
1571     httpheaderout = {};
1572     while(ic < imax):
1573      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1574      ic += 1;
1575    except AttributeError:
1576     pass;
1577   httpheaderout = fix_header_names(httpheaderout);
1578   if(isinstance(httpheadersentout, list)):
1579    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1580   httpheadersentout = fix_header_names(httpheadersentout);
1581   downloadsize = httpheaderout.get('Content-Length');
1582   if(downloadsize is not None):
1583    downloadsize = int(downloadsize);
1584   if downloadsize is None: downloadsize = 0;
1585   fulldatasize = 0;
1586   prevdownsize = 0;
1587   log.info("Downloading URL "+httpurl);
1588   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1589    tmpfilename = f.name;
1590    try:
1591     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
1592    except AttributeError:
1593     try:
1594      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1595     except ValueError:
1596      pass;
1597    except ValueError:
1598     pass;
1599    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1600    while True:
1601     databytes = geturls_text.read(buffersize);
1602     if not databytes: break;
1603     datasize = len(databytes);
1604     fulldatasize = datasize + fulldatasize;
1605     percentage = "";
1606     if(downloadsize>0):
1607      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1608     downloaddiff = fulldatasize - prevdownsize;
1609     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1610     prevdownsize = fulldatasize;
1611     f.write(databytes);
1612    f.close();
1613   geturls_text.close();
1614   exec_time_end = time.time();
1615   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1616   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1617   return returnval;
1618
1619 if(not havehttplib2):
1620  def download_from_url_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1621   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep)
1622   return returnval;
1623
1624 if(havehttplib2):
1625  def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1626   global geturls_download_sleep;
1627   if(sleep<0):
1628    sleep = geturls_download_sleep;
1629   if(not outfile=="-"):
1630    outpath = outpath.rstrip(os.path.sep);
1631    filepath = os.path.realpath(outpath+os.path.sep+outfile);
1632    if(not os.path.exists(outpath)):
1633     os.makedirs(outpath);
1634    if(os.path.exists(outpath) and os.path.isfile(outpath)):
1635     return False;
1636    if(os.path.exists(filepath) and os.path.isdir(filepath)):
1637     return False;
1638    pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1639    if(not pretmpfilename):
1640     return False;
1641    tmpfilename = pretmpfilename['Filename'];
1642    downloadsize = os.path.getsize(tmpfilename);
1643    fulldatasize = 0;
1644    log.info("Moving file "+tmpfilename+" to "+filepath);
1645    exec_time_start = time.time();
1646    shutil.move(tmpfilename, filepath);
1647    try:
1648     os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1649    except AttributeError:
1650     try:
1651      os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1652     except ValueError:
1653      pass;
1654    except ValueError:
1655     pass;
1656    exec_time_end = time.time();
1657    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1658    if(os.path.exists(tmpfilename)):
1659     os.remove(tmpfilename);
1660    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1661   if(outfile=="-" and sys.version[0]=="2"):
1662    pretmpfilename = download_from_url_file_with_httplib2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1663    if(not pretmpfilename):
1664     return False;
1665    tmpfilename = pretmpfilename['Filename'];
1666    downloadsize = os.path.getsize(tmpfilename);
1667    fulldatasize = 0;
1668    prevdownsize = 0;
1669    exec_time_start = time.time();
1670    with open(tmpfilename, 'rb') as ft:
1671     f = StringIO();
1672     while True:
1673      databytes = ft.read(buffersize[1]);
1674      if not databytes: break;
1675      datasize = len(databytes);
1676      fulldatasize = datasize + fulldatasize;
1677      percentage = "";
1678      if(downloadsize>0):
1679       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1680      downloaddiff = fulldatasize - prevdownsize;
1681      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1682      prevdownsize = fulldatasize;
1683      f.write(databytes);
1684     f.seek(0);
1685     fdata = f.getvalue();
1686     f.close();
1687     ft.close();
1688     os.remove(tmpfilename);
1689     exec_time_end = time.time();
1690     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1691    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1692   if(outfile=="-" and sys.version[0]>="3"):
1693    pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1694    tmpfilename = pretmpfilename['Filename'];
1695    downloadsize = os.path.getsize(tmpfilename);
1696    fulldatasize = 0;
1697    prevdownsize = 0;
1698    exec_time_start = time.time();
1699    with open(tmpfilename, 'rb') as ft:
1700     f = BytesIO();
1701     while True:
1702      databytes = ft.read(buffersize[1]);
1703      if not databytes: break;
1704      datasize = len(databytes);
1705      fulldatasize = datasize + fulldatasize;
1706      percentage = "";
1707      if(downloadsize>0):
1708       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1709      downloaddiff = fulldatasize - prevdownsize;
1710      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1711      prevdownsize = fulldatasize;
1712      f.write(databytes);
1713     f.seek(0);
1714     fdata = f.getvalue();
1715     f.close();
1716     ft.close();
1717     os.remove(tmpfilename);
1718     exec_time_end = time.time();
1719     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
1720    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1721   return returnval;
1722
1723 if(not havehttplib2):
1724  def download_from_url_to_file_with_httplib2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1725   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
1726   return returnval;
1727
1728 def download_from_url_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
1729  global geturls_download_sleep, havebrotli;
1730  if(sleep<0):
1731   sleep = geturls_download_sleep;
1732  urlparts = urlparse.urlparse(httpurl);
1733  if(isinstance(httpheaders, list)):
1734   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1735  httpheaders = fix_header_names(httpheaders);
1736  if(httpuseragent is not None):
1737   if('User-Agent' in httpheaders):
1738    httpheaders['User-Agent'] = httpuseragent;
1739   else:
1740    httpuseragent.update({'User-Agent': httpuseragent});
1741  if(httpreferer is not None):
1742   if('Referer' in httpheaders):
1743    httpheaders['Referer'] = httpreferer;
1744   else:
1745    httpuseragent.update({'Referer': httpreferer});
1746  if(urlparts.username is not None or urlparts.password is not None):
1747   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1748   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1749  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1750  if(isinstance(httpheaders, dict)):
1751   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1752  geturls_opener.addheaders = httpheaders;
1753  install_opener(geturls_opener);
1754  time.sleep(sleep);
1755  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1756  if(postdata is not None and not isinstance(postdata, dict)):
1757   postdata = urlencode(postdata);
1758  try:
1759   if(httpmethod=="GET"):
1760    geturls_request = Request(httpurl, headers=httpheaders);
1761    geturls_text = urlopen(geturls_request);
1762   elif(httpmethod=="POST"):
1763    geturls_request = Request(httpurl, headers=httpheaders);
1764    geturls_text = urlopen(geturls_request, data=postdata);
1765   else:
1766    geturls_request = Request(httpurl, headers=httpheaders);
1767    geturls_text = urlopen(geturls_request);
1768  except HTTPError as geturls_text_error:
1769   geturls_text = geturls_text_error;
1770   log.info("Error With URL "+httpurl);
1771  except URLError:
1772   log.info("Error With URL "+httpurl);
1773   return False;
1774  except socket.timeout:
1775   log.info("Error With URL "+httpurl);
1776   return False;
1777  httpcodeout = geturls_text.getcode();
1778  httpcodereason = geturls_text.reason;
1779  httpversionout = "1.1";
1780  httpmethodout = httpmethod;
1781  httpurlout = geturls_text.geturl();
1782  httpheaderout = geturls_text.headers;
1783  httpheadersentout = httpheaders;
1784  if(isinstance(httpheaderout, list)):
1785   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1786  if(sys.version[0]=="2"):
1787   try:
1788    prehttpheaderout = httpheaderout;
1789    httpheaderkeys = httpheaderout.keys();
1790    imax = len(httpheaderkeys);
1791    ic = 0;
1792    httpheaderout = {};
1793    while(ic < imax):
1794     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1795     ic += 1;
1796   except AttributeError:
1797    pass;
1798  httpheaderout = fix_header_names(httpheaderout);
1799  if(isinstance(httpheadersentout, list)):
1800   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1801  httpheadersentout = fix_header_names(httpheadersentout);
1802  log.info("Downloading URL "+httpurl);
1803  if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
1804   if(sys.version[0]=="2"):
1805    strbuf = StringIO(geturls_text.read());
1806   if(sys.version[0]>="3"):
1807    strbuf = BytesIO(geturls_text.read());
1808   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
1809   returnval_content = gzstrbuf.read()[:];
1810  if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
1811   returnval_content = geturls_text.read()[:];
1812  if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
1813   returnval_content = geturls_text.read()[:];
1814   returnval_content = brotli.decompress(returnval_content);
1815  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1816  geturls_text.close();
1817  return returnval;
1818
1819 def download_from_url_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
1820  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
1821  exec_time_start = time.time();
1822  myhash = hashlib.new("sha1");
1823  if(sys.version[0]=="2"):
1824   myhash.update(httpurl);
1825   myhash.update(str(buffersize));
1826   myhash.update(str(exec_time_start));
1827  if(sys.version[0]>="3"):
1828   myhash.update(httpurl.encode('utf-8'));
1829   myhash.update(str(buffersize).encode('utf-8'));
1830   myhash.update(str(exec_time_start).encode('utf-8'));
1831  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
1832  if(sleep<0):
1833   sleep = geturls_download_sleep;
1834  urlparts = urlparse.urlparse(httpurl);
1835  if(isinstance(httpheaders, list)):
1836   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1837  httpheaders = fix_header_names(httpheaders);
1838  if(httpuseragent is not None):
1839   if('User-Agent' in httpheaders):
1840    httpheaders['User-Agent'] = httpuseragent;
1841   else:
1842    httpuseragent.update({'User-Agent': httpuseragent});
1843  if(httpreferer is not None):
1844   if('Referer' in httpheaders):
1845    httpheaders['Referer'] = httpreferer;
1846   else:
1847    httpuseragent.update({'Referer': httpreferer});
1848  if(urlparts.username is not None or urlparts.password is not None):
1849   inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
1850   httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
1851  geturls_opener = build_opener(HTTPCookieProcessor(httpcookie));
1852  if(isinstance(httpheaders, dict)):
1853   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
1854  geturls_opener.addheaders = httpheaders;
1855  install_opener(geturls_opener);
1856  time.sleep(sleep);
1857  httpheaders = make_http_headers_from_list_to_dict(httpheaders);
1858  if(postdata is not None and not isinstance(postdata, dict)):
1859   postdata = urlencode(postdata);
1860  try:
1861   if(httpmethod=="GET"):
1862    geturls_request = Request(httpurl, headers=httpheaders);
1863    geturls_text = urlopen(geturls_request);
1864   elif(httpmethod=="POST"):
1865    geturls_request = Request(httpurl, headers=httpheaders);
1866    geturls_text = urlopen(geturls_request, data=postdata);
1867   else:
1868    geturls_request = Request(httpurl, headers=httpheaders);
1869    geturls_text = urlopen(geturls_request);
1870  except HTTPError as geturls_text_error:
1871   geturls_text = geturls_text_error;
1872   log.info("Error With URL "+httpurl);
1873  except URLError:
1874   log.info("Error With URL "+httpurl);
1875   return False;
1876  except socket.timeout:
1877   log.info("Error With URL "+httpurl);
1878   return False;
1879  httpcodeout = geturls_text.getcode();
1880  httpcodereason = geturls_text.reason;
1881  httpversionout = "1.1";
1882  httpmethodout = httpmethod;
1883  httpurlout = geturls_text.geturl();
1884  httpheaderout = geturls_text.headers;
1885  httpheadersentout = httpheaders;
1886  if(isinstance(httpheaderout, list)):
1887   httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
1888  if(sys.version[0]=="2"):
1889   try:
1890    prehttpheaderout = httpheaderout;
1891    httpheaderkeys = httpheaderout.keys();
1892    imax = len(httpheaderkeys);
1893    ic = 0;
1894    httpheaderout = {};
1895    while(ic < imax):
1896     httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
1897     ic += 1;
1898   except AttributeError:
1899    pass;
1900  httpheaderout = fix_header_names(httpheaderout);
1901  if(isinstance(httpheadersentout, list)):
1902   httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
1903  httpheadersentout = fix_header_names(httpheadersentout);
1904  downloadsize = httpheaderout.get('Content-Length');
1905  if(downloadsize is not None):
1906   downloadsize = int(downloadsize);
1907  if downloadsize is None: downloadsize = 0;
1908  fulldatasize = 0;
1909  prevdownsize = 0;
1910  log.info("Downloading URL "+httpurl);
1911  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
1912   tmpfilename = f.name;
1913   try:
1914    os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
1915   except AttributeError:
1916    try:
1917     os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1918    except ValueError:
1919     pass;
1920   except ValueError:
1921    pass;
1922   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
1923   while True:
1924    databytes = geturls_text.read(buffersize);
1925    if not databytes: break;
1926    datasize = len(databytes);
1927    fulldatasize = datasize + fulldatasize;
1928    percentage = "";
1929    if(downloadsize>0):
1930     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1931    downloaddiff = fulldatasize - prevdownsize;
1932    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1933    prevdownsize = fulldatasize;
1934    f.write(databytes);
1935   f.close();
1936  geturls_text.close();
1937  exec_time_end = time.time();
1938  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
1939  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
1940  return returnval;
1941
1942 def download_from_url_to_file_with_request(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
1943  global geturls_download_sleep;
1944  if(sleep<0):
1945   sleep = geturls_download_sleep;
1946  if(not outfile=="-"):
1947   outpath = outpath.rstrip(os.path.sep);
1948   filepath = os.path.realpath(outpath+os.path.sep+outfile);
1949   if(not os.path.exists(outpath)):
1950    os.makedirs(outpath);
1951   if(os.path.exists(outpath) and os.path.isfile(outpath)):
1952    return False;
1953   if(os.path.exists(filepath) and os.path.isdir(filepath)):
1954    return False;
1955   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1956   if(not pretmpfilename):
1957    return False;
1958   tmpfilename = pretmpfilename['Filename'];
1959   downloadsize = os.path.getsize(tmpfilename);
1960   fulldatasize = 0;
1961   log.info("Moving file "+tmpfilename+" to "+filepath);
1962   exec_time_start = time.time();
1963   shutil.move(tmpfilename, filepath);
1964   try:
1965    os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
1966   except AttributeError:
1967    try:
1968     os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
1969    except ValueError:
1970     pass;
1971   except ValueError:
1972    pass;
1973   exec_time_end = time.time();
1974   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
1975   if(os.path.exists(tmpfilename)):
1976    os.remove(tmpfilename);
1977   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent':pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
1978  if(outfile=="-" and sys.version[0]=="2"):
1979   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
1980   if(not pretmpfilename):
1981    return False;
1982   tmpfilename = pretmpfilename['Filename'];
1983   downloadsize = os.path.getsize(tmpfilename);
1984   fulldatasize = 0;
1985   prevdownsize = 0;
1986   exec_time_start = time.time();
1987   with open(tmpfilename, 'rb') as ft:
1988    f = StringIO();
1989    while True:
1990     databytes = ft.read(buffersize[1]);
1991     if not databytes: break;
1992     datasize = len(databytes);
1993     fulldatasize = datasize + fulldatasize;
1994     percentage = "";
1995     if(downloadsize>0):
1996      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
1997     downloaddiff = fulldatasize - prevdownsize;
1998     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
1999     prevdownsize = fulldatasize;
2000     f.write(databytes);
2001    f.seek(0);
2002    fdata = f.getvalue();
2003    f.close();
2004    ft.close();
2005    os.remove(tmpfilename);
2006    exec_time_end = time.time();
2007    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2008   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2009  if(outfile=="-" and sys.version[0]>="3"):
2010   pretmpfilename = download_from_url_file_with_request(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2011   tmpfilename = pretmpfilename['Filename'];
2012   downloadsize = os.path.getsize(tmpfilename);
2013   fulldatasize = 0;
2014   prevdownsize = 0;
2015   exec_time_start = time.time();
2016   with open(tmpfilename, 'rb') as ft:
2017    f = BytesIO();
2018    while True:
2019     databytes = ft.read(buffersize[1]);
2020     if not databytes: break;
2021     datasize = len(databytes);
2022     fulldatasize = datasize + fulldatasize;
2023     percentage = "";
2024     if(downloadsize>0):
2025      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2026     downloaddiff = fulldatasize - prevdownsize;
2027     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2028     prevdownsize = fulldatasize;
2029     f.write(databytes);
2030    f.seek(0);
2031    fdata = f.getvalue();
2032    f.close();
2033    ft.close();
2034    os.remove(tmpfilename);
2035    exec_time_end = time.time();
2036    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2037   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2038  return returnval;
2039
2040 if(haverequests):
2041  def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2042   global geturls_download_sleep, havebrotli;
2043   if(sleep<0):
2044    sleep = geturls_download_sleep;
2045   urlparts = urlparse.urlparse(httpurl);
2046   if(isinstance(httpheaders, list)):
2047    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2048   httpheaders = fix_header_names(httpheaders);
2049   if(httpuseragent is not None):
2050    if('User-Agent' in httpheaders):
2051     httpheaders['User-Agent'] = httpuseragent;
2052    else:
2053     httpuseragent.update({'User-Agent': httpuseragent});
2054   if(httpreferer is not None):
2055    if('Referer' in httpheaders):
2056     httpheaders['Referer'] = httpreferer;
2057    else:
2058     httpuseragent.update({'Referer': httpreferer});
2059   if(urlparts.username is not None or urlparts.password is not None):
2060    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2061    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2062   time.sleep(sleep);
2063   if(postdata is not None and not isinstance(postdata, dict)):
2064    postdata = urlencode(postdata);
2065   try:
2066    reqsession = requests.Session();
2067    if(httpmethod=="GET"):
2068     geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
2069    elif(httpmethod=="POST"):
2070     geturls_text = reqsession.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie, stream=True);
2071    else:
2072     geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
2073   except requests.exceptions.ConnectTimeout:
2074    log.info("Error With URL "+httpurl);
2075    return False;
2076   except requests.exceptions.ConnectError:
2077    log.info("Error With URL "+httpurl);
2078    return False;
2079   except socket.timeout:
2080    log.info("Error With URL "+httpurl);
2081    return False;
2082   httpcodeout = geturls_text.status_code;
2083   httpcodereason = geturls_text.reason;
2084   if(geturls_text.raw.version=="10"):
2085    httpversionout = "1.0";
2086   else:
2087    httpversionout = "1.1";
2088   httpmethodout = httpmethod;
2089   httpurlout = geturls_text.url;
2090   httpheaderout = geturls_text.headers;
2091   httpheadersentout = geturls_text.request.headers;
2092   if(isinstance(httpheaderout, list)):
2093    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2094   if(sys.version[0]=="2"):
2095    try:
2096     prehttpheaderout = httpheaderout;
2097     httpheaderkeys = httpheaderout.keys();
2098     imax = len(httpheaderkeys);
2099     ic = 0;
2100     httpheaderout = {};
2101     while(ic < imax):
2102      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2103      ic += 1;
2104    except AttributeError:
2105     pass;
2106   httpheaderout = fix_header_names(httpheaderout);
2107   if(isinstance(httpheadersentout, list)):
2108    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2109   httpheadersentout = fix_header_names(httpheadersentout);
2110   log.info("Downloading URL "+httpurl);
2111   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
2112    if(sys.version[0]=="2"):
2113     strbuf = StringIO(geturls_text.raw.read());
2114    if(sys.version[0]>="3"):
2115     strbuf = BytesIO(geturls_text.raw.read());
2116    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2117    returnval_content = gzstrbuf.read()[:];
2118   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
2119    returnval_content = geturls_text.raw.read()[:];
2120   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
2121    returnval_content = geturls_text.raw.read()[:];
2122    returnval_content = brotli.decompress(returnval_content);
2123   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2124   geturls_text.close();
2125   return returnval;
2126
2127 if(not haverequests):
2128  def download_from_url_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2129   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep)
2130   return returnval;
2131
2132 if(haverequests):
2133  def download_from_url_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2134   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2135   exec_time_start = time.time();
2136   myhash = hashlib.new("sha1");
2137   if(sys.version[0]=="2"):
2138    myhash.update(httpurl);
2139    myhash.update(str(buffersize));
2140    myhash.update(str(exec_time_start));
2141   if(sys.version[0]>="3"):
2142    myhash.update(httpurl.encode('utf-8'));
2143    myhash.update(str(buffersize).encode('utf-8'));
2144    myhash.update(str(exec_time_start).encode('utf-8'));
2145   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2146   if(sleep<0):
2147    sleep = geturls_download_sleep;
2148   urlparts = urlparse.urlparse(httpurl);
2149   if(isinstance(httpheaders, list)):
2150    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2151   httpheaders = fix_header_names(httpheaders);
2152   if(httpuseragent is not None):
2153    if('User-Agent' in httpheaders):
2154     httpheaders['User-Agent'] = httpuseragent;
2155    else:
2156     httpuseragent.update({'User-Agent': httpuseragent});
2157   if(httpreferer is not None):
2158    if('Referer' in httpheaders):
2159     httpheaders['Referer'] = httpreferer;
2160    else:
2161     httpuseragent.update({'Referer': httpreferer});
2162   if(urlparts.username is not None or urlparts.password is not None):
2163    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2164    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2165   time.sleep(sleep);
2166   if(postdata is not None and not isinstance(postdata, dict)):
2167    postdata = urlencode(postdata);
2168   try:
2169    reqsession = requests.Session();
2170    if(httpmethod=="GET"):
2171     geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
2172    elif(httpmethod=="POST"):
2173     geturls_text = reqsession.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie, stream=True);
2174    else:
2175     geturls_text = reqsession.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
2176   except requests.exceptions.ConnectTimeout:
2177    log.info("Error With URL "+httpurl);
2178    return False;
2179   except requests.exceptions.ConnectError:
2180    log.info("Error With URL "+httpurl);
2181    return False;
2182   except socket.timeout:
2183    log.info("Error With URL "+httpurl);
2184    return False;
2185   httpcodeout = geturls_text.status_code;
2186   httpcodereason = geturls_text.reason;
2187   if(geturls_text.raw.version=="10"):
2188    httpversionout = "1.0";
2189   else:
2190    httpversionout = "1.1";
2191   httpmethodout = httpmethod;
2192   httpurlout = geturls_text.url;
2193   httpheaderout = geturls_text.headers;
2194   httpheadersentout = geturls_text.request.headers;
2195   if(isinstance(httpheaderout, list)):
2196    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2197   if(sys.version[0]=="2"):
2198    try:
2199     prehttpheaderout = httpheaderout;
2200     httpheaderkeys = httpheaderout.keys();
2201     imax = len(httpheaderkeys);
2202     ic = 0;
2203     httpheaderout = {};
2204     while(ic < imax):
2205      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2206      ic += 1;
2207    except AttributeError:
2208     pass;
2209   httpheaderout = fix_header_names(httpheaderout);
2210   if(isinstance(httpheadersentout, list)):
2211    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2212   httpheadersentout = fix_header_names(httpheadersentout);
2213   downloadsize = int(httpheaderout.get('Content-Length'));
2214   if(downloadsize is not None):
2215    downloadsize = int(downloadsize);
2216   if downloadsize is None: downloadsize = 0;
2217   fulldatasize = 0;
2218   prevdownsize = 0;
2219   log.info("Downloading URL "+httpurl);
2220   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2221    tmpfilename = f.name;
2222    try:
2223     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
2224    except AttributeError:
2225     try:
2226      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2227     except ValueError:
2228      pass;
2229    except ValueError:
2230     pass;
2231    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2232    for databytes in geturls_text.iter_content(chunk_size=buffersize):
2233     datasize = len(databytes);
2234     fulldatasize = datasize + fulldatasize;
2235     percentage = "";
2236     if(downloadsize>0):
2237      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2238     downloaddiff = fulldatasize - prevdownsize;
2239     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2240     prevdownsize = fulldatasize;
2241     f.write(databytes);
2242    f.close();
2243   geturls_text.close();
2244   exec_time_end = time.time();
2245   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2246   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2247   return returnval;
2248
2249 if(not haverequests):
2250  def download_from_url_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2251   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep)
2252   return returnval;
2253
2254 if(haverequests):
2255  def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2256   global geturls_download_sleep;
2257   if(sleep<0):
2258    sleep = geturls_download_sleep;
2259   if(not outfile=="-"):
2260    outpath = outpath.rstrip(os.path.sep);
2261    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2262    if(not os.path.exists(outpath)):
2263     os.makedirs(outpath);
2264    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2265     return False;
2266    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2267     return False;
2268    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2269    if(not pretmpfilename):
2270     return False;
2271    tmpfilename = pretmpfilename['Filename'];
2272    downloadsize = os.path.getsize(tmpfilename);
2273    fulldatasize = 0;
2274    log.info("Moving file "+tmpfilename+" to "+filepath);
2275    exec_time_start = time.time();
2276    shutil.move(tmpfilename, filepath);
2277    try:
2278     os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2279    except AttributeError:
2280     try:
2281      os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2282     except ValueError:
2283      pass;
2284    except ValueError:
2285     pass;
2286    exec_time_end = time.time();
2287    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2288    if(os.path.exists(tmpfilename)):
2289     os.remove(tmpfilename);
2290    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2291   if(outfile=="-" and sys.version[0]=="2"):
2292    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2293    if(not pretmpfilename):
2294     return False;
2295    tmpfilename = pretmpfilename['Filename'];
2296    downloadsize = os.path.getsize(tmpfilename);
2297    fulldatasize = 0;
2298    prevdownsize = 0;
2299    exec_time_start = time.time();
2300    with open(tmpfilename, 'rb') as ft:
2301     f = StringIO();
2302     while True:
2303      databytes = ft.read(buffersize[1]);
2304      if not databytes: break;
2305      datasize = len(databytes);
2306      fulldatasize = datasize + fulldatasize;
2307      percentage = "";
2308      if(downloadsize>0):
2309       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2310      downloaddiff = fulldatasize - prevdownsize;
2311      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2312      prevdownsize = fulldatasize;
2313      f.write(databytes);
2314     f.seek(0);
2315     fdata = f.getvalue();
2316     f.close();
2317     ft.close();
2318     os.remove(tmpfilename);
2319     exec_time_end = time.time();
2320     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2321    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2322   if(outfile=="-" and sys.version[0]>="3"):
2323    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2324    tmpfilename = pretmpfilename['Filename'];
2325    downloadsize = os.path.getsize(tmpfilename);
2326    fulldatasize = 0;
2327    prevdownsize = 0;
2328    exec_time_start = time.time();
2329    with open(tmpfilename, 'rb') as ft:
2330     f = BytesIO();
2331     while True:
2332      databytes = ft.read(buffersize[1]);
2333      if not databytes: break;
2334      datasize = len(databytes);
2335      fulldatasize = datasize + fulldatasize;
2336      percentage = "";
2337      if(downloadsize>0):
2338       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2339      downloaddiff = fulldatasize - prevdownsize;
2340      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2341      prevdownsize = fulldatasize;
2342      f.write(databytes);
2343     f.seek(0);
2344     fdata = f.getvalue();
2345     f.close();
2346     ft.close();
2347     os.remove(tmpfilename);
2348     exec_time_end = time.time();
2349     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2350    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2351   return returnval;
2352
2353 if(not haverequests):
2354  def download_from_url_to_file_with_requests(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2355   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2356   return returnval;
2357
2358 if(havehttpx):
2359  def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2360   global geturls_download_sleep, havebrotli;
2361   if(sleep<0):
2362    sleep = geturls_download_sleep;
2363   urlparts = urlparse.urlparse(httpurl);
2364   if(isinstance(httpheaders, list)):
2365    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2366   httpheaders = fix_header_names(httpheaders);
2367   if(httpuseragent is not None):
2368    if('User-Agent' in httpheaders):
2369     httpheaders['User-Agent'] = httpuseragent;
2370    else:
2371     httpuseragent.update({'User-Agent': httpuseragent});
2372   if(httpreferer is not None):
2373    if('Referer' in httpheaders):
2374     httpheaders['Referer'] = httpreferer;
2375    else:
2376     httpuseragent.update({'Referer': httpreferer});
2377   if(urlparts.username is not None or urlparts.password is not None):
2378    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2379    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2380   time.sleep(sleep);
2381   if(postdata is not None and not isinstance(postdata, dict)):
2382    postdata = urlencode(postdata);
2383   try:
2384    if(httpmethod=="GET"):
2385     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
2386     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2387    elif(httpmethod=="POST"):
2388     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
2389     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
2390    else:
2391     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
2392     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2393   except httpx.ConnectTimeout:
2394    log.info("Error With URL "+httpurl);
2395    return False;
2396   except httpx.ConnectError:
2397    log.info("Error With URL "+httpurl);
2398    return False;
2399   except socket.timeout:
2400    log.info("Error With URL "+httpurl);
2401    return False;
2402   httpcodeout = geturls_text.status_code;
2403   httpcodereason = geturls_text.reason_phrase;
2404   httpversionout = geturls_text.http_version;
2405   httpmethodout = httpmethod;
2406   httpurlout = str(geturls_text.url);
2407   httpheaderout = geturls_text.headers;
2408   httpheadersentout = geturls_text.request.headers;
2409   if(isinstance(httpheaderout, list)):
2410    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2411   if(sys.version[0]=="2"):
2412    try:
2413     prehttpheaderout = httpheaderout;
2414     httpheaderkeys = httpheaderout.keys();
2415     imax = len(httpheaderkeys);
2416     ic = 0;
2417     httpheaderout = {};
2418     while(ic < imax):
2419      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2420      ic += 1;
2421    except AttributeError:
2422     pass;
2423   httpheaderout = fix_header_names(httpheaderout);
2424   if(isinstance(httpheadersentout, list)):
2425    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2426   httpheadersentout = fix_header_names(httpheadersentout);
2427   log.info("Downloading URL "+httpurl);
2428   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
2429    if(sys.version[0]=="2"):
2430     strbuf = StringIO(geturls_text.read());
2431    if(sys.version[0]>="3"):
2432     strbuf = BytesIO(geturls_text.read());
2433    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2434    returnval_content = gzstrbuf.read()[:];
2435   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
2436    returnval_content = geturls_text.read()[:];
2437   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
2438    returnval_content = geturls_text.read()[:];
2439    returnval_content = brotli.decompress(returnval_content);
2440   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2441   geturls_text.close();
2442   return returnval;
2443
2444 if(not havehttpx):
2445  def download_from_url_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2446   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep)
2447   return returnval;
2448
2449 if(havehttpx):
2450  def download_from_url_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2451   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2452   exec_time_start = time.time();
2453   myhash = hashlib.new("sha1");
2454   if(sys.version[0]=="2"):
2455    myhash.update(httpurl);
2456    myhash.update(str(buffersize));
2457    myhash.update(str(exec_time_start));
2458   if(sys.version[0]>="3"):
2459    myhash.update(httpurl.encode('utf-8'));
2460    myhash.update(str(buffersize).encode('utf-8'));
2461    myhash.update(str(exec_time_start).encode('utf-8'));
2462   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2463   if(sleep<0):
2464    sleep = geturls_download_sleep;
2465   urlparts = urlparse.urlparse(httpurl);
2466   if(isinstance(httpheaders, list)):
2467    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2468   httpheaders = fix_header_names(httpheaders);
2469   if(httpuseragent is not None):
2470    if('User-Agent' in httpheaders):
2471     httpheaders['User-Agent'] = httpuseragent;
2472    else:
2473     httpuseragent.update({'User-Agent': httpuseragent});
2474   if(httpreferer is not None):
2475    if('Referer' in httpheaders):
2476     httpheaders['Referer'] = httpreferer;
2477    else:
2478     httpuseragent.update({'Referer': httpreferer});
2479   if(urlparts.username is not None or urlparts.password is not None):
2480    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2481    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2482   time.sleep(sleep);
2483   if(postdata is not None and not isinstance(postdata, dict)):
2484    postdata = urlencode(postdata);
2485   try:
2486    if(httpmethod=="GET"):
2487     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
2488     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2489    elif(httpmethod=="POST"):
2490     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
2491     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
2492    else:
2493     httpx_pool = httpx.Client(http1=True, http2=False, trust_env=True);
2494     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2495   except httpx.ConnectTimeout:
2496    log.info("Error With URL "+httpurl);
2497    return False;
2498   except httpx.ConnectError:
2499    log.info("Error With URL "+httpurl);
2500    return False;
2501   except socket.timeout:
2502    log.info("Error With URL "+httpurl);
2503    return False;
2504   httpcodeout = geturls_text.status_code;
2505   httpcodereason = geturls_text.reason_phrase;
2506   httpversionout = geturls_text.http_version;
2507   httpmethodout = httpmethod;
2508   httpurlout = str(geturls_text.url);
2509   httpheaderout = geturls_text.headers;
2510   httpheadersentout = geturls_text.request.headers;
2511   if(isinstance(httpheaderout, list)):
2512    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2513   if(sys.version[0]=="2"):
2514    try:
2515     prehttpheaderout = httpheaderout;
2516     httpheaderkeys = httpheaderout.keys();
2517     imax = len(httpheaderkeys);
2518     ic = 0;
2519     httpheaderout = {};
2520     while(ic < imax):
2521      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2522      ic += 1;
2523    except AttributeError:
2524     pass;
2525   httpheaderout = fix_header_names(httpheaderout);
2526   if(isinstance(httpheadersentout, list)):
2527    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2528   httpheadersentout = fix_header_names(httpheadersentout);
2529   downloadsize = int(httpheaderout.get('Content-Length'));
2530   if(downloadsize is not None):
2531    downloadsize = int(downloadsize);
2532   if downloadsize is None: downloadsize = 0;
2533   fulldatasize = 0;
2534   prevdownsize = 0;
2535   log.info("Downloading URL "+httpurl);
2536   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2537    tmpfilename = f.name;
2538    try:
2539     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
2540    except AttributeError:
2541     try:
2542      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2543     except ValueError:
2544      pass;
2545    except ValueError:
2546     pass;
2547    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2548    for databytes in geturls_text.iter_content(chunk_size=buffersize):
2549     datasize = len(databytes);
2550     fulldatasize = datasize + fulldatasize;
2551     percentage = "";
2552     if(downloadsize>0):
2553      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2554     downloaddiff = fulldatasize - prevdownsize;
2555     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2556     prevdownsize = fulldatasize;
2557     f.write(databytes);
2558    f.close();
2559   geturls_text.close();
2560   exec_time_end = time.time();
2561   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2562   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2563   return returnval;
2564
2565 if(not havehttpx):
2566  def download_from_url_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2567   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep)
2568   return returnval;
2569
2570 if(havehttpx):
2571  def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2572   global geturls_download_sleep;
2573   if(sleep<0):
2574    sleep = geturls_download_sleep;
2575   if(not outfile=="-"):
2576    outpath = outpath.rstrip(os.path.sep);
2577    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2578    if(not os.path.exists(outpath)):
2579     os.makedirs(outpath);
2580    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2581     return False;
2582    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2583     return False;
2584    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2585    if(not pretmpfilename):
2586     return False;
2587    tmpfilename = pretmpfilename['Filename'];
2588    downloadsize = os.path.getsize(tmpfilename);
2589    fulldatasize = 0;
2590    log.info("Moving file "+tmpfilename+" to "+filepath);
2591    exec_time_start = time.time();
2592    shutil.move(tmpfilename, filepath);
2593    try:
2594     os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2595    except AttributeError:
2596     try:
2597      os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2598     except ValueError:
2599      pass;
2600    except ValueError:
2601     pass;
2602    exec_time_end = time.time();
2603    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2604    if(os.path.exists(tmpfilename)):
2605     os.remove(tmpfilename);
2606    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2607   if(outfile=="-" and sys.version[0]=="2"):
2608    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2609    if(not pretmpfilename):
2610     return False;
2611    tmpfilename = pretmpfilename['Filename'];
2612    downloadsize = os.path.getsize(tmpfilename);
2613    fulldatasize = 0;
2614    prevdownsize = 0;
2615    exec_time_start = time.time();
2616    with open(tmpfilename, 'rb') as ft:
2617     f = StringIO();
2618     while True:
2619      databytes = ft.read(buffersize[1]);
2620      if not databytes: break;
2621      datasize = len(databytes);
2622      fulldatasize = datasize + fulldatasize;
2623      percentage = "";
2624      if(downloadsize>0):
2625       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2626      downloaddiff = fulldatasize - prevdownsize;
2627      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2628      prevdownsize = fulldatasize;
2629      f.write(databytes);
2630     f.seek(0);
2631     fdata = f.getvalue();
2632     f.close();
2633     ft.close();
2634     os.remove(tmpfilename);
2635     exec_time_end = time.time();
2636     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2637    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2638   if(outfile=="-" and sys.version[0]>="3"):
2639    pretmpfilename = download_from_url_file_with_httpx(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2640    tmpfilename = pretmpfilename['Filename'];
2641    downloadsize = os.path.getsize(tmpfilename);
2642    fulldatasize = 0;
2643    prevdownsize = 0;
2644    exec_time_start = time.time();
2645    with open(tmpfilename, 'rb') as ft:
2646     f = BytesIO();
2647     while True:
2648      databytes = ft.read(buffersize[1]);
2649      if not databytes: break;
2650      datasize = len(databytes);
2651      fulldatasize = datasize + fulldatasize;
2652      percentage = "";
2653      if(downloadsize>0):
2654       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2655      downloaddiff = fulldatasize - prevdownsize;
2656      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2657      prevdownsize = fulldatasize;
2658      f.write(databytes);
2659     f.seek(0);
2660     fdata = f.getvalue();
2661     f.close();
2662     ft.close();
2663     os.remove(tmpfilename);
2664     exec_time_end = time.time();
2665     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2666    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2667   return returnval;
2668
2669 if(not havehttpx):
2670  def download_from_url_to_file_with_httpx(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2671   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2672   return returnval;
2673
2674 if(havehttpx):
2675  def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2676   global geturls_download_sleep, havebrotli;
2677   if(sleep<0):
2678    sleep = geturls_download_sleep;
2679   urlparts = urlparse.urlparse(httpurl);
2680   if(isinstance(httpheaders, list)):
2681    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2682   httpheaders = fix_header_names(httpheaders);
2683   if(httpuseragent is not None):
2684    if('User-Agent' in httpheaders):
2685     httpheaders['User-Agent'] = httpuseragent;
2686    else:
2687     httpuseragent.update({'User-Agent': httpuseragent});
2688   if(httpreferer is not None):
2689    if('Referer' in httpheaders):
2690     httpheaders['Referer'] = httpreferer;
2691    else:
2692     httpuseragent.update({'Referer': httpreferer});
2693   if(urlparts.username is not None or urlparts.password is not None):
2694    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2695    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2696   time.sleep(sleep);
2697   if(postdata is not None and not isinstance(postdata, dict)):
2698    postdata = urlencode(postdata);
2699   try:
2700    if(httpmethod=="GET"):
2701     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2702     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2703    elif(httpmethod=="POST"):
2704     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2705     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
2706    else:
2707     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2708     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2709   except httpx.ConnectTimeout:
2710    log.info("Error With URL "+httpurl);
2711    return False;
2712   except httpx.ConnectError:
2713    log.info("Error With URL "+httpurl);
2714    return False;
2715   except socket.timeout:
2716    log.info("Error With URL "+httpurl);
2717    return False;
2718   httpcodeout = geturls_text.status_code;
2719   httpcodereason = geturls_text.reason_phrase;
2720   httpversionout = geturls_text.http_version;
2721   httpmethodout = httpmethod;
2722   httpurlout = str(geturls_text.url);
2723   httpheaderout = geturls_text.headers;
2724   httpheadersentout = geturls_text.request.headers;
2725   if(isinstance(httpheaderout, list)):
2726    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2727   if(sys.version[0]=="2"):
2728    try:
2729     prehttpheaderout = httpheaderout;
2730     httpheaderkeys = httpheaderout.keys();
2731     imax = len(httpheaderkeys);
2732     ic = 0;
2733     httpheaderout = {};
2734     while(ic < imax):
2735      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2736      ic += 1;
2737    except AttributeError:
2738     pass;
2739   httpheaderout = fix_header_names(httpheaderout);
2740   if(isinstance(httpheadersentout, list)):
2741    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2742   httpheadersentout = fix_header_names(httpheadersentout);
2743   log.info("Downloading URL "+httpurl);
2744   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
2745    if(sys.version[0]=="2"):
2746     strbuf = StringIO(geturls_text.read());
2747    if(sys.version[0]>="3"):
2748     strbuf = BytesIO(geturls_text.read());
2749    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
2750    returnval_content = gzstrbuf.read()[:];
2751   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
2752    returnval_content = geturls_text.read()[:];
2753   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
2754    returnval_content = geturls_text.read()[:];
2755    returnval_content = brotli.decompress(returnval_content);
2756   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2757   geturls_text.close();
2758   return returnval;
2759
2760 if(not havehttpx):
2761  def download_from_url_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2762   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep)
2763   return returnval;
2764
2765 if(havehttpx):
2766  def download_from_url_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2767   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
2768   exec_time_start = time.time();
2769   myhash = hashlib.new("sha1");
2770   if(sys.version[0]=="2"):
2771    myhash.update(httpurl);
2772    myhash.update(str(buffersize));
2773    myhash.update(str(exec_time_start));
2774   if(sys.version[0]>="3"):
2775    myhash.update(httpurl.encode('utf-8'));
2776    myhash.update(str(buffersize).encode('utf-8'));
2777    myhash.update(str(exec_time_start).encode('utf-8'));
2778   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
2779   if(sleep<0):
2780    sleep = geturls_download_sleep;
2781   urlparts = urlparse.urlparse(httpurl);
2782   if(isinstance(httpheaders, list)):
2783    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2784   httpheaders = fix_header_names(httpheaders);
2785   if(httpuseragent is not None):
2786    if('User-Agent' in httpheaders):
2787     httpheaders['User-Agent'] = httpuseragent;
2788    else:
2789     httpuseragent.update({'User-Agent': httpuseragent});
2790   if(httpreferer is not None):
2791    if('Referer' in httpheaders):
2792     httpheaders['Referer'] = httpreferer;
2793    else:
2794     httpuseragent.update({'Referer': httpreferer});
2795   if(urlparts.username is not None or urlparts.password is not None):
2796    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
2797    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
2798   time.sleep(sleep);
2799   if(postdata is not None and not isinstance(postdata, dict)):
2800    postdata = urlencode(postdata);
2801   try:
2802    if(httpmethod=="GET"):
2803     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2804     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2805    elif(httpmethod=="POST"):
2806     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2807     geturls_text = httpx_pool.post(httpurl, data=postdata, headers=httpheaders, cookies=httpcookie);
2808    else:
2809     httpx_pool = httpx.Client(http1=True, http2=True, trust_env=True);
2810     geturls_text = httpx_pool.get(httpurl, headers=httpheaders, cookies=httpcookie);
2811   except httpx.ConnectTimeout:
2812    log.info("Error With URL "+httpurl);
2813    return False;
2814   except httpx.ConnectError:
2815    log.info("Error With URL "+httpurl);
2816    return False;
2817   except socket.timeout:
2818    log.info("Error With URL "+httpurl);
2819    return False;
2820   httpcodeout = geturls_text.status_code;
2821   httpcodereason = geturls_text.reason_phrase;
2822   httpversionout = geturls_text.http_version;
2823   httpmethodout = httpmethod;
2824   httpurlout = str(geturls_text.url);
2825   httpheaderout = geturls_text.headers;
2826   httpheadersentout = geturls_text.request.headers;
2827   if(isinstance(httpheaderout, list)):
2828    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
2829   if(sys.version[0]=="2"):
2830    try:
2831     prehttpheaderout = httpheaderout;
2832     httpheaderkeys = httpheaderout.keys();
2833     imax = len(httpheaderkeys);
2834     ic = 0;
2835     httpheaderout = {};
2836     while(ic < imax):
2837      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
2838      ic += 1;
2839    except AttributeError:
2840     pass;
2841   httpheaderout = fix_header_names(httpheaderout);
2842   if(isinstance(httpheadersentout, list)):
2843    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
2844   httpheadersentout = fix_header_names(httpheadersentout);
2845   downloadsize = int(httpheaderout.get('Content-Length'));
2846   if(downloadsize is not None):
2847    downloadsize = int(downloadsize);
2848   if downloadsize is None: downloadsize = 0;
2849   fulldatasize = 0;
2850   prevdownsize = 0;
2851   log.info("Downloading URL "+httpurl);
2852   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
2853    tmpfilename = f.name;
2854    try:
2855     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
2856    except AttributeError:
2857     try:
2858      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2859     except ValueError:
2860      pass;
2861    except ValueError:
2862     pass;
2863    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
2864    for databytes in geturls_text.iter_content(chunk_size=buffersize):
2865     datasize = len(databytes);
2866     fulldatasize = datasize + fulldatasize;
2867     percentage = "";
2868     if(downloadsize>0):
2869      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2870     downloaddiff = fulldatasize - prevdownsize;
2871     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2872     prevdownsize = fulldatasize;
2873     f.write(databytes);
2874    f.close();
2875   geturls_text.close();
2876   exec_time_end = time.time();
2877   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
2878   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
2879   return returnval;
2880
2881 if(not havehttpx):
2882  def download_from_url_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
2883   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep)
2884   return returnval;
2885
2886 if(havehttpx):
2887  def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2888   global geturls_download_sleep;
2889   if(sleep<0):
2890    sleep = geturls_download_sleep;
2891   if(not outfile=="-"):
2892    outpath = outpath.rstrip(os.path.sep);
2893    filepath = os.path.realpath(outpath+os.path.sep+outfile);
2894    if(not os.path.exists(outpath)):
2895     os.makedirs(outpath);
2896    if(os.path.exists(outpath) and os.path.isfile(outpath)):
2897     return False;
2898    if(os.path.exists(filepath) and os.path.isdir(filepath)):
2899     return False;
2900    pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2901    if(not pretmpfilename):
2902     return False;
2903    tmpfilename = pretmpfilename['Filename'];
2904    downloadsize = os.path.getsize(tmpfilename);
2905    fulldatasize = 0;
2906    log.info("Moving file "+tmpfilename+" to "+filepath);
2907    exec_time_start = time.time();
2908    shutil.move(tmpfilename, filepath);
2909    try:
2910     os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
2911    except AttributeError:
2912     try:
2913      os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
2914     except ValueError:
2915      pass;
2916    except ValueError:
2917     pass;
2918    exec_time_end = time.time();
2919    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
2920    if(os.path.exists(tmpfilename)):
2921     os.remove(tmpfilename);
2922    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2923   if(outfile=="-" and sys.version[0]=="2"):
2924    pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2925    if(not pretmpfilename):
2926     return False;
2927    tmpfilename = pretmpfilename['Filename'];
2928    downloadsize = os.path.getsize(tmpfilename);
2929    fulldatasize = 0;
2930    prevdownsize = 0;
2931    exec_time_start = time.time();
2932    with open(tmpfilename, 'rb') as ft:
2933     f = StringIO();
2934     while True:
2935      databytes = ft.read(buffersize[1]);
2936      if not databytes: break;
2937      datasize = len(databytes);
2938      fulldatasize = datasize + fulldatasize;
2939      percentage = "";
2940      if(downloadsize>0):
2941       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2942      downloaddiff = fulldatasize - prevdownsize;
2943      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2944      prevdownsize = fulldatasize;
2945      f.write(databytes);
2946     f.seek(0);
2947     fdata = f.getvalue();
2948     f.close();
2949     ft.close();
2950     os.remove(tmpfilename);
2951     exec_time_end = time.time();
2952     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2953    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2954   if(outfile=="-" and sys.version[0]>="3"):
2955    pretmpfilename = download_from_url_file_with_httpx2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
2956    tmpfilename = pretmpfilename['Filename'];
2957    downloadsize = os.path.getsize(tmpfilename);
2958    fulldatasize = 0;
2959    prevdownsize = 0;
2960    exec_time_start = time.time();
2961    with open(tmpfilename, 'rb') as ft:
2962     f = BytesIO();
2963     while True:
2964      databytes = ft.read(buffersize[1]);
2965      if not databytes: break;
2966      datasize = len(databytes);
2967      fulldatasize = datasize + fulldatasize;
2968      percentage = "";
2969      if(downloadsize>0):
2970       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
2971      downloaddiff = fulldatasize - prevdownsize;
2972      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
2973      prevdownsize = fulldatasize;
2974      f.write(databytes);
2975     f.seek(0);
2976     fdata = f.getvalue();
2977     f.close();
2978     ft.close();
2979     os.remove(tmpfilename);
2980     exec_time_end = time.time();
2981     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
2982    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
2983   return returnval;
2984
2985 if(not havehttpx):
2986  def download_from_url_to_file_with_httpx2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
2987   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
2988   return returnval;
2989
2990 if(havehttpcore):
2991  def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
2992   global geturls_download_sleep, havebrotli;
2993   if(sleep<0):
2994    sleep = geturls_download_sleep;
2995   urlparts = urlparse.urlparse(httpurl);
2996   if(isinstance(httpheaders, list)):
2997    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
2998   httpheaders = fix_header_names(httpheaders);
2999   if(httpuseragent is not None):
3000    if('User-Agent' in httpheaders):
3001     httpheaders['User-Agent'] = httpuseragent;
3002    else:
3003     httpuseragent.update({'User-Agent': httpuseragent});
3004   if(httpreferer is not None):
3005    if('Referer' in httpheaders):
3006     httpheaders['Referer'] = httpreferer;
3007    else:
3008     httpuseragent.update({'Referer': httpreferer});
3009   if(urlparts.username is not None or urlparts.password is not None):
3010    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3011    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3012   time.sleep(sleep);
3013   if(postdata is not None and not isinstance(postdata, dict)):
3014    postdata = urlencode(postdata);
3015   try:
3016    if(httpmethod=="GET"):
3017     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
3018     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
3019    elif(httpmethod=="POST"):
3020     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
3021     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
3022    else:
3023     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
3024     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
3025   except httpcore.ConnectTimeout:
3026    log.info("Error With URL "+httpurl);
3027    return False;
3028   except httpcore.ConnectError:
3029    log.info("Error With URL "+httpurl);
3030    return False;
3031   except socket.timeout:
3032    log.info("Error With URL "+httpurl);
3033    return False;
3034   httpcodeout = geturls_text.status;
3035   httpcodereason = http_status_to_reason(geturls_text.status);
3036   httpversionout = "1.1";
3037   httpmethodout = httpmethod;
3038   httpurlout = str(httpurl);
3039   httpheaderout = geturls_text.headers;
3040   httpheadersentout = httpheaders;
3041   if(isinstance(httpheaderout, list)):
3042    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
3043   if(sys.version[0]=="2"):
3044    try:
3045     prehttpheaderout = httpheaderout;
3046     httpheaderkeys = httpheaderout.keys();
3047     imax = len(httpheaderkeys);
3048     ic = 0;
3049     httpheaderout = {};
3050     while(ic < imax):
3051      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
3052      ic += 1;
3053    except AttributeError:
3054     pass;
3055   httpheaderout = fix_header_names(httpheaderout);
3056   if(isinstance(httpheadersentout, list)):
3057    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
3058   httpheadersentout = fix_header_names(httpheadersentout);
3059   log.info("Downloading URL "+httpurl);
3060   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
3061    if(sys.version[0]=="2"):
3062     strbuf = StringIO(geturls_text.read());
3063    if(sys.version[0]>="3"):
3064     strbuf = BytesIO(geturls_text.read());
3065    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
3066    returnval_content = gzstrbuf.read()[:];
3067   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
3068    returnval_content = geturls_text.read()[:];
3069   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
3070    returnval_content = geturls_text.read()[:];
3071    returnval_content = brotli.decompress(returnval_content);
3072   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
3073   geturls_text.close();
3074   return returnval;
3075
3076 if(not havehttpcore):
3077  def download_from_url_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3078   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep)
3079   return returnval;
3080
3081 if(havehttpcore):
3082  def download_from_url_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3083   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
3084   exec_time_start = time.time();
3085   myhash = hashlib.new("sha1");
3086   if(sys.version[0]=="2"):
3087    myhash.update(httpurl);
3088    myhash.update(str(buffersize));
3089    myhash.update(str(exec_time_start));
3090   if(sys.version[0]>="3"):
3091    myhash.update(httpurl.encode('utf-8'));
3092    myhash.update(str(buffersize).encode('utf-8'));
3093    myhash.update(str(exec_time_start).encode('utf-8'));
3094   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
3095   if(sleep<0):
3096    sleep = geturls_download_sleep;
3097   urlparts = urlparse.urlparse(httpurl);
3098   if(isinstance(httpheaders, list)):
3099    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3100   httpheaders = fix_header_names(httpheaders);
3101   if(httpuseragent is not None):
3102    if('User-Agent' in httpheaders):
3103     httpheaders['User-Agent'] = httpuseragent;
3104    else:
3105     httpuseragent.update({'User-Agent': httpuseragent});
3106   if(httpreferer is not None):
3107    if('Referer' in httpheaders):
3108     httpheaders['Referer'] = httpreferer;
3109    else:
3110     httpuseragent.update({'Referer': httpreferer});
3111   if(urlparts.username is not None or urlparts.password is not None):
3112    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3113    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3114   time.sleep(sleep);
3115   if(postdata is not None and not isinstance(postdata, dict)):
3116    postdata = urlencode(postdata);
3117   try:
3118    if(httpmethod=="GET"):
3119     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
3120     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
3121    elif(httpmethod=="POST"):
3122     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
3123     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
3124    else:
3125     httpx_pool = httpcore.ConnectionPool(http1=True, http2=False);
3126     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
3127   except httpcore.ConnectTimeout:
3128    log.info("Error With URL "+httpurl);
3129    return False;
3130   except httpcore.ConnectError:
3131    log.info("Error With URL "+httpurl);
3132    return False;
3133   except socket.timeout:
3134    log.info("Error With URL "+httpurl);
3135    return False;
3136   httpcodeout = geturls_text.status;
3137   httpcodereason = http_status_to_reason(geturls_text.status);
3138   httpversionout = "1.1";
3139   httpmethodout = httpmethod;
3140   httpurlout = str(httpurl);
3141   httpheaderout = geturls_text.headers;
3142   httpheadersentout = httpheaders;
3143   if(isinstance(httpheaderout, list)):
3144    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
3145   if(sys.version[0]=="2"):
3146    try:
3147     prehttpheaderout = httpheaderout;
3148     httpheaderkeys = httpheaderout.keys();
3149     imax = len(httpheaderkeys);
3150     ic = 0;
3151     httpheaderout = {};
3152     while(ic < imax):
3153      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
3154      ic += 1;
3155    except AttributeError:
3156     pass;
3157   httpheaderout = fix_header_names(httpheaderout);
3158   if(isinstance(httpheadersentout, list)):
3159    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
3160   httpheadersentout = fix_header_names(httpheadersentout);
3161   downloadsize = int(httpheaderout.get('Content-Length'));
3162   if(downloadsize is not None):
3163    downloadsize = int(downloadsize);
3164   if downloadsize is None: downloadsize = 0;
3165   fulldatasize = 0;
3166   prevdownsize = 0;
3167   log.info("Downloading URL "+httpurl);
3168   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
3169    tmpfilename = f.name;
3170    try:
3171     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
3172    except AttributeError:
3173     try:
3174      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3175     except ValueError:
3176      pass;
3177    except ValueError:
3178     pass;
3179    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
3180    for databytes in geturls_text.iter_content(chunk_size=buffersize):
3181     datasize = len(databytes);
3182     fulldatasize = datasize + fulldatasize;
3183     percentage = "";
3184     if(downloadsize>0):
3185      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3186     downloaddiff = fulldatasize - prevdownsize;
3187     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3188     prevdownsize = fulldatasize;
3189     f.write(databytes);
3190    f.close();
3191   geturls_text.close();
3192   exec_time_end = time.time();
3193   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
3194   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
3195   return returnval;
3196
3197 if(not havehttpcore):
3198  def download_from_url_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3199   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep)
3200   return returnval;
3201
3202 if(havehttpcore):
3203  def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3204   global geturls_download_sleep;
3205   if(sleep<0):
3206    sleep = geturls_download_sleep;
3207   if(not outfile=="-"):
3208    outpath = outpath.rstrip(os.path.sep);
3209    filepath = os.path.realpath(outpath+os.path.sep+outfile);
3210    if(not os.path.exists(outpath)):
3211     os.makedirs(outpath);
3212    if(os.path.exists(outpath) and os.path.isfile(outpath)):
3213     return False;
3214    if(os.path.exists(filepath) and os.path.isdir(filepath)):
3215     return False;
3216    pretmpfilename = download_from_url_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3217    if(not pretmpfilename):
3218     return False;
3219    tmpfilename = pretmpfilename['Filename'];
3220    downloadsize = os.path.getsize(tmpfilename);
3221    fulldatasize = 0;
3222    log.info("Moving file "+tmpfilename+" to "+filepath);
3223    exec_time_start = time.time();
3224    shutil.move(tmpfilename, filepath);
3225    try:
3226     os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3227    except AttributeError:
3228     try:
3229      os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3230     except ValueError:
3231      pass;
3232    except ValueError:
3233     pass;
3234    exec_time_end = time.time();
3235    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
3236    if(os.path.exists(tmpfilename)):
3237     os.remove(tmpfilename);
3238    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3239   if(outfile=="-" and sys.version[0]=="2"):
3240    pretmpfilename = download_from_url_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3241    if(not pretmpfilename):
3242     return False;
3243    tmpfilename = pretmpfilename['Filename'];
3244    downloadsize = os.path.getsize(tmpfilename);
3245    fulldatasize = 0;
3246    prevdownsize = 0;
3247    exec_time_start = time.time();
3248    with open(tmpfilename, 'rb') as ft:
3249     f = StringIO();
3250     while True:
3251      databytes = ft.read(buffersize[1]);
3252      if not databytes: break;
3253      datasize = len(databytes);
3254      fulldatasize = datasize + fulldatasize;
3255      percentage = "";
3256      if(downloadsize>0):
3257       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3258      downloaddiff = fulldatasize - prevdownsize;
3259      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3260      prevdownsize = fulldatasize;
3261      f.write(databytes);
3262     f.seek(0);
3263     fdata = f.getvalue();
3264     f.close();
3265     ft.close();
3266     os.remove(tmpfilename);
3267     exec_time_end = time.time();
3268     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3269    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3270   if(outfile=="-" and sys.version[0]>="3"):
3271    pretmpfilename = download_from_url_file_with_httpcore(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3272    tmpfilename = pretmpfilename['Filename'];
3273    downloadsize = os.path.getsize(tmpfilename);
3274    fulldatasize = 0;
3275    prevdownsize = 0;
3276    exec_time_start = time.time();
3277    with open(tmpfilename, 'rb') as ft:
3278     f = BytesIO();
3279     while True:
3280      databytes = ft.read(buffersize[1]);
3281      if not databytes: break;
3282      datasize = len(databytes);
3283      fulldatasize = datasize + fulldatasize;
3284      percentage = "";
3285      if(downloadsize>0):
3286       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3287      downloaddiff = fulldatasize - prevdownsize;
3288      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3289      prevdownsize = fulldatasize;
3290      f.write(databytes);
3291     f.seek(0);
3292     fdata = f.getvalue();
3293     f.close();
3294     ft.close();
3295     os.remove(tmpfilename);
3296     exec_time_end = time.time();
3297     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3298    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3299   return returnval;
3300
3301 if(not havehttpcore):
3302  def download_from_url_to_file_with_httpcore(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3303   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
3304   return returnval;
3305
3306 if(havehttpcore):
3307  def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3308   global geturls_download_sleep, havebrotli;
3309   if(sleep<0):
3310    sleep = geturls_download_sleep;
3311   urlparts = urlparse.urlparse(httpurl);
3312   if(isinstance(httpheaders, list)):
3313    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3314   httpheaders = fix_header_names(httpheaders);
3315   if(httpuseragent is not None):
3316    if('User-Agent' in httpheaders):
3317     httpheaders['User-Agent'] = httpuseragent;
3318    else:
3319     httpuseragent.update({'User-Agent': httpuseragent});
3320   if(httpreferer is not None):
3321    if('Referer' in httpheaders):
3322     httpheaders['Referer'] = httpreferer;
3323    else:
3324     httpuseragent.update({'Referer': httpreferer});
3325   if(urlparts.username is not None or urlparts.password is not None):
3326    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3327    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3328   time.sleep(sleep);
3329   if(postdata is not None and not isinstance(postdata, dict)):
3330    postdata = urlencode(postdata);
3331   try:
3332    if(httpmethod=="GET"):
3333     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
3334     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
3335    elif(httpmethod=="POST"):
3336     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
3337     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
3338    else:
3339     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
3340     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
3341   except httpcore.ConnectTimeout:
3342    log.info("Error With URL "+httpurl);
3343    return False;
3344   except httpcore.ConnectError:
3345    log.info("Error With URL "+httpurl);
3346    return False;
3347   except socket.timeout:
3348    log.info("Error With URL "+httpurl);
3349    return False;
3350   httpcodeout = geturls_text.status;
3351   httpcodereason = geturls_text.reason_phrase;
3352   httpversionout = "1.1";
3353   httpmethodout = httpmethod;
3354   httpurlout = str(httpurl);
3355   httpheaderout = geturls_text.headers;
3356   httpheadersentout = httpheaders;
3357   if(isinstance(httpheaderout, list)):
3358    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
3359   if(sys.version[0]=="2"):
3360    try:
3361     prehttpheaderout = httpheaderout;
3362     httpheaderkeys = httpheaderout.keys();
3363     imax = len(httpheaderkeys);
3364     ic = 0;
3365     httpheaderout = {};
3366     while(ic < imax):
3367      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
3368      ic += 1;
3369    except AttributeError:
3370     pass;
3371   httpheaderout = fix_header_names(httpheaderout);
3372   if(isinstance(httpheadersentout, list)):
3373    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
3374   httpheadersentout = fix_header_names(httpheadersentout);
3375   log.info("Downloading URL "+httpurl);
3376   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
3377    if(sys.version[0]=="2"):
3378     strbuf = StringIO(geturls_text.read());
3379    if(sys.version[0]>="3"):
3380     strbuf = BytesIO(geturls_text.read());
3381    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
3382    returnval_content = gzstrbuf.read()[:];
3383   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
3384    returnval_content = geturls_text.read()[:];
3385   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
3386    returnval_content = geturls_text.read()[:];
3387    returnval_content = brotli.decompress(returnval_content);
3388   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
3389   geturls_text.close();
3390   return returnval;
3391
3392 if(not havehttpcore):
3393  def download_from_url_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3394   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep)
3395   return returnval;
3396
3397 if(havehttpcore):
3398  def download_from_url_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3399   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
3400   exec_time_start = time.time();
3401   myhash = hashlib.new("sha1");
3402   if(sys.version[0]=="2"):
3403    myhash.update(httpurl);
3404    myhash.update(str(buffersize));
3405    myhash.update(str(exec_time_start));
3406   if(sys.version[0]>="3"):
3407    myhash.update(httpurl.encode('utf-8'));
3408    myhash.update(str(buffersize).encode('utf-8'));
3409    myhash.update(str(exec_time_start).encode('utf-8'));
3410   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
3411   if(sleep<0):
3412    sleep = geturls_download_sleep;
3413   urlparts = urlparse.urlparse(httpurl);
3414   if(isinstance(httpheaders, list)):
3415    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3416   httpheaders = fix_header_names(httpheaders);
3417   if(httpuseragent is not None):
3418    if('User-Agent' in httpheaders):
3419     httpheaders['User-Agent'] = httpuseragent;
3420    else:
3421     httpuseragent.update({'User-Agent': httpuseragent});
3422   if(httpreferer is not None):
3423    if('Referer' in httpheaders):
3424     httpheaders['Referer'] = httpreferer;
3425    else:
3426     httpuseragent.update({'Referer': httpreferer});
3427   if(urlparts.username is not None or urlparts.password is not None):
3428    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3429    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3430   time.sleep(sleep);
3431   if(postdata is not None and not isinstance(postdata, dict)):
3432    postdata = urlencode(postdata);
3433   try:
3434    if(httpmethod=="GET"):
3435     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
3436     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
3437    elif(httpmethod=="POST"):
3438     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
3439     geturls_text = httpx_pool.request("GET", httpurl, data=postdata, headers=httpheaders);
3440    else:
3441     httpx_pool = httpcore.ConnectionPool(http1=True, http2=True);
3442     geturls_text = httpx_pool.request("GET", httpurl, headers=httpheaders);
3443   except httpcore.ConnectTimeout:
3444    log.info("Error With URL "+httpurl);
3445    return False;
3446   except httpcore.ConnectError:
3447    log.info("Error With URL "+httpurl);
3448    return False;
3449   except socket.timeout:
3450    log.info("Error With URL "+httpurl);
3451    return False;
3452   httpcodeout = geturls_text.status;
3453   httpcodereason = geturls_text.reason_phrase;
3454   httpversionout = "1.1";
3455   httpmethodout = httpmethod;
3456   httpurlout = str(httpurl);
3457   httpheaderout = geturls_text.headers;
3458   httpheadersentout = httpheaders;
3459   if(isinstance(httpheaderout, list)):
3460    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
3461   if(sys.version[0]=="2"):
3462    try:
3463     prehttpheaderout = httpheaderout;
3464     httpheaderkeys = httpheaderout.keys();
3465     imax = len(httpheaderkeys);
3466     ic = 0;
3467     httpheaderout = {};
3468     while(ic < imax):
3469      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
3470      ic += 1;
3471    except AttributeError:
3472     pass;
3473   httpheaderout = fix_header_names(httpheaderout);
3474   if(isinstance(httpheadersentout, list)):
3475    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
3476   httpheadersentout = fix_header_names(httpheadersentout);
3477   downloadsize = int(httpheaderout.get('Content-Length'));
3478   if(downloadsize is not None):
3479    downloadsize = int(downloadsize);
3480   if downloadsize is None: downloadsize = 0;
3481   fulldatasize = 0;
3482   prevdownsize = 0;
3483   log.info("Downloading URL "+httpurl);
3484   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
3485    tmpfilename = f.name;
3486    try:
3487     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
3488    except AttributeError:
3489     try:
3490      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3491     except ValueError:
3492      pass;
3493    except ValueError:
3494     pass;
3495    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
3496    for databytes in geturls_text.iter_content(chunk_size=buffersize):
3497     datasize = len(databytes);
3498     fulldatasize = datasize + fulldatasize;
3499     percentage = "";
3500     if(downloadsize>0):
3501      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3502     downloaddiff = fulldatasize - prevdownsize;
3503     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3504     prevdownsize = fulldatasize;
3505     f.write(databytes);
3506    f.close();
3507   geturls_text.close();
3508   exec_time_end = time.time();
3509   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
3510   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
3511   return returnval;
3512
3513 if(not havehttpcore):
3514  def download_from_url_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3515   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep)
3516   return returnval;
3517
3518 if(havehttpcore):
3519  def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3520   global geturls_download_sleep;
3521   if(sleep<0):
3522    sleep = geturls_download_sleep;
3523   if(not outfile=="-"):
3524    outpath = outpath.rstrip(os.path.sep);
3525    filepath = os.path.realpath(outpath+os.path.sep+outfile);
3526    if(not os.path.exists(outpath)):
3527     os.makedirs(outpath);
3528    if(os.path.exists(outpath) and os.path.isfile(outpath)):
3529     return False;
3530    if(os.path.exists(filepath) and os.path.isdir(filepath)):
3531     return False;
3532    pretmpfilename = download_from_url_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3533    if(not pretmpfilename):
3534     return False;
3535    tmpfilename = pretmpfilename['Filename'];
3536    downloadsize = os.path.getsize(tmpfilename);
3537    fulldatasize = 0;
3538    log.info("Moving file "+tmpfilename+" to "+filepath);
3539    exec_time_start = time.time();
3540    shutil.move(tmpfilename, filepath);
3541    try:
3542     os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3543    except AttributeError:
3544     try:
3545      os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3546     except ValueError:
3547      pass;
3548    except ValueError:
3549     pass;
3550    exec_time_end = time.time();
3551    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
3552    if(os.path.exists(tmpfilename)):
3553     os.remove(tmpfilename);
3554    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3555   if(outfile=="-" and sys.version[0]=="2"):
3556    pretmpfilename = download_from_url_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3557    if(not pretmpfilename):
3558     return False;
3559    tmpfilename = pretmpfilename['Filename'];
3560    downloadsize = os.path.getsize(tmpfilename);
3561    fulldatasize = 0;
3562    prevdownsize = 0;
3563    exec_time_start = time.time();
3564    with open(tmpfilename, 'rb') as ft:
3565     f = StringIO();
3566     while True:
3567      databytes = ft.read(buffersize[1]);
3568      if not databytes: break;
3569      datasize = len(databytes);
3570      fulldatasize = datasize + fulldatasize;
3571      percentage = "";
3572      if(downloadsize>0):
3573       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3574      downloaddiff = fulldatasize - prevdownsize;
3575      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3576      prevdownsize = fulldatasize;
3577      f.write(databytes);
3578     f.seek(0);
3579     fdata = f.getvalue();
3580     f.close();
3581     ft.close();
3582     os.remove(tmpfilename);
3583     exec_time_end = time.time();
3584     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3585    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3586   if(outfile=="-" and sys.version[0]>="3"):
3587    pretmpfilename = download_from_url_file_with_httpcore2(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3588    tmpfilename = pretmpfilename['Filename'];
3589    downloadsize = os.path.getsize(tmpfilename);
3590    fulldatasize = 0;
3591    prevdownsize = 0;
3592    exec_time_start = time.time();
3593    with open(tmpfilename, 'rb') as ft:
3594     f = BytesIO();
3595     while True:
3596      databytes = ft.read(buffersize[1]);
3597      if not databytes: break;
3598      datasize = len(databytes);
3599      fulldatasize = datasize + fulldatasize;
3600      percentage = "";
3601      if(downloadsize>0):
3602       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3603      downloaddiff = fulldatasize - prevdownsize;
3604      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3605      prevdownsize = fulldatasize;
3606      f.write(databytes);
3607     f.seek(0);
3608     fdata = f.getvalue();
3609     f.close();
3610     ft.close();
3611     os.remove(tmpfilename);
3612     exec_time_end = time.time();
3613     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3614    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3615   return returnval;
3616
3617 if(not havehttpx):
3618  def download_from_url_to_file_with_httpcore2(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3619   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
3620   return returnval;
3621
3622 if(haveurllib3):
3623  def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3624   global geturls_download_sleep, havebrotli;
3625   if(sleep<0):
3626    sleep = geturls_download_sleep;
3627   urlparts = urlparse.urlparse(httpurl);
3628   if(isinstance(httpheaders, list)):
3629    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3630   httpheaders = fix_header_names(httpheaders);
3631   if(httpuseragent is not None):
3632    if('User-Agent' in httpheaders):
3633     httpheaders['User-Agent'] = httpuseragent;
3634    else:
3635     httpuseragent.update({'User-Agent': httpuseragent});
3636   if(httpreferer is not None):
3637    if('Referer' in httpheaders):
3638     httpheaders['Referer'] = httpreferer;
3639    else:
3640     httpuseragent.update({'Referer': httpreferer});
3641   if(urlparts.username is not None or urlparts.password is not None):
3642    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3643    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3644   time.sleep(sleep);
3645   urllib_pool = urllib3.PoolManager(headers=httpheaders);
3646   if(postdata is not None and not isinstance(postdata, dict)):
3647    postdata = urlencode(postdata);
3648   try:
3649    if(httpmethod=="GET"):
3650     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
3651    elif(httpmethod=="POST"):
3652     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
3653    else:
3654     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
3655   except urllib3.exceptions.ConnectTimeoutError:
3656    log.info("Error With URL "+httpurl);
3657    return False;
3658   except urllib3.exceptions.ConnectError:
3659    log.info("Error With URL "+httpurl);
3660    return False;
3661   except urllib3.exceptions.MaxRetryError:
3662    log.info("Error With URL "+httpurl);
3663    return False;
3664   except socket.timeout:
3665    log.info("Error With URL "+httpurl);
3666    return False;
3667   httpcodeout = geturls_text.status;
3668   httpcodereason = geturls_text.reason;
3669   httpversionout = "1.1";
3670   httpmethodout = httpmethod;
3671   httpurlout = geturls_text.geturl();
3672   httpheaderout = geturls_text.info();
3673   httpheadersentout = httpheaders;
3674   if(isinstance(httpheaderout, list)):
3675    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
3676   if(sys.version[0]=="2"):
3677    try:
3678     prehttpheaderout = httpheaderout;
3679     httpheaderkeys = httpheaderout.keys();
3680     imax = len(httpheaderkeys);
3681     ic = 0;
3682     httpheaderout = {};
3683     while(ic < imax):
3684      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
3685      ic += 1;
3686    except AttributeError:
3687     pass;
3688   httpheaderout = fix_header_names(httpheaderout);
3689   if(isinstance(httpheadersentout, list)):
3690    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
3691   httpheadersentout = fix_header_names(httpheadersentout);
3692   log.info("Downloading URL "+httpurl);
3693   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
3694    if(sys.version[0]=="2"):
3695     strbuf = StringIO(geturls_text.read());
3696    if(sys.version[0]>="3"):
3697     strbuf = BytesIO(geturls_text.read());
3698    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
3699    returnval_content = gzstrbuf.read()[:];
3700   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
3701    returnval_content = geturls_text.read()[:];
3702   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
3703    returnval_content = geturls_text.read()[:];
3704    returnval_content = brotli.decompress(returnval_content);
3705   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
3706   geturls_text.close();
3707   return returnval;
3708
3709 if(not haveurllib3):
3710  def download_from_url_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3711   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep)
3712   return returnval;
3713
3714 if(haveurllib3):
3715  def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3716   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
3717   exec_time_start = time.time();
3718   myhash = hashlib.new("sha1");
3719   if(sys.version[0]=="2"):
3720    myhash.update(httpurl);
3721    myhash.update(str(buffersize));
3722    myhash.update(str(exec_time_start));
3723   if(sys.version[0]>="3"):
3724    myhash.update(httpurl.encode('utf-8'));
3725    myhash.update(str(buffersize).encode('utf-8'));
3726    myhash.update(str(exec_time_start).encode('utf-8'));
3727   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
3728   if(sleep<0):
3729    sleep = geturls_download_sleep;
3730   urlparts = urlparse.urlparse(httpurl);
3731   if(isinstance(httpheaders, list)):
3732    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3733   httpheaders = fix_header_names(httpheaders);
3734   if(httpuseragent is not None):
3735    if('User-Agent' in httpheaders):
3736     httpheaders['User-Agent'] = httpuseragent;
3737    else:
3738     httpuseragent.update({'User-Agent': httpuseragent});
3739   if(httpreferer is not None):
3740    if('Referer' in httpheaders):
3741     httpheaders['Referer'] = httpreferer;
3742    else:
3743     httpuseragent.update({'Referer': httpreferer});
3744   if(urlparts.username is not None or urlparts.password is not None):
3745    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3746    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3747   time.sleep(sleep);
3748   urllib_pool = urllib3.PoolManager(headers=httpheaders);
3749   if(postdata is not None and not isinstance(postdata, dict)):
3750    postdata = urlencode(postdata);
3751   try:
3752    if(httpmethod=="GET"):
3753     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
3754    elif(httpmethod=="POST"):
3755     geturls_text = geturls_text = urllib_pool.request("POST", httpurl, body=postdata, headers=httpheaders, preload_content=False);
3756    else:
3757     geturls_text = geturls_text = urllib_pool.request("GET", httpurl, headers=httpheaders, preload_content=False);
3758   except urllib3.exceptions.ConnectTimeoutError:
3759    log.info("Error With URL "+httpurl);
3760    return False;
3761   except urllib3.exceptions.ConnectError:
3762    log.info("Error With URL "+httpurl);
3763    return False;
3764   except urllib3.exceptions.MaxRetryError:
3765    log.info("Error With URL "+httpurl);
3766    return False;
3767   except socket.timeout:
3768    log.info("Error With URL "+httpurl);
3769    return False;
3770   httpcodeout = geturls_text.status;
3771   httpcodereason = geturls_text.reason;
3772   httpversionout = "1.1";
3773   httpmethodout = httpmethod;
3774   httpurlout = geturls_text.geturl();
3775   httpheaderout = geturls_text.info();
3776   httpheadersentout = httpheaders;
3777   if(isinstance(httpheaderout, list)):
3778    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
3779   if(sys.version[0]=="2"):
3780    try:
3781     prehttpheaderout = httpheaderout;
3782     httpheaderkeys = httpheaderout.keys();
3783     imax = len(httpheaderkeys);
3784     ic = 0;
3785     httpheaderout = {};
3786     while(ic < imax):
3787      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
3788      ic += 1;
3789    except AttributeError:
3790     pass;
3791   httpheaderout = fix_header_names(httpheaderout);
3792   if(isinstance(httpheadersentout, list)):
3793    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
3794   httpheadersentout = fix_header_names(httpheadersentout);
3795   downloadsize = int(geturls_text.headers.get('Content-Length'));
3796   if(downloadsize is not None):
3797    downloadsize = int(downloadsize);
3798   if downloadsize is None: downloadsize = 0;
3799   fulldatasize = 0;
3800   prevdownsize = 0;
3801   log.info("Downloading URL "+httpurl);
3802   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
3803    tmpfilename = f.name;
3804    try:
3805     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
3806    except AttributeError:
3807     try:
3808      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3809     except ValueError:
3810      pass;
3811    except ValueError:
3812     pass;
3813    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
3814    while True:
3815     databytes = geturls_text.read(buffersize);
3816     if not databytes: break;
3817     datasize = len(databytes);
3818     fulldatasize = datasize + fulldatasize;
3819     percentage = "";
3820     if(downloadsize>0):
3821      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3822     downloaddiff = fulldatasize - prevdownsize;
3823     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3824     prevdownsize = fulldatasize;
3825     f.write(databytes);
3826    f.close();
3827   geturls_text.close();
3828   exec_time_end = time.time();
3829   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
3830   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
3831   return returnval;
3832
3833 if(not haveurllib3):
3834  def download_from_url_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
3835   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep)
3836   return returnval;
3837
3838 if(haveurllib3):
3839  def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3840   global geturls_download_sleep;
3841   if(sleep<0):
3842    sleep = geturls_download_sleep;
3843   if(not outfile=="-"):
3844    outpath = outpath.rstrip(os.path.sep);
3845    filepath = os.path.realpath(outpath+os.path.sep+outfile);
3846    if(not os.path.exists(outpath)):
3847     os.makedirs(outpath);
3848    if(os.path.exists(outpath) and os.path.isfile(outpath)):
3849     return False;
3850    if(os.path.exists(filepath) and os.path.isdir(filepath)):
3851     return False;
3852    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3853    if(not pretmpfilename):
3854     return False;
3855    tmpfilename = pretmpfilename['Filename'];
3856    downloadsize = os.path.getsize(tmpfilename);
3857    fulldatasize = 0;
3858    log.info("Moving file "+tmpfilename+" to "+filepath);
3859    exec_time_start = time.time();
3860    shutil.move(tmpfilename, filepath);
3861    try:
3862     os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
3863    except AttributeError:
3864     try:
3865      os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
3866     except ValueError:
3867      pass;
3868    except ValueError:
3869     pass;
3870    exec_time_end = time.time();
3871    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
3872    if(os.path.exists(tmpfilename)):
3873     os.remove(tmpfilename);
3874    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3875   if(outfile=="-" and sys.version[0]=="2"):
3876    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3877    if(not pretmpfilename):
3878     return False;
3879    tmpfilename = pretmpfilename['Filename'];
3880    downloadsize = os.path.getsize(tmpfilename);
3881    fulldatasize = 0;
3882    prevdownsize = 0;
3883    exec_time_start = time.time();
3884    with open(tmpfilename, 'rb') as ft:
3885     f = StringIO();
3886     while True:
3887      databytes = ft.read(buffersize[1]);
3888      if not databytes: break;
3889      datasize = len(databytes);
3890      fulldatasize = datasize + fulldatasize;
3891      percentage = "";
3892      if(downloadsize>0):
3893       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3894      downloaddiff = fulldatasize - prevdownsize;
3895      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3896      prevdownsize = fulldatasize;
3897      f.write(databytes);
3898     f.seek(0);
3899     fdata = f.getvalue();
3900     f.close();
3901     ft.close();
3902     os.remove(tmpfilename);
3903     exec_time_end = time.time();
3904     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3905    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3906   if(outfile=="-" and sys.version[0]>="3"):
3907    pretmpfilename = download_from_url_file_with_request3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
3908    tmpfilename = pretmpfilename['Filename'];
3909    downloadsize = os.path.getsize(tmpfilename);
3910    fulldatasize = 0;
3911    prevdownsize = 0;
3912    exec_time_start = time.time();
3913    with open(tmpfilename, 'rb') as ft:
3914     f = BytesIO();
3915     while True:
3916      databytes = ft.read(buffersize[1]);
3917      if not databytes: break;
3918      datasize = len(databytes);
3919      fulldatasize = datasize + fulldatasize;
3920      percentage = "";
3921      if(downloadsize>0):
3922       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
3923      downloaddiff = fulldatasize - prevdownsize;
3924      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
3925      prevdownsize = fulldatasize;
3926      f.write(databytes);
3927     f.seek(0);
3928     fdata = f.getvalue();
3929     f.close();
3930     ft.close();
3931     os.remove(tmpfilename);
3932     exec_time_end = time.time();
3933     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
3934    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
3935   return returnval;
3936
3937 if(not haveurllib3):
3938  def download_from_url_to_file_with_request3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
3939   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
3940   return returnval;
3941
3942 if(haveurllib3):
3943  def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
3944   global geturls_download_sleep, havebrotli;
3945   if(sleep<0):
3946    sleep = geturls_download_sleep;
3947   urlparts = urlparse.urlparse(httpurl);
3948   if(isinstance(httpheaders, list)):
3949    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
3950   httpheaders = fix_header_names(httpheaders);
3951   if(httpuseragent is not None):
3952    if('User-Agent' in httpheaders):
3953     httpheaders['User-Agent'] = httpuseragent;
3954    else:
3955     httpuseragent.update({'User-Agent': httpuseragent});
3956   if(httpreferer is not None):
3957    if('Referer' in httpheaders):
3958     httpheaders['Referer'] = httpreferer;
3959    else:
3960     httpuseragent.update({'Referer': httpreferer});
3961   if(urlparts.username is not None or urlparts.password is not None):
3962    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
3963    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
3964   time.sleep(sleep);
3965   urllib_pool = urllib3.PoolManager(headers=httpheaders);
3966   if(postdata is not None and not isinstance(postdata, dict)):
3967    postdata = urlencode(postdata);
3968   try:
3969    if(httpmethod=="GET"):
3970     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
3971    elif(httpmethod=="POST"):
3972     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
3973    else:
3974     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
3975   except urllib3.exceptions.ConnectTimeoutError:
3976    log.info("Error With URL "+httpurl);
3977    return False;
3978   except urllib3.exceptions.ConnectError:
3979    log.info("Error With URL "+httpurl);
3980    return False;
3981   except urllib3.exceptions.MaxRetryError:
3982    log.info("Error With URL "+httpurl);
3983    return False;
3984   except socket.timeout:
3985    log.info("Error With URL "+httpurl);
3986    return False;
3987   httpcodeout = geturls_text.status;
3988   httpcodereason = geturls_text.reason;
3989   httpversionout = "1.1";
3990   httpmethodout = httpmethod;
3991   httpurlout = geturls_text.geturl();
3992   httpheaderout = geturls_text.info();
3993   httpheadersentout = httpheaders;
3994   if(isinstance(httpheaderout, list)):
3995    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
3996   if(sys.version[0]=="2"):
3997    try:
3998     prehttpheaderout = httpheaderout;
3999     httpheaderkeys = httpheaderout.keys();
4000     imax = len(httpheaderkeys);
4001     ic = 0;
4002     httpheaderout = {};
4003     while(ic < imax):
4004      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
4005      ic += 1;
4006    except AttributeError:
4007     pass;
4008   httpheaderout = fix_header_names(httpheaderout);
4009   if(isinstance(httpheadersentout, list)):
4010    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
4011   httpheadersentout = fix_header_names(httpheadersentout);
4012   log.info("Downloading URL "+httpurl);
4013   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
4014    if(sys.version[0]=="2"):
4015     strbuf = StringIO(geturls_text.read());
4016    if(sys.version[0]>="3"):
4017     strbuf = BytesIO(geturls_text.read());
4018    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
4019    returnval_content = gzstrbuf.read()[:];
4020   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
4021    returnval_content = geturls_text.read()[:];
4022   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
4023    returnval_content = geturls_text.read()[:];
4024    returnval_content = brotli.decompress(returnval_content);
4025   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
4026   geturls_text.close();
4027   return returnval;
4028
4029 if(not haveurllib3):
4030  def download_from_url_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
4031   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep)
4032   return returnval;
4033
4034 if(haveurllib3):
4035  def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
4036   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
4037   exec_time_start = time.time();
4038   myhash = hashlib.new("sha1");
4039   if(sys.version[0]=="2"):
4040    myhash.update(httpurl);
4041    myhash.update(str(buffersize));
4042    myhash.update(str(exec_time_start));
4043   if(sys.version[0]>="3"):
4044    myhash.update(httpurl.encode('utf-8'));
4045    myhash.update(str(buffersize).encode('utf-8'));
4046    myhash.update(str(exec_time_start).encode('utf-8'));
4047   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
4048   if(sleep<0):
4049    sleep = geturls_download_sleep;
4050   urlparts = urlparse.urlparse(httpurl);
4051   if(isinstance(httpheaders, list)):
4052    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4053   httpheaders = fix_header_names(httpheaders);
4054   if(httpuseragent is not None):
4055    if('User-Agent' in httpheaders):
4056     httpheaders['User-Agent'] = httpuseragent;
4057    else:
4058     httpuseragent.update({'User-Agent': httpuseragent});
4059   if(httpreferer is not None):
4060    if('Referer' in httpheaders):
4061     httpheaders['Referer'] = httpreferer;
4062    else:
4063     httpuseragent.update({'Referer': httpreferer});
4064   if(urlparts.username is not None or urlparts.password is not None):
4065    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
4066    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
4067   time.sleep(sleep);
4068   urllib_pool = urllib3.PoolManager(headers=httpheaders);
4069   if(postdata is not None and not isinstance(postdata, dict)):
4070    postdata = urlencode(postdata);
4071   try:
4072    if(httpmethod=="GET"):
4073     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
4074    elif(httpmethod=="POST"):
4075     geturls_text = urllib_pool.urlopen("GET", httpurl, body=postdata, headers=httpheaders, preload_content=False);
4076    else:
4077     geturls_text = urllib_pool.urlopen("GET", httpurl, headers=httpheaders, preload_content=False);
4078   except urllib3.exceptions.ConnectTimeoutError:
4079    log.info("Error With URL "+httpurl);
4080    return False;
4081   except urllib3.exceptions.ConnectError:
4082    log.info("Error With URL "+httpurl);
4083    return False;
4084   except urllib3.exceptions.MaxRetryError:
4085    log.info("Error With URL "+httpurl);
4086    return False;
4087   except socket.timeout:
4088    log.info("Error With URL "+httpurl);
4089    return False;
4090   httpcodeout = geturls_text.status;
4091   httpcodereason = geturls_text.reason;
4092   httpversionout = "1.1";
4093   httpmethodout = httpmethod;
4094   httpurlout = geturls_text.geturl();
4095   httpheaderout = geturls_text.info();
4096   httpheadersentout = httpheaders;
4097   if(isinstance(httpheaderout, list)):
4098    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
4099   if(sys.version[0]=="2"):
4100    try:
4101     prehttpheaderout = httpheaderout;
4102     httpheaderkeys = httpheaderout.keys();
4103     imax = len(httpheaderkeys);
4104     ic = 0;
4105     httpheaderout = {};
4106     while(ic < imax):
4107      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
4108      ic += 1;
4109    except AttributeError:
4110     pass;
4111   httpheaderout = fix_header_names(httpheaderout);
4112   if(isinstance(httpheadersentout, list)):
4113    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
4114   httpheadersentout = fix_header_names(httpheadersentout);
4115   downloadsize = int(geturls_text.headers.get('Content-Length'));
4116   if(downloadsize is not None):
4117    downloadsize = int(downloadsize);
4118   if downloadsize is None: downloadsize = 0;
4119   fulldatasize = 0;
4120   prevdownsize = 0;
4121   log.info("Downloading URL "+httpurl);
4122   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
4123    tmpfilename = f.name;
4124    try:
4125     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
4126    except AttributeError:
4127     try:
4128      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4129     except ValueError:
4130      pass;
4131    except ValueError:
4132     pass;
4133    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
4134    while True:
4135     databytes = geturls_text.read(buffersize);
4136     if not databytes: break;
4137     datasize = len(databytes);
4138     fulldatasize = datasize + fulldatasize;
4139     percentage = "";
4140     if(downloadsize>0):
4141      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4142     downloaddiff = fulldatasize - prevdownsize;
4143     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4144     prevdownsize = fulldatasize;
4145     f.write(databytes);
4146    f.close();
4147   geturls_text.close();
4148   exec_time_end = time.time();
4149   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
4150   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
4151   return returnval;
4152
4153 if(not haveurllib3):
4154  def download_from_url_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
4155   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep)
4156   return returnval;
4157
4158 if(haveurllib3):
4159  def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
4160   global geturls_download_sleep;
4161   if(sleep<0):
4162    sleep = geturls_download_sleep;
4163   if(not outfile=="-"):
4164    outpath = outpath.rstrip(os.path.sep);
4165    filepath = os.path.realpath(outpath+os.path.sep+outfile);
4166    if(not os.path.exists(outpath)):
4167     os.makedirs(outpath);
4168    if(os.path.exists(outpath) and os.path.isfile(outpath)):
4169     return False;
4170    if(os.path.exists(filepath) and os.path.isdir(filepath)):
4171     return False;
4172    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
4173    if(not pretmpfilename):
4174     return False;
4175    tmpfilename = pretmpfilename['Filename'];
4176    downloadsize = os.path.getsize(tmpfilename);
4177    fulldatasize = 0;
4178    log.info("Moving file "+tmpfilename+" to "+filepath);
4179    exec_time_start = time.time();
4180    shutil.move(tmpfilename, filepath);
4181    try:
4182     os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
4183    except AttributeError:
4184     try:
4185      os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4186     except ValueError:
4187      pass;
4188    except ValueError:
4189     pass;
4190    exec_time_end = time.time();
4191    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
4192    if(os.path.exists(tmpfilename)):
4193     os.remove(tmpfilename);
4194    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4195   if(outfile=="-" and sys.version[0]=="2"):
4196    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
4197    if(not pretmpfilename):
4198     return False;
4199    tmpfilename = pretmpfilename['Filename'];
4200    downloadsize = os.path.getsize(tmpfilename);
4201    fulldatasize = 0;
4202    prevdownsize = 0;
4203    exec_time_start = time.time();
4204    with open(tmpfilename, 'rb') as ft:
4205     f = StringIO();
4206     while True:
4207      databytes = ft.read(buffersize[1]);
4208      if not databytes: break;
4209      datasize = len(databytes);
4210      fulldatasize = datasize + fulldatasize;
4211      percentage = "";
4212      if(downloadsize>0):
4213       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4214      downloaddiff = fulldatasize - prevdownsize;
4215      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4216      prevdownsize = fulldatasize;
4217      f.write(databytes);
4218     f.seek(0);
4219     fdata = f.getvalue();
4220     f.close();
4221     ft.close();
4222     os.remove(tmpfilename);
4223     exec_time_end = time.time();
4224     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4225    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4226   if(outfile=="-" and sys.version[0]>="3"):
4227    pretmpfilename = download_from_url_file_with_urllib3(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
4228    tmpfilename = pretmpfilename['Filename'];
4229    downloadsize = os.path.getsize(tmpfilename);
4230    fulldatasize = 0;
4231    prevdownsize = 0;
4232    exec_time_start = time.time();
4233    with open(tmpfilename, 'rb') as ft:
4234     f = BytesIO();
4235     while True:
4236      databytes = ft.read(buffersize[1]);
4237      if not databytes: break;
4238      datasize = len(databytes);
4239      fulldatasize = datasize + fulldatasize;
4240      percentage = "";
4241      if(downloadsize>0):
4242       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4243      downloaddiff = fulldatasize - prevdownsize;
4244      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4245      prevdownsize = fulldatasize;
4246      f.write(databytes);
4247     f.seek(0);
4248     fdata = f.getvalue();
4249     f.close();
4250     ft.close();
4251     os.remove(tmpfilename);
4252     exec_time_end = time.time();
4253     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4254    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4255   return returnval;
4256
4257 if(not haveurllib3):
4258  def download_from_url_to_file_with_urllib3(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
4259   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
4260   return returnval;
4261
4262 if(havemechanize):
4263  def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
4264   global geturls_download_sleep, havebrotli;
4265   if(sleep<0):
4266    sleep = geturls_download_sleep;
4267   urlparts = urlparse.urlparse(httpurl);
4268   if(isinstance(httpheaders, list)):
4269    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4270   httpheaders = fix_header_names(httpheaders);
4271   if(httpuseragent is not None):
4272    if('User-Agent' in httpheaders):
4273     httpheaders['User-Agent'] = httpuseragent;
4274    else:
4275     httpuseragent.update({'User-Agent': httpuseragent});
4276   if(httpreferer is not None):
4277    if('Referer' in httpheaders):
4278     httpheaders['Referer'] = httpreferer;
4279    else:
4280     httpuseragent.update({'Referer': httpreferer});
4281   if(urlparts.username is not None or urlparts.password is not None):
4282    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
4283    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
4284   geturls_opener = mechanize.Browser();
4285   if(isinstance(httpheaders, dict)):
4286    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
4287   time.sleep(sleep);
4288   geturls_opener.addheaders = httpheaders;
4289   geturls_opener.set_cookiejar(httpcookie);
4290   geturls_opener.set_handle_robots(False);
4291   if(postdata is not None and not isinstance(postdata, dict)):
4292    postdata = urlencode(postdata);
4293   try:
4294    if(httpmethod=="GET"):
4295     geturls_text = geturls_opener.open(httpurl);
4296    elif(httpmethod=="POST"):
4297     geturls_text = geturls_opener.open(httpurl, data=postdata);
4298    else:
4299     geturls_text = geturls_opener.open(httpurl);
4300   except mechanize.HTTPError as geturls_text_error:
4301    geturls_text = geturls_text_error;
4302    log.info("Error With URL "+httpurl);
4303   except URLError:
4304    log.info("Error With URL "+httpurl);
4305    return False;
4306   except socket.timeout:
4307    log.info("Error With URL "+httpurl);
4308    return False;
4309   httpcodeout = geturls_text.code;
4310   httpcodereason = http_status_to_reason(geturls_text.code);
4311   httpversionout = "1.1";
4312   httpmethodout = httpmethod;
4313   httpurlout = geturls_text.geturl();
4314   httpheaderout = geturls_text.info();
4315   reqhead = geturls_opener.request;
4316   httpheadersentout = reqhead.header_items();
4317   if(isinstance(httpheaderout, list)):
4318    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
4319   if(sys.version[0]=="2"):
4320    try:
4321     prehttpheaderout = httpheaderout;
4322     httpheaderkeys = httpheaderout.keys();
4323     imax = len(httpheaderkeys);
4324     ic = 0;
4325     httpheaderout = {};
4326     while(ic < imax):
4327      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
4328      ic += 1;
4329    except AttributeError:
4330     pass;
4331   httpheaderout = fix_header_names(httpheaderout);
4332   if(isinstance(httpheadersentout, list)):
4333    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
4334   httpheadersentout = fix_header_names(httpheadersentout);
4335   log.info("Downloading URL "+httpurl);
4336   if(httpheaderout.get("Content-Encoding")=="gzip" or httpheaderout.get("Content-Encoding")=="deflate"):
4337    if(sys.version[0]=="2"):
4338     strbuf = StringIO(geturls_text.read());
4339    if(sys.version[0]>="3"):
4340     strbuf = BytesIO(geturls_text.read());
4341    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
4342    returnval_content = gzstrbuf.read()[:];
4343   if(httpheaderout.get("Content-Encoding")!="gzip" and httpheaderout.get("Content-Encoding")!="deflate" and httpheaderout.get("Content-Encoding")!="br"):
4344    returnval_content = geturls_text.read()[:];
4345   if(httpheaderout.get("Content-Encoding")=="br" and havebrotli):
4346    returnval_content = geturls_text.read()[:];
4347    returnval_content = brotli.decompress(returnval_content);
4348   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
4349   geturls_text.close();
4350   return returnval;
4351
4352 if(not havemechanize):
4353  def download_from_url_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
4354   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, sleep)
4355   return returnval;
4356
4357 if(havemechanize):
4358  def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
4359   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
4360   exec_time_start = time.time();
4361   myhash = hashlib.new("sha1");
4362   if(sys.version[0]=="2"):
4363    myhash.update(httpurl);
4364    myhash.update(str(buffersize));
4365    myhash.update(str(exec_time_start));
4366   if(sys.version[0]>="3"):
4367    myhash.update(httpurl.encode('utf-8'));
4368    myhash.update(str(buffersize).encode('utf-8'));
4369    myhash.update(str(exec_time_start).encode('utf-8'));
4370   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
4371   if(sleep<0):
4372    sleep = geturls_download_sleep;
4373   urlparts = urlparse.urlparse(httpurl);
4374   if(isinstance(httpheaders, list)):
4375    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4376   httpheaders = fix_header_names(httpheaders);
4377   if(httpuseragent is not None):
4378    if('User-Agent' in httpheaders):
4379     httpheaders['User-Agent'] = httpuseragent;
4380    else:
4381     httpuseragent.update({'User-Agent': httpuseragent});
4382   if(httpreferer is not None):
4383    if('Referer' in httpheaders):
4384     httpheaders['Referer'] = httpreferer;
4385    else:
4386     httpuseragent.update({'Referer': httpreferer});
4387   if(urlparts.username is not None or urlparts.password is not None):
4388    inurlencode = b64encode(str(urlparts.username+":"+urlparts.password).encode()).decode("UTF-8");
4389    httpheaders.update( { 'Authorization': "Basic "+inurlencode } );
4390   geturls_opener = mechanize.Browser();
4391   if(isinstance(httpheaders, dict)):
4392    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
4393   time.sleep(sleep);
4394   geturls_opener.addheaders = httpheaders;
4395   geturls_opener.set_cookiejar(httpcookie);
4396   geturls_opener.set_handle_robots(False);
4397   if(postdata is not None and not isinstance(postdata, dict)):
4398    postdata = urlencode(postdata);
4399   try:
4400    if(httpmethod=="GET"):
4401     geturls_text = geturls_opener.open(httpurl);
4402    elif(httpmethod=="POST"):
4403     geturls_text = geturls_opener.open(httpurl, data=postdata);
4404    else:
4405     geturls_text = geturls_opener.open(httpurl);
4406   except mechanize.HTTPError as geturls_text_error:
4407    geturls_text = geturls_text_error;
4408    log.info("Error With URL "+httpurl);
4409   except URLError:
4410    log.info("Error With URL "+httpurl);
4411    return False;
4412   except socket.timeout:
4413    log.info("Error With URL "+httpurl);
4414    return False;
4415   httpcodeout = geturls_text.code;
4416   httpcodereason = http_status_to_reason(geturls_text.code);
4417   httpversionout = "1.1";
4418   httpmethodout = httpmethod;
4419   httpurlout = geturls_text.geturl();
4420   httpheaderout = geturls_text.info();
4421   reqhead = geturls_opener.request;
4422   httpheadersentout = reqhead.header_items();
4423   if(isinstance(httpheaderout, list)):
4424    httpheaderout = dict(make_http_headers_from_list_to_dict(httpheaderout));
4425   if(sys.version[0]=="2"):
4426    try:
4427     prehttpheaderout = httpheaderout;
4428     httpheaderkeys = httpheaderout.keys();
4429     imax = len(httpheaderkeys);
4430     ic = 0;
4431     httpheaderout = {};
4432     while(ic < imax):
4433      httpheaderout.update({httpheaderkeys[ic]: prehttpheaderout[httpheaderkeys[ic]]});
4434      ic += 1;
4435    except AttributeError:
4436     pass;
4437   httpheaderout = fix_header_names(httpheaderout);
4438   if(isinstance(httpheadersentout, list)):
4439    httpheadersentout = dict(make_http_headers_from_list_to_dict(httpheadersentout));
4440   httpheadersentout = fix_header_names(httpheadersentout);
4441   downloadsize = int(httpheaderout.get('Content-Length'));
4442   if(downloadsize is not None):
4443    downloadsize = int(downloadsize);
4444   if downloadsize is None: downloadsize = 0;
4445   fulldatasize = 0;
4446   prevdownsize = 0;
4447   log.info("Downloading URL "+httpurl);
4448   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
4449    tmpfilename = f.name;
4450    try:
4451     os.utime(tmpfilename, (time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(httpheaderout.get('Last-Modified')).timetuple())));
4452    except AttributeError:
4453     try:
4454      os.utime(tmpfilename, (time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(httpheaderout.get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4455     except ValueError:
4456      pass;
4457    except ValueError:
4458     pass;
4459    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': httpheaderout, 'Version': httpversionout, 'Method': httpmethodout, 'HeadersSent': httpheadersentout, 'URL': httpurlout, 'Code': httpcodeout, 'Reason': httpcodereason};
4460    while True:
4461     databytes = geturls_text.read(buffersize);
4462     if not databytes: break;
4463     datasize = len(databytes);
4464     fulldatasize = datasize + fulldatasize;
4465     percentage = "";
4466     if(downloadsize>0):
4467      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4468     downloaddiff = fulldatasize - prevdownsize;
4469     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4470     prevdownsize = fulldatasize;
4471     f.write(databytes);
4472    f.close();
4473   geturls_text.close();
4474   exec_time_end = time.time();
4475   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
4476   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
4477   return returnval;
4478
4479 if(not havemechanize):
4480  def download_from_url_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
4481   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, sleep)
4482   return returnval;
4483
4484 if(havemechanize):
4485  def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
4486   global geturls_download_sleep;
4487   if(sleep<0):
4488    sleep = geturls_download_sleep;
4489   if(not outfile=="-"):
4490    outpath = outpath.rstrip(os.path.sep);
4491    filepath = os.path.realpath(outpath+os.path.sep+outfile);
4492    if(not os.path.exists(outpath)):
4493     os.makedirs(outpath);
4494    if(os.path.exists(outpath) and os.path.isfile(outpath)):
4495     return False;
4496    if(os.path.exists(filepath) and os.path.isdir(filepath)):
4497     return False;
4498    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
4499    if(not pretmpfilename):
4500     return False;
4501    tmpfilename = pretmpfilename['Filename'];
4502    downloadsize = os.path.getsize(tmpfilename);
4503    fulldatasize = 0;
4504    log.info("Moving file "+tmpfilename+" to "+filepath);
4505    exec_time_start = time.time();
4506    shutil.move(tmpfilename, filepath);
4507    try:
4508     os.utime(filepath, (time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple()), time.mktime(email.utils.parsedate_to_datetime(pretmpfilename.get('Headers').get('Last-Modified')).timetuple())));
4509    except AttributeError:
4510     try:
4511      os.utime(filepath, (time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple()), time.mktime(datetime.datetime.strptime(pretmpfilename.get('Headers').get('Last-Modified'), "%a, %d %b %Y %H:%M:%S %Z").timetuple())));
4512     except ValueError:
4513      pass;
4514    except ValueError:
4515     pass;
4516    exec_time_end = time.time();
4517    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
4518    if(os.path.exists(tmpfilename)):
4519     os.remove(tmpfilename);
4520    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4521   if(outfile=="-" and sys.version[0]=="2"):
4522    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
4523    if(not pretmpfilename):
4524     return False;
4525    tmpfilename = pretmpfilename['Filename'];
4526    downloadsize = os.path.getsize(tmpfilename);
4527    fulldatasize = 0;
4528    prevdownsize = 0;
4529    exec_time_start = time.time();
4530    with open(tmpfilename, 'rb') as ft:
4531     f = StringIO();
4532     while True:
4533      databytes = ft.read(buffersize[1]);
4534      if not databytes: break;
4535      datasize = len(databytes);
4536      fulldatasize = datasize + fulldatasize;
4537      percentage = "";
4538      if(downloadsize>0):
4539       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4540      downloaddiff = fulldatasize - prevdownsize;
4541      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4542      prevdownsize = fulldatasize;
4543      f.write(databytes);
4544     f.seek(0);
4545     fdata = f.getvalue();
4546     f.close();
4547     ft.close();
4548     os.remove(tmpfilename);
4549     exec_time_end = time.time();
4550     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4551    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4552   if(outfile=="-" and sys.version[0]>="3"):
4553    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
4554    tmpfilename = pretmpfilename['Filename'];
4555    downloadsize = os.path.getsize(tmpfilename);
4556    fulldatasize = 0;
4557    prevdownsize = 0;
4558    exec_time_start = time.time();
4559    with open(tmpfilename, 'rb') as ft:
4560     f = BytesIO();
4561     while True:
4562      databytes = ft.read(buffersize[1]);
4563      if not databytes: break;
4564      datasize = len(databytes);
4565      fulldatasize = datasize + fulldatasize;
4566      percentage = "";
4567      if(downloadsize>0):
4568       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4569      downloaddiff = fulldatasize - prevdownsize;
4570      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4571      prevdownsize = fulldatasize;
4572      f.write(databytes);
4573     f.seek(0);
4574     fdata = f.getvalue();
4575     f.close();
4576     ft.close();
4577     os.remove(tmpfilename);
4578     exec_time_end = time.time();
4579     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4580    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': httpmethod, 'HeadersSent': ['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4581   return returnval;
4582
4583 if(not havemechanize):
4584  def download_from_url_to_file_with_mechanize(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
4585   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize, outfile, outpath, sleep)
4586   return returnval;
4587
4588 def download_file_from_ftp_file(url):
4589  urlparts = urlparse.urlparse(url);
4590  file_name = os.path.basename(urlparts.path);
4591  file_dir = os.path.dirname(urlparts.path);
4592  if(urlparts.username is not None):
4593   ftp_username = urlparts.username;
4594  else:
4595   ftp_username = "anonymous";
4596  if(urlparts.password is not None):
4597   ftp_password = urlparts.password;
4598  elif(urlparts.password is None and urlparts.username=="anonymous"):
4599   ftp_password = "anonymous";
4600  else:
4601   ftp_password = "";
4602  if(urlparts.scheme=="ftp"):
4603   ftp = FTP();
4604  elif(urlparts.scheme=="ftps"):
4605   ftp = FTP_TLS();
4606  else:
4607   return False;
4608  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
4609   return False;
4610  ftp_port = urlparts.port;
4611  if(urlparts.port is None):
4612   ftp_port = 21;
4613  try:
4614   ftp.connect(urlparts.hostname, ftp_port);
4615  except socket.gaierror:
4616   log.info("Error With URL "+httpurl);
4617   return False;
4618  except socket.timeout:
4619   log.info("Error With URL "+httpurl);
4620   return False;
4621  ftp.login(urlparts.username, urlparts.password);
4622  if(urlparts.scheme=="ftps"):
4623   ftp.prot_p();
4624  ftpfile = BytesIO();
4625  ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
4626  #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
4627  ftp.close();
4628  ftpfile.seek(0, 0);
4629  return ftpfile;
4630
4631 def download_file_from_ftp_string(url):
4632  ftpfile = download_file_from_ftp_file(url);
4633  return ftpfile.read();
4634
4635 def download_from_url_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
4636  global geturls_download_sleep, havebrotli;
4637  if(sleep<0):
4638   sleep = geturls_download_sleep;
4639  urlparts = urlparse.urlparse(httpurl);
4640  if(isinstance(httpheaders, list)):
4641   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4642  httpheaders = fix_header_names(httpheaders);
4643  if(httpuseragent is not None):
4644   if('User-Agent' in httpheaders):
4645    httpheaders['User-Agent'] = httpuseragent;
4646   else:
4647    httpuseragent.update({'User-Agent': httpuseragent});
4648  if(httpreferer is not None):
4649   if('Referer' in httpheaders):
4650    httpheaders['Referer'] = httpreferer;
4651   else:
4652    httpuseragent.update({'Referer': httpreferer});
4653  if(isinstance(httpheaders, dict)):
4654   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
4655  time.sleep(sleep);
4656  geturls_text = download_file_from_ftp_file(httpurl);
4657  if(not geturls_text):
4658   return False;
4659  log.info("Downloading URL "+httpurl);
4660  returnval_content = geturls_text.read()[:];
4661  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
4662  geturls_text.close();
4663  return returnval;
4664
4665 def download_from_url_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
4666  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
4667  exec_time_start = time.time();
4668  myhash = hashlib.new("sha1");
4669  if(sys.version[0]=="2"):
4670   myhash.update(httpurl);
4671   myhash.update(str(buffersize));
4672   myhash.update(str(exec_time_start));
4673  if(sys.version[0]>="3"):
4674   myhash.update(httpurl.encode('utf-8'));
4675   myhash.update(str(buffersize).encode('utf-8'));
4676   myhash.update(str(exec_time_start).encode('utf-8'));
4677  newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
4678  if(sleep<0):
4679   sleep = geturls_download_sleep;
4680  urlparts = urlparse.urlparse(httpurl);
4681  if(isinstance(httpheaders, list)):
4682   httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4683  httpheaders = fix_header_names(httpheaders);
4684  if(httpuseragent is not None):
4685   if('User-Agent' in httpheaders):
4686    httpheaders['User-Agent'] = httpuseragent;
4687   else:
4688    httpuseragent.update({'User-Agent': httpuseragent});
4689  if(httpreferer is not None):
4690   if('Referer' in httpheaders):
4691    httpheaders['Referer'] = httpreferer;
4692   else:
4693    httpuseragent.update({'Referer': httpreferer});
4694  if(isinstance(httpheaders, dict)):
4695   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
4696  time.sleep(sleep);
4697  geturls_text = download_file_from_ftp_file(httpurl);
4698  if(not geturls_text):
4699   return False;
4700  geturls_text.seek(0, 2);
4701  downloadsize = geturls_text.tell();
4702  geturls_text.seek(0, 0);
4703  if(downloadsize is not None):
4704   downloadsize = int(downloadsize);
4705  if downloadsize is None: downloadsize = 0;
4706  fulldatasize = 0;
4707  prevdownsize = 0;
4708  log.info("Downloading URL "+httpurl);
4709  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
4710   tmpfilename = f.name;
4711   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
4712   while True:
4713    databytes = geturls_text.read(buffersize);
4714    if not databytes: break;
4715    datasize = len(databytes);
4716    fulldatasize = datasize + fulldatasize;
4717    percentage = "";
4718    if(downloadsize>0):
4719     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4720    downloaddiff = fulldatasize - prevdownsize;
4721    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4722    prevdownsize = fulldatasize;
4723    f.write(databytes);
4724   f.close();
4725  geturls_text.close();
4726  exec_time_end = time.time();
4727  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
4728  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
4729  return returnval;
4730
4731 def download_from_url_to_file_with_ftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
4732  global geturls_download_sleep;
4733  if(sleep<0):
4734   sleep = geturls_download_sleep;
4735  if(not outfile=="-"):
4736   outpath = outpath.rstrip(os.path.sep);
4737   filepath = os.path.realpath(outpath+os.path.sep+outfile);
4738   if(not os.path.exists(outpath)):
4739    os.makedirs(outpath);
4740   if(os.path.exists(outpath) and os.path.isfile(outpath)):
4741    return False;
4742   if(os.path.exists(filepath) and os.path.isdir(filepath)):
4743    return False;
4744   pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
4745   if(not pretmpfilename):
4746    return False;
4747   tmpfilename = pretmpfilename['Filename'];
4748   downloadsize = os.path.getsize(tmpfilename);
4749   fulldatasize = 0;
4750   log.info("Moving file "+tmpfilename+" to "+filepath);
4751   exec_time_start = time.time();
4752   shutil.move(tmpfilename, filepath);
4753   exec_time_end = time.time();
4754   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
4755   if(os.path.exists(tmpfilename)):
4756    os.remove(tmpfilename);
4757   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4758  if(outfile=="-" and sys.version[0]=="2"):
4759   pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
4760   if(not pretmpfilename):
4761    return False;
4762   tmpfilename = pretmpfilename['Filename'];
4763   downloadsize = os.path.getsize(tmpfilename);
4764   fulldatasize = 0;
4765   prevdownsize = 0;
4766   exec_time_start = time.time();
4767   with open(tmpfilename, 'rb') as ft:
4768    f = StringIO();
4769    while True:
4770     databytes = ft.read(buffersize[1]);
4771     if not databytes: break;
4772     datasize = len(databytes);
4773     fulldatasize = datasize + fulldatasize;
4774     percentage = "";
4775     if(downloadsize>0):
4776      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4777     downloaddiff = fulldatasize - prevdownsize;
4778     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4779     prevdownsize = fulldatasize;
4780     f.write(databytes);
4781    f.seek(0);
4782    fdata = f.getvalue();
4783    f.close();
4784    ft.close();
4785    os.remove(tmpfilename);
4786    exec_time_end = time.time();
4787    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4788   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4789  if(outfile=="-" and sys.version[0]>="3"):
4790   pretmpfilename = download_from_url_file_with_ftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
4791   tmpfilename = pretmpfilename['Filename'];
4792   downloadsize = os.path.getsize(tmpfilename);
4793   fulldatasize = 0;
4794   prevdownsize = 0;
4795   exec_time_start = time.time();
4796   with open(tmpfilename, 'rb') as ft:
4797    f = BytesIO();
4798    while True:
4799     databytes = ft.read(buffersize[1]);
4800     if not databytes: break;
4801     datasize = len(databytes);
4802     fulldatasize = datasize + fulldatasize;
4803     percentage = "";
4804     if(downloadsize>0):
4805      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
4806     downloaddiff = fulldatasize - prevdownsize;
4807     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
4808     prevdownsize = fulldatasize;
4809     f.write(databytes);
4810    f.seek(0);
4811    fdata = f.getvalue();
4812    f.close();
4813    ft.close();
4814    os.remove(tmpfilename);
4815    exec_time_end = time.time();
4816    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
4817   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
4818  return returnval;
4819
4820 def upload_file_to_ftp_file(ftpfile, url):
4821  urlparts = urlparse.urlparse(url);
4822  file_name = os.path.basename(urlparts.path);
4823  file_dir = os.path.dirname(urlparts.path);
4824  if(urlparts.username is not None):
4825   ftp_username = urlparts.username;
4826  else:
4827   ftp_username = "anonymous";
4828  if(urlparts.password is not None):
4829   ftp_password = urlparts.password;
4830  elif(urlparts.password is None and urlparts.username=="anonymous"):
4831   ftp_password = "anonymous";
4832  else:
4833   ftp_password = "";
4834  if(urlparts.scheme=="ftp"):
4835   ftp = FTP();
4836  elif(urlparts.scheme=="ftps"):
4837   ftp = FTP_TLS();
4838  else:
4839   return False;
4840  if(urlparts.scheme=="http" or urlparts.scheme=="https"):
4841   return False;
4842  ftp_port = urlparts.port;
4843  if(urlparts.port is None):
4844   ftp_port = 21;
4845  try:
4846   ftp.connect(urlparts.hostname, ftp_port);
4847  except socket.gaierror:
4848   log.info("Error With URL "+httpurl);
4849   return False;
4850  except socket.timeout:
4851   log.info("Error With URL "+httpurl);
4852   return False;
4853  ftp.login(urlparts.username, urlparts.password);
4854  if(urlparts.scheme=="ftps"):
4855   ftp.prot_p();
4856  ftp.storbinary("STOR "+urlparts.path, ftpfile);
4857  ftp.close();
4858  ftpfile.seek(0, 0);
4859  return ftpfile;
4860
4861 def upload_file_to_ftp_string(ftpstring, url):
4862  ftpfileo = BytesIO(ftpstring);
4863  ftpfile = upload_file_to_ftp_file(ftpfileo, url);
4864  ftpfileo.close();
4865  return ftpfile;
4866
4867 if(haveparamiko):
4868  def download_file_from_sftp_file(url):
4869   urlparts = urlparse.urlparse(url);
4870   file_name = os.path.basename(urlparts.path);
4871   file_dir = os.path.dirname(urlparts.path);
4872   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
4873    return False;
4874   sftp_port = urlparts.port;
4875   if(urlparts.port is None):
4876    sftp_port = 22;
4877   else:
4878    sftp_port = urlparts.port;
4879   if(urlparts.username is not None):
4880    sftp_username = urlparts.username;
4881   else:
4882    sftp_username = "anonymous";
4883   if(urlparts.password is not None):
4884    sftp_password = urlparts.password;
4885   elif(urlparts.password is None and urlparts.username=="anonymous"):
4886    sftp_password = "anonymous";
4887   else:
4888    sftp_password = "";
4889   if(urlparts.scheme!="sftp"):
4890    return False;
4891   ssh = paramiko.SSHClient();
4892   ssh.load_system_host_keys();
4893   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
4894   try:
4895    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
4896   except paramiko.ssh_exception.SSHException:
4897    return False;
4898   except socket.gaierror:
4899    log.info("Error With URL "+httpurl);
4900    return False;
4901   except socket.timeout:
4902    log.info("Error With URL "+httpurl);
4903    return False;
4904   sftp = ssh.open_sftp();
4905   sftpfile = BytesIO();
4906   sftp.getfo(urlparts.path, sftpfile);
4907   sftp.close();
4908   ssh.close();
4909   sftpfile.seek(0, 0);
4910   return sftpfile;
4911 else:
4912  def download_file_from_sftp_file(url):
4913   return False;
4914
4915 if(haveparamiko):
4916  def download_file_from_sftp_string(url):
4917   sftpfile = download_file_from_sftp_file(url);
4918   return sftpfile.read();
4919 else:
4920  def download_file_from_ftp_string(url):
4921   return False;
4922
4923 if(haveparamiko):
4924  def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
4925   global geturls_download_sleep, havebrotli;
4926   if(sleep<0):
4927    sleep = geturls_download_sleep;
4928   urlparts = urlparse.urlparse(httpurl);
4929   if(isinstance(httpheaders, list)):
4930    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4931   httpheaders = fix_header_names(httpheaders);
4932   if(httpuseragent is not None):
4933    if('User-Agent' in httpheaders):
4934     httpheaders['User-Agent'] = httpuseragent;
4935    else:
4936     httpuseragent.update({'User-Agent': httpuseragent});
4937   if(httpreferer is not None):
4938    if('Referer' in httpheaders):
4939     httpheaders['Referer'] = httpreferer;
4940    else:
4941     httpuseragent.update({'Referer': httpreferer});
4942   if(isinstance(httpheaders, dict)):
4943    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
4944   time.sleep(sleep);
4945   geturls_text = download_file_from_sftp_file(httpurl);
4946   if(not geturls_text):
4947    return False;
4948   log.info("Downloading URL "+httpurl);
4949   returnval_content = geturls_text.read()[:];
4950   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
4951   geturls_text.close();
4952   return returnval;
4953
4954 if(not haveparamiko):
4955  def download_from_url_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
4956   return False;
4957
4958 if(haveparamiko):
4959  def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
4960   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
4961   exec_time_start = time.time();
4962   myhash = hashlib.new("sha1");
4963   if(sys.version[0]=="2"):
4964    myhash.update(httpurl);
4965    myhash.update(str(buffersize));
4966    myhash.update(str(exec_time_start));
4967   if(sys.version[0]>="3"):
4968    myhash.update(httpurl.encode('utf-8'));
4969    myhash.update(str(buffersize).encode('utf-8'));
4970    myhash.update(str(exec_time_start).encode('utf-8'));
4971   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
4972   if(sleep<0):
4973    sleep = geturls_download_sleep;
4974   urlparts = urlparse.urlparse(httpurl);
4975   if(isinstance(httpheaders, list)):
4976    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
4977   httpheaders = fix_header_names(httpheaders);
4978   if(httpuseragent is not None):
4979    if('User-Agent' in httpheaders):
4980     httpheaders['User-Agent'] = httpuseragent;
4981    else:
4982     httpuseragent.update({'User-Agent': httpuseragent});
4983   if(httpreferer is not None):
4984    if('Referer' in httpheaders):
4985     httpheaders['Referer'] = httpreferer;
4986    else:
4987     httpuseragent.update({'Referer': httpreferer});
4988   if(isinstance(httpheaders, dict)):
4989    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
4990   time.sleep(sleep);
4991   geturls_text = download_file_from_sftp_file(httpurl);
4992   if(not geturls_text):
4993    return False;
4994   geturls_text.seek(0, 2);
4995   downloadsize = geturls_text.tell();
4996   geturls_text.seek(0, 0);
4997   if(downloadsize is not None):
4998    downloadsize = int(downloadsize);
4999   if downloadsize is None: downloadsize = 0;
5000   fulldatasize = 0;
5001   prevdownsize = 0;
5002   log.info("Downloading URL "+httpurl);
5003   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
5004    tmpfilename = f.name;
5005    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
5006    while True:
5007     databytes = geturls_text.read(buffersize);
5008     if not databytes: break;
5009     datasize = len(databytes);
5010     fulldatasize = datasize + fulldatasize;
5011     percentage = "";
5012     if(downloadsize>0):
5013      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
5014     downloaddiff = fulldatasize - prevdownsize;
5015     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
5016     prevdownsize = fulldatasize;
5017     f.write(databytes);
5018    f.close();
5019   geturls_text.close();
5020   exec_time_end = time.time();
5021   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
5022   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
5023   return returnval;
5024
5025 if(not haveparamiko):
5026  def download_from_url_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
5027   return False;
5028
5029 if(haveparamiko):
5030  def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
5031   global geturls_download_sleep;
5032   if(sleep<0):
5033    sleep = geturls_download_sleep;
5034   if(not outfile=="-"):
5035    outpath = outpath.rstrip(os.path.sep);
5036    filepath = os.path.realpath(outpath+os.path.sep+outfile);
5037    if(not os.path.exists(outpath)):
5038     os.makedirs(outpath);
5039    if(os.path.exists(outpath) and os.path.isfile(outpath)):
5040     return False;
5041    if(os.path.exists(filepath) and os.path.isdir(filepath)):
5042     return False;
5043    pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
5044    if(not pretmpfilename):
5045     return False;
5046    tmpfilename = pretmpfilename['Filename'];
5047    downloadsize = os.path.getsize(tmpfilename);
5048    fulldatasize = 0;
5049    log.info("Moving file "+tmpfilename+" to "+filepath);
5050    exec_time_start = time.time();
5051    shutil.move(tmpfilename, filepath);
5052    exec_time_end = time.time();
5053    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
5054    if(os.path.exists(tmpfilename)):
5055     os.remove(tmpfilename);
5056    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
5057   if(outfile=="-" and sys.version[0]=="2"):
5058    pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
5059    if(not pretmpfilename):
5060     return False;
5061    tmpfilename = pretmpfilename['Filename'];
5062    downloadsize = os.path.getsize(tmpfilename);
5063    fulldatasize = 0;
5064    prevdownsize = 0;
5065    exec_time_start = time.time();
5066    with open(tmpfilename, 'rb') as ft:
5067     f = StringIO();
5068     while True:
5069      databytes = ft.read(buffersize[1]);
5070      if not databytes: break;
5071      datasize = len(databytes);
5072      fulldatasize = datasize + fulldatasize;
5073      percentage = "";
5074      if(downloadsize>0):
5075       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
5076      downloaddiff = fulldatasize - prevdownsize;
5077      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
5078      prevdownsize = fulldatasize;
5079      f.write(databytes);
5080     f.seek(0);
5081     fdata = f.getvalue();
5082     f.close();
5083     ft.close();
5084     os.remove(tmpfilename);
5085     exec_time_end = time.time();
5086     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
5087    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
5088   if(outfile=="-" and sys.version[0]>="3"):
5089    pretmpfilename = download_from_url_file_with_sftp(httpurl, httpheaders, httpuseragent, httpreferer, httpcookie, httpmethod, postdata, buffersize[0], sleep);
5090    tmpfilename = pretmpfilename['Filename'];
5091    downloadsize = os.path.getsize(tmpfilename);
5092    fulldatasize = 0;
5093    prevdownsize = 0;
5094    exec_time_start = time.time();
5095    with open(tmpfilename, 'rb') as ft:
5096     f = BytesIO();
5097     while True:
5098      databytes = ft.read(buffersize[1]);
5099      if not databytes: break;
5100      datasize = len(databytes);
5101      fulldatasize = datasize + fulldatasize;
5102      percentage = "";
5103      if(downloadsize>0):
5104       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
5105      downloaddiff = fulldatasize - prevdownsize;
5106      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
5107      prevdownsize = fulldatasize;
5108      f.write(databytes);
5109     f.seek(0);
5110     fdata = f.getvalue();
5111     f.close();
5112     ft.close();
5113     os.remove(tmpfilename);
5114     exec_time_end = time.time();
5115     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
5116    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
5117   return returnval;
5118
5119 if(not haveparamiko):
5120  def download_from_url_to_file_with_sftp(httpurl, httpheaders=geturls_headers, httpuseragent=None, httpreferer=None, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
5121   return False;
5122
5123 if(haveparamiko):
5124  def upload_file_to_sftp_file(sftpfile, url):
5125   urlparts = urlparse.urlparse(url);
5126   file_name = os.path.basename(urlparts.path);
5127   file_dir = os.path.dirname(urlparts.path);
5128   sftp_port = urlparts.port;
5129   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
5130    return False;
5131   if(urlparts.port is None):
5132    sftp_port = 22;
5133   else:
5134    sftp_port = urlparts.port;
5135   if(urlparts.username is not None):
5136    sftp_username = urlparts.username;
5137   else:
5138    sftp_username = "anonymous";
5139   if(urlparts.password is not None):
5140    sftp_password = urlparts.password;
5141   elif(urlparts.password is None and urlparts.username=="anonymous"):
5142    sftp_password = "anonymous";
5143   else:
5144    sftp_password = "";
5145   if(urlparts.scheme!="sftp"):
5146    return False;
5147   ssh = paramiko.SSHClient();
5148   ssh.load_system_host_keys();
5149   ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
5150   try:
5151    ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
5152   except paramiko.ssh_exception.SSHException:
5153    return False;
5154   except socket.gaierror:
5155    log.info("Error With URL "+httpurl);
5156    return False;
5157   except socket.timeout:
5158    log.info("Error With URL "+httpurl);
5159    return False;
5160   sftp = ssh.open_sftp();
5161   sftp.putfo(sftpfile, urlparts.path);
5162   sftp.close();
5163   ssh.close();
5164   sftpfile.seek(0, 0);
5165   return sftpfile;
5166 else:
5167  def upload_file_to_sftp_file(sftpfile, url):
5168   return False;
5169
5170 if(haveparamiko):
5171  def upload_file_to_sftp_string(sftpstring, url):
5172   sftpfileo = BytesIO(sftpstring);
5173   sftpfile = upload_file_to_sftp_files(ftpfileo, url);
5174   sftpfileo.close();
5175   return sftpfile;
5176 else:
5177  def upload_file_to_sftp_string(url):
5178   return False;
5179
5180
5181 if(havepysftp):
5182  def download_file_from_pysftp_file(url):
5183   urlparts = urlparse.urlparse(url);
5184   file_name = os.path.basename(urlparts.path);
5185   file_dir = os.path.dirname(urlparts.path);
5186   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
5187    return False;
5188   sftp_port = urlparts.port;
5189   if(urlparts.port is None):
5190    sftp_port = 22;
5191   else:
5192    sftp_port = urlparts.port;
5193   if(urlparts.username is not None):
5194    sftp_username = urlparts.username;
5195   else:
5196    sftp_username = "anonymous";
5197   if(urlparts.password is not None):
5198    sftp_password = urlparts.password;
5199   elif(urlparts.password is None and urlparts.username=="anonymous"):
5200    sftp_password = "anonymous";
5201   else:
5202    sftp_password = "";
5203   if(urlparts.scheme!="sftp"):
5204    return False;
5205   try:
5206    pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
5207   except paramiko.ssh_exception.SSHException:
5208    return False;
5209   except socket.gaierror:
5210    log.info("Error With URL "+httpurl);
5211    return False;
5212   except socket.timeout:
5213    log.info("Error With URL "+httpurl);
5214    return False;
5215   sftp = ssh.open_sftp();
5216   sftpfile = BytesIO();
5217   sftp.getfo(urlparts.path, sftpfile);
5218   sftp.close();
5219   ssh.close();
5220   sftpfile.seek(0, 0);
5221   return sftpfile;
5222 else:
5223  def download_file_from_pysftp_file(url):
5224   return False;
5225
5226 if(havepysftp):
5227  def download_file_from_pysftp_string(url):
5228   sftpfile = download_file_from_pysftp_file(url);
5229   return sftpfile.read();
5230 else:
5231  def download_file_from_ftp_string(url):
5232   return False;
5233
5234 if(havepysftp):
5235  def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
5236   global geturls_download_sleep, havebrotli;
5237   if(sleep<0):
5238    sleep = geturls_download_sleep;
5239   urlparts = urlparse.urlparse(httpurl);
5240   if(isinstance(httpheaders, list)):
5241    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
5242   httpheaders = fix_header_names(httpheaders);
5243   if(isinstance(httpheaders, dict)):
5244    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
5245   time.sleep(sleep);
5246   geturls_text = download_file_from_pysftp_file(httpurl);
5247   if(not geturls_text):
5248    return False;
5249   log.info("Downloading URL "+httpurl);
5250   returnval_content = geturls_text.read()[:];
5251   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
5252   geturls_text.close();
5253   return returnval;
5254
5255 if(not havepysftp):
5256  def download_from_url_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, sleep=-1):
5257   return False;
5258
5259 if(havepysftp):
5260  def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
5261   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
5262   exec_time_start = time.time();
5263   myhash = hashlib.new("sha1");
5264   if(sys.version[0]=="2"):
5265    myhash.update(httpurl);
5266    myhash.update(str(buffersize));
5267    myhash.update(str(exec_time_start));
5268   if(sys.version[0]>="3"):
5269    myhash.update(httpurl.encode('utf-8'));
5270    myhash.update(str(buffersize).encode('utf-8'));
5271    myhash.update(str(exec_time_start).encode('utf-8'));
5272   newtmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
5273   if(sleep<0):
5274    sleep = geturls_download_sleep;
5275   urlparts = urlparse.urlparse(httpurl);
5276   if(isinstance(httpheaders, list)):
5277    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
5278   httpheaders = fix_header_names(httpheaders);
5279   if(isinstance(httpheaders, dict)):
5280    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
5281   time.sleep(sleep);
5282   geturls_text = download_file_from_pysftp_file(httpurl);
5283   if(not geturls_text):
5284    return False;
5285   geturls_text.seek(0, 2);
5286   downloadsize = geturls_text.tell();
5287   geturls_text.seek(0, 0);
5288   if(downloadsize is not None):
5289    downloadsize = int(downloadsize);
5290   if downloadsize is None: downloadsize = 0;
5291   fulldatasize = 0;
5292   prevdownsize = 0;
5293   log.info("Downloading URL "+httpurl);
5294   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=newtmpfilesuffix, delete=False) as f:
5295    tmpfilename = f.name;
5296    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': None, 'Version': None, 'Method': None, 'HeadersSent': None, 'URL': httpurl, 'Code': None};
5297    while True:
5298     databytes = geturls_text.read(buffersize);
5299     if not databytes: break;
5300     datasize = len(databytes);
5301     fulldatasize = datasize + fulldatasize;
5302     percentage = "";
5303     if(downloadsize>0):
5304      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
5305     downloaddiff = fulldatasize - prevdownsize;
5306     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
5307     prevdownsize = fulldatasize;
5308     f.write(databytes);
5309    f.close();
5310   geturls_text.close();
5311   exec_time_end = time.time();
5312   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
5313   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
5314   return returnval;
5315
5316 if(not havepysftp):
5317  def download_from_url_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, buffersize=524288, sleep=-1):
5318   return False;
5319
5320 if(havepysftp):
5321  def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
5322   global geturls_download_sleep;
5323   if(sleep<0):
5324    sleep = geturls_download_sleep;
5325   if(not outfile=="-"):
5326    outpath = outpath.rstrip(os.path.sep);
5327    filepath = os.path.realpath(outpath+os.path.sep+outfile);
5328    if(not os.path.exists(outpath)):
5329     os.makedirs(outpath);
5330    if(os.path.exists(outpath) and os.path.isfile(outpath)):
5331     return False;
5332    if(os.path.exists(filepath) and os.path.isdir(filepath)):
5333     return False;
5334    pretmpfilename = download_from_url_file_with_pysftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
5335    if(not pretmpfilename):
5336     return False;
5337    tmpfilename = pretmpfilename['Filename'];
5338    downloadsize = os.path.getsize(tmpfilename);
5339    fulldatasize = 0;
5340    log.info("Moving file "+tmpfilename+" to "+filepath);
5341    exec_time_start = time.time();
5342    shutil.move(tmpfilename, filepath);
5343    exec_time_end = time.time();
5344    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
5345    if(os.path.exists(tmpfilename)):
5346     os.remove(tmpfilename);
5347    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
5348   if(outfile=="-" and sys.version[0]=="2"):
5349    pretmpfilename = download_from_url_file_with_pysftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
5350    if(not pretmpfilename):
5351     return False;
5352    tmpfilename = pretmpfilename['Filename'];
5353    downloadsize = os.path.getsize(tmpfilename);
5354    fulldatasize = 0;
5355    prevdownsize = 0;
5356    exec_time_start = time.time();
5357    with open(tmpfilename, 'rb') as ft:
5358     f = StringIO();
5359     while True:
5360      databytes = ft.read(buffersize[1]);
5361      if not databytes: break;
5362      datasize = len(databytes);
5363      fulldatasize = datasize + fulldatasize;
5364      percentage = "";
5365      if(downloadsize>0):
5366       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
5367      downloaddiff = fulldatasize - prevdownsize;
5368      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
5369      prevdownsize = fulldatasize;
5370      f.write(databytes);
5371     f.seek(0);
5372     fdata = f.getvalue();
5373     f.close();
5374     ft.close();
5375     os.remove(tmpfilename);
5376     exec_time_end = time.time();
5377     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
5378    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
5379   if(outfile=="-" and sys.version[0]>="3"):
5380    pretmpfilename = download_from_url_file_with_pysftp(httpurl, httpheaders, httpcookie, httpmethod, postdata, buffersize[0], sleep);
5381    tmpfilename = pretmpfilename['Filename'];
5382    downloadsize = os.path.getsize(tmpfilename);
5383    fulldatasize = 0;
5384    prevdownsize = 0;
5385    exec_time_start = time.time();
5386    with open(tmpfilename, 'rb') as ft:
5387     f = BytesIO();
5388     while True:
5389      databytes = ft.read(buffersize[1]);
5390      if not databytes: break;
5391      datasize = len(databytes);
5392      fulldatasize = datasize + fulldatasize;
5393      percentage = "";
5394      if(downloadsize>0):
5395       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
5396      downloaddiff = fulldatasize - prevdownsize;
5397      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
5398      prevdownsize = fulldatasize;
5399      f.write(databytes);
5400     f.seek(0);
5401     fdata = f.getvalue();
5402     f.close();
5403     ft.close();
5404     os.remove(tmpfilename);
5405     exec_time_end = time.time();
5406     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
5407    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'Version': pretmpfilename['Version'], 'Method': pretmpfilename['Method'], 'Method': None, 'HeadersSent': pretmpfilename['HeadersSent'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code'], 'Reason': pretmpfilename['Reason']};
5408   return returnval;
5409
5410 if(not havepysftp):
5411  def download_from_url_to_file_with_pysftp(httpurl, httpheaders=geturls_headers, httpcookie=geturls_cj, httpmethod="GET", postdata=None, outfile="-", outpath=os.getcwd(), buffersize=[524288, 524288], sleep=-1):
5412   return False;
5413
5414 if(havepysftp):
5415  def upload_file_to_pysftp_file(sftpfile, url):
5416   urlparts = urlparse.urlparse(url);
5417   file_name = os.path.basename(urlparts.path);
5418   file_dir = os.path.dirname(urlparts.path);
5419   sftp_port = urlparts.port;
5420   if(urlparts.scheme=="http" or urlparts.scheme=="https"):
5421    return False;
5422   if(urlparts.port is None):
5423    sftp_port = 22;
5424   else:
5425    sftp_port = urlparts.port;
5426   if(urlparts.username is not None):
5427    sftp_username = urlparts.username;
5428   else:
5429    sftp_username = "anonymous";
5430   if(urlparts.password is not None):
5431    sftp_password = urlparts.password;
5432   elif(urlparts.password is None and urlparts.username=="anonymous"):
5433    sftp_password = "anonymous";
5434   else:
5435    sftp_password = "";
5436   if(urlparts.scheme!="sftp"):
5437    return False;
5438   try:
5439    pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
5440   except paramiko.ssh_exception.SSHException:
5441    return False;
5442   except socket.gaierror:
5443    log.info("Error With URL "+httpurl);
5444    return False;
5445   except socket.timeout:
5446    log.info("Error With URL "+httpurl);
5447    return False;
5448   sftp = ssh.open_sftp();
5449   sftp.putfo(sftpfile, urlparts.path);
5450   sftp.close();
5451   ssh.close();
5452   sftpfile.seek(0, 0);
5453   return sftpfile;
5454 else:
5455  def upload_file_to_pysftp_file(sftpfile, url):
5456   return False;
5457
5458 if(havepysftp):
5459  def upload_file_to_pysftp_string(sftpstring, url):
5460   sftpfileo = BytesIO(sftpstring);
5461   sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
5462   sftpfileo.close();
5463   return sftpfile;
5464 else:
5465  def upload_file_to_pysftp_string(url):
5466   return False;