pymotherless.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016 Cool Dude 2k - http://idb.berlios.de/
  13     Copyright 2016 Game Maker 2k - http://intdb.sourceforge.net/
  14     Copyright 2016 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  15
  16     $FileInfo: pymotherless.py - Last Update: 6/1/2016 Ver. 0.4.7 RC 1 - Author: cooldude2k $
  17 '''
  18
  19 from __future__ import division, absolute_import, print_function;
  20 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, datetime, argparse, cgi, subprocess, imp;
  21 import logging as log;
  22 haverequests = False;
  23 try:
  24  imp.find_module('requests');
  25  haverequests = True;
  26  import requests;
  27 except ImportError:
  28  haverequests = False;
  29 havemechanize = False;
  30 try:
  31  imp.find_module('mechanize');
  32  havemechanize = True;
  33  import mechanize;
  34 except ImportError:
  35  havemechanize = False;
  36 if(sys.version[0]=="2"):
  37  try:
  38   from cStringIO import StringIO;
  39  except ImportError:
  40   from StringIO import StringIO;
  41  # From http://python-future.org/compatible_idioms.html
  42  from urlparse import urlparse;
  43  from urllib import urlencode;
  44  from urllib2 import urlopen, Request, HTTPError;
  45  import urllib2, urlparse, cookielib;
  46 if(sys.version[0]>="3"):
  47  from io import StringIO, BytesIO;
  48  # From http://python-future.org/compatible_idioms.html
  49  from urllib.parse import urlparse, urlencode
  50  from urllib.request import urlopen, Request
  51  from urllib.error import HTTPError
  52  import urllib.request as urllib2;
  53  import urllib.parse as urlparse;
  54  import http.cookiejar as cookielib;
  55
  56 __program_name__ = "PyMotherless";
  57 __project__ = __program_name__;
  58 __project_url__ = "https://github.com/GameMaker2k/PyMotherless";
  59 __version_info__ = (0, 4, 7, "RC 1", 1);
  60 __version_date_info__ = (2016, 6, 1, "RC 1", 1);
  61 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
  62 if(__version_info__[4]!=None):
  63  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
  64 if(__version_info__[4]==None):
  65  __version_date_plusrc__ = __version_date__;
  66 if(__version_info__[3]!=None):
  67  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
  68 if(__version_info__[3]==None):
  69  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
  70
  71 geturls_cj = cookielib.CookieJar();
  72 geturls_ua_firefox_windows7 = "Mozilla/5.0 (Windows NT 6.1; rv:44.0) Gecko/20100101 Firefox/44.0";
  73 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 (Windows NT 6.1; rv:42.0) Gecko/20100101 Firefox/42.0 SeaMonkey/2.39";
  74 geturls_ua_chrome_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36";
  75 geturls_ua_chromium_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chromium/48.0.2564.116 Chrome/48.0.2564.116 Safari/537.36";
  76 geturls_ua_midori_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/538.15 (KHTML, like Gecko) Chrome/18.0.1025.133 Safari/538.15 Midori/0.5";
  77 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko";
  78 geturls_ua_pymotherless_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
  79 if(platform.python_implementation()!=""):
  80  geturls_ua_pymotherless_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=platform.python_implementation(), pyver=platform.python_version(), proname=__project__, prover=__version__);
  81 if(platform.python_implementation()==""):
  82  geturls_ua_pymotherless_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp="Python", pyver=platform.python_version(), proname=__project__, prover=__version__);
  83 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
  84 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
  85 geturls_ua = geturls_ua_firefox_windows7;
  86 geturls_headers_firefox_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  87 geturls_headers_seamonkey_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  88 geturls_headers_chrome_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  89 geturls_headers_chromium_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  90 geturls_headers_midori_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_midori_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  91 geturls_headers_internet_explorer_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  92 geturls_headers_pymotherless_python = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_pymotherless_python, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  93 geturls_headers_pymotherless_python_alt = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_pymotherless_python_alt, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  94 geturls_headers_googlebot_google = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  95 geturls_headers_googlebot_google_old = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  96 geturls_headers = geturls_headers_firefox_windows7;
  97 geturls_download_sleep = 0;
  98
  99 def add_url_param(url, **params):
 100  n=3;
 101  parts = list(urlparse.urlsplit(url));
 102  d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 103  d.update(params);
 104  parts[n]=urlencode(d);
 105  return urlparse.urlunsplit(parts);
 106
 107 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 108 def which_exec(execfile):
 109  for path in os.environ["PATH"].split(":"):
 110   if os.path.exists(path + "/" + execfile):
 111    return path + "/" + execfile;
 112
 113 def listize(varlist):
 114  il = 0;
 115  ix = len(varlist);
 116  ilx = 1;
 117  newlistreg = {};
 118  newlistrev = {};
 119  newlistfull = {};
 120  while(il < ix):
 121   newlistreg.update({ilx: varlist[il]});
 122   newlistrev.update({varlist[il]: ilx});
 123   ilx = ilx + 1;
 124   il = il + 1;
 125  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 126  return newlistfull;
 127
 128 def twolistize(varlist):
 129  il = 0;
 130  ix = len(varlist);
 131  ilx = 1;
 132  newlistnamereg = {};
 133  newlistnamerev = {};
 134  newlistdescreg = {};
 135  newlistdescrev = {};
 136  newlistfull = {};
 137  while(il < ix):
 138   newlistnamereg.update({ilx: varlist[il][0].strip()});
 139   newlistnamerev.update({varlist[il][0].strip(): ilx});
 140   newlistdescreg.update({ilx: varlist[il][1].strip()});
 141   newlistdescrev.update({varlist[il][1].strip(): ilx});
 142   ilx = ilx + 1;
 143   il = il + 1;
 144  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 145  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 146  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 147  return newlistfull;
 148
 149 def arglistize(proexec, *varlist):
 150  il = 0;
 151  ix = len(varlist);
 152  ilx = 1;
 153  newarglist = [proexec];
 154  while(il < ix):
 155   if varlist[il][0] is not None:
 156    newarglist.append(varlist[il][0]);
 157   if varlist[il][1] is not None:
 158    newarglist.append(varlist[il][1]);
 159   il = il + 1;
 160  return newarglist;
 161
 162 # get_readable_size by Lipis
 163 # http://stackoverflow.com/posts/14998888/revisions
 164 def get_readable_size(bytes, precision=1, unit="IEC"):
 165  unit = unit.upper();
 166  if(unit!="IEC" and unit!="SI"):
 167   unit = "IEC";
 168  if(unit=="IEC"):
 169   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 170   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 171   unitsize = 1024.0;
 172  if(unit=="SI"):
 173   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 174   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 175   unitsize = 1000.0;
 176  return_val = {};
 177  orgbytes = bytes;
 178  for unit in units:
 179   if abs(bytes) < unitsize:
 180    strformat = "%3."+str(precision)+"f%s";
 181    pre_return_val = (strformat % (bytes, unit));
 182    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 183    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 184    alt_return_val = pre_return_val.split();
 185    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 186    return return_val;
 187   bytes /= unitsize;
 188  strformat = "%."+str(precision)+"f%s";
 189  pre_return_val = (strformat % (bytes, "YiB"));
 190  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 191  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 192  alt_return_val = pre_return_val.split();
 193  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 194  return return_val;
 195
 196 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 197  unit = unit.upper();
 198  usehashtypes = usehashtypes.lower();
 199  getfilesize = os.path.getsize(infile);
 200  return_val = get_readable_size(getfilesize, precision, unit);
 201  if(usehashes==True):
 202   hashtypelist = usehashtypes.split(",");
 203   openfile = open(infile, "rb");
 204   filecontents = openfile.read();
 205   openfile.close();
 206   listnumcount = 0;
 207   listnumend = len(hashtypelist);
 208   while(listnumcount < listnumend):
 209    hashtypelistlow = hashtypelist[listnumcount].strip();
 210    hashtypelistup = hashtypelistlow.upper();
 211    filehash = hashlib.new(hashtypelistup);
 212    filehash.update(filecontents);
 213    filegethash = filehash.hexdigest();
 214    return_val.update({hashtypelistup: filegethash});
 215    listnumcount += 1;
 216  return return_val;
 217
 218 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 219  unit = unit.upper();
 220  usehashtypes = usehashtypes.lower();
 221  getfilesize = len(instring);
 222  return_val = get_readable_size(getfilesize, precision, unit);
 223  if(usehashes==True):
 224   hashtypelist = usehashtypes.split(",");
 225   listnumcount = 0;
 226   listnumend = len(hashtypelist);
 227   while(listnumcount < listnumend):
 228    hashtypelistlow = hashtypelist[listnumcount].strip();
 229    hashtypelistup = hashtypelistlow.upper();
 230    filehash = hashlib.new(hashtypelistup);
 231    if(sys.version[0]=="2"):
 232     filehash.update(instring);
 233    if(sys.version[0]>="3"):
 234     filehash.update(instring.encode('utf-8'));
 235    filegethash = filehash.hexdigest();
 236    return_val.update({hashtypelistup: filegethash});
 237    listnumcount += 1;
 238  return return_val;
 239
 240 def make_http_headers_from_dict_to_list(headers={'Referer': "http://motherless.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 241  if isinstance(headers, dict):
 242   returnval = [];
 243   if(sys.version[0]=="2"):
 244    for headkey, headvalue in headers.iteritems():
 245     returnval.append((headkey, headvalue));
 246   if(sys.version[0]>="3"):
 247    for headkey, headvalue in headers.items():
 248     returnval.append((headkey, headvalue));
 249  elif isinstance(headers, list):
 250   returnval = headers;
 251  else:
 252   returnval = False;
 253  return returnval;
 254
 255 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://motherless.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 256  if isinstance(headers, dict):
 257   returnval = [];
 258   if(sys.version[0]=="2"):
 259    for headkey, headvalue in headers.iteritems():
 260     returnval.append(headkey+": "+headvalue);
 261   if(sys.version[0]>="3"):
 262    for headkey, headvalue in headers.items():
 263     returnval.append(headkey+": "+headvalue);
 264  elif isinstance(headers, list):
 265   returnval = headers;
 266  else:
 267   returnval = False;
 268  return returnval;
 269
 270 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://motherless.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", "gzip, deflate"), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 271  if isinstance(headers, list):
 272   returnval = {};
 273   mli = 0;
 274   mlil = len(headers);
 275   while(mli<mlil):
 276    returnval.update({headers[mli][0]: headers[mli][1]});
 277    mli = mli + 1;
 278  elif isinstance(headers, dict):
 279   returnval = headers;
 280  else:
 281   returnval = False;
 282  return returnval;
 283
 284 def download_from_url(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1):
 285  global geturls_download_sleep, haverequests;
 286  if(sleep<0):
 287   sleep = geturls_download_sleep;
 288  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 289   httplibuse = "urllib";
 290  if(haverequests==False and httplibuse=="requests"):
 291   httplibuse = "urllib";
 292  if(havemechanize==False and httplibuse=="mechanize"):
 293   httplibuse = "urllib";
 294  if(httplibuse=="urllib"):
 295   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep);
 296  elif(httplibuse=="requests"):
 297   returnval = download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep);
 298  elif(httplibuse=="mechanize"):
 299   returnval = download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep);
 300  else:
 301   returnval = False;
 302  return returnval;
 303
 304 def download_from_url_file(httpurl, httpheaders, httpcookie, httplibuse="urllib", buffersize=262144, sleep=-1):
 305  global geturls_download_sleep, haverequests;
 306  if(sleep<0):
 307   sleep = geturls_download_sleep;
 308  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 309   httplibuse = "urllib";
 310  if(haverequests==False and httplibuse=="requests"):
 311   httplibuse = "urllib";
 312  if(havemechanize==False and httplibuse=="mechanize"):
 313   httplibuse = "urllib";
 314  if(httplibuse=="urllib"):
 315   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep);
 316  elif(httplibuse=="requests"):
 317   returnval = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize, sleep);
 318  elif(httplibuse=="mechanize"):
 319   returnval = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize, sleep);
 320  else:
 321   returnval = False;
 322  return returnval;
 323
 324 def download_from_url_to_file(httpurl, httpheaders, httpcookie, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=[262144, 262144], sleep=-1):
 325  global geturls_download_sleep, haverequests;
 326  if(sleep<0):
 327   sleep = geturls_download_sleep;
 328  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 329   httplibuse = "urllib";
 330  if(haverequests==False and httplibuse=="requests"):
 331   httplibuse = "urllib";
 332  if(havemechanize==False and httplibuse=="mechanize"):
 333   httplibuse = "urllib";
 334  if(httplibuse=="urllib"):
 335   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 336  elif(httplibuse=="requests"):
 337   returnval = download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 338  elif(httplibuse=="mechanize"):
 339   returnval = download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 340  else:
 341   returnval = False;
 342  return returnval;
 343
 344 def download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep=-1):
 345  global geturls_download_sleep;
 346  if(sleep<0):
 347   sleep = geturls_download_sleep;
 348  geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(httpcookie));
 349  if(isinstance(httpheaders, dict)):
 350   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 351  geturls_opener.addheaders = httpheaders;
 352  time.sleep(sleep);
 353  geturls_text = geturls_opener.open(httpurl);
 354  log.info("Downloading URL "+httpurl);
 355  if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 356   if(sys.version[0]=="2"):
 357    strbuf = StringIO(geturls_text.read());
 358   if(sys.version[0]>="3"):
 359    strbuf = BytesIO(geturls_text.read());
 360   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 361   returnval_content = gzstrbuf.read()[:];
 362  if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 363   returnval_content = geturls_text.read()[:];
 364  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 365  geturls_text.close();
 366  return returnval;
 367
 368 def download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize=262144, sleep=-1):
 369  global geturls_download_sleep;
 370  if(sleep<0):
 371   sleep = geturls_download_sleep;
 372  geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(httpcookie));
 373  if(isinstance(httpheaders, dict)):
 374   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 375  geturls_opener.addheaders = httpheaders;
 376  time.sleep(sleep);
 377  geturls_text = geturls_opener.open(httpurl);
 378  downloadsize = geturls_text.info().get('Content-Length');
 379  if(downloadsize is not None):
 380   downloadsize = int(downloadsize);
 381  if downloadsize is None: downloadsize = 0;
 382  fulldatasize = 0;
 383  log.info("Downloading URL "+httpurl);
 384  with tempfile.NamedTemporaryFile('wb+', prefix="pywwwget-", delete=False) as f:
 385   tmpfilename = f.name;
 386   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 387   while True:
 388    databytes = geturls_text.read(buffersize);
 389    if not databytes: break;
 390    datasize = len(databytes);
 391    fulldatasize = datasize + fulldatasize;
 392    percentage = "";
 393    if(downloadsize>0):
 394     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 395    log.info("Downloading "+str(fulldatasize)+" / "+str(downloadsize)+" bytes. "+str(percentage)+" done.");
 396    f.write(databytes);
 397   f.close();
 398  geturls_text.close();
 399  returnval.update({'Filesize': os.path.getsize(tmpfilename)});
 400  return returnval;
 401
 402 def download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[262144, 262144], sleep=-1):
 403  global geturls_download_sleep;
 404  if(sleep<0):
 405   sleep = geturls_download_sleep;
 406  if(not outfile=="-"):
 407   outpath = outpath.rstrip(os.path.sep);
 408   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 409   if(not os.path.exists(outpath)):
 410    os.makedirs(outpath);
 411   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 412    return False;
 413   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 414    return False;
 415   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 416   tmpfilename = pretmpfilename['Filename'];
 417   downloadsize = os.path.getsize(tmpfilename);
 418   fulldatasize = 0;
 419   log.info("Moving file "+tmpfilename+" to "+filepath);
 420   shutil.move(tmpfilename, filepath);
 421   if(os.path.exists(tmpfilename)==True):
 422    os.remove(tmpfilename);
 423   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 424  if(outfile=="-" and sys.version[0]=="2"):
 425   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 426   tmpfilename = pretmpfilename['Filename'];
 427   downloadsize = os.path.getsize(tmpfilename);
 428   fulldatasize = 0;
 429   with open(tmpfilename, 'rb') as ft:
 430    f = StringIO();
 431    while True:
 432     databytes = ft.read(buffersize[1]);
 433     if not databytes: break;
 434     datasize = len(databytes);
 435     fulldatasize = datasize + fulldatasize;
 436     percentage = "";
 437     if(downloadsize>0):
 438      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 439     log.info("Copying "+str(fulldatasize)+" / "+str(downloadsize)+" bytes. "+str(percentage)+" done.");
 440     f.write(databytes);
 441    f.seek(0);
 442    fdata = f.getvalue();
 443    f.close();
 444    ft.close();
 445    os.remove(tmpfilename);
 446   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 447  if(outfile=="-" and sys.version[0]>="3"):
 448   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 449   tmpfilename = pretmpfilename['Filename'];
 450   downloadsize = os.path.getsize(tmpfilename);
 451   fulldatasize = 0;
 452   with open(tmpfilename, 'rb') as ft:
 453    f = BytesIO();
 454    while True:
 455     databytes = ft.read(buffersize[1]);
 456     if not databytes: break;
 457     datasize = len(databytes);
 458     fulldatasize = datasize + fulldatasize;
 459     percentage = "";
 460     if(downloadsize>0):
 461      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 462     log.info("Copying "+str(fulldatasize)+" / "+str(downloadsize)+" bytes. "+str(percentage)+" done.");
 463     f.write(databytes);
 464    f.seek(0);
 465    fdata = f.getvalue();
 466    f.close();
 467    ft.close();
 468    os.remove(tmpfilename);
 469   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 470  return returnval;
 471
 472 if(haverequests==True):
 473  def download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep=-1):
 474   global geturls_download_sleep;
 475   if(sleep<0):
 476    sleep = geturls_download_sleep;
 477   if(isinstance(httpheaders, list)):
 478    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 479   time.sleep(sleep);
 480   geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
 481   log.info("Downloading URL "+httpurl);
 482   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
 483    if(sys.version[0]=="2"):
 484     strbuf = StringIO(geturls_text.content);
 485    if(sys.version[0]>="3"):
 486     strbuf = BytesIO(geturls_text.content);
 487    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 488    returnval_content = gzstrbuf.content[:];
 489   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
 490    returnval_content = geturls_text.content[:];
 491   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
 492   geturls_text.close();
 493   return returnval;
 494
 495 if(haverequests==False):
 496  def download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep=-1):
 497   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep)
 498   return returnval;
 499
 500 if(haverequests==True):
 501  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize=262144, sleep=-1):
 502   global geturls_download_sleep;
 503   if(sleep<0):
 504    sleep = geturls_download_sleep;
 505   if(isinstance(httpheaders, list)):
 506    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 507   time.sleep(sleep);
 508   geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
 509   downloadsize = int(geturls_text.headers.get('Content-Length'));
 510   if(downloadsize is not None):
 511    downloadsize = int(downloadsize);
 512   if downloadsize is None: downloadsize = 0;
 513   fulldatasize = 0;
 514   log.info("Downloading URL "+httpurl);
 515   with tempfile.NamedTemporaryFile('wb+', prefix="pywwwget-", delete=False) as f:
 516    tmpfilename = f.name;
 517    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
 518    for databytes in geturls_text.iter_content(chunk_size=buffersize):
 519     datasize = len(databytes);
 520     fulldatasize = datasize + fulldatasize;
 521     percentage = "";
 522     if(downloadsize>0):
 523      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 524     log.info("Downloading "+str(fulldatasize)+" / "+str(downloadsize)+" bytes. "+str(percentage)+" done.");
 525     f.write(databytes);
 526    f.close();
 527   geturls_text.close();
 528   returnval.update({'Filesize': os.path.getsize(tmpfilename)});
 529   return returnval;
 530
 531 if(haverequests==False):
 532  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize=262144, sleep=-1):
 533   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep)
 534   return returnval;
 535
 536 if(haverequests==True):
 537  def download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[262144, 262144], sleep=-1):
 538   global geturls_download_sleep;
 539   if(sleep<0):
 540    sleep = geturls_download_sleep;
 541   if(not outfile=="-"):
 542    outpath = outpath.rstrip(os.path.sep);
 543    filepath = os.path.realpath(outpath+os.path.sep+outfile);
 544    if(not os.path.exists(outpath)):
 545     os.makedirs(outpath);
 546    if(os.path.exists(outpath) and os.path.isfile(outpath)):
 547     return False;
 548    if(os.path.exists(filepath) and os.path.isdir(filepath)):
 549     return False;
 550    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 551    tmpfilename = pretmpfilename['Filename'];
 552    downloadsize = os.path.getsize(tmpfilename);
 553    fulldatasize = 0;
 554    log.info("Moving file "+tmpfilename+" to "+filepath);
 555    shutil.move(tmpfilename, filepath);
 556    if(os.path.exists(tmpfilename)==True):
 557     os.remove(tmpfilename);
 558    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 559   if(outfile=="-" and sys.version[0]=="2"):
 560    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 561    tmpfilename = pretmpfilename['Filename'];
 562    downloadsize = os.path.getsize(tmpfilename);
 563    fulldatasize = 0;
 564    with open(tmpfilename, 'rb') as ft:
 565     f = StringIO();
 566     while True:
 567      databytes = ft.read(buffersize[1]);
 568      if not databytes: break;
 569      datasize = len(databytes);
 570      fulldatasize = datasize + fulldatasize;
 571      percentage = "";
 572      if(downloadsize>0):
 573       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 574      log.info("Copying "+str(fulldatasize)+" / "+str(downloadsize)+" bytes. "+str(percentage)+" done.");
 575      f.write(databytes);
 576     f.seek(0);
 577     fdata = f.getvalue();
 578     f.close();
 579     ft.close();
 580     os.remove(tmpfilename);
 581    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 582   if(outfile=="-" and sys.version[0]>="3"):
 583    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 584    tmpfilename = pretmpfilename['Filename'];
 585    downloadsize = os.path.getsize(tmpfilename);
 586    fulldatasize = 0;
 587    with open(tmpfilename, 'rb') as ft:
 588     f = BytesIO();
 589     while True:
 590      databytes = ft.read(buffersize[1]);
 591      if not databytes: break;
 592      datasize = len(databytes);
 593      fulldatasize = datasize + fulldatasize;
 594      percentage = "";
 595      if(downloadsize>0):
 596       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 597      log.info("Copying "+str(fulldatasize)+" / "+str(downloadsize)+" bytes. "+str(percentage)+" done.");
 598      f.write(databytes);
 599     f.seek(0);
 600     fdata = f.getvalue();
 601     f.close();
 602     ft.close();
 603     os.remove(tmpfilename);
 604    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 605   return returnval;
 606
 607 if(haverequests==False):
 608  def download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[262144, 262144], sleep=-1):
 609   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, outfile, outpath, sleep)
 610   return returnval;
 611
 612 if(havemechanize==True):
 613  def download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep=-1):
 614   global geturls_download_sleep;
 615   if(sleep<0):
 616    sleep = geturls_download_sleep;
 617   geturls_opener = mechanize.Browser();
 618   if(isinstance(httpheaders, dict)):
 619    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 620   time.sleep(sleep);
 621   geturls_opener.addheaders = httpheaders;
 622   geturls_opener.set_cookiejar(httpcookie);
 623   geturls_opener.set_handle_robots(False);
 624   geturls_text = geturls_opener.open(httpurl);
 625   log.info("Downloading URL "+httpurl);
 626   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 627    if(sys.version[0]=="2"):
 628     strbuf = StringIO(geturls_text.read());
 629    if(sys.version[0]>="3"):
 630     strbuf = BytesIO(geturls_text.read());
 631    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 632    returnval_content = gzstrbuf.read()[:];
 633   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 634    returnval_content = geturls_text.read()[:];
 635   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
 636   geturls_text.close();
 637   return returnval;
 638
 639 if(havemechanize==False):
 640  def download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep=-1):
 641   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep)
 642   return returnval;
 643
 644 if(havemechanize==True):
 645  def download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize=262144, sleep=-1):
 646   global geturls_download_sleep;
 647   if(sleep<0):
 648    sleep = geturls_download_sleep;
 649   geturls_opener = mechanize.Browser();
 650   if(isinstance(httpheaders, dict)):
 651    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 652   time.sleep(sleep);
 653   geturls_opener.addheaders = httpheaders;
 654   geturls_opener.set_cookiejar(httpcookie);
 655   geturls_opener.set_handle_robots(False);
 656   geturls_text = geturls_opener.open(httpurl);
 657   downloadsize = int(geturls_text.info().get('Content-Length'));
 658   if(downloadsize is not None):
 659    downloadsize = int(downloadsize);
 660   if downloadsize is None: downloadsize = 0;
 661   fulldatasize = 0;
 662   log.info("Downloading URL "+httpurl);
 663   with tempfile.NamedTemporaryFile('wb+', prefix="pywwwget-", delete=False) as f:
 664    tmpfilename = f.name;
 665    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
 666    while True:
 667     databytes = geturls_text.read(buffersize);
 668     if not databytes: break;
 669     datasize = len(databytes);
 670     fulldatasize = datasize + fulldatasize;
 671     percentage = "";
 672     if(downloadsize>0):
 673      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 674     log.info("Downloading "+str(fulldatasize)+" / "+str(downloadsize)+" bytes. "+str(percentage)+" done.");
 675     f.write(databytes);
 676    f.close();
 677   geturls_text.close();
 678   returnval.update({'Filesize': os.path.getsize(tmpfilename)});
 679   return returnval;
 680
 681 if(havemechanize==False):
 682  def download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize=262144, sleep=-1):
 683   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep)
 684   return returnval;
 685
 686 if(havemechanize==True):
 687  def download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[262144, 262144], sleep=-1):
 688   global geturls_download_sleep;
 689   if(sleep<0):
 690    sleep = geturls_download_sleep;
 691   if(not outfile=="-"):
 692    outpath = outpath.rstrip(os.path.sep);
 693    filepath = os.path.realpath(outpath+os.path.sep+outfile);
 694    if(not os.path.exists(outpath)):
 695     os.makedirs(outpath);
 696    if(os.path.exists(outpath) and os.path.isfile(outpath)):
 697     return False;
 698    if(os.path.exists(filepath) and os.path.isdir(filepath)):
 699     return False;
 700    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 701    tmpfilename = pretmpfilename['Filename'];
 702    downloadsize = os.path.getsize(tmpfilename);
 703    fulldatasize = 0;
 704    log.info("Moving file "+tmpfilename+" to "+filepath);
 705    shutil.move(tmpfilename, filepath);
 706    if(os.path.exists(tmpfilename)==True):
 707     os.remove(tmpfilename);
 708    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 709   if(outfile=="-" and sys.version[0]=="2"):
 710    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 711    tmpfilename = pretmpfilename['Filename'];
 712    downloadsize = os.path.getsize(tmpfilename);
 713    fulldatasize = 0;
 714    with open(tmpfilename, 'rb') as ft:
 715     f = StringIO();
 716     while True:
 717      databytes = ft.read(buffersize[1]);
 718      if not databytes: break;
 719      datasize = len(databytes);
 720      fulldatasize = datasize + fulldatasize;
 721      percentage = "";
 722      if(downloadsize>0):
 723       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 724      log.info("Copying "+str(fulldatasize)+" / "+str(downloadsize)+" bytes. "+str(percentage)+" done.");
 725      f.write(databytes);
 726     f.seek(0);
 727     fdata = f.getvalue();
 728     f.close();
 729     ft.close();
 730     os.remove(tmpfilename);
 731    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 732   if(outfile=="-" and sys.version[0]>="3"):
 733    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 734    tmpfilename = pretmpfilename['Filename'];
 735    downloadsize = os.path.getsize(tmpfilename);
 736    fulldatasize = 0;
 737    with open(tmpfilename, 'rb') as ft:
 738     f = BytesIO();
 739     while True:
 740      databytes = ft.read(buffersize[1]);
 741      if not databytes: break;
 742      datasize = len(databytes);
 743      fulldatasize = datasize + fulldatasize;
 744      percentage = "";
 745      if(downloadsize>0):
 746       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 747      log.info("Copying "+str(fulldatasize)+" / "+str(downloadsize)+" bytes. "+str(percentage)+" done.");
 748      f.write(databytes);
 749     f.seek(0);
 750     fdata = f.getvalue();
 751     f.close();
 752     ft.close();
 753     os.remove(tmpfilename);
 754    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 755   return returnval;
 756
 757 if(havemechanize==False):
 758  def download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=[262144, 262144], sleep=-1):
 759   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, outfile, outpath, sleep)
 760   return returnval;
 761
 762 def get_motherless_number_of_pages(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
 763  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
 764  mrtext = premrtext['Content'];
 765  if(sys.version[0]>="3"):
 766   mrtext = mrtext.decode('ascii', 'replace');
 767  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
 768  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
 769  try:
 770   returnval = int(mlesspagenum[-1][0]);
 771  except:
 772   returnval = 1;
 773  return returnval;
 774
 775 def get_motherless_link_type(httpurl):
 776  mregex_gettitle = re.escape("http://cdn")+"([1-4])"+re.escape(".");
 777  httpurl = re.sub(mregex_gettitle, "http://cdn.", httpurl);
 778  mlessvidqstr = urlparse.parse_qs(urlparse.urlparse(httpurl).query);
 779  mlessvidid_parts = urlparse.urlparse(httpurl);
 780  mlessvidid = mlessvidid_parts.path.split("/");
 781  returnval = False;
 782  if(mlessvidid[1]=="videos" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 783   returnval = "gallery";
 784  if(mlessvidid[1]=="images" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 785   returnval = "gallery";
 786  if(mlessvidid[1]=="galleries" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 787   returnval = "gallery";
 788  if(mlessvidid[1]=="videos" and len(mlessvidid)==2):
 789   returnval = "sample-videos";
 790  if(mlessvidid[1]=="images" and len(mlessvidid)==2):
 791   returnval = "sample-images";
 792  if(mlessvidid[1]=="galleries" and len(mlessvidid)==2):
 793   returnval = "sample-galleries";
 794  if(mlessvidid[1]=="" and len(mlessvidid)==2):
 795   returnval = "sample";
 796  if(mlessvidid[1]=="groups" and len(mlessvidid)==2):
 797   returnval = "group";
 798  if(mlessvidid[1]=="groups" and len(mlessvidid)==3 and mlessvidid[2]=="search"):
 799   returnval = "group";
 800  if(mlessvidid[1]=="groups" and len(mlessvidid)==4 and mlessvidid[2]=="category"):
 801   returnval = "group";
 802  if(mlessvidid[1]=="live" and len(mlessvidid)==3 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images")):
 803   returnval = "gallery";
 804  if(mlessvidid[1]=="u" and len(mlessvidid)==3):
 805   returnval = "gallery";
 806  if(mlessvidid[1]=="f" and len(mlessvidid)==4 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images" or mlessvidid[2]=="galleries")):
 807   returnval = "gallery";
 808  if(mlessvidid[1]=="galleries" and len(mlessvidid)==4 and mlessvidid[2]=="member"):
 809   returnval = "gallery";
 810  if(mlessvidid[1]=="galleries" and len(mlessvidid)==5 and mlessvidid[2]=="member" and (mlessvidid[4]=="created" or mlessvidid[4]=="viewed" or mlessvidid[4]=="favorited" or mlessvidid[4]=="commented")):
 811   returnval = "gallery";
 812  if(mlessvidid[1]=="gv" and len(mlessvidid)==3):
 813   returnval = "gallery";
 814  if(mlessvidid[1]=="gi" and len(mlessvidid)==3):
 815   returnval = "gallery";
 816  if(mlessvidid[1]=="gm" and len(mlessvidid)==3):
 817   returnval = "member";
 818  if(mlessvidid[1]=="term" and len(mlessvidid)==3 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images" or mlessvidid[2]=="galleries")):
 819   returnval = "gallery";
 820  if(mlessvidid[1]=="g" and len(mlessvidid)==4 and re.match("^([0-9A-F]+)$", mlessvidid[3])):
 821   returnval = "file";
 822  if(mlessvidid[1]=="random" and len(mlessvidid)==3 and (mlessvidid[2]=="video" or mlessvidid[2]=="image")):
 823   returnval = "file";
 824  if(re.match("^V", mlessvidid[1]) and len(mlessvidid)==2):
 825   returnval = "board";
 826  if(re.match("^H", mlessvidid[1]) and len(mlessvidid)==2):
 827   returnval = "gallery";
 828  if(re.match("^G", mlessvidid[1]) and len(mlessvidid)==2):
 829   returnval = "gallery";
 830  if(re.match("^G", mlessvidid[1]) and len(mlessvidid)==3):
 831   returnval = "file";
 832  if(re.match("^g", mlessvidid[1]) and len(mlessvidid)==4 and re.match("^([0-9A-F]+)$", mlessvidid[3])):
 833   returnval = "file";
 834  if(mlessvidid[1]=="members" and len(mlessvidid)==2):
 835   returnval = "member";
 836  if(mlessvidid[1]=="members" and len(mlessvidid)==3 and mlessvidid[2]=="search"):
 837   returnval = "member";
 838  if(mlessvidid[1]=="members" and len(mlessvidid)==4 and (mlessvidid[2]=="uploader" or mlessvidid[2]=="viewed" or mlessvidid[2]=="social" or mlessvidid[2]=="favorited" or mlessvidid[2]=="commented" or mlessvidid[2]=="mentioned" or mlessvidid[2]=="verified")):
 839   returnval = "member";
 840  if(mlessvidid[1]=="girls" and len(mlessvidid)==2):
 841   returnval = "girls";
 842  if(mlessvidid[1]=="referers" and len(mlessvidid)==2):
 843   returnval = "referers";
 844  if(mlessvidid[1]=="about" and len(mlessvidid)==2):
 845   returnval = "team";
 846  if(mlessvidid_parts.netloc=="cdn.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn.thumbs.motherlessmedia.com"):
 847   returnval = "download";
 848  if(mlessvidid_parts.netloc=="cdn1.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn1.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn1.thumbs.motherlessmedia.com"):
 849   returnval = "download";
 850  if(mlessvidid_parts.netloc=="cdn2.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn2.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn2.thumbs.motherlessmedia.com"):
 851   returnval = "download";
 852  if(mlessvidid_parts.netloc=="cdn3.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn3.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn3.thumbs.motherlessmedia.com"):
 853   returnval = "download";
 854  if(mlessvidid_parts.netloc=="cdn4.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn4.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn4.thumbs.motherlessmedia.com"):
 855   returnval = "download";
 856  if(returnval==False and len(mlessvidid)==2 and re.match("^([0-9A-F]+)$", mlessvidid[1])):
 857   returnval = "file";
 858  return returnval;
 859
 860 def get_motherless_user_info(username, motherless_serv=None):
 861  motherless_subdomain = "cdn.";
 862  if(motherless_serv<5 or motherless_serv>0):
 863   motherless_subdomain = "cdn"+str(motherless_serv)+".";
 864  returnval = {'username': username};
 865  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg").path));
 866  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
 867  returnval.update({'orginurl': "http://motherless.com/m/"+username});
 868  returnval.update({'orginurltype': "profile"});
 869  returnval.update({'url': "http://motherless.com/m/"+username});
 870  returnval.update({'urltype': "profile"});
 871  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg"});
 872  returnval.update({'avatarfullfilename': avatarfilenameext});
 873  returnval.update({'avatarfilename': avatarfilename});
 874  returnval.update({'avatarextension': avatarfileextension});
 875  return returnval;
 876
 877 def get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", motherless_serv=None):
 878  motherless_subdomain = "cdn.";
 879  if(motherless_serv<5 or motherless_serv>0):
 880   motherless_subdomain = "cdn"+str(motherless_serv)+".";
 881  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
 882  mrtext = premrtext['Content'];
 883  if(sys.version[0]>="3"):
 884   mrtext = mrtext.decode('ascii', 'replace');
 885  mregex_gettitle = re.escape("http://cdn")+"([1-4])"+re.escape(".");
 886  mrtext = re.sub(mregex_gettitle, "http://cdn.", mrtext);
 887  mregex_gettitle = re.escape("<title>")+"(.*)"+re.escape(" - MOTHERLESS.COM</title>");
 888  mlesstitle = re.findall(mregex_gettitle, mrtext);
 889  mregex_geturlone = re.escape("__fileurl = '")+'?\'?([^"\'>]*)'+re.escape("';");
 890  mlesslinkone = re.findall(mregex_geturlone, mrtext);
 891  mregex_geturlonetype = re.escape("http://cdn.")+"(images|videos)"+re.escape(".motherlessmedia.com/")+"(images|videos)"+re.escape("/")+"([\w\/\?\&\=\.\-]+)";
 892  mlesslinkonetype = re.findall(mregex_geturlonetype, mrtext);
 893  mregex_geturltwo = re.escape("<meta property=\"og:image\" content=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
 894  mlesslinktwo = re.findall(mregex_geturltwo, mrtext);
 895  mregex_geturltwotype = re.escape("http://cdn.")+"(images|thumbs)"+re.escape(".motherlessmedia.com/")+"(images|thumbs)"+re.escape("/")+"([\w\/\?\&\=\.\-]+)";
 896  mlesslinktwotype = re.findall(mregex_geturltwotype, mrtext);
 897  filenameext = os.path.basename(urlparse.urljoin(mlesslinkone[0], urlparse.urlparse(mlesslinkone[0]).path));
 898  filename, fileextension = os.path.splitext(filenameext);
 899  thumbfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[0], urlparse.urlparse(mlesslinktwo[0]).path));
 900  thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
 901  mregex_getuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
 902  mlessuname = re.findall(mregex_getuname, mrtext);
 903  mlessuname = mlessuname[0];
 904  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">\n            <img");
 905  mlessurlname = re.findall(mregex_geturlname, mrtext);
 906  mlessurlname = mlessurlname[0].replace("/m/", "");
 907  mregex_getavatar = re.escape("<img\n    src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n    class=\"avatar avatar-small\"");
 908  mlessavatar = re.findall(mregex_getavatar, mrtext);
 909  mlessavatar = mlessavatar[0];
 910  avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar, urlparse.urlparse(mlessavatar).path));
 911  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
 912  mregex_getviews = re.escape("<strong>Views</strong>\n")+"(\s+|\t+)([0-9\,]+)(\s+\t+)"+re.escape("</h2>");
 913  mlessviews = re.findall(mregex_getviews, mrtext);
 914  mregex_getfavorited = re.escape("<strong>Favorited</strong>\n")+"(\s+\t+)([0-9\,]+)(\s+\t+)"+re.escape("</h2>");
 915  mlessfavorited = re.findall(mregex_getfavorited, mrtext);
 916  if(mlesslinkonetype[0][1]=="images"):
 917   thumbnailaltpart = thumbfilename+"-zoom"+thumbfileextension;
 918   thumbnailalt = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+thumbnailaltpart;
 919   thumbnailaltfilenameext = os.path.basename(urlparse.urljoin(thumbnailalt, urlparse.urlparse(thumbnailalt).path));
 920   thumbnailaltfilename, thumbnailaltfileextension = os.path.splitext(thumbnailaltfilenameext);
 921  if(mlesslinkonetype[0][1]=="videos"):
 922   thumbnailaltpart = thumbfilename+"-small"+thumbfileextension;
 923   thumbnailalt = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+thumbnailaltpart;
 924   thumbnailaltfilenameext = os.path.basename(urlparse.urljoin(thumbnailalt, urlparse.urlparse(thumbnailalt).path));
 925   thumbnailaltfilename, thumbnailaltfileextension = os.path.splitext(thumbnailaltfilenameext);
 926  returnval = False;
 927  mlessurltype = get_motherless_link_type(mlesslinkone[0]);
 928  if(mlesslinkonetype[0][1]=="images"):
 929   returnval = {'type': mlesslinkonetype[0][1], 'urltype': mlessurltype, 'url': mlesslinkone[0], 'orginurl': httpurl, 'orginurltype': get_motherless_link_type(httpurl), 'thumbnail': mlesslinktwo[0].replace("images", "thumbs"), 'thumbnailalt': thumbnailalt+"?from_helper", 'title': mlesstitle[0], 'fullfilename': filenameext, 'filename': filename, 'extension': fileextension, 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'thumbnailaltfullfilename': thumbnailaltpart, 'thumbnailaltfilename': thumbnailaltfilename, 'thumbnailaltextension': thumbnailaltfileextension, 'userinfo': get_motherless_user_info(mlessuname), 'username': mlessuname, 'avatarurl': mlessavatar, 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlessviews[0][1].replace(',', '')), 'numberoffavorites': int(mlessfavorited[0][1].replace(',', ''))};
 930  if(mlesslinkonetype[0][1]=="videos"):
 931   returnval = {'type': mlesslinkonetype[0][1], 'url': mlesslinkone[0], 'orginurl': httpurl, 'orginurltype': get_motherless_link_type(httpurl), 'thumbnail': mlesslinktwo[0].replace("images", "thumbs"), 'thumbnailalt': thumbnailalt+"?from_helper", 'title': mlesstitle[0], 'fullfilename': filenameext, 'filename': filename, 'extension': fileextension, 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'thumbnailaltfullfilename': thumbnailaltpart, 'thumbnailaltfilename': thumbnailaltfilename, 'thumbnailaltextension': thumbnailaltfileextension, 'userinfo': get_motherless_user_info(mlessuname), 'username': mlessuname, 'avatarurl': mlessavatar, 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlessviews[0][1].replace(',', '')), 'numberoffavorites': int(mlessfavorited[0][1].replace(',', ''))};
 932  return returnval;
 933
 934 def get_motherless_links_from_url(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
 935  returnval = False;
 936  if(get_motherless_link_type(httpurl)=="download"):
 937   urlparts = urlparse.urlparse(httpurl);
 938   filewithext = os.path.split(urlparts.path);
 939   if(filewithext[0]=="/images"):
 940    filewithoutext = os.path.splitext(filewithext[1])[0];
 941    returnval = get_motherless_links("http://motherless.com/"+filewithoutext, httpheaders, httpcookie, httplibuse="urllib");
 942   if(filewithext[0]=="/thumbs"):
 943    filewithoutext = os.path.splitext(filewithext[1])[0];
 944    filewithoutext = filewithoutext.replace("-zoom", "");
 945    filewithoutext = filewithoutext.replace("-small", "");
 946    filewithoutext = filewithoutext.replace("-strip", "");
 947    returnval = get_motherless_links("http://motherless.com/"+filewithoutext, httpheaders, httpcookie, httplibuse="urllib");
 948  return returnval;
 949
 950 def get_motherless_external_links(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
 951  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
 952  mrtext = premrtext['Content'];
 953  if(sys.version[0]>="3"):
 954   mrtext = mrtext.decode('ascii', 'replace');
 955  mregex_geturlinternal = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"outbound\" rel=\"nofollow\" class=\"pop\" target=\"_blank\" title=\"motherless link\">");
 956  mlesslinkinternal = re.findall(mregex_geturlinternal, mrtext);
 957  mregex_geturlexternal = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"outbound\" rel=\"nofollow\" class=\"pop\" target=\"_blank\" title=\"external link\">");
 958  mlesslinkexternal = re.findall(mregex_geturlexternal, mrtext);
 959  returnvalone = None;
 960  if(len(mlesslinkinternal)>1):
 961   mli = 0;
 962   mlil = len(mlesslinkinternal);
 963   returnvalone = {'numoflinks': mlil};
 964   returnvalone.update({'numofalllinks': len(mlesslinkinternal)});
 965   returnvalone.update({'orginurl': httpurl});
 966   returnvalone.update({'orginurltype': get_motherless_link_type(httpurl)});
 967   mlessrooturltype = get_motherless_link_type(httpurl);
 968   returnvalone.update({'urltype': mlessrooturltype});
 969   while(mli<mlil):
 970    mlessurltype = get_motherless_link_type(mlesslinkinternal[mli]);
 971    returnvalone.update({mli: {'urltype': mlessurltype, 'url': mlesslinkinternal[mli]} });
 972    mli = mli + 1;
 973  returnvaltwo = None;
 974  if(len(mlesslinkexternal)>1):
 975   mli = 0;
 976   mlil = len(mlesslinkexternal);
 977   returnvaltwo = {'numoflinks': mlil};
 978   returnvaltwo.update({'numofalllinks': len(mlesslinkexternal)});
 979   returnvaltwo.update({'orginurl': httpurl});
 980   returnvaltwo.update({'orginurltype': get_motherless_link_type(httpurl)});
 981   mlessrooturltype = get_motherless_link_type(httpurl);
 982   returnvaltwo.update({'urltype': mlessrooturltype});
 983   while(mli<mlil):
 984    returnvaltwo.update({mli: {'urltype': "external", 'url': mlesslinkexternal[mli]} });
 985    mli = mli + 1;
 986  if(returnvalone==None and returnvaltwo==None):
 987   returnval = {'internal': None, 'external': None};
 988  if(not returnvalone==None and not returnvaltwo==None):
 989   returnval = {'internal': returnvalone, 'external': returnvaltwo};
 990  if(not returnvalone==None and returnvaltwo==None):
 991   returnval = {'internal': returnvalone, 'external': None};
 992  if(returnvalone==None and not returnvaltwo==None):
 993   returnval = {'internal': None, 'external': returnvaltwo};
 994  return returnval;
 995
 996 def get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1], motherless_serv=None):
 997  motherless_subdomain = "cdn.";
 998  if(motherless_serv<5 or motherless_serv>0):
 999   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1000  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1001  mrtext = premrtext['Content'];
1002  if(sys.version[0]>="3"):
1003   mrtext = mrtext.decode('ascii', 'replace');
1004  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1005  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1006  try:
1007   lastpage = int(mlesspagenum[-1][0]);
1008  except:
1009   lastpage = 1;
1010  if(page>lastpage):
1011   page = lastpage;
1012  httpurl = add_url_param(httpurl, page=str(page));
1013  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1014  mrtext = premrtext['Content'];
1015  if(sys.version[0]>="3"):
1016   mrtext = mrtext.decode('ascii', 'replace');
1017  mregex_geturlone = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"img-container\" target=\"_self\">");
1018  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext);
1019  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1020  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext_tmp);
1021  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1022  mlesslinkone = re.findall(mregex_geturlone, mrtext_tmp);
1023  mregex_geturltwo = re.escape("<img class=\"static\" src=\"")+'?\'?([^"\'>]*)'+re.escape("\" data-strip-src=\"")+'?\'?([^"\'>]*)'+re.escape("\" alt=\"")+'?\'?([^">]*)'+re.escape("\" />");
1024  mlesslinktwo = re.findall(mregex_geturltwo, mrtext);
1025  mregex_getcaption = re.escape("<h2 class=\"caption title\">")+"(.*)"+re.escape("</h2>");
1026  mlesscaption = re.findall(mregex_getcaption, mrtext);
1027  mregex_gethits = re.escape("<div class=\"caption right\">")+"([0-9\,]+)"+re.escape(" Hits</div>");
1028  mlesshits = re.findall(mregex_gethits, mrtext);
1029  mregex_getuserinfo = re.escape("<a class=\"caption left\" href=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1030  mlessuname = re.findall(mregex_getuserinfo, mrtext);
1031  if(getlinks[1]>len(mlesslinkone) or getlinks[1]==-1):
1032   getlinks[1] = len(mlesslinkone);
1033  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1034   tmpgetlinks0 = getlinks[0];
1035   tmpgetlinks1 = getlinks[1];
1036   getlinks[0] = tmpgetlinks1;
1037   getlinks[1] = tmpgetlinks0;
1038  if(getlinks[0]<0):
1039   getlinks[0] = 0;
1040  mli = getlinks[0];
1041  mlil = getlinks[1];
1042  returnval = {'pages': lastpage};
1043  returnval.update({'curpage': page});
1044  returnval.update({'numoflinks': mlil});
1045  returnval.update({'numofalllinks': len(mlesslinkone)});
1046  returnval.update({'orginurl': httpurl});
1047  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1048  mlessrooturltype = get_motherless_link_type(httpurl);
1049  returnval.update({'urltype': mlessrooturltype});
1050  while(mli<mlil):
1051   mlessuname[mli] = mlessuname[mli].replace("/m/", "");
1052   stripfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][1], urlparse.urlparse(mlesslinktwo[mli][1]).path));
1053   stripfilename, stripfileextension = os.path.splitext(stripfilenameext);
1054   thumbfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][0], urlparse.urlparse(mlesslinktwo[mli][0]).path));
1055   thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1056   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlesslinkone[mli]);
1057   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg").path));
1058   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1059   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli], 'thumbnail': mlesslinktwo[mli][0], 'strip': mlesslinktwo[mli][1], 'title': mlesslinktwo[mli][2], 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'stripfullfilename': stripfilenameext, 'stripfilename': stripfilename, 'stripextension': stripfileextension, 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlesshits[mli].replace(',', ''))} });
1060   mli = mli + 1;
1061  return returnval;
1062
1063 def get_motherless_random_links(httpheaders, httpcookie, httplibuse="urllib", linktype="video", getlinks=[0, 80]):
1064  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1065   tmpgetlinks0 = getlinks[0];
1066   tmpgetlinks1 = getlinks[1];
1067   getlinks[0] = tmpgetlinks1;
1068   getlinks[1] = tmpgetlinks0;
1069  if(getlinks[0]<0):
1070   getlinks[0] = 0;
1071  mli = getlinks[0];
1072  mlil = getlinks[1];
1073  if(linktype=="image"):
1074   returnval = {'pages': 1};
1075   returnval.update({'curpage': 1});
1076   returnval.update({'numoflinks': 80});
1077   returnval.update({'numofalllinks': mlil});
1078   returnval.update({'orginurl': "http://motherless.com/random/image"});
1079   returnval.update({'orginurltype': "gallery"});
1080   returnval.update({'urltype': "gallery"});
1081   while(mli<mlil):
1082    get_links = get_motherless_links("http://motherless.com/random/image", httpheaders, httpcookie, httplibuse);
1083    returnval.update({mli: {'urltype': get_motherless_link_type("http://motherless.com/"+get_links['filename']), 'url': "http://motherless.com/"+get_links['filename'], 'thumbnail': get_links['thumbnail'], 'strip': get_links['thumbnailalt'], 'title': get_links['title'], 'thumbfullfilename': get_links['thumbfullfilename'], 'thumbfilename': get_links['thumbfilename'], 'thumbextension': get_links['thumbextension'], 'stripfullfilename': get_links['thumbnailaltfullfilename'], 'stripfilename': get_links['thumbnailaltextension'], 'stripextension': get_links['thumbnailaltfilename'], 'userinfo': get_motherless_user_info(get_links['username']), 'username': get_links['username'], 'avatarurl': get_links['avatarurl'], 'avatarfullfilename': get_links['avatarfullfilename'], 'avatarfilename': get_links['avatarfilename'], 'avatarextension': get_links['avatarextension']} });
1084    mli = mli + 1;
1085  if(linktype=="video"):
1086   returnval = {'pages': 1};
1087   returnval.update({'curpage': 1});
1088   returnval.update({'numoflinks': 80});
1089   returnval.update({'numofalllinks': mlil});
1090   returnval.update({'orginurl': "http://motherless.com/random/video"});
1091   returnval.update({'orginurltype': "gallery"});
1092   returnval.update({'urltype': "gallery"});
1093   while(mli<mlil):
1094    get_links = get_motherless_links("http://motherless.com/random/video", httpheaders, httpcookie, httplibuse);
1095    returnval.update({mli: {'urltype': get_motherless_link_type("http://motherless.com/"+get_links['filename']), 'url': "http://motherless.com/"+get_links['filename'], 'thumbnail': get_links['thumbnail'], 'strip': get_links['thumbnailalt'], 'title': get_links['title'], 'thumbfullfilename': get_links['thumbfullfilename'], 'thumbfilename': get_links['thumbfilename'], 'thumbextension': get_links['thumbextension'], 'stripfullfilename': get_links['thumbnailaltfullfilename'], 'stripfilename': get_links['thumbnailaltextension'], 'stripextension': get_links['thumbnailaltfilename'], 'userinfo': get_motherless_user_info(get_links['username']), 'username': get_links['username'], 'avatarurl': get_links['avatarurl'], 'avatarfullfilename': get_links['avatarfullfilename'], 'avatarfilename': get_links['avatarfilename'], 'avatarextension': get_links['avatarextension']} });
1096    mli = mli + 1;
1097  return returnval;
1098
1099 def get_motherless_random_links_alt(httpheaders, httpcookie, httplibuse="urllib", linktype="video", getlinks=[0, 80]):
1100  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1101   tmpgetlinks0 = getlinks[0];
1102   tmpgetlinks1 = getlinks[1];
1103   getlinks[0] = tmpgetlinks1;
1104   getlinks[1] = tmpgetlinks0;
1105  if(getlinks[0]<0):
1106   getlinks[0] = 0;
1107  mli = getlinks[0];
1108  mlil = getlinks[1];
1109  if(linktype=="image"):
1110   returnval = {'pages': 1};
1111   returnval.update({'curpage': 1});
1112   returnval.update({'numoflinks': 80});
1113   returnval.update({'numofalllinks': mlil});
1114   returnval.update({'orginurl': "http://motherless.com/random/image"});
1115   returnval.update({'orginurltype': "gallery"});
1116   returnval.update({'urltype': "gallery"});
1117   while(mli<mlil):
1118    get_links = get_motherless_links("http://motherless.com/random/image", httpheaders, httpcookie, httplibuse);
1119    returnval.update({mli: get_links});
1120    mli = mli + 1;
1121  if(linktype=="video"):
1122   returnval = {'pages': 1};
1123   returnval.update({'curpage': 1});
1124   returnval.update({'numoflinks': 80});
1125   returnval.update({'numofalllinks': mlil});
1126   returnval.update({'orginurl': "http://motherless.com/random/video"});
1127   returnval.update({'orginurltype': "gallery"});
1128   returnval.update({'urltype': "gallery"});
1129   while(mli<mlil):
1130    get_links = get_motherless_links("http://motherless.com/random/video", httpheaders, httpcookie, httplibuse);
1131    returnval.update({mli: get_links});
1132    mli = mli + 1;
1133  return returnval;
1134
1135 def get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1], motherless_serv=None):
1136  motherless_subdomain = "cdn.";
1137  if(motherless_serv<5 or motherless_serv>0):
1138   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1139  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1140  mrtext = premrtext['Content'];
1141  if(sys.version[0]>="3"):
1142   mrtext = mrtext.decode('ascii', 'replace');
1143  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext);
1144  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1145  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext_tmp);
1146  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1147  mregex_geturlone = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" title=\"motherless link\">");
1148  mlesslinkone = re.findall(mregex_geturlone, mrtext_tmp);
1149  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"post-username special-member op-member\" title=\"op\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1150  mlessurlname = re.findall(mregex_geturlname, mrtext);
1151  mlessavaturl = "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessurlname[0][1]+"-avatar.jpg";
1152  avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavaturl, urlparse.urlparse(mlessavaturl).path));
1153  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1154  if(getlinks[1]>len(mlesslinkone) or getlinks[1]==-1):
1155   getlinks[1] = len(mlesslinkone);
1156  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1157   tmpgetlinks0 = getlinks[0];
1158   tmpgetlinks1 = getlinks[1];
1159   getlinks[0] = tmpgetlinks1;
1160   getlinks[1] = tmpgetlinks0;
1161  if(getlinks[0]<0):
1162   getlinks[0] = 0;
1163  mli = getlinks[0];
1164  mlil = getlinks[1];
1165  returnval = {'numoflinks': mlil};
1166  returnval.update({'numofalllinks': len(mlesslinkone)});
1167  returnval.update({'orginurl': httpurl});
1168  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1169  mlessrooturltype = get_motherless_link_type(httpurl);
1170  returnval.update({'urltype': mlessrooturltype});
1171  returnval.update({'userinfo': get_motherless_user_info(mlessurlname[0][1])});
1172  returnval.update({'username': mlessurlname[0][1]});
1173  returnval.update({'avatarurl': mlessavaturl});
1174  returnval.update({'avatarfullfilename': avatarfilenameext});
1175  returnval.update({'avatarfilename': avatarfilename});
1176  returnval.update({'avatarextension': avatarfileextension});
1177  while(mli<mlil):
1178   mlessurltype = get_motherless_link_type("http://motherless.com"+mlesslinkone[mli]);
1179   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli]} });
1180   mli = mli + 1;
1181  return returnval;
1182
1183 def get_motherless_boards_posts(httpurl, httpheaders, httpcookie, httplibuse="urllib", getposts=[0, -1], motherless_serv=None):
1184  motherless_subdomain = "cdn.";
1185  if(motherless_serv<5 or motherless_serv>0):
1186   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1187  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1188  mrtext = premrtext['Content'];
1189  if(sys.version[0]>="3"):
1190   mrtext = mrtext.decode('ascii', 'replace');
1191  mregex_getposts = "(\t+)"+re.escape("<p>")+"(.+?)"+re.escape("</p>")+"(\t+)";
1192  mlessposts = re.findall(mregex_getposts, mrtext);
1193  mregex_getuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1194  mlessuname = re.findall(mregex_getuname, mrtext);
1195  mregex_getopuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"post-username special-member op-member\" title=\"op\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1196  mlessopuname = re.findall(mregex_getopuname, mrtext);
1197  if(getposts[1]>len(mlessposts) or getposts[1]==-1):
1198   getposts[1] = len(mlessposts);
1199  if(getposts[0]>getposts[1] and not getposts[1]==-1):
1200   tmpgetposts0 = getposts[0];
1201   tmpgetposts1 = getposts[1];
1202   getposts[0] = tmpgetposts1;
1203   getposts[1] = tmpgetposts0;
1204  if(getposts[0]<0):
1205   getposts[0] = 0;
1206  mli = getposts[0];
1207  mlil = getposts[1];
1208  returnval = {'numofposts': mlil};
1209  returnval.update({'numofallposts': len(mlessposts)});
1210  returnval.update({'orginurl': httpurl});
1211  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1212  mlessrooturltype = get_motherless_link_type(httpurl);
1213  returnval.update({'urltype': mlessrooturltype});
1214  returnval.update({'userinfo': get_motherless_user_info(mlessopuname[0][1])});
1215  returnval.update({'username': mlessopuname[0][1]});
1216  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg").path));
1217  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1218  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg"});
1219  returnval.update({'avatarfullfilename': avatarfilenameext});
1220  returnval.update({'avatarfilename': avatarfilename});
1221  returnval.update({'avatarextension': avatarfileextension});
1222  while(mli<mlil):
1223   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg").path));
1224   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1225   returnval.update({mli: {'post': mlessposts[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1226   mli = mli + 1;
1227  return returnval;
1228
1229 def get_motherless_links_comments(httpurl, httpheaders, httpcookie, httplibuse="urllib", getposts=[0, -1], motherless_serv=None):
1230  motherless_subdomain = "cdn.";
1231  if(motherless_serv<5 or motherless_serv>0):
1232   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1233  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1234  mrtext = premrtext['Content'];
1235  if(sys.version[0]>="3"):
1236   mrtext = mrtext.decode('ascii', 'replace');
1237  mregex_getposts = re.escape("<div style=\"text-align: justify;\">\n")+"(\t+)(.+?)"+re.escape("</div>");
1238  mlessposts = re.findall(mregex_getposts, mrtext);
1239  mregex_getuname = re.escape("<div class=\"media-comment\" id=\"")+'?\'?([^"\'>]*)'+re.escape("\" rel=\"0\" rev=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1240  mlessuname = re.findall(mregex_getuname, mrtext);
1241  mregex_getopuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1242  mlessopuname = re.findall(mregex_getopuname, mrtext);
1243  if(getposts[1]>len(mlessposts) or getposts[1]==-1):
1244   getposts[1] = len(mlessposts);
1245  if(getposts[0]>getposts[1] and not getposts[1]==-1):
1246   tmpgetposts0 = getposts[0];
1247   tmpgetposts1 = getposts[1];
1248   getposts[0] = tmpgetposts1;
1249   getposts[1] = tmpgetposts0;
1250  if(getposts[0]<0):
1251   getposts[0] = 0;
1252  mli = getposts[0];
1253  mlil = getposts[1];
1254  returnval = {'numofposts': mlil};
1255  returnval.update({'numofallposts': len(mlessposts)});
1256  returnval.update({'orginurl': httpurl});
1257  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1258  mlessrooturltype = get_motherless_link_type(httpurl);
1259  returnval.update({'urltype': mlessrooturltype});
1260  returnval.update({'userinfo': get_motherless_user_info(mlessopuname[0])});
1261  returnval.update({'username': mlessopuname[0]});
1262  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg").path));
1263  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1264  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg"});
1265  returnval.update({'avatarfullfilename': avatarfilenameext});
1266  returnval.update({'avatarfilename': avatarfilename});
1267  returnval.update({'avatarextension': avatarfileextension});
1268  while(mli<mlil):
1269   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg").path));
1270   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1271   returnval.update({mli: {'post': mlessposts[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli][1]), 'username': mlessuname[mli][1], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1272   mli = mli + 1;
1273  return returnval;
1274
1275 def get_motherless_search_members(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1276  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1277  mrtext = premrtext['Content'];
1278  if(sys.version[0]>="3"):
1279   mrtext = mrtext.decode('ascii', 'replace');
1280  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1281  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1282  try:
1283   lastpage = int(mlesspagenum[-1][0]);
1284  except:
1285   lastpage = 1;
1286  if(page>lastpage):
1287   page = lastpage;
1288  httpurl = add_url_param(httpurl, page=str(page));
1289  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1290  mrtext = premrtext['Content'];
1291  if(sys.version[0]>="3"):
1292   mrtext = mrtext.decode('ascii', 'replace');
1293  mregex_getuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1294  mlessuname = re.findall(mregex_getuname, mrtext);
1295  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">\n            <img");
1296  mlessurlname = re.findall(mregex_geturlname, mrtext);
1297  mregex_getavatar = re.escape("<img\n    src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n    class=\"avatar avatar-small\"");
1298  mlessavatar = re.findall(mregex_getavatar, mrtext);
1299  if(getlinks[1]>len(mlessuname) or getlinks[1]==-1):
1300   getlinks[1] = len(mlessuname);
1301  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1302   tmpgetlinks0 = getlinks[0];
1303   tmpgetlinks1 = getlinks[1];
1304   getlinks[0] = tmpgetlinks1;
1305   getlinks[1] = tmpgetlinks0;
1306  if(getlinks[0]<0):
1307   getlinks[0] = 0;
1308  mli = getlinks[0];
1309  mlil = getlinks[1];
1310  returnval = {'numoflinks': mlil};
1311  returnval.update({'numofalllinks': len(mlessuname)});
1312  returnval.update({'pages': lastpage});
1313  returnval.update({'curpage': page});
1314  returnval.update({'orginurl': httpurl});
1315  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1316  mlessrooturltype = get_motherless_link_type(httpurl);
1317  returnval.update({'urltype': mlessrooturltype});
1318  while(mli<mlil):
1319   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli], urlparse.urlparse(mlessavatar[mli]).path));
1320   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1321   mlessurlname[mli] = mlessurlname[mli].replace("/m/", "");
1322   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessurlname[mli]);
1323   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessurlname[mli], 'userinfo': get_motherless_user_info(mlessurlname[mli]), 'username': mlessuname[mli], 'avatarurl': mlessavatar[mli], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1324   mli = mli + 1;
1325  return returnval;
1326
1327 def get_motherless_girls(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1328  premrtext = download_from_url("http://motherless.com/girls", httpheaders, httpcookie, httplibuse);
1329  mrtext = premrtext['Content'];
1330  if(sys.version[0]>="3"):
1331   mrtext = mrtext.decode('ascii', 'replace');
1332  mregex_getuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"")+'?\'?([^"\'>]*)'+re.escape("\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1333  mlessuname = re.findall(mregex_getuname, mrtext);
1334  mregex_geturlname = re.escape("\n\t\t\t\t\t\t<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">");
1335  mlessurlname = re.findall(mregex_geturlname, mrtext);
1336  if(getlinks[1]>len(mlessuname) or getlinks[1]==-1):
1337   getlinks[1] = len(mlessuname);
1338  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1339   tmpgetlinks0 = getlinks[0];
1340   tmpgetlinks1 = getlinks[1];
1341   getlinks[0] = tmpgetlinks1;
1342   getlinks[1] = tmpgetlinks0;
1343  if(getlinks[0]<0):
1344   getlinks[0] = 0;
1345  mli = getlinks[0];
1346  mlil = getlinks[1];
1347  returnval = {'numoflinks': mlil};
1348  returnval.update({'numofalllinks': len(mlessuname)});
1349  returnval.update({'orginurl': "http://motherless.com/girls"});
1350  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/girls")});
1351  mlessrooturltype = get_motherless_link_type("http://motherless.com/girls");
1352  returnval.update({'urltype': mlessrooturltype});
1353  while(mli<mlil):
1354   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessuname[mli][0], urlparse.urlparse(mlessuname[mli][0]).path));
1355   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1356   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessuname[mli][1]);
1357   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessuname[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli][1]), 'username': mlessuname[mli][1], 'avatarurl': mlessuname[mli][0], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1358   mli = mli + 1;
1359  return returnval;
1360
1361 def get_motherless_team(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1362  premrtext = download_from_url("http://motherless.com/about", httpheaders, httpcookie, httplibuse);
1363  mrtext = premrtext['Content'];
1364  if(sys.version[0]>="3"):
1365   mrtext = mrtext.decode('ascii', 'replace');
1366  mregex_getuname = re.escape("<div class=\"about-us-member\">\n                         <a href=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1367  mlessuname = re.findall(mregex_getuname, mrtext);
1368  mlessuname_odd = mlessuname[1::2];
1369  mlessuname_even = mlessuname[::2];
1370  mregex_getavatar = re.escape("<img\n        src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n        class=\"avatar avatar-")+"(full|medium)"+re.escape("\"");
1371  mlessavatar = re.findall(mregex_getavatar, mrtext);
1372  if(getlinks[1]>len(mlessuname_odd) or getlinks[1]==-1):
1373   getlinks[1] = len(mlessuname_odd);
1374  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1375   tmpgetlinks0 = getlinks[0];
1376   tmpgetlinks1 = getlinks[1];
1377   getlinks[0] = tmpgetlinks1;
1378   getlinks[1] = tmpgetlinks0;
1379  if(getlinks[0]<0):
1380   getlinks[0] = 0;
1381  mli = getlinks[0];
1382  mlil = getlinks[1];
1383  returnval = {'numoflinks': mlil};
1384  returnval.update({'numofalllinks': len(mlessuname)});
1385  returnval.update({'orginurl': "http://motherless.com/about"});
1386  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/about")});
1387  mlessrooturltype = get_motherless_link_type("http://motherless.com/about");
1388  returnval.update({'urltype': mlessrooturltype});
1389  while(mli<mlil):
1390   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli][0], urlparse.urlparse(mlessavatar[mli][0]).path));
1391   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1392   mlessuname_odd[mli] = mlessuname_odd[mli].replace("/m/", "");
1393   mlessuname_even[mli] = mlessuname_even[mli].replace("/m/", "");
1394   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessuname_odd[mli]);
1395   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessuname_odd[mli], 'userinfo': get_motherless_user_info(mlessuname_odd[mli]), 'username': mlessuname_odd[mli], 'avatarurl': mlessavatar[mli][0], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1396   mli = mli + 1;
1397  return returnval;
1398
1399 def get_motherless_top_referrers(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1400  premrtext = download_from_url("http://motherless.com/referers", httpheaders, httpcookie, httplibuse);
1401  mrtext = premrtext['Content'];
1402  if(sys.version[0]>="3"):
1403   mrtext = mrtext.decode('ascii', 'replace');
1404  mregex_geturlname = "([0-9]+)"+re.escape(". <a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"pop\" target=\"_blank\" rel=\"nofollow\">\n                                            ")+"(.*)"+re.escape("                                   </a>");
1405  mlessurlname = re.findall(mregex_geturlname, mrtext);
1406  if(getlinks[1]>len(mlessurlname) or getlinks[1]==-1):
1407   getlinks[1] = len(mlessurlname);
1408  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1409   tmpgetlinks0 = getlinks[0];
1410   tmpgetlinks1 = getlinks[1];
1411   getlinks[0] = tmpgetlinks1;
1412   getlinks[1] = tmpgetlinks0;
1413  if(getlinks[0]<0):
1414   getlinks[0] = 0;
1415  mli = getlinks[0];
1416  mlil = getlinks[1];
1417  returnval = {'numoflinks': mlil};
1418  returnval.update({'numofalllinks': len(mlessurlname)});
1419  returnval.update({'orginurl': "http://motherless.com/referers"});
1420  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/referers")});
1421  mlessrooturltype = get_motherless_link_type("http://motherless.com/referers");
1422  returnval.update({'urltype': mlessrooturltype});
1423  while(mli<mlil):
1424   returnval.update({mli: {'urltype': "referer-links", 'url': mlessurlname[mli][1], 'title': mlessurlname[mli][2]} });
1425   mli = mli + 1;
1426  return returnval;
1427
1428 def get_motherless_top_referers(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1429  return get_motherless_top_referrers(httpheaders, httpcookie, httplibuse, getlinks);
1430
1431 def get_motherless_groups(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1432  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1433  mrtext = premrtext['Content'];
1434  if(sys.version[0]>="3"):
1435   mrtext = mrtext.decode('ascii', 'replace');
1436  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1437  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1438  try:
1439   lastpage = int(mlesspagenum[-1][0]);
1440  except:
1441   lastpage = 1;
1442  if(page>lastpage):
1443   page = lastpage;
1444  httpurl = add_url_param(httpurl, page=str(page));
1445  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1446  mrtext = premrtext['Content'];
1447  if(sys.version[0]>="3"):
1448   mrtext = mrtext.decode('ascii', 'replace');
1449  mregex_getavatar = re.escape("<img\n        src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n        class=\"avatar avatar-")+"(full|medium)"+re.escape("\"");
1450  mlessavatar = re.findall(mregex_getavatar, mrtext);
1451  mregex_getgroups = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"grunge motherless-red\">\n")+"(.*)"+re.escape("</a>");
1452  mlessgroups = re.findall(mregex_getgroups, mrtext);
1453  if(getlinks[1]>len(mlessgroups) or getlinks[1]==-1):
1454   getlinks[1] = len(mlessgroups);
1455  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1456   tmpgetlinks0 = getlinks[0];
1457   tmpgetlinks1 = getlinks[1];
1458   getlinks[0] = tmpgetlinks1;
1459   getlinks[1] = tmpgetlinks0;
1460  if(getlinks[0]<0):
1461   getlinks[0] = 0;
1462  mli = getlinks[0];
1463  mlil = getlinks[1];
1464  returnval = {'pages': lastpage};
1465  returnval.update({'curpage': page});
1466  returnval.update({'numoflinks': mlil});
1467  returnval.update({'numofalllinks': len(mlessgroups)});
1468  returnval.update({'orginurl': httpurl});
1469  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1470  while(mli<mlil):
1471   thumbfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli][0], urlparse.urlparse(mlessavatar[mli][0]).path));
1472   thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1473   returnval.update({mli: {'urltype': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0]), 'url': "http://motherless.com"+mlessgroups[mli][0], 'urltype-image': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gi/")), 'url-image': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gi/"), 'urltype-video': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gv/")), 'url-video': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gv/"), 'urltype-member': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gm/")), 'url-member': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gm/"), 'thumbnail': mlessavatar[mli][0], 'title': mlessgroups[mli][1].strip(), 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension} });
1474   mli = mli + 1;
1475  return returnval;
1476
1477 def get_motherless_sample_links(httpheaders, httpcookie, httplibuse="urllib", numoflinks=10, urltype="video"):
1478  if(urltype=="video"):
1479   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/videos", 'orginurltype': get_motherless_link_type("http://motherless.com/videos"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "video", [0, numoflinks]) } };
1480  if(urltype=="image"):
1481   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/images", 'orginurltype': get_motherless_link_type("http://motherless.com/images"), 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "image", [0, numoflinks]) } };
1482  if(urltype=="gallery"):
1483   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/galleries", 'orginurltype': get_motherless_link_type("http://motherless.com/galleries"), 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1484  if(urltype=="all"):
1485   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/", 'orginurltype': get_motherless_link_type("http://motherless.com/"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "video", [0, numoflinks]) }, 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "image", [0, numoflinks]) }, 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1486  return returnval;
1487
1488 def get_motherless_link_by_type(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1489  returnval = False;
1490  if(get_motherless_link_type(httpurl)=="file"):
1491   returnval = get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse);
1492  if(get_motherless_link_type(httpurl)=="gallery"):
1493   returnval = get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse, page);
1494  if(get_motherless_link_type(httpurl)=="sample-videos"):
1495   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "video");
1496  if(get_motherless_link_type(httpurl)=="sample-images"):
1497   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "image");
1498  if(get_motherless_link_type(httpurl)=="sample-galleries"):
1499   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "gallery");
1500  if(get_motherless_link_type(httpurl)=="sample" or get_motherless_link_type(httpurl)=="sample-all"):
1501   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "all");
1502  if(get_motherless_link_type(httpurl)=="board"):
1503   returnval = get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse);
1504  if(get_motherless_link_type(httpurl)=="member"):
1505   returnval = get_motherless_search_members(httpurl, httpheaders, httpcookie, httplibuse, page);
1506  if(get_motherless_link_type(httpurl)=="girls"):
1507   returnval = get_motherless_girls(httpheaders, httpcookie, httplibuse);
1508  if(get_motherless_link_type(httpurl)=="download"):
1509   returnval = httpurl;
1510  return returnval;
1511
1512 def view_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", viewerpro="mpv", prearg=[], proarg=[]):
1513  commandlist = [viewerpro] + prearg;
1514  commandlist = commandlist + [get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse)['url']];
1515  commandlist = commandlist + proarg;
1516  mpvplaylistp = subprocess.Popen(commandlist, stdout=subprocess.PIPE, stderr=subprocess.PIPE);
1517  mpvplayout, mpvplayerr = mpvplaylistp.communicate();
1518  return True;
1519
1520 def download_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=[262144, 262144], outfile="-", outpath=os.getcwd(), usetitlename=False):
1521  global geturls_download_sleep;
1522  if(sleep<0):
1523   sleep = geturls_download_sleep;
1524  mlessurl = get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse);
1525  outputname = mlessurl['fullfilename'];
1526  outpath = outpath.rstrip(os.path.sep);
1527  if(usetitlename==True):
1528   outputname = mlessurl['title'];
1529  if(usetitlename=="-" and outfile=="-"):
1530   outputname = "-";
1531  if(usetitlename=="-" and not outfile=="-"):
1532   outputname = outfile;
1533  returnval = download_from_url_to_file(mlessurl['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep);
1534  return returnval;
1535
1536 def download_motherless_links_by_type(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=[262144, 262144], outfile="-", outpath=os.getcwd(), usetitlename=False, page=1, getlinks=[0, -1]):
1537  global geturls_download_sleep;
1538  if(sleep<0):
1539   sleep = geturls_download_sleep;
1540  mlessurl = get_motherless_link_by_type(httpurl, httpheaders, httpcookie, httplibuse, page);
1541  if(mlessurl['urltype']=="download"):
1542   outputname = mlessurl['fullfilename'];
1543   outpathname = outpath.rstrip(os.path.sep);
1544   if(usetitlename==True):
1545    outputname = mlessurl['title'];
1546   if(usetitlename=="-" and outfile[mli]=="-"):
1547    outputname = "-";
1548   if(usetitlename=="-" and not outfile[mli]=="-"):
1549    outputname = outfile;
1550   returnval = download_from_url_to_file(mlessurl['url'], httpheaders, httpcookie, httplibuse, outputname, outpathname, buffersize, sleep);
1551  if(not mlessurl['urltype']=="download"):
1552   returnval = mlessurl;
1553  return returnval;
1554
1555 def download_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=[262144, 262144], outfile="-", outpath=os.getcwd(), usetitlename=False, page=1, getlinks=[0, -1]):
1556  global geturls_download_sleep;
1557  if(sleep<0):
1558   sleep = geturls_download_sleep;
1559  mlessgalleries = get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse, page, getlinks);
1560  mli = 0;
1561  mlil = mlessgalleries['numoflinks'];
1562  returnval = {'pages': mlessgalleries['pages']};
1563  returnval.update({'numoflists': mlessgalleries['numoflinks']});
1564  returnval.update({'curpage': mlessgalleries['curpage']});
1565  returnval.update({'numoflinks': mlessgalleries['numoflinks']});
1566  returnval.update({'numofalllinks': mlessgalleries['numofalllinks']});
1567  returnval.update({'orginurl': httpurl});
1568  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1569  while(mli<mlil):
1570   mlesslink = get_motherless_links(mlessgalleries[mli]['url'], httpheaders, httpcookie, httplibuse);
1571   outputname = mlesslink['fullfilename'];
1572   outpath = outpath.rstrip(os.path.sep);
1573   if(usetitlename==True):
1574    outputname = mlesslink['title'];
1575   if(usetitlename=="-" and outfile=="-"):
1576    outputname = "-";
1577   if(usetitlename=="-" and not outfile=="-"):
1578    outputname = outfile;
1579   returnval.update({mli: {'download': download_from_url_to_file(mlesslink['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep), 'linkinfo': mlesslink, 'outputfile': outputname} });
1580   mli = mli + 1;
1581  return returnval;
1582
1583 def download_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=[262144, 262144], outfile="-", outpath=os.getcwd(), usetitlename=False, getlinks=[0, -1]):
1584  global geturls_download_sleep;
1585  if(sleep<0):
1586   sleep = geturls_download_sleep;
1587  mlessgalleries = get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse, getlinks);
1588  mli = 0;
1589  mlil = mlessgalleries['numoflinks'];
1590  returnval = {'numoflists': mlessgalleries['numoflinks']};
1591  returnval.update({'numofalllinks': mlessgalleries['numofalllinks']});
1592  returnval.update({'orginurl': httpurl});
1593  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1594  while(mli<mlil):
1595   mlesslink = get_motherless_links(mlessgalleries[mli]['url'], httpheaders, httpcookie, httplibuse);
1596   outputname = mlesslink['fullfilename'];
1597   outpath = outpath.rstrip(os.path.sep);
1598   if(usetitlename==True):
1599    outputname = mlesslink['title'];
1600   if(usetitlename=="-" and outfile=="-"):
1601    outputname = "-";
1602   if(usetitlename=="-" and not outfile=="-"):
1603    outputname = outfile;
1604   returnval.update({mli: {'download': download_from_url_to_file(mlesslink['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep), 'linkinfo': mlesslink, 'outputfile': outputname} });
1605   mli = mli + 1;
1606  return returnval;
1607