pymotherless.py

   1 #!/usr/bin/env python
   2
   3 '''
   4     This program is free software; you can redistribute it and/or modify
   5     it under the terms of the Revised BSD License.
   6
   7     This program is distributed in the hope that it will be useful,
   8     but WITHOUT ANY WARRANTY; without even the implied warranty of
   9     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10     Revised BSD License for more details.
  11
  12     Copyright 2016 Cool Dude 2k - http://idb.berlios.de/
  13     Copyright 2016 Game Maker 2k - http://intdb.sourceforge.net/
  14     Copyright 2016 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
  15
  16     $FileInfo: pymotherless.py - Last Update: 6/16/2016 Ver. 0.4.7 RC 1 - Author: cooldude2k $
  17 '''
  18
  19 from __future__ import division, absolute_import, print_function;
  20 import re, os, sys, hashlib, shutil, platform, tempfile, urllib, gzip, time, argparse, cgi, subprocess, imp;
  21 import logging as log;
  22 haverequests = False;
  23 try:
  24  imp.find_module('requests');
  25  haverequests = True;
  26  import requests;
  27 except ImportError:
  28  haverequests = False;
  29 havemechanize = False;
  30 try:
  31  imp.find_module('mechanize');
  32  havemechanize = True;
  33  import mechanize;
  34 except ImportError:
  35  havemechanize = False;
  36 if(sys.version[0]=="2"):
  37  try:
  38   from cStringIO import StringIO;
  39  except ImportError:
  40   from StringIO import StringIO;
  41  # From http://python-future.org/compatible_idioms.html
  42  from urlparse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin;
  43  from urllib import urlencode;
  44  from urllib2 import urlopen, Request, HTTPError;
  45  import urllib2, urlparse, cookielib;
  46 if(sys.version[0]>="3"):
  47  from io import StringIO, BytesIO;
  48  # From http://python-future.org/compatible_idioms.html
  49  from urllib.parse import urlparse, urlunparse, urlsplit, urlunsplit, urljoin, urlencode;
  50  from urllib.request import urlopen, Request;
  51  from urllib.error import HTTPError;
  52  import urllib.request as urllib2;
  53  import urllib.parse as urlparse;
  54  import http.cookiejar as cookielib;
  55
  56 __program_name__ = "PyMotherless";
  57 __project__ = __program_name__;
  58 __project_url__ = "https://github.com/GameMaker2k/PyMotherless";
  59 __version_info__ = (0, 4, 7, "RC 1", 1);
  60 __version_date_info__ = (2016, 6, 16, "RC 1", 1);
  61 __version_date__ = str(__version_date_info__[0])+"."+str(__version_date_info__[1]).zfill(2)+"."+str(__version_date_info__[2]).zfill(2);
  62 if(__version_info__[4]!=None):
  63  __version_date_plusrc__ = __version_date__+"-"+str(__version_date_info__[4]);
  64 if(__version_info__[4]==None):
  65  __version_date_plusrc__ = __version_date__;
  66 if(__version_info__[3]!=None):
  67  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2])+" "+str(__version_info__[3]);
  68 if(__version_info__[3]==None):
  69  __version__ = str(__version_info__[0])+"."+str(__version_info__[1])+"."+str(__version_info__[2]);
  70
  71 tmpfileprefix = "py"+str(sys.version_info[0])+"motherless"+str(__version_info__[0])+"-";
  72 tmpfilesuffix = "-";
  73 pytempdir = tempfile.gettempdir();
  74 defbuffersize = 524288;
  75 defbuffersizedub = [defbuffersize, defbuffersize];
  76
  77 geturls_cj = cookielib.CookieJar();
  78 geturls_ua_firefox_windows7 = "Mozilla/5.0 (Windows NT 6.1; rv:47.0) Gecko/20100101 Firefox/47.0";
  79 geturls_ua_seamonkey_windows7 = "Mozilla/5.0 (Windows NT 6.1; rv:43.0) Gecko/20100101 Firefox/43.0 SeaMonkey/2.40";
  80 geturls_ua_chrome_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36";
  81 geturls_ua_chromium_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chromium/50.0.2661.102 Chrome/50.0.2661.102 Safari/537.36";
  82 geturls_ua_midori_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/538.15 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/538.15 Midori/0.5";
  83 geturls_ua_palemoon_windows7 = "Mozilla/5.0 (Windows NT 6.1; rv:45.9) Gecko/20100101 Goanna/2.1 Firefox/45.9 PaleMoon/26.3.0";
  84 geturls_ua_opera_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36 OPR/38.0.2220.31";
  85 geturls_ua_vivaldi_windows7 = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36 Vivaldi/1.2.490.43";
  86 geturls_ua_internet_explorer_windows7 = "Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko";
  87 geturls_ua_pymotherless_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
  88 if(platform.python_implementation()!=""):
  89  geturls_ua_pymotherless_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=platform.python_implementation(), pyver=platform.python_version(), proname=__project__, prover=__version__);
  90 if(platform.python_implementation()==""):
  91  geturls_ua_pymotherless_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp="Python", pyver=platform.python_version(), proname=__project__, prover=__version__);
  92 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
  93 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
  94 geturls_ua = geturls_ua_firefox_windows7;
  95 geturls_headers_firefox_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_firefox_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  96 geturls_headers_seamonkey_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_seamonkey_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  97 geturls_headers_chrome_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_chrome_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  98 geturls_headers_chromium_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_chromium_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
  99 geturls_headers_midori_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_midori_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 100 geturls_headers_palemoon_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_palemoon_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 101 geturls_headers_opera_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_opera_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 102 geturls_headers_vivaldi_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_vivaldi_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 103 geturls_headers_internet_explorer_windows7 = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_internet_explorer_windows7, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 104 geturls_headers_pymotherless_python = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_pymotherless_python, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 105 geturls_headers_pymotherless_python_alt = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_pymotherless_python_alt, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 106 geturls_headers_googlebot_google = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 107 geturls_headers_googlebot_google_old = {'Referer': "http://motherless.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
 108 geturls_headers = geturls_headers_firefox_windows7;
 109 geturls_download_sleep = 0;
 110
 111 def add_url_param(url, **params):
 112  n=3;
 113  parts = list(urlparse.urlsplit(url));
 114  d = dict(cgi.parse_qsl(parts[n])); # use cgi.parse_qs for list values
 115  d.update(params);
 116  parts[n]=urlencode(d);
 117  return urlparse.urlunsplit(parts);
 118
 119 os.environ["PATH"] = os.environ["PATH"] + os.pathsep + os.path.dirname(os.path.realpath(__file__)) + os.pathsep + os.getcwd();
 120 def which_exec(execfile):
 121  for path in os.environ["PATH"].split(":"):
 122   if os.path.exists(path + "/" + execfile):
 123    return path + "/" + execfile;
 124
 125 def listize(varlist):
 126  il = 0;
 127  ix = len(varlist);
 128  ilx = 1;
 129  newlistreg = {};
 130  newlistrev = {};
 131  newlistfull = {};
 132  while(il < ix):
 133   newlistreg.update({ilx: varlist[il]});
 134   newlistrev.update({varlist[il]: ilx});
 135   ilx = ilx + 1;
 136   il = il + 1;
 137  newlistfull = {1: newlistreg, 2: newlistrev, 'reg': newlistreg, 'rev': newlistrev};
 138  return newlistfull;
 139
 140 def twolistize(varlist):
 141  il = 0;
 142  ix = len(varlist);
 143  ilx = 1;
 144  newlistnamereg = {};
 145  newlistnamerev = {};
 146  newlistdescreg = {};
 147  newlistdescrev = {};
 148  newlistfull = {};
 149  while(il < ix):
 150   newlistnamereg.update({ilx: varlist[il][0].strip()});
 151   newlistnamerev.update({varlist[il][0].strip(): ilx});
 152   newlistdescreg.update({ilx: varlist[il][1].strip()});
 153   newlistdescrev.update({varlist[il][1].strip(): ilx});
 154   ilx = ilx + 1;
 155   il = il + 1;
 156  newlistnametmp = {1: newlistnamereg, 2: newlistnamerev, 'reg': newlistnamereg, 'rev': newlistnamerev};
 157  newlistdesctmp = {1: newlistdescreg, 2: newlistdescrev, 'reg': newlistdescreg, 'rev': newlistdescrev};
 158  newlistfull = {1: newlistnametmp, 2: newlistdesctmp, 'name': newlistnametmp, 'desc': newlistdesctmp}
 159  return newlistfull;
 160
 161 def arglistize(proexec, *varlist):
 162  il = 0;
 163  ix = len(varlist);
 164  ilx = 1;
 165  newarglist = [proexec];
 166  while(il < ix):
 167   if varlist[il][0] is not None:
 168    newarglist.append(varlist[il][0]);
 169   if varlist[il][1] is not None:
 170    newarglist.append(varlist[il][1]);
 171   il = il + 1;
 172  return newarglist;
 173
 174 # hms_string by ArcGIS Python Recipes
 175 # https://arcpy.wordpress.com/2012/04/20/146/
 176 def hms_string(sec_elapsed):
 177  h = int(sec_elapsed / (60 * 60));
 178  m = int((sec_elapsed % (60 * 60)) / 60);
 179  s = sec_elapsed % 60.0;
 180  return "{}:{:>02}:{:>05.2f}".format(h, m, s);
 181
 182 # get_readable_size by Lipis
 183 # http://stackoverflow.com/posts/14998888/revisions
 184 def get_readable_size(bytes, precision=1, unit="IEC"):
 185  unit = unit.upper();
 186  if(unit!="IEC" and unit!="SI"):
 187   unit = "IEC";
 188  if(unit=="IEC"):
 189   units = [" B"," KiB"," MiB"," GiB"," TiB"," PiB"," EiB"," ZiB"];
 190   unitswos = ["B","KiB","MiB","GiB","TiB","PiB","EiB","ZiB"];
 191   unitsize = 1024.0;
 192  if(unit=="SI"):
 193   units = [" B"," kB"," MB"," GB"," TB"," PB"," EB"," ZB"];
 194   unitswos = ["B","kB","MB","GB","TB","PB","EB","ZB"];
 195   unitsize = 1000.0;
 196  return_val = {};
 197  orgbytes = bytes;
 198  for unit in units:
 199   if abs(bytes) < unitsize:
 200    strformat = "%3."+str(precision)+"f%s";
 201    pre_return_val = (strformat % (bytes, unit));
 202    pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 203    pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 204    alt_return_val = pre_return_val.split();
 205    return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 206    return return_val;
 207   bytes /= unitsize;
 208  strformat = "%."+str(precision)+"f%s";
 209  pre_return_val = (strformat % (bytes, "YiB"));
 210  pre_return_val = re.sub(r"([0]+) ([A-Za-z]+)", r" \2", pre_return_val);
 211  pre_return_val = re.sub(r"\. ([A-Za-z]+)", r" \1", pre_return_val);
 212  alt_return_val = pre_return_val.split();
 213  return_val = {'Bytes': orgbytes, 'ReadableWithSuffix': pre_return_val, 'ReadableWithoutSuffix': alt_return_val[0], 'ReadableSuffix': alt_return_val[1]}
 214  return return_val;
 215
 216 def get_readable_size_from_file(infile, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 217  unit = unit.upper();
 218  usehashtypes = usehashtypes.lower();
 219  getfilesize = os.path.getsize(infile);
 220  return_val = get_readable_size(getfilesize, precision, unit);
 221  if(usehashes==True):
 222   hashtypelist = usehashtypes.split(",");
 223   openfile = open(infile, "rb");
 224   filecontents = openfile.read();
 225   openfile.close();
 226   listnumcount = 0;
 227   listnumend = len(hashtypelist);
 228   while(listnumcount < listnumend):
 229    hashtypelistlow = hashtypelist[listnumcount].strip();
 230    hashtypelistup = hashtypelistlow.upper();
 231    filehash = hashlib.new(hashtypelistup);
 232    filehash.update(filecontents);
 233    filegethash = filehash.hexdigest();
 234    return_val.update({hashtypelistup: filegethash});
 235    listnumcount += 1;
 236  return return_val;
 237
 238 def get_readable_size_from_string(instring, precision=1, unit="IEC", usehashes=False, usehashtypes="md5,sha1"):
 239  unit = unit.upper();
 240  usehashtypes = usehashtypes.lower();
 241  getfilesize = len(instring);
 242  return_val = get_readable_size(getfilesize, precision, unit);
 243  if(usehashes==True):
 244   hashtypelist = usehashtypes.split(",");
 245   listnumcount = 0;
 246   listnumend = len(hashtypelist);
 247   while(listnumcount < listnumend):
 248    hashtypelistlow = hashtypelist[listnumcount].strip();
 249    hashtypelistup = hashtypelistlow.upper();
 250    filehash = hashlib.new(hashtypelistup);
 251    if(sys.version[0]=="2"):
 252     filehash.update(instring);
 253    if(sys.version[0]>="3"):
 254     filehash.update(instring.encode('utf-8'));
 255    filegethash = filehash.hexdigest();
 256    return_val.update({hashtypelistup: filegethash});
 257    listnumcount += 1;
 258  return return_val;
 259
 260 def make_http_headers_from_dict_to_list(headers={'Referer': "http://motherless.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 261  if isinstance(headers, dict):
 262   returnval = [];
 263   if(sys.version[0]=="2"):
 264    for headkey, headvalue in headers.iteritems():
 265     returnval.append((headkey, headvalue));
 266   if(sys.version[0]>="3"):
 267    for headkey, headvalue in headers.items():
 268     returnval.append((headkey, headvalue));
 269  elif isinstance(headers, list):
 270   returnval = headers;
 271  else:
 272   returnval = False;
 273  return returnval;
 274
 275 def make_http_headers_from_dict_to_pycurl(headers={'Referer': "http://motherless.com/", 'User-Agent': geturls_ua, 'Accept-Encoding': "gzip, deflate", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"}):
 276  if isinstance(headers, dict):
 277   returnval = [];
 278   if(sys.version[0]=="2"):
 279    for headkey, headvalue in headers.iteritems():
 280     returnval.append(headkey+": "+headvalue);
 281   if(sys.version[0]>="3"):
 282    for headkey, headvalue in headers.items():
 283     returnval.append(headkey+": "+headvalue);
 284  elif isinstance(headers, list):
 285   returnval = headers;
 286  else:
 287   returnval = False;
 288  return returnval;
 289
 290 def make_http_headers_from_list_to_dict(headers=[("Referer", "http://motherless.com/"), ("User-Agent", geturls_ua), ("Accept-Encoding", "gzip, deflate"), ("Accept-Language", "en-US,en;q=0.8,en-CA,en-GB;q=0.6"), ("Accept-Charset", "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7"), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Connection", "close")]):
 291  if isinstance(headers, list):
 292   returnval = {};
 293   mli = 0;
 294   mlil = len(headers);
 295   while(mli<mlil):
 296    returnval.update({headers[mli][0]: headers[mli][1]});
 297    mli = mli + 1;
 298  elif isinstance(headers, dict):
 299   returnval = headers;
 300  else:
 301   returnval = False;
 302  return returnval;
 303
 304 def get_httplib_support(checkvalue=None):
 305  global haverequests, havemechanize;
 306  returnval = [];
 307  returnval.append("urllib");
 308  if(haverequests==True):
 309   returnval.append("requests");
 310  if(havemechanize==True):
 311   returnval.append("mechanize");
 312  if(not checkvalue==None):
 313   if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 314    checkvalue = "urllib";
 315   if(checkvalue in returnval):
 316    returnval = True;
 317   else:
 318    returnval = False;
 319  return returnval;
 320
 321 def check_httplib_support(checkvalue="urllib"):
 322  if(checkvalue=="urllib1" or checkvalue=="urllib2"):
 323   checkvalue = "urllib";
 324  returnval = get_httplib_support(checkvalue);
 325  return returnval;
 326
 327 def get_httplib_support_list():
 328  returnval = get_httplib_support(None);
 329  return returnval;
 330
 331 def download_from_url(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1):
 332  global geturls_download_sleep, haverequests, havemechanize;
 333  if(sleep<0):
 334   sleep = geturls_download_sleep;
 335  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 336   httplibuse = "urllib";
 337  if(haverequests==False and httplibuse=="requests"):
 338   httplibuse = "urllib";
 339  if(havemechanize==False and httplibuse=="mechanize"):
 340   httplibuse = "urllib";
 341  if(httplibuse=="urllib"):
 342   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep);
 343  elif(httplibuse=="requests"):
 344   returnval = download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep);
 345  elif(httplibuse=="mechanize"):
 346   returnval = download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep);
 347  else:
 348   returnval = False;
 349  return returnval;
 350
 351 def download_from_url_file(httpurl, httpheaders, httpcookie, httplibuse="urllib", buffersize=defbuffersize, sleep=-1):
 352  global geturls_download_sleep, haverequests, havemechanize;
 353  if(sleep<0):
 354   sleep = geturls_download_sleep;
 355  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 356   httplibuse = "urllib";
 357  if(haverequests==False and httplibuse=="requests"):
 358   httplibuse = "urllib";
 359  if(havemechanize==False and httplibuse=="mechanize"):
 360   httplibuse = "urllib";
 361  if(httplibuse=="urllib"):
 362   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep);
 363  elif(httplibuse=="requests"):
 364   returnval = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize, sleep);
 365  elif(httplibuse=="mechanize"):
 366   returnval = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize, sleep);
 367  else:
 368   returnval = False;
 369  return returnval;
 370
 371 def download_from_url_to_file(httpurl, httpheaders, httpcookie, httplibuse="urllib", outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 372  global geturls_download_sleep, haverequests, havemechanize;
 373  if(sleep<0):
 374   sleep = geturls_download_sleep;
 375  if(httplibuse=="urllib1" or httplibuse=="urllib2"):
 376   httplibuse = "urllib";
 377  if(haverequests==False and httplibuse=="requests"):
 378   httplibuse = "urllib";
 379  if(havemechanize==False and httplibuse=="mechanize"):
 380   httplibuse = "urllib";
 381  if(httplibuse=="urllib"):
 382   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 383  elif(httplibuse=="requests"):
 384   returnval = download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 385  elif(httplibuse=="mechanize"):
 386   returnval = download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile, outpath, buffersize, sleep);
 387  else:
 388   returnval = False;
 389  return returnval;
 390
 391 def download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep=-1):
 392  global geturls_download_sleep;
 393  if(sleep<0):
 394   sleep = geturls_download_sleep;
 395  geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(httpcookie));
 396  if(isinstance(httpheaders, dict)):
 397   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 398  geturls_opener.addheaders = httpheaders;
 399  time.sleep(sleep);
 400  geturls_text = geturls_opener.open(httpurl);
 401  log.info("Downloading URL "+httpurl);
 402  if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 403   if(sys.version[0]=="2"):
 404    strbuf = StringIO(geturls_text.read());
 405   if(sys.version[0]>="3"):
 406    strbuf = BytesIO(geturls_text.read());
 407   gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 408   returnval_content = gzstrbuf.read()[:];
 409  if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 410   returnval_content = geturls_text.read()[:];
 411  returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 412  geturls_text.close();
 413  return returnval;
 414
 415 def download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 416  global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 417  exec_time_start = time.time();
 418  myhash = hashlib.new("sha1");
 419  myhash.update(httpurl);
 420  myhash.update(str(buffersize));
 421  myhash.update(str(exec_time_start));
 422  tmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 423  if(sleep<0):
 424   sleep = geturls_download_sleep;
 425  geturls_opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(httpcookie));
 426  if(isinstance(httpheaders, dict)):
 427   httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 428  geturls_opener.addheaders = httpheaders;
 429  time.sleep(sleep);
 430  geturls_text = geturls_opener.open(httpurl);
 431  downloadsize = geturls_text.info().get('Content-Length');
 432  if(downloadsize is not None):
 433   downloadsize = int(downloadsize);
 434  if downloadsize is None: downloadsize = 0;
 435  fulldatasize = 0;
 436  prevdownsize = 0;
 437  log.info("Downloading URL "+httpurl);
 438  with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=tmpfilesuffix, delete=False) as f:
 439   tmpfilename = f.name;
 440   returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.getcode()};
 441   while True:
 442    databytes = geturls_text.read(buffersize);
 443    if not databytes: break;
 444    datasize = len(databytes);
 445    fulldatasize = datasize + fulldatasize;
 446    percentage = "";
 447    if(downloadsize>0):
 448     percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 449    downloaddiff = fulldatasize - prevdownsize;
 450    log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 451    prevdownsize = fulldatasize;
 452    f.write(databytes);
 453   f.close();
 454  geturls_text.close();
 455  exec_time_end = time.time();
 456  log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 457  returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 458  return returnval;
 459
 460 def download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 461  global geturls_download_sleep;
 462  if(sleep<0):
 463   sleep = geturls_download_sleep;
 464  if(not outfile=="-"):
 465   outpath = outpath.rstrip(os.path.sep);
 466   filepath = os.path.realpath(outpath+os.path.sep+outfile);
 467   if(not os.path.exists(outpath)):
 468    os.makedirs(outpath);
 469   if(os.path.exists(outpath) and os.path.isfile(outpath)):
 470    return False;
 471   if(os.path.exists(filepath) and os.path.isdir(filepath)):
 472    return False;
 473   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 474   tmpfilename = pretmpfilename['Filename'];
 475   downloadsize = os.path.getsize(tmpfilename);
 476   fulldatasize = 0;
 477   log.info("Moving file "+tmpfilename+" to "+filepath);
 478   exec_time_start = time.time();
 479   shutil.move(tmpfilename, filepath);
 480   exec_time_end = time.time();
 481   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 482   if(os.path.exists(tmpfilename)==True):
 483    os.remove(tmpfilename);
 484   returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 485  if(outfile=="-" and sys.version[0]=="2"):
 486   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 487   tmpfilename = pretmpfilename['Filename'];
 488   downloadsize = os.path.getsize(tmpfilename);
 489   fulldatasize = 0;
 490   prevdownsize = 0;
 491   exec_time_start = time.time();
 492   with open(tmpfilename, 'rb') as ft:
 493    f = StringIO();
 494    while True:
 495     databytes = ft.read(buffersize[1]);
 496     if not databytes: break;
 497     datasize = len(databytes);
 498     fulldatasize = datasize + fulldatasize;
 499     percentage = "";
 500     if(downloadsize>0):
 501      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 502     downloaddiff = fulldatasize - prevdownsize;
 503     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 504     prevdownsize = fulldatasize;
 505     f.write(databytes);
 506    f.seek(0);
 507    fdata = f.getvalue();
 508    f.close();
 509    ft.close();
 510    os.remove(tmpfilename);
 511    exec_time_end = time.time();
 512    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 513   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 514  if(outfile=="-" and sys.version[0]>="3"):
 515   pretmpfilename = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 516   tmpfilename = pretmpfilename['Filename'];
 517   downloadsize = os.path.getsize(tmpfilename);
 518   fulldatasize = 0;
 519   prevdownsize = 0;
 520   exec_time_start = time.time();
 521   with open(tmpfilename, 'rb') as ft:
 522    f = BytesIO();
 523    while True:
 524     databytes = ft.read(buffersize[1]);
 525     if not databytes: break;
 526     datasize = len(databytes);
 527     fulldatasize = datasize + fulldatasize;
 528     percentage = "";
 529     if(downloadsize>0):
 530      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 531     downloaddiff = fulldatasize - prevdownsize;
 532     log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 533     prevdownsize = fulldatasize;
 534     f.write(databytes);
 535    f.seek(0);
 536    fdata = f.getvalue();
 537    f.close();
 538    ft.close();
 539    os.remove(tmpfilename);
 540    exec_time_end = time.time();
 541    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 542   returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 543  return returnval;
 544
 545 if(haverequests==True):
 546  def download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep=-1):
 547   global geturls_download_sleep;
 548   if(sleep<0):
 549    sleep = geturls_download_sleep;
 550   if(isinstance(httpheaders, list)):
 551    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 552   time.sleep(sleep);
 553   geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie);
 554   log.info("Downloading URL "+httpurl);
 555   if(geturls_text.headers.get('Content-Type')=="gzip" or geturls_text.headers.get('Content-Type')=="deflate"):
 556    if(sys.version[0]=="2"):
 557     strbuf = StringIO(geturls_text.content);
 558    if(sys.version[0]>="3"):
 559     strbuf = BytesIO(geturls_text.content);
 560    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 561    returnval_content = gzstrbuf.content[:];
 562   if(geturls_text.headers.get('Content-Type')!="gzip" and geturls_text.headers.get('Content-Type')!="deflate"):
 563    returnval_content = geturls_text.content[:];
 564   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.headers), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
 565   geturls_text.close();
 566   return returnval;
 567
 568 if(haverequests==False):
 569  def download_from_url_with_requests(httpurl, httpheaders, httpcookie, sleep=-1):
 570   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep)
 571   return returnval;
 572
 573 if(haverequests==True):
 574  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 575   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 576   exec_time_start = time.time();
 577   myhash = hashlib.new("sha1");
 578   myhash.update(httpurl);
 579   myhash.update(str(buffersize));
 580   myhash.update(str(exec_time_start));
 581   tmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 582   if(sleep<0):
 583    sleep = geturls_download_sleep;
 584   if(isinstance(httpheaders, list)):
 585    httpheaders = make_http_headers_from_list_to_dict(httpheaders);
 586   time.sleep(sleep);
 587   geturls_text = requests.get(httpurl, headers=httpheaders, cookies=httpcookie, stream=True);
 588   downloadsize = int(geturls_text.headers.get('Content-Length'));
 589   if(downloadsize is not None):
 590    downloadsize = int(downloadsize);
 591   if downloadsize is None: downloadsize = 0;
 592   fulldatasize = 0;
 593   prevdownsize = 0;
 594   log.info("Downloading URL "+httpurl);
 595   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=tmpfilesuffix, delete=False) as f:
 596    tmpfilename = f.name;
 597    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.headers), 'URL': geturls_text.url, 'Code': geturls_text.status_code};
 598    for databytes in geturls_text.iter_content(chunk_size=buffersize):
 599     datasize = len(databytes);
 600     fulldatasize = datasize + fulldatasize;
 601     percentage = "";
 602     if(downloadsize>0):
 603      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 604     downloaddiff = fulldatasize - prevdownsize;
 605     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 606     prevdownsize = fulldatasize;
 607     f.write(databytes);
 608    f.close();
 609   geturls_text.close();
 610   exec_time_end = time.time();
 611   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 612   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 613   return returnval;
 614
 615 if(haverequests==False):
 616  def download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 617   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep)
 618   return returnval;
 619
 620 if(haverequests==True):
 621  def download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 622   global geturls_download_sleep;
 623   if(sleep<0):
 624    sleep = geturls_download_sleep;
 625   if(not outfile=="-"):
 626    outpath = outpath.rstrip(os.path.sep);
 627    filepath = os.path.realpath(outpath+os.path.sep+outfile);
 628    if(not os.path.exists(outpath)):
 629     os.makedirs(outpath);
 630    if(os.path.exists(outpath) and os.path.isfile(outpath)):
 631     return False;
 632    if(os.path.exists(filepath) and os.path.isdir(filepath)):
 633     return False;
 634    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 635    tmpfilename = pretmpfilename['Filename'];
 636    downloadsize = os.path.getsize(tmpfilename);
 637    fulldatasize = 0;
 638    log.info("Moving file "+tmpfilename+" to "+filepath);
 639    exec_time_start = time.time();
 640    shutil.move(tmpfilename, filepath);
 641    exec_time_end = time.time();
 642    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 643    if(os.path.exists(tmpfilename)==True):
 644     os.remove(tmpfilename);
 645    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 646   if(outfile=="-" and sys.version[0]=="2"):
 647    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 648    tmpfilename = pretmpfilename['Filename'];
 649    downloadsize = os.path.getsize(tmpfilename);
 650    fulldatasize = 0;
 651    prevdownsize = 0;
 652    exec_time_start = time.time();
 653    with open(tmpfilename, 'rb') as ft:
 654     f = StringIO();
 655     while True:
 656      databytes = ft.read(buffersize[1]);
 657      if not databytes: break;
 658      datasize = len(databytes);
 659      fulldatasize = datasize + fulldatasize;
 660      percentage = "";
 661      if(downloadsize>0):
 662       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 663      downloaddiff = fulldatasize - prevdownsize;
 664      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 665      prevdownsize = fulldatasize;
 666      f.write(databytes);
 667     f.seek(0);
 668     fdata = f.getvalue();
 669     f.close();
 670     ft.close();
 671     os.remove(tmpfilename);
 672     exec_time_end = time.time();
 673     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 674    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 675   if(outfile=="-" and sys.version[0]>="3"):
 676    pretmpfilename = download_from_url_file_with_requests(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 677    tmpfilename = pretmpfilename['Filename'];
 678    downloadsize = os.path.getsize(tmpfilename);
 679    fulldatasize = 0;
 680    prevdownsize = 0;
 681    exec_time_start = time.time();
 682    with open(tmpfilename, 'rb') as ft:
 683     f = BytesIO();
 684     while True:
 685      databytes = ft.read(buffersize[1]);
 686      if not databytes: break;
 687      datasize = len(databytes);
 688      fulldatasize = datasize + fulldatasize;
 689      percentage = "";
 690      if(downloadsize>0):
 691       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 692      downloaddiff = fulldatasize - prevdownsize;
 693      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 694      prevdownsize = fulldatasize;
 695      f.write(databytes);
 696     f.seek(0);
 697     fdata = f.getvalue();
 698     f.close();
 699     ft.close();
 700     os.remove(tmpfilename);
 701     exec_time_end = time.time();
 702     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 703    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 704   return returnval;
 705
 706 if(haverequests==False):
 707  def download_from_url_to_file_with_requests(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 708   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, outfile, outpath, sleep)
 709   return returnval;
 710
 711 if(havemechanize==True):
 712  def download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep=-1):
 713   global geturls_download_sleep;
 714   if(sleep<0):
 715    sleep = geturls_download_sleep;
 716   geturls_opener = mechanize.Browser();
 717   if(isinstance(httpheaders, dict)):
 718    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 719   time.sleep(sleep);
 720   geturls_opener.addheaders = httpheaders;
 721   geturls_opener.set_cookiejar(httpcookie);
 722   geturls_opener.set_handle_robots(False);
 723   geturls_text = geturls_opener.open(httpurl);
 724   log.info("Downloading URL "+httpurl);
 725   if(geturls_text.info().get("Content-Encoding")=="gzip" or geturls_text.info().get("Content-Encoding")=="deflate"):
 726    if(sys.version[0]=="2"):
 727     strbuf = StringIO(geturls_text.read());
 728    if(sys.version[0]>="3"):
 729     strbuf = BytesIO(geturls_text.read());
 730    gzstrbuf = gzip.GzipFile(fileobj=strbuf);
 731    returnval_content = gzstrbuf.read()[:];
 732   if(geturls_text.info().get("Content-Encoding")!="gzip" and geturls_text.info().get("Content-Encoding")!="deflate"):
 733    returnval_content = geturls_text.read()[:];
 734   returnval = {'Type': "Content", 'Content': returnval_content, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
 735   geturls_text.close();
 736   return returnval;
 737
 738 if(havemechanize==False):
 739  def download_from_url_with_mechanize(httpurl, httpheaders, httpcookie, sleep=-1):
 740   returnval = download_from_url_with_urllib(httpurl, httpheaders, httpcookie, sleep)
 741   return returnval;
 742
 743 if(havemechanize==True):
 744  def download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 745   global geturls_download_sleep, tmpfileprefix, tmpfilesuffix;
 746   exec_time_start = time.time();
 747   myhash = hashlib.new("sha1");
 748   myhash.update(httpurl);
 749   myhash.update(str(buffersize));
 750   myhash.update(str(exec_time_start));
 751   tmpfilesuffix = tmpfilesuffix + str(myhash.hexdigest());
 752   if(sleep<0):
 753    sleep = geturls_download_sleep;
 754   geturls_opener = mechanize.Browser();
 755   if(isinstance(httpheaders, dict)):
 756    httpheaders = make_http_headers_from_dict_to_list(httpheaders);
 757   time.sleep(sleep);
 758   geturls_opener.addheaders = httpheaders;
 759   geturls_opener.set_cookiejar(httpcookie);
 760   geturls_opener.set_handle_robots(False);
 761   geturls_text = geturls_opener.open(httpurl);
 762   downloadsize = int(geturls_text.info().get('Content-Length'));
 763   if(downloadsize is not None):
 764    downloadsize = int(downloadsize);
 765   if downloadsize is None: downloadsize = 0;
 766   fulldatasize = 0;
 767   prevdownsize = 0;
 768   log.info("Downloading URL "+httpurl);
 769   with tempfile.NamedTemporaryFile('wb+', prefix=tmpfileprefix, suffix=tmpfilesuffix, delete=False) as f:
 770    tmpfilename = f.name;
 771    returnval = {'Type': "File", 'Filename': tmpfilename, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'Headers': dict(geturls_text.info()), 'URL': geturls_text.geturl(), 'Code': geturls_text.code};
 772    while True:
 773     databytes = geturls_text.read(buffersize);
 774     if not databytes: break;
 775     datasize = len(databytes);
 776     fulldatasize = datasize + fulldatasize;
 777     percentage = "";
 778     if(downloadsize>0):
 779      percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 780     downloaddiff = fulldatasize - prevdownsize;
 781     log.info("Downloading "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Downloaded "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 782     prevdownsize = fulldatasize;
 783     f.write(databytes);
 784    f.close();
 785   geturls_text.close();
 786   exec_time_end = time.time();
 787   log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to download file.");
 788   returnval.update({'Filesize': os.path.getsize(tmpfilename), 'DownloadTime': float(exec_time_start - exec_time_end), 'DownloadTimeReadable': hms_string(exec_time_start - exec_time_end)});
 789   return returnval;
 790
 791 if(havemechanize==False):
 792  def download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize=defbuffersize, sleep=-1):
 793   returnval = download_from_url_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, sleep)
 794   return returnval;
 795
 796 if(havemechanize==True):
 797  def download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 798   global geturls_download_sleep;
 799   if(sleep<0):
 800    sleep = geturls_download_sleep;
 801   if(not outfile=="-"):
 802    outpath = outpath.rstrip(os.path.sep);
 803    filepath = os.path.realpath(outpath+os.path.sep+outfile);
 804    if(not os.path.exists(outpath)):
 805     os.makedirs(outpath);
 806    if(os.path.exists(outpath) and os.path.isfile(outpath)):
 807     return False;
 808    if(os.path.exists(filepath) and os.path.isdir(filepath)):
 809     return False;
 810    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 811    tmpfilename = pretmpfilename['Filename'];
 812    downloadsize = os.path.getsize(tmpfilename);
 813    fulldatasize = 0;
 814    log.info("Moving file "+tmpfilename+" to "+filepath);
 815    exec_time_start = time.time();
 816    log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to move file.");
 817    shutil.move(tmpfilename, filepath);
 818    if(os.path.exists(tmpfilename)==True):
 819     os.remove(tmpfilename);
 820    returnval = {'Type': "File", 'Filename': filepath, 'Filesize': downloadsize, 'FilesizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 821   if(outfile=="-" and sys.version[0]=="2"):
 822    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 823    tmpfilename = pretmpfilename['Filename'];
 824    downloadsize = os.path.getsize(tmpfilename);
 825    fulldatasize = 0;
 826    prevdownsize = 0;
 827    exec_time_start = time.time();
 828    with open(tmpfilename, 'rb') as ft:
 829     f = StringIO();
 830     while True:
 831      databytes = ft.read(buffersize[1]);
 832      if not databytes: break;
 833      datasize = len(databytes);
 834      fulldatasize = datasize + fulldatasize;
 835      percentage = "";
 836      if(downloadsize>0):
 837       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 838      downloaddiff = fulldatasize - prevdownsize;
 839      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 840      prevdownsize = fulldatasize;
 841      f.write(databytes);
 842     f.seek(0);
 843     fdata = f.getvalue();
 844     f.close();
 845     ft.close();
 846     os.remove(tmpfilename);
 847     exec_time_end = time.time();
 848     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 849    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 850   if(outfile=="-" and sys.version[0]>="3"):
 851    pretmpfilename = download_from_url_file_with_mechanize(httpurl, httpheaders, httpcookie, buffersize[0], sleep);
 852    tmpfilename = pretmpfilename['Filename'];
 853    downloadsize = os.path.getsize(tmpfilename);
 854    fulldatasize = 0;
 855    prevdownsize = 0;
 856    exec_time_start = time.time();
 857    with open(tmpfilename, 'rb') as ft:
 858     f = BytesIO();
 859     while True:
 860      databytes = ft.read(buffersize[1]);
 861      if not databytes: break;
 862      datasize = len(databytes);
 863      fulldatasize = datasize + fulldatasize;
 864      percentage = "";
 865      if(downloadsize>0):
 866       percentage = str("{0:.2f}".format(float(float(fulldatasize / downloadsize) * 100))).rstrip('0').rstrip('.')+"%";
 867      downloaddiff = fulldatasize - prevdownsize;
 868      log.info("Copying "+get_readable_size(fulldatasize, 2, "SI")['ReadableWithSuffix']+" / "+get_readable_size(downloadsize, 2, "SI")['ReadableWithSuffix']+" "+str(percentage)+" / Copied "+get_readable_size(downloaddiff, 2, "IEC")['ReadableWithSuffix']);
 869      prevdownsize = fulldatasize;
 870      f.write(databytes);
 871     f.seek(0);
 872     fdata = f.getvalue();
 873     f.close();
 874     ft.close();
 875     os.remove(tmpfilename);
 876     exec_time_end = time.time();
 877     log.info("It took "+hms_string(exec_time_start - exec_time_end)+" to copy file.");
 878    returnval = {'Type': "Content", 'Content': fdata, 'Contentsize': downloadsize, 'ContentsizeAlt': {'IEC': get_readable_size(downloadsize, 2, "IEC"), 'SI': get_readable_size(downloadsize, 2, "SI")}, 'DownloadTime': pretmpfilename['DownloadTime'], 'DownloadTimeReadable': pretmpfilename['DownloadTimeReadable'], 'MoveFileTime': float(exec_time_start - exec_time_end), 'MoveFileTimeReadable': hms_string(exec_time_start - exec_time_end), 'Headers': pretmpfilename['Headers'], 'URL': pretmpfilename['URL'], 'Code': pretmpfilename['Code']};
 879   return returnval;
 880
 881 if(havemechanize==False):
 882  def download_from_url_to_file_with_mechanize(httpurl, httpheaders, httpcookie, outfile="-", outpath=os.getcwd(), buffersize=defbuffersizedub, sleep=-1):
 883   returnval = download_from_url_to_file_with_urllib(httpurl, httpheaders, httpcookie, buffersize, outfile, outpath, sleep)
 884   return returnval;
 885
 886 def check_motherless_url(httpurl):
 887  returnval = {};
 888  httpurl.replace("https://", "http://");
 889  if(not httpurl.startswith("http")):
 890   httpurl = "http://"+httpurl;
 891  urlparts = urlsplit(httpurl);
 892  urltype = None;
 893  domaintype = None;
 894  urlservtype = None;
 895  if(urlparts.netloc=="motherless.com" or urlparts.netloc=="www.motherless.com"):
 896   urltype = "motherless-link";
 897   domaintype = "motherless";
 898   urlservtype = "www";
 899  if(urlparts.netloc=="cdn.avatars.motherlessmedia.com" or urlparts.netloc=="cdn1.avatars.motherlessmedia.com" or urlparts.netloc=="cdn2.avatars.motherlessmedia.com" or urlparts.netloc=="cdn3.avatars.motherlessmedia.com" or urlparts.netloc=="cdn4.avatars.motherlessmedia.com"):
 900   urltype = "motherless-avatar";
 901   domaintype = "motherlessmedia";
 902  if(urlparts.netloc=="cdn.images.motherlessmedia.com" or urlparts.netloc=="cdn1.images.motherlessmedia.com" or urlparts.netloc=="cdn2.images.motherlessmedia.com" or urlparts.netloc=="cdn3.images.motherlessmedia.com" or urlparts.netloc=="cdn4.images.motherlessmedia.com"):
 903   urltype = "motherless-image";
 904   domaintype = "motherlessmedia";
 905  if(urlparts.netloc=="cdn.videos.motherlessmedia.com" or urlparts.netloc=="cdn1.videos.motherlessmedia.com" or urlparts.netloc=="cdn2.videos.motherlessmedia.com" or urlparts.netloc=="cdn3.videos.motherlessmedia.com" or urlparts.netloc=="cdn4.videos.motherlessmedia.com"):
 906   urltype = "motherless-video";
 907   domaintype = "motherlessmedia";
 908  if(urlparts.netloc=="cdn.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn1.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn2.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn3.thumbs.motherlessmedia.com" or urlparts.netloc=="cdn4.thumbs.motherlessmedia.com"):
 909   urltype = "motherless-thumbnail";
 910   domaintype = "motherlessmedia";
 911  if(urlparts.netloc=="cdn.avatars.motherlessmedia.com" or urlparts.netloc=="cdn.images.motherlessmedia.com" or urlparts.netloc=="cdn.videos.motherlessmedia.com" or urlparts.netloc=="cdn.thumbs.motherlessmedia.com"):
 912   urlservtype = "cdn";
 913  if(urlparts.netloc=="cdn1.avatars.motherlessmedia.com" or urlparts.netloc=="cdn1.images.motherlessmedia.com" or urlparts.netloc=="cdn1.videos.motherlessmedia.com" or urlparts.netloc=="cdn1.thumbs.motherlessmedia.com"):
 914   urlservtype = "cdn1";
 915  if(urlparts.netloc=="cdn2.avatars.motherlessmedia.com" or urlparts.netloc=="cdn2.images.motherlessmedia.com" or urlparts.netloc=="cdn2.videos.motherlessmedia.com" or urlparts.netloc=="cdn2.thumbs.motherlessmedia.com"):
 916   urlservtype = "cdn2";
 917  if(urlparts.netloc=="cdn3.avatars.motherlessmedia.com" or urlparts.netloc=="cdn3.images.motherlessmedia.com" or urlparts.netloc=="cdn3.videos.motherlessmedia.com" or urlparts.netloc=="cdn3.thumbs.motherlessmedia.com"):
 918   urlservtype = "cdn3";
 919  if(urlparts.netloc=="cdn4.avatars.motherlessmedia.com" or urlparts.netloc=="cdn4.images.motherlessmedia.com" or urlparts.netloc=="cdn4.videos.motherlessmedia.com" or urlparts.netloc=="cdn4.thumbs.motherlessmedia.com"):
 920   urlservtype = "cdn4";
 921  if(urltype==None or domaintype==None or urlservtype==None):
 922   return False;
 923  returnval.update({'url': urlparts.geturl(), 'originalurl': httpurl, 'urltype': urltype, 'urlserv': urlservtype, 'domaintype': domaintype, 'urlinfo': {'scheme': urlparts.scheme, 'netloc': urlparts.netloc, 'path': urlparts.path, 'query': urlparts.query, 'fragment': urlparts.fragment}});
 924  return returnval;
 925
 926 def fix_motherless_url(httpurl):
 927  tmpreturnval = check_motherless_url(httpurl);
 928  if(tmpreturnval==False):
 929   returnval = False
 930  else:
 931   returnval = tmpreturnval['url'];
 932  return returnval;
 933
 934 def get_motherless_link_type(httpurl):
 935  httpurl = fix_motherless_url(httpurl);
 936  if(httpurl==False):
 937   return False;
 938  mregex_gettitle = re.escape("http://cdn")+"([1-4])"+re.escape(".");
 939  httpurl = re.sub(mregex_gettitle, "http://cdn.", httpurl);
 940  mlessvidqstr = urlparse.parse_qs(urlparse.urlparse(httpurl).query);
 941  mlessvidid_parts = urlparse.urlparse(httpurl);
 942  mlessvidid = mlessvidid_parts.path.split("/");
 943  returnval = False;
 944  if(mlessvidid[1]=="videos" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 945   returnval = "gallery";
 946  if(mlessvidid[1]=="images" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 947   returnval = "gallery";
 948  if(mlessvidid[1]=="galleries" and len(mlessvidid)==3 and (mlessvidid[2]=="recent" or mlessvidid[2]=="favorited" or mlessvidid[2]=="viewed" or mlessvidid[2]=="commented" or mlessvidid[2]=="popular")):
 949   returnval = "gallery";
 950  if(mlessvidid[1]=="videos" and len(mlessvidid)==2):
 951   returnval = "sample-videos";
 952  if(mlessvidid[1]=="images" and len(mlessvidid)==2):
 953   returnval = "sample-images";
 954  if(mlessvidid[1]=="galleries" and len(mlessvidid)==2):
 955   returnval = "sample-galleries";
 956  if(mlessvidid[1]=="" and len(mlessvidid)==2):
 957   returnval = "sample";
 958  if(mlessvidid[1]=="groups" and len(mlessvidid)==2):
 959   returnval = "group";
 960  if(mlessvidid[1]=="groups" and len(mlessvidid)==3 and mlessvidid[2]=="search"):
 961   returnval = "group";
 962  if(mlessvidid[1]=="groups" and len(mlessvidid)==4 and mlessvidid[2]=="category"):
 963   returnval = "group";
 964  if(mlessvidid[1]=="live" and len(mlessvidid)==3 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images")):
 965   returnval = "gallery";
 966  if(mlessvidid[1]=="u" and len(mlessvidid)==3):
 967   returnval = "gallery";
 968  if(mlessvidid[1]=="f" and len(mlessvidid)==4 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images" or mlessvidid[2]=="galleries")):
 969   returnval = "gallery";
 970  if(mlessvidid[1]=="galleries" and len(mlessvidid)==4 and mlessvidid[2]=="member"):
 971   returnval = "gallery";
 972  if(mlessvidid[1]=="galleries" and len(mlessvidid)==5 and mlessvidid[2]=="member" and (mlessvidid[4]=="created" or mlessvidid[4]=="viewed" or mlessvidid[4]=="favorited" or mlessvidid[4]=="commented")):
 973   returnval = "gallery";
 974  if(mlessvidid[1]=="gv" and len(mlessvidid)==3):
 975   returnval = "gallery";
 976  if(mlessvidid[1]=="gi" and len(mlessvidid)==3):
 977   returnval = "gallery";
 978  if(mlessvidid[1]=="gm" and len(mlessvidid)==3):
 979   returnval = "member";
 980  if(mlessvidid[1]=="term" and len(mlessvidid)==3 and (mlessvidid[2]=="videos" or mlessvidid[2]=="images" or mlessvidid[2]=="galleries")):
 981   returnval = "gallery";
 982  if(mlessvidid[1]=="g" and len(mlessvidid)==4 and re.match("^([0-9A-F]+)$", mlessvidid[3])):
 983   returnval = "link";
 984  if(mlessvidid[1]=="random" and len(mlessvidid)==3 and (mlessvidid[2]=="video" or mlessvidid[2]=="image")):
 985   returnval = "link";
 986  if(re.match("^V", mlessvidid[1]) and len(mlessvidid)==2):
 987   returnval = "board";
 988  if(re.match("^H", mlessvidid[1]) and len(mlessvidid)==2):
 989   returnval = "gallery";
 990  if(re.match("^G", mlessvidid[1]) and len(mlessvidid)==2):
 991   returnval = "gallery";
 992  if(re.match("^G", mlessvidid[1]) and len(mlessvidid)==3):
 993   returnval = "link";
 994  if(re.match("^g", mlessvidid[1]) and len(mlessvidid)==4 and re.match("^([0-9A-F]+)$", mlessvidid[3])):
 995   returnval = "link";
 996  if(mlessvidid[1]=="members" and len(mlessvidid)==2):
 997   returnval = "member";
 998  if(mlessvidid[1]=="members" and len(mlessvidid)==3 and mlessvidid[2]=="search"):
 999   returnval = "member";
1000  if(mlessvidid[1]=="members" and len(mlessvidid)==4 and (mlessvidid[2]=="uploader" or mlessvidid[2]=="viewed" or mlessvidid[2]=="social" or mlessvidid[2]=="favorited" or mlessvidid[2]=="commented" or mlessvidid[2]=="mentioned" or mlessvidid[2]=="verified")):
1001   returnval = "member";
1002  if(mlessvidid[1]=="girls" and len(mlessvidid)==2):
1003   returnval = "girls";
1004  if(mlessvidid[1]=="referers" and len(mlessvidid)==2):
1005   returnval = "referers";
1006  if(mlessvidid[1]=="about" and len(mlessvidid)==2):
1007   returnval = "team";
1008  if(mlessvidid_parts.netloc=="cdn.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn.thumbs.motherlessmedia.com"):
1009   returnval = "download";
1010  if(mlessvidid_parts.netloc=="cdn1.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn1.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn1.thumbs.motherlessmedia.com"):
1011   returnval = "download";
1012  if(mlessvidid_parts.netloc=="cdn2.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn2.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn2.thumbs.motherlessmedia.com"):
1013   returnval = "download";
1014  if(mlessvidid_parts.netloc=="cdn3.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn3.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn3.thumbs.motherlessmedia.com"):
1015   returnval = "download";
1016  if(mlessvidid_parts.netloc=="cdn4.images.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn4.videos.motherlessmedia.com" or mlessvidid_parts.netloc=="cdn4.thumbs.motherlessmedia.com"):
1017   returnval = "download";
1018  if(returnval==False and len(mlessvidid)==2 and re.match("^([0-9A-F]+)$", mlessvidid[1])):
1019   returnval = "link";
1020  return returnval;
1021
1022 def get_motherless_link_type_alt(httpurl):
1023  httpurl = fix_motherless_url(httpurl);
1024  returnval = {'urlinfo': check_motherless_url(httpurl), 'motherlessinfo': get_motherless_link_type(httpurl)};
1025  return returnval;
1026
1027 def get_motherless_number_of_pages(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
1028  httpurl = fix_motherless_url(httpurl);
1029  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1030  mrtext = premrtext['Content'];
1031  if(sys.version[0]>="3"):
1032   mrtext = mrtext.decode('ascii', 'replace');
1033  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1034  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1035  try:
1036   returnval = int(mlesspagenum[-1][0]);
1037  except:
1038   returnval = 1;
1039  return returnval;
1040
1041 def get_motherless_user_info(username, motherless_serv=None):
1042  motherless_subdomain = "cdn.";
1043  if(motherless_serv==None):
1044   motherless_serv = "";
1045  if(motherless_serv<5 or motherless_serv>0):
1046   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1047  returnval = {'username': username};
1048  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg").path));
1049  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1050  returnval.update({'orginurl': "http://motherless.com/m/"+username});
1051  returnval.update({'orginurltype': "profile"});
1052  returnval.update({'url': "http://motherless.com/m/"+username});
1053  returnval.update({'urltype': "profile"});
1054  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+username+"-avatar.jpg"});
1055  returnval.update({'avatarfullfilename': avatarfilenameext});
1056  returnval.update({'avatarfilename': avatarfilename});
1057  returnval.update({'avatarextension': avatarfileextension});
1058  return returnval;
1059
1060 def get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", motherless_serv=None):
1061  httpurl = fix_motherless_url(httpurl);
1062  if(get_motherless_link_type(httpurl)!="link"):
1063   returnval = False;
1064   return returnval;
1065  motherless_subdomain = "cdn.";
1066  if(motherless_serv==None):
1067   motherless_serv = "";
1068  if(motherless_serv<5 or motherless_serv>0):
1069   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1070  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1071  mrtext = premrtext['Content'];
1072  if(sys.version[0]>="3"):
1073   mrtext = mrtext.decode('ascii', 'replace');
1074  mregex_gettitle = re.escape("http://cdn")+"([1-4])"+re.escape(".");
1075  mrtext = re.sub(mregex_gettitle, "http://cdn.", mrtext);
1076  mregex_gettitle = re.escape("<title>")+"(.*)"+re.escape(" - MOTHERLESS.COM</title>");
1077  mlesstitle = re.findall(mregex_gettitle, mrtext);
1078  mregex_geturlone = re.escape("__fileurl = '")+'?\'?([^"\'>]*)'+re.escape("';");
1079  mlesslinkone = re.findall(mregex_geturlone, mrtext);
1080  mregex_geturlonetype = re.escape("http://cdn.")+"(images|videos)"+re.escape(".motherlessmedia.com/")+"(images|videos)"+re.escape("/")+"([\w\/\?\&\=\.\-]+)";
1081  mlesslinkonetype = re.findall(mregex_geturlonetype, mrtext);
1082  mregex_geturltwo = re.escape("<meta property=\"og:image\" content=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1083  mlesslinktwo = re.findall(mregex_geturltwo, mrtext);
1084  mregex_geturltwotype = re.escape("http://cdn.")+"(images|thumbs)"+re.escape(".motherlessmedia.com/")+"(images|thumbs)"+re.escape("/")+"([\w\/\?\&\=\.\-]+)";
1085  mlesslinktwotype = re.findall(mregex_geturltwotype, mrtext);
1086  filenameext = os.path.basename(urlparse.urljoin(mlesslinkone[0], urlparse.urlparse(mlesslinkone[0]).path));
1087  filename, fileextension = os.path.splitext(filenameext);
1088  thumbfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[0], urlparse.urlparse(mlesslinktwo[0]).path));
1089  thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1090  mregex_getuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1091  mlessuname = re.findall(mregex_getuname, mrtext);
1092  mlessuname = mlessuname[0];
1093  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">\n            <img");
1094  mlessurlname = re.findall(mregex_geturlname, mrtext);
1095  mlessurlname = mlessurlname[0].replace("/m/", "");
1096  mregex_getavatar = re.escape("<img\n    src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n    class=\"avatar avatar-small\"");
1097  mlessavatar = re.findall(mregex_getavatar, mrtext);
1098  mlessavatar = mlessavatar[0];
1099  avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar, urlparse.urlparse(mlessavatar).path));
1100  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1101  mregex_getviews = re.escape("<strong>Views</strong>\n")+"(\s+|\t+)([0-9\,]+)(\s+\t+)"+re.escape("</h2>");
1102  mlessviews = re.findall(mregex_getviews, mrtext);
1103  mregex_getfavorited = re.escape("<strong>Favorited</strong>\n")+"(\s+\t+)([0-9\,]+)(\s+\t+)"+re.escape("</h2>");
1104  mlessfavorited = re.findall(mregex_getfavorited, mrtext);
1105  if(mlesslinkonetype[0][1]=="images"):
1106   thumbnailaltpart = thumbfilename+"-zoom"+thumbfileextension;
1107   thumbnailalt = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+thumbnailaltpart;
1108   thumbnailaltfilenameext = os.path.basename(urlparse.urljoin(thumbnailalt, urlparse.urlparse(thumbnailalt).path));
1109   thumbnailaltfilename, thumbnailaltfileextension = os.path.splitext(thumbnailaltfilenameext);
1110  if(mlesslinkonetype[0][1]=="videos"):
1111   thumbnailaltpart = thumbfilename+"-small"+thumbfileextension;
1112   thumbnailalt = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+thumbnailaltpart;
1113   thumbnailaltfilenameext = os.path.basename(urlparse.urljoin(thumbnailalt, urlparse.urlparse(thumbnailalt).path));
1114   thumbnailaltfilename, thumbnailaltfileextension = os.path.splitext(thumbnailaltfilenameext);
1115   stripimgpart = thumbfilename+"-strip"+thumbfileextension;
1116   stripimg = "http://"+motherless_subdomain+"thumbs.motherlessmedia.com/thumbs/"+stripimgpart;
1117   stripimgfilenameext = os.path.basename(urlparse.urljoin(stripimg, urlparse.urlparse(stripimg).path));
1118   stripimgfilename, stripimgfileextension = os.path.splitext(stripimgfilenameext);
1119  returnval = False;
1120  mlessurltype = get_motherless_link_type(mlesslinkone[0]);
1121  if(mlesslinkonetype[0][1]=="images"):
1122   returnval = {'type': mlesslinkonetype[0][1], 'urltype': mlessurltype, 'url': mlesslinkone[0], 'orginurl': httpurl, 'orginurltype': get_motherless_link_type(httpurl), 'thumbnail': mlesslinktwo[0].replace("images", "thumbs"), 'thumbnailalt': thumbnailalt+"?from_helper", 'title': mlesstitle[0], 'fullfilename': filenameext, 'filename': filename, 'extension': fileextension, 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'thumbnailaltfullfilename': thumbnailaltpart, 'thumbnailaltfilename': thumbnailaltfilename, 'thumbnailaltextension': thumbnailaltfileextension, 'userinfo': get_motherless_user_info(mlessuname), 'username': mlessuname, 'avatarurl': mlessavatar, 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlessviews[0][1].replace(',', '')), 'numberoffavorites': int(mlessfavorited[0][1].replace(',', ''))};
1123  if(mlesslinkonetype[0][1]=="videos"):
1124   returnval = {'type': mlesslinkonetype[0][1], 'url': mlesslinkone[0], 'orginurl': httpurl, 'orginurltype': get_motherless_link_type(httpurl), 'thumbnail': thumbnailalt+"?from_helper", 'thumbnailalt': mlesslinktwo[0].replace("images", "thumbs"), 'strip': stripimg+"?from_helper", 'title': mlesstitle[0], 'fullfilename': filenameext, 'filename': filename, 'extension': fileextension, 'thumbfullfilename': thumbnailaltpart, 'thumbfilename': thumbnailaltfilename, 'thumbextension': thumbnailaltfileextension, 'thumbnailaltfullfilename': thumbfilenameext, 'thumbnailaltfilename': thumbfilename, 'thumbnailaltextension': thumbfileextension, 'stripfullfilename': stripimgfilenameext, 'stripfilename': stripimgfilename, 'stripextension': stripimgfileextension, 'userinfo': get_motherless_user_info(mlessuname), 'username': mlessuname, 'avatarurl': mlessavatar, 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlessviews[0][1].replace(',', '')), 'numberoffavorites': int(mlessfavorited[0][1].replace(',', ''))};
1125  return returnval;
1126
1127 def get_motherless_links_from_url(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
1128  httpurl = fix_motherless_url(httpurl);
1129  if(get_motherless_link_type(httpurl)!="download"):
1130   returnval = False;
1131   return returnval;
1132  returnval = False;
1133  if(get_motherless_link_type(httpurl)=="download"):
1134   urlparts = urlparse.urlparse(httpurl);
1135   filewithext = os.path.split(urlparts.path);
1136   if(filewithext[0]=="/images"):
1137    filewithoutext = os.path.splitext(filewithext[1])[0];
1138    returnval = get_motherless_links("http://motherless.com/"+filewithoutext, httpheaders, httpcookie, httplibuse="urllib");
1139   if(filewithext[0]=="/thumbs"):
1140    filewithoutext = os.path.splitext(filewithext[1])[0];
1141    filewithoutext = filewithoutext.replace("-zoom", "");
1142    filewithoutext = filewithoutext.replace("-small", "");
1143    filewithoutext = filewithoutext.replace("-strip", "");
1144    returnval = get_motherless_links("http://motherless.com/"+filewithoutext, httpheaders, httpcookie, httplibuse="urllib");
1145  return returnval;
1146
1147 def get_motherless_external_links(httpurl, httpheaders, httpcookie, httplibuse="urllib"):
1148  httpurl = fix_motherless_url(httpurl);
1149  if(get_motherless_link_type(httpurl)!="board" and get_motherless_link_type(httpurl)!="link"):
1150   returnval = False;
1151   return returnval;
1152  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1153  mrtext = premrtext['Content'];
1154  if(sys.version[0]>="3"):
1155   mrtext = mrtext.decode('ascii', 'replace');
1156  mregex_geturlinternal = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"outbound\" rel=\"nofollow\" class=\"pop\" target=\"_blank\" title=\"motherless link\">");
1157  mlesslinkinternal = re.findall(mregex_geturlinternal, mrtext);
1158  mregex_geturlexternal = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"outbound\" rel=\"nofollow\" class=\"pop\" target=\"_blank\" title=\"external link\">");
1159  mlesslinkexternal = re.findall(mregex_geturlexternal, mrtext);
1160  returnvalone = None;
1161  if(len(mlesslinkinternal)>1):
1162   mli = 0;
1163   mlil = len(mlesslinkinternal);
1164   returnvalone = {'numoflinks': mlil};
1165   returnvalone.update({'numofalllinks': len(mlesslinkinternal)});
1166   returnvalone.update({'orginurl': httpurl});
1167   returnvalone.update({'orginurltype': get_motherless_link_type(httpurl)});
1168   mlessrooturltype = get_motherless_link_type(httpurl);
1169   returnvalone.update({'urltype': mlessrooturltype});
1170   while(mli<mlil):
1171    mlessurltype = get_motherless_link_type(mlesslinkinternal[mli]);
1172    returnvalone.update({mli: {'urltype': mlessurltype, 'url': mlesslinkinternal[mli]} });
1173    mli = mli + 1;
1174  returnvaltwo = None;
1175  if(len(mlesslinkexternal)>1):
1176   mli = 0;
1177   mlil = len(mlesslinkexternal);
1178   returnvaltwo = {'numoflinks': mlil};
1179   returnvaltwo.update({'numofalllinks': len(mlesslinkexternal)});
1180   returnvaltwo.update({'orginurl': httpurl});
1181   returnvaltwo.update({'orginurltype': get_motherless_link_type(httpurl)});
1182   mlessrooturltype = get_motherless_link_type(httpurl);
1183   returnvaltwo.update({'urltype': mlessrooturltype});
1184   while(mli<mlil):
1185    returnvaltwo.update({mli: {'urltype': "external", 'url': mlesslinkexternal[mli]} });
1186    mli = mli + 1;
1187  if(returnvalone==None and returnvaltwo==None):
1188   returnval = {'internal': None, 'external': None};
1189  if(not returnvalone==None and not returnvaltwo==None):
1190   returnval = {'internal': returnvalone, 'external': returnvaltwo};
1191  if(not returnvalone==None and returnvaltwo==None):
1192   returnval = {'internal': returnvalone, 'external': None};
1193  if(returnvalone==None and not returnvaltwo==None):
1194   returnval = {'internal': None, 'external': returnvaltwo};
1195  return returnval;
1196
1197 def get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1], motherless_serv=None):
1198  httpurl = fix_motherless_url(httpurl);
1199  if(get_motherless_link_type(httpurl)!="gallery"):
1200   returnval = False;
1201   return returnval;
1202  motherless_subdomain = "cdn.";
1203  if(motherless_serv==None):
1204   motherless_serv = "";
1205  if(motherless_serv<5 or motherless_serv>0):
1206   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1207  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1208  mrtext = premrtext['Content'];
1209  if(sys.version[0]>="3"):
1210   mrtext = mrtext.decode('ascii', 'replace');
1211  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1212  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1213  try:
1214   lastpage = int(mlesspagenum[-1][0]);
1215  except:
1216   lastpage = 1;
1217  if(page>lastpage):
1218   page = lastpage;
1219  httpurl = add_url_param(httpurl, page=str(page));
1220  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1221  mrtext = premrtext['Content'];
1222  if(sys.version[0]>="3"):
1223   mrtext = mrtext.decode('ascii', 'replace');
1224  mregex_geturlone = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"img-container\" target=\"_self\">");
1225  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext);
1226  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1227  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext_tmp);
1228  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1229  mlesslinkone = re.findall(mregex_geturlone, mrtext_tmp);
1230  mregex_geturltwo = re.escape("<img class=\"static\" src=\"")+'?\'?([^"\'>]*)'+re.escape("\" data-strip-src=\"")+'?\'?([^"\'>]*)'+re.escape("\" alt=\"")+'?\'?([^">]*)'+re.escape("\" />");
1231  mlesslinktwo = re.findall(mregex_geturltwo, mrtext);
1232  mregex_getcaption = re.escape("<h2 class=\"caption title\">")+"(.*)"+re.escape("</h2>");
1233  mlesscaption = re.findall(mregex_getcaption, mrtext);
1234  mregex_gethits = re.escape("<div class=\"caption right\">")+"([0-9\,]+)"+re.escape(" Hits</div>");
1235  mlesshits = re.findall(mregex_gethits, mrtext);
1236  mregex_getuserinfo = re.escape("<a class=\"caption left\" href=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1237  mlessuname = re.findall(mregex_getuserinfo, mrtext);
1238  mregex_getmediainfo = re.escape("data-size=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-is-premium=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-codename=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-mediatype=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-gallery-codename=\"")+'?\'?([^"\'>]*)'+re.escape("\" ")+re.escape("data-frames=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1239  mlessmediainfo = re.findall(mregex_getmediainfo, mrtext);
1240  if(getlinks[1]>len(mlesslinkone) or getlinks[1]==-1):
1241   getlinks[1] = len(mlesslinkone);
1242  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1243   tmpgetlinks0 = getlinks[0];
1244   tmpgetlinks1 = getlinks[1];
1245   getlinks[0] = tmpgetlinks1;
1246   getlinks[1] = tmpgetlinks0;
1247  if(getlinks[0]<0):
1248   getlinks[0] = 0;
1249  mli = getlinks[0];
1250  mlil = getlinks[1];
1251  returnval = {'pages': lastpage};
1252  returnval.update({'curpage': page});
1253  returnval.update({'numoflinks': mlil});
1254  returnval.update({'numofalllinks': len(mlesslinkone)});
1255  returnval.update({'orginurl': httpurl});
1256  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1257  mlessrooturltype = get_motherless_link_type(httpurl);
1258  returnval.update({'urltype': mlessrooturltype});
1259  while(mli<mlil):
1260   mlessuname[mli] = mlessuname[mli].replace("/m/", "");
1261   stripfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][1], urlparse.urlparse(mlesslinktwo[mli][1]).path));
1262   stripfilename, stripfileextension = os.path.splitext(stripfilenameext);
1263   thumbfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][0], urlparse.urlparse(mlesslinktwo[mli][0]).path));
1264   thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1265   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlesslinkone[mli]);
1266   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg").path));
1267   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1268   if(mlessmediainfo[mli][3]=="image"):
1269    returnval.update({mli: {'type': "images", 'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli], 'thumbnail': mlesslinktwo[mli][0], 'thumbnailalt': mlesslinktwo[mli][1], 'title': mlesslinktwo[mli][2], 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'thumbnailaltfullfilename': stripfilenameext, 'thumbnailaltfilename': stripfilename, 'thumbnailaltextension': stripfileextension, 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlesshits[mli].replace(',', ''))} });
1270   if(mlessmediainfo[mli][3]=="video"):
1271    altthumbnail = mlesslinktwo[mli][1].replace('-strip', '');
1272    altfilenameext = os.path.basename(urlparse.urljoin(mlesslinktwo[mli][1], urlparse.urlparse(mlesslinktwo[mli][1]).path));
1273    altfilename, altfileextension = os.path.splitext(altfilenameext);
1274    returnval.update({mli: {'type': "videos", 'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli], 'thumbnail': mlesslinktwo[mli][0], 'strip': mlesslinktwo[mli][1], 'thumbnailalt': altthumbnail, 'title': mlesslinktwo[mli][2], 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension, 'stripfullfilename': stripfilenameext, 'stripfilename': stripfilename, 'stripextension': stripfileextension, 'thumbnailaltfullfilename': altfilenameext, 'thumbnailaltfilename': altfilename, 'thumbnailaltextension': altfileextension, 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension, 'numberofviews': int(mlesshits[mli].replace(',', ''))} });
1275   mli = mli + 1;
1276  return returnval;
1277
1278 def get_motherless_random_links(httpheaders, httpcookie, httplibuse="urllib", linktype="video", getlinks=[0, 80]):
1279  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1280   tmpgetlinks0 = getlinks[0];
1281   tmpgetlinks1 = getlinks[1];
1282   getlinks[0] = tmpgetlinks1;
1283   getlinks[1] = tmpgetlinks0;
1284  if(getlinks[0]<0):
1285   getlinks[0] = 0;
1286  mli = getlinks[0];
1287  mlil = getlinks[1];
1288  if(linktype=="image"):
1289   returnval = {'pages': 1};
1290   returnval.update({'curpage': 1});
1291   returnval.update({'numoflinks': 80});
1292   returnval.update({'numofalllinks': mlil});
1293   returnval.update({'orginurl': "http://motherless.com/random/image"});
1294   returnval.update({'orginurltype': "gallery"});
1295   returnval.update({'urltype': "gallery"});
1296   while(mli<mlil):
1297    get_links = get_motherless_links("http://motherless.com/random/image", httpheaders, httpcookie, httplibuse);
1298    returnval.update({mli: {'type': "images", 'urltype': get_motherless_link_type("http://motherless.com/"+get_links['filename']), 'url': "http://motherless.com/"+get_links['filename'], 'thumbnail': get_links['thumbnail'], 'strip': get_links['thumbnailalt'], 'title': get_links['title'], 'thumbfullfilename': get_links['thumbfullfilename'], 'thumbfilename': get_links['thumbfilename'], 'thumbextension': get_links['thumbextension'], 'stripfullfilename': get_links['thumbnailaltfullfilename'], 'stripfilename': get_links['thumbnailaltextension'], 'stripextension': get_links['thumbnailaltfilename'], 'userinfo': get_motherless_user_info(get_links['username']), 'username': get_links['username'], 'avatarurl': get_links['avatarurl'], 'avatarfullfilename': get_links['avatarfullfilename'], 'avatarfilename': get_links['avatarfilename'], 'avatarextension': get_links['avatarextension']} });
1299    mli = mli + 1;
1300  if(linktype=="video"):
1301   returnval = {'pages': 1};
1302   returnval.update({'curpage': 1});
1303   returnval.update({'numoflinks': 80});
1304   returnval.update({'numofalllinks': mlil});
1305   returnval.update({'orginurl': "http://motherless.com/random/video"});
1306   returnval.update({'orginurltype': "gallery"});
1307   returnval.update({'urltype': "gallery"});
1308   while(mli<mlil):
1309    get_links = get_motherless_links("http://motherless.com/random/video", httpheaders, httpcookie, httplibuse);
1310    returnval.update({mli: {'type': "videos", 'urltype': get_motherless_link_type("http://motherless.com/"+get_links['filename']), 'url': "http://motherless.com/"+get_links['filename'], 'thumbnail': get_links['thumbnail'], 'strip': get_links['thumbnailalt'], 'title': get_links['title'], 'thumbfullfilename': get_links['thumbfullfilename'], 'thumbfilename': get_links['thumbfilename'], 'thumbextension': get_links['thumbextension'], 'stripfullfilename': get_links['thumbnailaltfullfilename'], 'stripfilename': get_links['thumbnailaltextension'], 'stripextension': get_links['thumbnailaltfilename'], 'userinfo': get_motherless_user_info(get_links['username']), 'username': get_links['username'], 'avatarurl': get_links['avatarurl'], 'avatarfullfilename': get_links['avatarfullfilename'], 'avatarfilename': get_links['avatarfilename'], 'avatarextension': get_links['avatarextension']} });
1311    mli = mli + 1;
1312  return returnval;
1313
1314 def get_motherless_random_links_alt(httpheaders, httpcookie, httplibuse="urllib", linktype="video", getlinks=[0, 80]):
1315  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1316   tmpgetlinks0 = getlinks[0];
1317   tmpgetlinks1 = getlinks[1];
1318   getlinks[0] = tmpgetlinks1;
1319   getlinks[1] = tmpgetlinks0;
1320  if(getlinks[0]<0):
1321   getlinks[0] = 0;
1322  mli = getlinks[0];
1323  mlil = getlinks[1];
1324  if(linktype=="image"):
1325   returnval = {'pages': 1};
1326   returnval.update({'curpage': 1});
1327   returnval.update({'numoflinks': 80});
1328   returnval.update({'numofalllinks': mlil});
1329   returnval.update({'orginurl': "http://motherless.com/random/image"});
1330   returnval.update({'orginurltype': "gallery"});
1331   returnval.update({'urltype': "gallery"});
1332   while(mli<mlil):
1333    get_links = get_motherless_links("http://motherless.com/random/image", httpheaders, httpcookie, httplibuse);
1334    returnval.update({mli: get_links});
1335    mli = mli + 1;
1336  if(linktype=="video"):
1337   returnval = {'pages': 1};
1338   returnval.update({'curpage': 1});
1339   returnval.update({'numoflinks': 80});
1340   returnval.update({'numofalllinks': mlil});
1341   returnval.update({'orginurl': "http://motherless.com/random/video"});
1342   returnval.update({'orginurltype': "gallery"});
1343   returnval.update({'urltype': "gallery"});
1344   while(mli<mlil):
1345    get_links = get_motherless_links("http://motherless.com/random/video", httpheaders, httpcookie, httplibuse);
1346    returnval.update({mli: get_links});
1347    mli = mli + 1;
1348  return returnval;
1349
1350 def get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1], motherless_serv=None):
1351  httpurl = fix_motherless_url(httpurl);
1352  if(get_motherless_link_type(httpurl)!="board"):
1353   returnval = False;
1354   return returnval;
1355  motherless_subdomain = "cdn.";
1356  if(motherless_serv==None):
1357   motherless_serv = "";
1358  if(motherless_serv<5 or motherless_serv>0):
1359   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1360  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1361  mrtext = premrtext['Content'];
1362  if(sys.version[0]>="3"):
1363   mrtext = mrtext.decode('ascii', 'replace');
1364  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext);
1365  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1366  mrtext_tmp = re.sub(re.escape("http://motherless.com"), "", mrtext_tmp);
1367  mrtext_tmp = re.sub(re.escape("http://www.motherless.com"), "", mrtext_tmp);
1368  mregex_geturlone = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" title=\"motherless link\">");
1369  mlesslinkone = re.findall(mregex_geturlone, mrtext_tmp);
1370  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"post-username special-member op-member\" title=\"op\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1371  mlessurlname = re.findall(mregex_geturlname, mrtext);
1372  mlessavaturl = "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessurlname[0][1]+"-avatar.jpg";
1373  avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavaturl, urlparse.urlparse(mlessavaturl).path));
1374  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1375  if(getlinks[1]>len(mlesslinkone) or getlinks[1]==-1):
1376   getlinks[1] = len(mlesslinkone);
1377  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1378   tmpgetlinks0 = getlinks[0];
1379   tmpgetlinks1 = getlinks[1];
1380   getlinks[0] = tmpgetlinks1;
1381   getlinks[1] = tmpgetlinks0;
1382  if(getlinks[0]<0):
1383   getlinks[0] = 0;
1384  mli = getlinks[0];
1385  mlil = getlinks[1];
1386  returnval = {'numoflinks': mlil};
1387  returnval.update({'numofalllinks': len(mlesslinkone)});
1388  returnval.update({'orginurl': httpurl});
1389  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1390  mlessrooturltype = get_motherless_link_type(httpurl);
1391  returnval.update({'urltype': mlessrooturltype});
1392  returnval.update({'userinfo': get_motherless_user_info(mlessurlname[0][1])});
1393  returnval.update({'username': mlessurlname[0][1]});
1394  returnval.update({'avatarurl': mlessavaturl});
1395  returnval.update({'avatarfullfilename': avatarfilenameext});
1396  returnval.update({'avatarfilename': avatarfilename});
1397  returnval.update({'avatarextension': avatarfileextension});
1398  while(mli<mlil):
1399   mlessurltype = get_motherless_link_type("http://motherless.com"+mlesslinkone[mli]);
1400   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com"+mlesslinkone[mli]} });
1401   mli = mli + 1;
1402  return returnval;
1403
1404 def get_motherless_boards_posts(httpurl, httpheaders, httpcookie, httplibuse="urllib", getposts=[0, -1], motherless_serv=None):
1405  httpurl = fix_motherless_url(httpurl);
1406  if(get_motherless_link_type(httpurl)!="board"):
1407   returnval = False;
1408   return returnval;
1409  motherless_subdomain = "cdn.";
1410  if(motherless_serv==None):
1411   motherless_serv = "";
1412  if(motherless_serv<5 or motherless_serv>0):
1413   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1414  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1415  mrtext = premrtext['Content'];
1416  if(sys.version[0]>="3"):
1417   mrtext = mrtext.decode('ascii', 'replace');
1418  mregex_getposts = "(\t+)"+re.escape("<p>")+"(.+?)"+re.escape("</p>")+"(\t+)";
1419  mlessposts = re.findall(mregex_getposts, mrtext);
1420  mregex_getuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1421  mlessuname = re.findall(mregex_getuname, mrtext);
1422  mregex_getopuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"post-username special-member op-member\" title=\"op\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1423  mlessopuname = re.findall(mregex_getopuname, mrtext);
1424  if(getposts[1]>len(mlessposts) or getposts[1]==-1):
1425   getposts[1] = len(mlessposts);
1426  if(getposts[0]>getposts[1] and not getposts[1]==-1):
1427   tmpgetposts0 = getposts[0];
1428   tmpgetposts1 = getposts[1];
1429   getposts[0] = tmpgetposts1;
1430   getposts[1] = tmpgetposts0;
1431  if(getposts[0]<0):
1432   getposts[0] = 0;
1433  mli = getposts[0];
1434  mlil = getposts[1];
1435  returnval = {'numofposts': mlil};
1436  returnval.update({'numofallposts': len(mlessposts)});
1437  returnval.update({'orginurl': httpurl});
1438  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1439  mlessrooturltype = get_motherless_link_type(httpurl);
1440  returnval.update({'urltype': mlessrooturltype});
1441  returnval.update({'userinfo': get_motherless_user_info(mlessopuname[0][1])});
1442  returnval.update({'username': mlessopuname[0][1]});
1443  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg").path));
1444  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1445  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0][1]+"-avatar.jpg"});
1446  returnval.update({'avatarfullfilename': avatarfilenameext});
1447  returnval.update({'avatarfilename': avatarfilename});
1448  returnval.update({'avatarextension': avatarfileextension});
1449  while(mli<mlil):
1450   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg").path));
1451   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1452   returnval.update({mli: {'post': mlessposts[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli]), 'username': mlessuname[mli], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1453   mli = mli + 1;
1454  return returnval;
1455
1456 def get_motherless_links_comments(httpurl, httpheaders, httpcookie, httplibuse="urllib", getposts=[0, -1], motherless_serv=None):
1457  httpurl = fix_motherless_url(httpurl);
1458  if(get_motherless_link_type(httpurl)!="link"):
1459   returnval = False;
1460   return returnval;
1461  motherless_subdomain = "cdn.";
1462  if(motherless_serv==None):
1463   motherless_serv = "";
1464  if(motherless_serv<5 or motherless_serv>0):
1465   motherless_subdomain = "cdn"+str(motherless_serv)+".";
1466  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1467  mrtext = premrtext['Content'];
1468  if(sys.version[0]>="3"):
1469   mrtext = mrtext.decode('ascii', 'replace');
1470  mregex_getposts = re.escape("<div style=\"text-align: justify;\">\n")+"(\t+)(.+?)"+re.escape("</div>");
1471  mlessposts = re.findall(mregex_getposts, mrtext);
1472  mregex_getuname = re.escape("<div class=\"media-comment\" id=\"")+'?\'?([^"\'>]*)'+re.escape("\" rel=\"0\" rev=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1473  mlessuname = re.findall(mregex_getuname, mrtext);
1474  mregex_getopuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1475  mlessopuname = re.findall(mregex_getopuname, mrtext);
1476  if(getposts[1]>len(mlessposts) or getposts[1]==-1):
1477   getposts[1] = len(mlessposts);
1478  if(getposts[0]>getposts[1] and not getposts[1]==-1):
1479   tmpgetposts0 = getposts[0];
1480   tmpgetposts1 = getposts[1];
1481   getposts[0] = tmpgetposts1;
1482   getposts[1] = tmpgetposts0;
1483  if(getposts[0]<0):
1484   getposts[0] = 0;
1485  mli = getposts[0];
1486  mlil = getposts[1];
1487  returnval = {'numofposts': mlil};
1488  returnval.update({'numofallposts': len(mlessposts)});
1489  returnval.update({'orginurl': httpurl});
1490  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1491  mlessrooturltype = get_motherless_link_type(httpurl);
1492  returnval.update({'urltype': mlessrooturltype});
1493  returnval.update({'userinfo': get_motherless_user_info(mlessopuname[0])});
1494  returnval.update({'username': mlessopuname[0]});
1495  avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg").path));
1496  avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1497  returnval.update({'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessopuname[0]+"-avatar.jpg"});
1498  returnval.update({'avatarfullfilename': avatarfilenameext});
1499  returnval.update({'avatarfilename': avatarfilename});
1500  returnval.update({'avatarextension': avatarfileextension});
1501  while(mli<mlil):
1502   avatarfilenameext = os.path.basename(urlparse.urljoin("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg", urlparse.urlparse("http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg").path));
1503   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1504   returnval.update({mli: {'post': mlessposts[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli][1]), 'username': mlessuname[mli][1], 'avatarurl': "http://"+motherless_subdomain+"avatars.motherlessmedia.com/thumbs/"+mlessuname[mli][1]+"-avatar.jpg", 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1505   mli = mli + 1;
1506  return returnval;
1507
1508 def get_motherless_search_members(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1509  httpurl = fix_motherless_url(httpurl);
1510  if(get_motherless_link_type(httpurl)!="member"):
1511   returnval = False;
1512   return returnval;
1513  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1514  mrtext = premrtext['Content'];
1515  if(sys.version[0]>="3"):
1516   mrtext = mrtext.decode('ascii', 'replace');
1517  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1518  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1519  try:
1520   lastpage = int(mlesspagenum[-1][0]);
1521  except:
1522   lastpage = 1;
1523  if(page>lastpage):
1524   page = lastpage;
1525  httpurl = add_url_param(httpurl, page=str(page));
1526  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1527  mrtext = premrtext['Content'];
1528  if(sys.version[0]>="3"):
1529   mrtext = mrtext.decode('ascii', 'replace');
1530  mregex_getuname = re.escape("<tr rel=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1531  mlessuname = re.findall(mregex_getuname, mrtext);
1532  mregex_geturlname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">\n            <img");
1533  mlessurlname = re.findall(mregex_geturlname, mrtext);
1534  mregex_getavatar = re.escape("<img\n    src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n    class=\"avatar avatar-small\"");
1535  mlessavatar = re.findall(mregex_getavatar, mrtext);
1536  if(getlinks[1]>len(mlessuname) or getlinks[1]==-1):
1537   getlinks[1] = len(mlessuname);
1538  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1539   tmpgetlinks0 = getlinks[0];
1540   tmpgetlinks1 = getlinks[1];
1541   getlinks[0] = tmpgetlinks1;
1542   getlinks[1] = tmpgetlinks0;
1543  if(getlinks[0]<0):
1544   getlinks[0] = 0;
1545  mli = getlinks[0];
1546  mlil = getlinks[1];
1547  returnval = {'numoflinks': mlil};
1548  returnval.update({'numofalllinks': len(mlessuname)});
1549  returnval.update({'pages': lastpage});
1550  returnval.update({'curpage': page});
1551  returnval.update({'orginurl': httpurl});
1552  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1553  mlessrooturltype = get_motherless_link_type(httpurl);
1554  returnval.update({'urltype': mlessrooturltype});
1555  while(mli<mlil):
1556   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli], urlparse.urlparse(mlessavatar[mli]).path));
1557   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1558   mlessurlname[mli] = mlessurlname[mli].replace("/m/", "");
1559   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessurlname[mli]);
1560   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessurlname[mli], 'userinfo': get_motherless_user_info(mlessurlname[mli]), 'username': mlessuname[mli], 'avatarurl': mlessavatar[mli], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1561   mli = mli + 1;
1562  return returnval;
1563
1564 def get_motherless_girls(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1565  premrtext = download_from_url("http://motherless.com/girls", httpheaders, httpcookie, httplibuse);
1566  mrtext = premrtext['Content'];
1567  if(sys.version[0]>="3"):
1568   mrtext = mrtext.decode('ascii', 'replace');
1569  mregex_getuname = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" rev=\"")+'?\'?([^"\'>]*)'+re.escape("\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">");
1570  mlessuname = re.findall(mregex_getuname, mrtext);
1571  mregex_geturlname = re.escape("\n\t\t\t\t\t\t<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" target=\"_blank\">");
1572  mlessurlname = re.findall(mregex_geturlname, mrtext);
1573  if(getlinks[1]>len(mlessuname) or getlinks[1]==-1):
1574   getlinks[1] = len(mlessuname);
1575  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1576   tmpgetlinks0 = getlinks[0];
1577   tmpgetlinks1 = getlinks[1];
1578   getlinks[0] = tmpgetlinks1;
1579   getlinks[1] = tmpgetlinks0;
1580  if(getlinks[0]<0):
1581   getlinks[0] = 0;
1582  mli = getlinks[0];
1583  mlil = getlinks[1];
1584  returnval = {'numoflinks': mlil};
1585  returnval.update({'numofalllinks': len(mlessuname)});
1586  returnval.update({'orginurl': "http://motherless.com/girls"});
1587  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/girls")});
1588  mlessrooturltype = get_motherless_link_type("http://motherless.com/girls");
1589  returnval.update({'urltype': mlessrooturltype});
1590  while(mli<mlil):
1591   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessuname[mli][0], urlparse.urlparse(mlessuname[mli][0]).path));
1592   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1593   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessuname[mli][1]);
1594   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessuname[mli][1], 'userinfo': get_motherless_user_info(mlessuname[mli][1]), 'username': mlessuname[mli][1], 'avatarurl': mlessuname[mli][0], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1595   mli = mli + 1;
1596  return returnval;
1597
1598 def get_motherless_team(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1599  premrtext = download_from_url("http://motherless.com/about", httpheaders, httpcookie, httplibuse);
1600  mrtext = premrtext['Content'];
1601  if(sys.version[0]>="3"):
1602   mrtext = mrtext.decode('ascii', 'replace');
1603  mregex_getuname = re.escape("<div class=\"about-us-member\">\n                         <a href=\"")+'?\'?([^"\'>]*)'+re.escape("\"");
1604  mlessuname = re.findall(mregex_getuname, mrtext);
1605  mlessuname_odd = mlessuname[1::2];
1606  mlessuname_even = mlessuname[::2];
1607  mregex_getavatar = re.escape("<img\n        src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n        class=\"avatar avatar-")+"(full|medium)"+re.escape("\"");
1608  mlessavatar = re.findall(mregex_getavatar, mrtext);
1609  if(getlinks[1]>len(mlessuname_odd) or getlinks[1]==-1):
1610   getlinks[1] = len(mlessuname_odd);
1611  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1612   tmpgetlinks0 = getlinks[0];
1613   tmpgetlinks1 = getlinks[1];
1614   getlinks[0] = tmpgetlinks1;
1615   getlinks[1] = tmpgetlinks0;
1616  if(getlinks[0]<0):
1617   getlinks[0] = 0;
1618  mli = getlinks[0];
1619  mlil = getlinks[1];
1620  returnval = {'numoflinks': mlil};
1621  returnval.update({'numofalllinks': len(mlessuname)});
1622  returnval.update({'orginurl': "http://motherless.com/about"});
1623  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/about")});
1624  mlessrooturltype = get_motherless_link_type("http://motherless.com/about");
1625  returnval.update({'urltype': mlessrooturltype});
1626  while(mli<mlil):
1627   avatarfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli][0], urlparse.urlparse(mlessavatar[mli][0]).path));
1628   avatarfilename, avatarfileextension = os.path.splitext(avatarfilenameext);
1629   mlessuname_odd[mli] = mlessuname_odd[mli].replace("/m/", "");
1630   mlessuname_even[mli] = mlessuname_even[mli].replace("/m/", "");
1631   mlessurltype = get_motherless_link_type("http://motherless.com/"+mlessuname_odd[mli]);
1632   returnval.update({mli: {'urltype': mlessurltype, 'url': "http://motherless.com/"+mlessuname_odd[mli], 'userinfo': get_motherless_user_info(mlessuname_odd[mli]), 'username': mlessuname_odd[mli], 'avatarurl': mlessavatar[mli][0], 'avatarfullfilename': avatarfilenameext, 'avatarfilename': avatarfilename, 'avatarextension': avatarfileextension} });
1633   mli = mli + 1;
1634  return returnval;
1635
1636 def get_motherless_top_referrers(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1637  premrtext = download_from_url("http://motherless.com/referers", httpheaders, httpcookie, httplibuse);
1638  mrtext = premrtext['Content'];
1639  if(sys.version[0]>="3"):
1640   mrtext = mrtext.decode('ascii', 'replace');
1641  mregex_geturlname = "([0-9]+)"+re.escape(". <a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"pop\" target=\"_blank\" rel=\"nofollow\">\n                                            ")+"(.*)"+re.escape("                                   </a>");
1642  mlessurlname = re.findall(mregex_geturlname, mrtext);
1643  if(getlinks[1]>len(mlessurlname) or getlinks[1]==-1):
1644   getlinks[1] = len(mlessurlname);
1645  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1646   tmpgetlinks0 = getlinks[0];
1647   tmpgetlinks1 = getlinks[1];
1648   getlinks[0] = tmpgetlinks1;
1649   getlinks[1] = tmpgetlinks0;
1650  if(getlinks[0]<0):
1651   getlinks[0] = 0;
1652  mli = getlinks[0];
1653  mlil = getlinks[1];
1654  returnval = {'numoflinks': mlil};
1655  returnval.update({'numofalllinks': len(mlessurlname)});
1656  returnval.update({'orginurl': "http://motherless.com/referers"});
1657  returnval.update({'orginurltype': get_motherless_link_type("http://motherless.com/referers")});
1658  mlessrooturltype = get_motherless_link_type("http://motherless.com/referers");
1659  returnval.update({'urltype': mlessrooturltype});
1660  while(mli<mlil):
1661   returnval.update({mli: {'urltype': "referer-links", 'url': mlessurlname[mli][1], 'title': mlessurlname[mli][2]} });
1662   mli = mli + 1;
1663  return returnval;
1664
1665 def get_motherless_top_referers(httpheaders, httpcookie, httplibuse="urllib", getlinks=[0, -1]):
1666  return get_motherless_top_referrers(httpheaders, httpcookie, httplibuse, getlinks);
1667
1668 def get_motherless_groups(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1669  httpurl = fix_motherless_url(httpurl);
1670  if(get_motherless_link_type(httpurl)!="group"):
1671   returnval = False;
1672   return returnval;
1673  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1674  mrtext = premrtext['Content'];
1675  if(sys.version[0]>="3"):
1676   mrtext = mrtext.decode('ascii', 'replace');
1677  mregex_getpagenum = re.escape("\" class=\"pop\" rel=\"")+'?\'?([^"\'>]*)'+re.escape("\">")+"([0-9]+)"+re.escape("</a>");
1678  mlesspagenum = re.findall(mregex_getpagenum, mrtext);
1679  try:
1680   lastpage = int(mlesspagenum[-1][0]);
1681  except:
1682   lastpage = 1;
1683  if(page>lastpage):
1684   page = lastpage;
1685  httpurl = add_url_param(httpurl, page=str(page));
1686  premrtext = download_from_url(httpurl, httpheaders, httpcookie, httplibuse);
1687  mrtext = premrtext['Content'];
1688  if(sys.version[0]>="3"):
1689   mrtext = mrtext.decode('ascii', 'replace');
1690  mregex_getavatar = re.escape("<img\n        src=\"")+'?\'?([^"\'>]*)'+re.escape("\"\n        class=\"avatar avatar-")+"(full|medium)"+re.escape("\"");
1691  mlessavatar = re.findall(mregex_getavatar, mrtext);
1692  mregex_getgroups = re.escape("<a href=\"")+'?\'?([^"\'>]*)'+re.escape("\" class=\"grunge motherless-red\">\n")+"(.*)"+re.escape("</a>");
1693  mlessgroups = re.findall(mregex_getgroups, mrtext);
1694  if(getlinks[1]>len(mlessgroups) or getlinks[1]==-1):
1695   getlinks[1] = len(mlessgroups);
1696  if(getlinks[0]>getlinks[1] and not getlinks[1]==-1):
1697   tmpgetlinks0 = getlinks[0];
1698   tmpgetlinks1 = getlinks[1];
1699   getlinks[0] = tmpgetlinks1;
1700   getlinks[1] = tmpgetlinks0;
1701  if(getlinks[0]<0):
1702   getlinks[0] = 0;
1703  mli = getlinks[0];
1704  mlil = getlinks[1];
1705  returnval = {'pages': lastpage};
1706  returnval.update({'curpage': page});
1707  returnval.update({'numoflinks': mlil});
1708  returnval.update({'numofalllinks': len(mlessgroups)});
1709  returnval.update({'orginurl': httpurl});
1710  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1711  while(mli<mlil):
1712   thumbfilenameext = os.path.basename(urlparse.urljoin(mlessavatar[mli][0], urlparse.urlparse(mlessavatar[mli][0]).path));
1713   thumbfilename, thumbfileextension = os.path.splitext(thumbfilenameext);
1714   returnval.update({mli: {'urltype': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0]), 'url': "http://motherless.com"+mlessgroups[mli][0], 'urltype-image': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gi/")), 'url-image': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gi/"), 'urltype-video': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gv/")), 'url-video': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gv/"), 'urltype-member': get_motherless_link_type("http://motherless.com/"+mlessgroups[mli][0].replace("/g/", "/gm/")), 'url-member': "http://motherless.com"+mlessgroups[mli][0].replace("/g/", "/gm/"), 'thumbnail': mlessavatar[mli][0], 'title': mlessgroups[mli][1].strip(), 'thumbfullfilename': thumbfilenameext, 'thumbfilename': thumbfilename, 'thumbextension': thumbfileextension} });
1715   mli = mli + 1;
1716  return returnval;
1717
1718 def get_motherless_sample_links(httpheaders, httpcookie, httplibuse="urllib", numoflinks=10, urltype="video", enablerand=True):
1719  if(urltype=="video" and enablerand==True):
1720   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/videos", 'orginurltype': get_motherless_link_type("http://motherless.com/videos"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "video", [0, numoflinks]) } };
1721  if(urltype=="video" and enablerand==False):
1722   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/videos", 'orginurltype': get_motherless_link_type("http://motherless.com/videos"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1723  if(urltype=="image" and enablerand==True):
1724   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/images", 'orginurltype': get_motherless_link_type("http://motherless.com/images"), 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "image", [0, numoflinks]) } };
1725  if(urltype=="image" and enablerand==False):
1726   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/images", 'orginurltype': get_motherless_link_type("http://motherless.com/images"), 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1727  if(urltype=="gallery"):
1728   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/galleries", 'orginurltype': get_motherless_link_type("http://motherless.com/galleries"), 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1729  if(urltype=="all" and enablerand==True):
1730   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/", 'orginurltype': get_motherless_link_type("http://motherless.com/"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "video", [0, numoflinks]) }, 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'random': get_motherless_random_links(httpheaders, httpcookie, httplibuse, "image", [0, numoflinks]) }, 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1731  if(urltype=="all" and enablerand==False):
1732   returnval = {'numoflinks': numoflinks, 'orginurl': "http://motherless.com/", 'orginurltype': get_motherless_link_type("http://motherless.com/"), 'videos': {'recent': get_motherless_galleries_links("http://motherless.com/videos/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/videos/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/videos/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/videos/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/videos/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/videos", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) }, 'images': {'recent': get_motherless_galleries_links("http://motherless.com/images/recent", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/images/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/images/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/images/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'popular': get_motherless_galleries_links("http://motherless.com/images/popular", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'live': get_motherless_galleries_links("http://motherless.com/live/images", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) }, 'galleries': {'updated': get_motherless_galleries_links("http://motherless.com/galleries/updated", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'created': get_motherless_galleries_links("http://motherless.com/galleries/created", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'viewed': get_motherless_galleries_links("http://motherless.com/galleries/viewed", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'favorited': get_motherless_galleries_links("http://motherless.com/galleries/favorited", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]), 'commented': get_motherless_galleries_links("http://motherless.com/galleries/commented", httpheaders, httpcookie, httplibuse, 1, [0, numoflinks]) } };
1733  return returnval;
1734
1735 def get_motherless_link_by_type(httpurl, httpheaders, httpcookie, httplibuse="urllib", page=1, getlinks=[0, -1]):
1736  httpurl = fix_motherless_url(httpurl);
1737  returnval = False;
1738  if(get_motherless_link_type(httpurl)=="link"):
1739   returnval = get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse);
1740  if(get_motherless_link_type(httpurl)=="gallery"):
1741   returnval = get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse, page);
1742  if(get_motherless_link_type(httpurl)=="sample-videos"):
1743   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "video");
1744  if(get_motherless_link_type(httpurl)=="sample-images"):
1745   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "image");
1746  if(get_motherless_link_type(httpurl)=="sample-galleries"):
1747   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "gallery");
1748  if(get_motherless_link_type(httpurl)=="sample" or get_motherless_link_type(httpurl)=="sample-all"):
1749   returnval = get_motherless_sample_links(httpheaders, httpcookie, httplibuse, 10, "all");
1750  if(get_motherless_link_type(httpurl)=="board"):
1751   returnval = get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse);
1752  if(get_motherless_link_type(httpurl)=="member"):
1753   returnval = get_motherless_search_members(httpurl, httpheaders, httpcookie, httplibuse, page);
1754  if(get_motherless_link_type(httpurl)=="girls"):
1755   returnval = get_motherless_girls(httpheaders, httpcookie, httplibuse);
1756  if(get_motherless_link_type(httpurl)=="download"):
1757   returnval = httpurl;
1758  return returnval;
1759
1760 def view_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", viewerpro="mpv", prearg=[], proarg=[]):
1761  httpurl = fix_motherless_url(httpurl);
1762  commandlist = [viewerpro] + prearg;
1763  commandlist = commandlist + [get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse)['url']];
1764  commandlist = commandlist + proarg;
1765  mpvplaylistp = subprocess.Popen(commandlist, stdout=subprocess.PIPE, stderr=subprocess.PIPE);
1766  mpvplayout, mpvplayerr = mpvplaylistp.communicate();
1767  return True;
1768
1769 def download_motherless_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=defbuffersizedub, outfile="-", outpath=os.getcwd(), usetitlename=False):
1770  global geturls_download_sleep;
1771  httpurl = fix_motherless_url(httpurl);
1772  if(sleep<0):
1773   sleep = geturls_download_sleep;
1774  mlessurl = get_motherless_links(httpurl, httpheaders, httpcookie, httplibuse);
1775  outputname = mlessurl['fullfilename'];
1776  outpath = outpath.rstrip(os.path.sep);
1777  if(usetitlename==True):
1778   outputname = mlessurl['title'];
1779  if(usetitlename=="-" and outfile=="-"):
1780   outputname = "-";
1781  if(usetitlename=="-" and not outfile=="-"):
1782   outputname = outfile;
1783  returnval = download_from_url_to_file(mlessurl['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep);
1784  return returnval;
1785
1786 def download_motherless_links_by_type(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=defbuffersizedub, outfile="-", outpath=os.getcwd(), usetitlename=False, page=1, getlinks=[0, -1]):
1787  global geturls_download_sleep;
1788  httpurl = fix_motherless_url(httpurl);
1789  if(sleep<0):
1790   sleep = geturls_download_sleep;
1791  mlessurl = get_motherless_link_by_type(httpurl, httpheaders, httpcookie, httplibuse, page);
1792  if(mlessurl['urltype']=="download"):
1793   outputname = mlessurl['fullfilename'];
1794   outpathname = outpath.rstrip(os.path.sep);
1795   if(usetitlename==True):
1796    outputname = mlessurl['title'];
1797   if(usetitlename=="-" and outfile[mli]=="-"):
1798    outputname = "-";
1799   if(usetitlename=="-" and not outfile[mli]=="-"):
1800    outputname = outfile;
1801   returnval = download_from_url_to_file(mlessurl['url'], httpheaders, httpcookie, httplibuse, outputname, outpathname, buffersize, sleep);
1802  if(not mlessurl['urltype']=="download"):
1803   returnval = mlessurl;
1804  return returnval;
1805
1806 def download_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=defbuffersizedub, outfile="-", outpath=os.getcwd(), usetitlename=False, page=1, getlinks=[0, -1]):
1807  global geturls_download_sleep;
1808  httpurl = fix_motherless_url(httpurl);
1809  if(sleep<0):
1810   sleep = geturls_download_sleep;
1811  mlessgalleries = get_motherless_galleries_links(httpurl, httpheaders, httpcookie, httplibuse, page, getlinks);
1812  mli = 0;
1813  mlil = mlessgalleries['numoflinks'];
1814  returnval = {'pages': mlessgalleries['pages']};
1815  returnval.update({'numoflists': mlessgalleries['numoflinks']});
1816  returnval.update({'curpage': mlessgalleries['curpage']});
1817  returnval.update({'numoflinks': mlessgalleries['numoflinks']});
1818  returnval.update({'numofalllinks': mlessgalleries['numofalllinks']});
1819  returnval.update({'orginurl': httpurl});
1820  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1821  while(mli<mlil):
1822   mlesslink = get_motherless_links(mlessgalleries[mli]['url'], httpheaders, httpcookie, httplibuse);
1823   outputname = mlesslink['fullfilename'];
1824   outpath = outpath.rstrip(os.path.sep);
1825   if(usetitlename==True):
1826    outputname = mlesslink['title'];
1827   if(usetitlename=="-" and outfile=="-"):
1828    outputname = "-";
1829   if(usetitlename=="-" and not outfile=="-"):
1830    outputname = outfile;
1831   returnval.update({mli: {'download': download_from_url_to_file(mlesslink['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep), 'linkinfo': mlesslink, 'outputfile': outputname} });
1832   mli = mli + 1;
1833  return returnval;
1834
1835 def download_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse="urllib", sleep=-1, buffersize=defbuffersizedub, outfile="-", outpath=os.getcwd(), usetitlename=False, getlinks=[0, -1]):
1836  global geturls_download_sleep;
1837  httpurl = fix_motherless_url(httpurl);
1838  if(sleep<0):
1839   sleep = geturls_download_sleep;
1840  mlessgalleries = get_motherless_boards_links(httpurl, httpheaders, httpcookie, httplibuse, getlinks);
1841  mli = 0;
1842  mlil = mlessgalleries['numoflinks'];
1843  returnval = {'numoflists': mlessgalleries['numoflinks']};
1844  returnval.update({'numofalllinks': mlessgalleries['numofalllinks']});
1845  returnval.update({'orginurl': httpurl});
1846  returnval.update({'orginurltype': get_motherless_link_type(httpurl)});
1847  while(mli<mlil):
1848   mlesslink = get_motherless_links(mlessgalleries[mli]['url'], httpheaders, httpcookie, httplibuse);
1849   outputname = mlesslink['fullfilename'];
1850   outpath = outpath.rstrip(os.path.sep);
1851   if(usetitlename==True):
1852    outputname = mlesslink['title'];
1853   if(usetitlename=="-" and outfile=="-"):
1854    outputname = "-";
1855   if(usetitlename=="-" and not outfile=="-"):
1856    outputname = outfile;
1857   returnval.update({mli: {'download': download_from_url_to_file(mlesslink['url'], httpheaders, httpcookie, httplibuse, outputname, outpath, buffersize, sleep), 'linkinfo': mlesslink, 'outputfile': outputname} });
1858   mli = mli + 1;
1859  return returnval;
1860