lib/urlwatch/html2txt.py

   1 #!/usr/bin/python
   2 # Convert HTML data to plaintext using Lynx, html2text or a regex
   3 # Requirements: Either lynx (default) or html2text or simply Python (for regex)
   4 # This file is part of urlwatch
   5 #
   6 # Copyright (c) 2009 Thomas Perl <thp@thpinfo.com>
   7 # All rights reserved.
   8 #
   9 # Redistribution and use in source and binary forms, with or without
  10 # modification, are permitted provided that the following conditions
  11 # are met:
  12 # 1. Redistributions of source code must retain the above copyright
  13 #    notice, this list of conditions and the following disclaimer.
  14 # 2. Redistributions in binary form must reproduce the above copyright
  15 #    notice, this list of conditions and the following disclaimer in the
  16 #    documentation and/or other materials provided with the distribution.
  17 # 3. The name of the author may not be used to endorse or promote products
  18 #    derived from this software without specific prior written permission.
  19 #
  20 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  21 # IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  22 # OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  23 # IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  24 # INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  25 # NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  29 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31
  32 def html2text(data, method='lynx'):
  33     """
  34     Convert a string consisting of HTML to plain text
  35     for easy difference checking.
  36
  37     Method may be one of:
  38      'lynx' (default) - Use "lynx -dump" for conversion
  39      'html2text'      - Use "html2text -nobs" for conversion
  40      're'             - A simple regex-based HTML tag stripper
  41
  42     Dependencies: apt-get install lynx html2text
  43     """
  44     if method == 're':
  45         import re
  46         stripped_tags = re.sub(r'<[^>]*>', '', data)
  47         d = '\n'.join((l.rstrip() for l in stripped_tags.splitlines() if l.strip() != ''))
  48         return d
  49
  50     if method == 'lynx':
  51         cmd = ['lynx', '-dump', '-stdin']
  52     elif method == 'html2text':
  53         cmd = ['html2text', '-nobs']
  54     else:
  55         return data
  56
  57     import subprocess
  58     html2text = subprocess.Popen(cmd, stdin=subprocess.PIPE, \
  59             stdout=subprocess.PIPE)
  60     (stdout, stderr) = html2text.communicate(data)
  61     return stdout
  62
  63
  64 if __name__ == '__main__':
  65     import sys
  66
  67     if len(sys.argv) == 2:
  68         print html2text(open(sys.argv[1]).read())
  69     else:
  70         print 'Usage: %s document.html' % (sys.argv[0])
  71         sys.exit(1)
  72