lib.py

   1 # -*- coding: utf-8 -*-
   2
   3 from ConfigParser import SafeConfigParser
   4 import codecs
   5
   6 from twython import Twython
   7 import twitter_text
   8 import MySQLdb
   9 from jinja2 import Environment, FileSystemLoader
  10
  11 class ReadNewTweets:
  12
  13     """We use the Twython module to issue a GET statuses/user_timeline
  14     request. The tweets of the response are returned by the method
  15     read_user_timeline as list of Unicode strings.
  16
  17     These are some API 1.1 docs about the GET statuses/user_timeline requests:
  18     https://dev.twitter.com/docs/api/1.1/get/statuses/user_timeline
  19
  20     In our tests we found that OAUTH authentication is a prerequisite for GET
  21     statuses/timeline requests. The Twython docs are vague about this, cf the
  22     comments of the get_user_timeline.py example: "We won't authenticate for
  23     this, but sometimes it's necessary"
  24     https://raw.github.com/ryanmcgrath/twython/master/core_examples/get_user_timeline.py
  25
  26     This class expects three arguments:
  27     1. The location of a configuration file with the necessary data for
  28     authentication. We use ConfigParser to read the configuration file. See
  29     config.sample.ini file.
  30     2. A twitter screen_name: "The screen name of the user for whom to return
  31     results for"
  32     3. A GET statuses/user_timeline count parameter: "Specifies the number of
  33     tweets to try and retrieve [...]".
  34
  35     The read_user_timeline method will return the tweets of the request
  36     response formatted as a list of Unicode type instances, Unicode strings,
  37     u'☺☻'.  We will often use this type of tweet list...
  38
  39     """
  40
  41     def __init__(self, config_file, screen_name, count):
  42         """config_file is a string and should point to a configuration file
  43         with the necessary credentials for OAUTH. We use ConfigParser to read
  44         the configuration file.
  45
  46         screen_name is a string and corresponds to the user whose timeline we
  47         will read.
  48
  49         count is an integer and corresponds to the number of tweets we will
  50         read.
  51
  52         At the moment, the HTML jinja2 template expects 70 tweets. The count
  53         argument may be overkill...
  54
  55         """
  56         self.config_file = config_file
  57         self.screen_name = screen_name
  58         self.count = count
  59
  60     def read_config_file(self):
  61         """We use ConfigParser to read the configuration file with the
  62         necessary credentials for OAUTH.
  63
  64         """
  65         parser = SafeConfigParser()
  66         parser.read('config.ini')
  67
  68         self.app_key = parser.get('Twitter_OAUTH', 'app_key')
  69         self.app_secret = parser.get('Twitter_OAUTH', 'app_secret')
  70         self.oauth_token = parser.get('Twitter_OAUTH', 'oauth_token')
  71         self.oauth_token_secret = parser.get('Twitter_OAUTH', 'oauth_token_secret')
  72
  73     def read_user_timeline(self):
  74         """We connect to twitter, and issue a GET statuses/user_timeline
  75         request with self.screen_name and self.count parameters.
  76
  77         A response's u'text' field corresponds to a "tweet".  See the API 1.1
  78         docs for a map of a typical response:
  79         https://dev.twitter.com/docs/api/1.1/get/statuses/user_timeline
  80
  81         This method's return value is some tweets packaged as list of Unicode
  82         strings.
  83
  84         """
  85         twitter = Twython(app_key=self.app_key, app_secret=self.app_secret,
  86                 oauth_token=self.oauth_token,
  87                 oauth_token_secret=self.oauth_token_secret)
  88
  89         tweets = twitter.getUserTimeline(screen_name=self.screen_name, count=self.count)
  90
  91         list_of_tweets = []
  92         for tweet in tweets:
  93             list_of_tweets.append(tweet[u'text'])
  94
  95         return list_of_tweets
  96
  97 class WriteNewTweetsToDB:
  98
  99     """We write some tweets to a database.
 100
 101     This class expects two arguments:
 102     1. The location of a configuration file with the usual MySQL connection
 103     stuff (host, user, pwd, db name). We use ConfigParser to read the
 104     configuration file. See config.sample.ini file.
 105     2. A list of some tweets as say returned by the read_user_timeline method
 106     of ReadNewTweets.
 107
 108     The list_of_tweets strings will be written to the database of the
 109     configuration file.
 110
 111     This class is not really Twitter-dependent and the strings of Argument #2
 112     could be anything...
 113
 114     """
 115
 116     def __init__(self, config_file, list_of_tweets):
 117         """config_file is a string and should point to a configuration file
 118         with the usual MySQL connection stuff. See config.sample.ini file...
 119
 120         We use ConfigParser to read the configuration file.
 121
 122         list_of_tweets is a list as say generated by the read_user_timeline
 123         method of the ReadNewTweets class.
 124
 125         """
 126         self.config_file = config_file
 127         self.list_of_tweets = list_of_tweets
 128
 129     def read_config_file(self):
 130         """We use ConfigParser to fetch the usual MySQL connection stuff"""
 131         parser = SafeConfigParser()
 132         parser.read(self.config_file)
 133
 134         self.db_host = parser.get('MySQL', 'host')
 135         self.db_user = parser.get('MySQL', 'user')
 136         self.db_password = parser.get('MySQL', 'password')
 137         self.db_database = parser.get('MySQL', 'database')
 138
 139     def write_tweets_to_db(self):
 140         """We write some tweets to the database.
 141
 142         We loop over the self.list_of_tweets list, and write each tweet to the
 143         database.
 144
 145         For the Unicode tweaks, we read the following:
 146         http://dev.mysql.com/doc/refman/5.0/en/charset-applications.html
 147         http://stackoverflow.com/questions/8365660/python-mysql-unicode-and-encoding
 148
 149         """
 150         connection = MySQLdb.connect(host = self.db_host,
 151                 user = self.db_user,
 152                 passwd = self.db_password,
 153                 db = self.db_database,
 154                 charset = 'utf8',
 155                 use_unicode = True)
 156
 157         cursor = connection.cursor()
 158         sql_statement = "CREATE TABLE IF NOT EXISTS tweets (Id INT PRIMARY KEY AUTO_INCREMENT, tweet VARCHAR(140))"
 159         cursor.execute(sql_statement)
 160
 161         for tweet in self.list_of_tweets:
 162             cursor.execute('INSERT INTO tweets(tweet) VALUES(%s)', (tweet,))
 163
 164 class ReadTweetsFromDB:
 165
 166     """We read some tweets from a database and return a list of tweets.
 167
 168     This class expects one argument:
 169     1. The location of a configuration file with the usual MySQL connection
 170     stuff (host, user, pwd, db name). We use ConfigParser to read the
 171     configuration file. See config.sample.ini file.
 172
 173     The database has been created by say the write_tweets_to_db method of the
 174     WriteTweetsToDB class.
 175
 176     The return value of read_tweets_from_db is a list of tweets as in the
 177     read_user_timeline method of the ReadNewTweets class.
 178
 179     """
 180
 181     def __init__(self, config_file):
 182         """config_file is a string and should point to a configuration file
 183         with the usual MySQL connection stuff. See config.sample.ini file...
 184
 185         We use ConfigParser to read the configuration file.
 186
 187         """
 188         self.config_file = config_file
 189
 190     def read_config_file(self):
 191         """We use ConfigParser to fetch the usual MySQL connection stuff"""
 192         parser = SafeConfigParser()
 193         parser.read(self.config_file)
 194
 195         self.db_host = parser.get('MySQL', 'host')
 196         self.db_user = parser.get('MySQL', 'user')
 197         self.db_password = parser.get('MySQL', 'password')
 198         self.db_database = parser.get('MySQL', 'database')
 199
 200     def read_tweets_from_db(self):
 201         """We connect to the database and retrieve some tweets.
 202
 203         This method's return value is a list of tweets as in the
 204         read_user_timeline method of the ReadNewTweets class.
 205
 206         """
 207         connection = MySQLdb.connect(host = self.db_host,
 208                 user = self.db_user,
 209                 passwd = self.db_password,
 210                 db = self.db_database,
 211                 charset = 'utf8',
 212                 use_unicode = True)
 213
 214         cursor = connection.cursor()
 215         cursor.execute("SELECT * FROM tweets")
 216
 217         tweets = cursor.fetchall()
 218         list_of_tweets = []
 219         for row in tweets:
 220             list_of_tweets.append(row[1])
 221
 222         return list_of_tweets
 223
 224 class WriteTweetsToHTMLFile:
 225
 226     """We write some tweets to a HTML file via Jinja templating.
 227
 228     This class expects four arguments:
 229     1. A string that points to a location in the file system where Jinja
 230     templates are stored. We use the "FileSystemLoader" Jinja loader. The
 231     Jinja documentation says: "The loader takes the path to the templates as
 232     string" http://jinja.pocoo.org/docs/api/#jinja2.FileSystemLoader
 233     2. A string that points to a Jinja template file in the location specified
 234     by Argument #1.
 235     3. A list of some tweets as say returned by the read_tweets_from_db method
 236     of ReadTweetsFromDB.  FWIW read_user_timeline of ReadNewTweets returns the
 237     same type of tweet lists...
 238     4. A string that points to a location in the file system where we can
 239     store our HTML.
 240
 241     If it doesn't exist already, the file html_file will be written anew.
 242
 243     """
 244
 245     def __init__(self, template_search_path, template_name, list_of_tweets,
 246             html_file):
 247         """template_search_path is a Jinja templates folder; and template_name
 248         is a template therein.
 249
 250         list_of_tweets is our list of tweets
 251
 252         If it doesn't exist already, the file html_file will be created and
 253         our Jinja-generated HTML will be written to it.
 254         """
 255
 256         self.template_search_path = template_search_path
 257         self.template_name = template_name
 258         self.list_of_tweets = list_of_tweets
 259         self.html_file = html_file
 260
 261     def autolink_list_of_tweets(self):
 262         """We auto link the plain text aspects of tweets. See "Twitter text
 263         conformance":
 264         https://github.com/twitter/twitter-text-conformance
 265
 266         We use the twitter-text-py module:
 267         pip install twitter-text-py
 268
 269         twitter-text-py documentation says the following about autolinking:
 270         "Add @<a></a>@ tags around the usernames, lists, hashtags and URLs in
 271         the provided text [...]"
 272         https://raw.github.com/dryan/twitter-text-py/master/README.textile
 273
 274         We need to do this in order to render the tweets on the browser...
 275         """
 276
 277         self.list_of_tweets_autolinked = []
 278         for tweet in self.list_of_tweets:
 279             my_autolink = twitter_text.Autolink(tweet)
 280             self.list_of_tweets_autolinked.append(my_autolink.auto_link())
 281
 282     def write_html_file(self):
 283         """We write the autolinked tweets to a HTML file via Jinja templating.
 284
 285         If it doesn't exist already a new file will be created and our HTML
 286         will be written to it. The new file's name is self.html_file
 287
 288         We pass the list_of_tweets_autolinked list to Jinja's render() method
 289         for templates.
 290
 291         See also our Jinja template: template.html
 292         """
 293
 294         env = Environment(loader=FileSystemLoader(self.template_search_path))
 295         template = env.get_template(self.template_name)
 296         f = codecs.open(self.html_file, mode="w", encoding='utf-8')
 297         f.write(template.render(list_of_tweets=self.list_of_tweets_autolinked))