From 7541f04110b536c8fafb15daaf0ee58c12c31727 Mon Sep 17 00:00:00 2001 From: Kagamin Date: Mon, 9 Feb 2009 17:37:44 +0300 Subject: [PATCH] =?utf8?q?basichttp=20server=20(=D0=B7=D0=B0=D0=B1=D1=8B?= =?utf8?q?=D0=BB=20git=20add=20.)?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit --- daemon.shit/board.py | 30 ++ daemon.shit/chan.py | 21 + daemon.shit/chanpool.py | 19 + daemon.shit/chanspyd | 136 +++++++ daemon.shit/config.py | 15 + daemon.shit/configs/config.yaml | 20 + daemon.shit/my_thread.py | 20 + daemon.shit/plugins/__init__.py | 0 daemon.shit/plugins/plugin_wakaba.py | 30 ++ daemon.shit/plugins/wakaba_thread.py | 49 +++ daemon.shit/post.py | 14 + daemon.shit/services.py | 74 ++++ gui.shit/chans.py | 33 ++ gui.shit/client.py | 39 ++ gui.shit/glades/main.glade | 108 +++++ gui.shit/gui.py | 92 +++++ other/updateArchive.py | 658 +++++++++++++++++++++++++++++++ prepare_for_commit.sh | 7 + server/chanspyd | 127 ++++++ server/config.py | 19 + server/configs/config.yaml | 28 ++ server/htdocs/blank.html | 10 + server/htdocs/chans/dobrochan/index.html | 20 + server/htdocs/chans/iichan/index.html | 11 + server/htdocs/chans/index.html | 11 + server/htdocs/index.html | 17 + server/htdocs/list_template.html | 9 + server/htdocs/settings.html | 13 + server/htdocs/styles/style.css | 19 + server/server.py | 42 ++ server/static_generation.py | 76 ++++ 31 files changed, 1767 insertions(+) create mode 100644 daemon.shit/board.py create mode 100644 daemon.shit/chan.py create mode 100644 daemon.shit/chanpool.py create mode 100755 daemon.shit/chanspyd create mode 100644 daemon.shit/config.py create mode 100644 daemon.shit/configs/config.yaml create mode 100644 daemon.shit/my_thread.py create mode 100644 daemon.shit/plugins/__init__.py create mode 100644 daemon.shit/plugins/plugin_wakaba.py create mode 100644 daemon.shit/plugins/wakaba_thread.py create mode 100644 daemon.shit/post.py create mode 100644 daemon.shit/services.py create mode 100755 gui.shit/chans.py create mode 100644 gui.shit/client.py create mode 100644 gui.shit/glades/main.glade create mode 100644 gui.shit/gui.py create mode 100644 other/updateArchive.py create mode 100755 prepare_for_commit.sh create mode 100755 server/chanspyd create mode 100644 server/config.py create mode 100644 server/configs/config.yaml create mode 100644 server/htdocs/blank.html create mode 100644 server/htdocs/chans/dobrochan/index.html create mode 100644 server/htdocs/chans/iichan/index.html create mode 100644 server/htdocs/chans/index.html create mode 100644 server/htdocs/index.html create mode 100644 server/htdocs/list_template.html create mode 100644 server/htdocs/settings.html create mode 100644 server/htdocs/styles/style.css create mode 100644 server/server.py create mode 100644 server/static_generation.py diff --git a/daemon.shit/board.py b/daemon.shit/board.py new file mode 100644 index 0000000..f5b5a51 --- /dev/null +++ b/daemon.shit/board.py @@ -0,0 +1,30 @@ +import dbus +import dbus.service + +INTERFACE = 'py.chans.dbus' + +class Board(dbus.service.Object): + """Board class. Replacing by board plugins.""" + + def __init__(self, bus_name, obj_path, name, base_uri): + dbus.service.Object.__init__(self, bus_name, obj_path) + self.bus_name = bus_name + self.obj_path = obj_path + self.name = name + self.threads = [] + self.board_uri = '%s/%s/' %(base_uri, self.name) + + @dbus.service.method(dbus_interface=INTERFACE, in_signature='', out_signature='') + def refresh(self): + """Parsing board threads.""" + + pass + + @dbus.service.method(dbus_interface=INTERFACE, in_signature='', out_signature='as') + def get_threads(self): + """Get threads: ['43244', '43245', '43246'].""" + + threads = [] + for thread in self.threads: + threads.append(thread.id) + return threads diff --git a/daemon.shit/chan.py b/daemon.shit/chan.py new file mode 100644 index 0000000..7a71200 --- /dev/null +++ b/daemon.shit/chan.py @@ -0,0 +1,21 @@ +import dbus.service + +INTERFACE = 'py.chans.dbus' + +class Chan(dbus.service.Object): + """Chan class. Containing boards.""" + + def __init__(self, bus_name, obj_path, name, base_uri, boards): + dbus.service.Object.__init__(self, bus_name, obj_path) + self.name = name + self.base_uri = base_uri + self.boards = boards + + @dbus.service.method(dbus_interface=INTERFACE, in_signature='', out_signature='as') + def get_boards(self): + """Get chan's boards names: ['b', 'i', 's']. Can be used by /pool/chan/board.""" + + boards = [] + for board in self.boards: + boards.append(board.name) + return boards diff --git a/daemon.shit/chanpool.py b/daemon.shit/chanpool.py new file mode 100644 index 0000000..f88fb1d --- /dev/null +++ b/daemon.shit/chanpool.py @@ -0,0 +1,19 @@ +import dbus.service + +INTERFACE = 'py.chans.dbus' + +class ChanPool(dbus.service.Object): + """ChanPool -- class that containing all chans.""" + + def __init__(self, bus_name, obj_path, chans): + dbus.service.Object.__init__(self, bus_name, obj_path) + self.chans = chans + + @dbus.service.method(dbus_interface=INTERFACE, in_signature='', out_signature='as') + def get_chans(self): + """Returning array of chan names. Then it can be used by /pool/chan_name.""" + + chans = [] + for chan in self.chans: + chans.append(chan.name) + return chans diff --git a/daemon.shit/chanspyd b/daemon.shit/chanspyd new file mode 100755 index 0000000..dfe3f8b --- /dev/null +++ b/daemon.shit/chanspyd @@ -0,0 +1,136 @@ +#!/usr/bin/env python + +############################################### +# chanspyd - chanspy daemon +# Copyright (C) 2008 anonymous +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +############################################### + +import sys +import os +import signal + +import config +import services + +#---------------------------------------------- +def main(): + """Main function. Write pid file and run service.""" + + if os.path.exists(PID): + sys.stderr.write('chanspyd\'s already executed. If not then remove %s.\n' %PID) + sys.exit(2) + + if write_pid: + try: + f = open(PID, 'w') + f.write(str(os.getpid())) + f.close() + except: + sys.stderr.write('Can\'t write pid to file.\n') + sys.exit(2) + signal.signal(signal.SIGTERM, sigTermCB) + + services.start(logfile, chans) + +#---------------------------------------------- +def help(): + """Print help message.""" + + help = ('Usage: %s [OPTIONS]' %sys.argv[0]) +\ + '\n\nOptions:\n' +\ + '-h --help help\n' +\ + '-d --daemon run in background\n' +\ + '-k --kill kill daemon' + print(help) + +#---------------------------------------------- +def error(): + """Invalid option.""" + + error = ('%s: invalid option.\n' %sys.argv[0]) +\ + 'Try `%s --help\' for more information.\n' %sys.argv[0] + sys.stderr.write(error) + sys.exit(1) + +#---------------------------------------------- +def kill(): + """Kill process by pid.""" + + try: + f = open(PID) + pid = f.read() + f.close() + except: + sys.stderr.write('chanspyd is not running\n') + sys.exit(2) + else: + try: + os.kill(int(pid), signal.SIGTERM) + except: + sys.stderr.write('Cat\'t kill %s.\n' %pid) + sys.exit(2) + +#---------------------------------------------- +def sigTermCB(signum, frame): + """Removing pid and exiting on SIGTERM.""" + + try: + os.remove(PID) + finally: + services.stop() + +############################################### +# default settings +config_path = 'configs/config.yaml' +write_pid = False +PID = 'chanspyd.pid' +chans = [] + +# reading options +options = config.read_config(config_path) +try: + write_pid = options['daemon']['write_pid'] + PID = options['daemon']['PID'] + chans = options['chans'] +except: + pass + +logfile = '/dev/stdout' + +if len(sys.argv) == 2: + if sys.argv[1] == '-h' or sys.argv[1] == '--help': + help() + + elif sys.argv[1] == '-d' or sys.argv[1] == '--daemon' : + try: logfile = options['daemon']['logfile'] + except: logfile = 'chanspyd.log' + + pid = os.fork() + if pid == 0: + os.setsid() + pid = os.fork() + if pid == 0: + main() + + elif sys.argv[1] == '-k' or sys.argv[1] == '--kill': + kill() + + else: + error() +elif len(sys.argv) == 1: + main() +else: + error() diff --git a/daemon.shit/config.py b/daemon.shit/config.py new file mode 100644 index 0000000..a938756 --- /dev/null +++ b/daemon.shit/config.py @@ -0,0 +1,15 @@ +import sys +import yaml + +def read_config(path): + """Read yaml config.""" + + options = {} + try: + file = open(path) + except: + sys.stderr.write('Can\'t find config file at %s/%s. Default settings will be load.\n' %(sys.path[0], path)) + else: + options = yaml.load(file) + + return options diff --git a/daemon.shit/configs/config.yaml b/daemon.shit/configs/config.yaml new file mode 100644 index 0000000..4f6f833 --- /dev/null +++ b/daemon.shit/configs/config.yaml @@ -0,0 +1,20 @@ +daemon: + write_pid: False + PID: chanspyd.pid + logfile: chanspyd.log + +chans: + - 2ch: + base_uri: http://2ch.ru + boards: + - wakaba: [b, s, r] + + - iichan: + base_uri: http://iichan.ru + boards: + - wakaba: [b, s, r] + + - dobrochan: + base_uri: http://dobrochan.ru + boards: + - wakaba: [b, r, hr, it, mu, oe, w, a, sw, hau, d] diff --git a/daemon.shit/my_thread.py b/daemon.shit/my_thread.py new file mode 100644 index 0000000..6c3ea0d --- /dev/null +++ b/daemon.shit/my_thread.py @@ -0,0 +1,20 @@ +import dbus.service + +INTERFACE = 'py.chans.dbus' + +class Thread(dbus.service.Object): + + def __init__(self, bus_name, obj_path, board, id): + dbus.service.Object.__init__(self, bus_name, obj_path) + self.bus_name = bus_name + self.obj_path = obj_path + self.board = board + self.id = id + self.posts = [] + + @dbus.service.method(dbus_interface=INTERFACE, in_signature='', out_signature='aa{ss}') + def get_posts_dicts(self): + posts_dicts = [] + for post in self.posts: + posts_dicts.append(post.post_dict) + return posts_dicts diff --git a/daemon.shit/plugins/__init__.py b/daemon.shit/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/daemon.shit/plugins/plugin_wakaba.py b/daemon.shit/plugins/plugin_wakaba.py new file mode 100644 index 0000000..954b324 --- /dev/null +++ b/daemon.shit/plugins/plugin_wakaba.py @@ -0,0 +1,30 @@ +import dbus.service + +from board import Board +from plugins.wakaba_thread import WakabaThread + +import urllib +import re + +INTERFACE = 'py.chans.dbus' + +class MetaBoard(Board): + """Wakaba board class.""" + + def __init__(self, bus_name, obj_path, name, base_uri): + Board.__init__(self, bus_name, obj_path, name, base_uri) + + #TODO: _normal_ parsing needs + @dbus.service.method(dbus_interface=INTERFACE, in_signature='', out_signature='') + def refresh(self): + """Parsing board threads.""" + + def get_threads_ids(): + data = urllib.urlopen(self.board_uri).read() + threads = re.findall('
.*?', post_data)[0] + post['date'] = '' + post['name'] = '' + post['title'] = '' + post['is_sage'] = '' + img_src = re.findall('', post_data) + if img_src: + post['img_src'] = img_src[0] + else: + post['img_src'] = '' + post['text'] = re.findall('
(.*?)
', post_data)[0] + return post + + def get_msg_tables(data): + return re.findall('
(.*?)
', data) + + data = urllib.urlopen('%s/%s/res/%s.html' %(self.board.base_uri, self.board.name, self.id)).read() + + first_post = re.findall('(
. +############################################### + +import sys + +import gui +import client + +try: + client.get_chans() +except: + sys.stderr.write('Server is not running.\n') + sys.exit(1) + +main_gui = gui.GUI('glades/main.glade') +main_gui.main() diff --git a/gui.shit/client.py b/gui.shit/client.py new file mode 100644 index 0000000..fc5b19f --- /dev/null +++ b/gui.shit/client.py @@ -0,0 +1,39 @@ +import dbus + +INTERFACE = 'py.chans.dbus' +bus = dbus.SessionBus() + +#---------------------------------------------- +def get_chans(): + """Loading chans from /pool via get_chans().""" + + pool_object = bus.get_object(INTERFACE, '/pool') + pool = dbus.Interface(pool_object, INTERFACE) + + return pool.get_chans() + +#---------------------------------------------- +def get_boards(chan): + """Loading boards from /pool/chan via get_boards().""" + + chan_object = bus.get_object(INTERFACE, '/pool/'+chan) + chan = dbus.Interface(chan_object, INTERFACE) + + return chan.get_boards() + +#---------------------------------------------- +def refresh_board(chan, board): + """Parsing board threads.""" + + board_object = bus.get_object(INTERFACE, '/pool/'+chan+'/'+board) + board = dbus.Interface(board_object, INTERFACE) + board.refresh() + +#---------------------------------------------- +def get_threads(chan, board): + """Loading boards from /pool/chan/board via get_threads().""" + + board_object = bus.get_object(INTERFACE, '/pool/'+chan+'/'+board) + board = dbus.Interface(board_object, INTERFACE) + + return board.get_threads() diff --git a/gui.shit/glades/main.glade b/gui.shit/glades/main.glade new file mode 100644 index 0000000..acf5c68 --- /dev/null +++ b/gui.shit/glades/main.glade @@ -0,0 +1,108 @@ + + + + + + True + chanspy + center + 800 + 600 + + + + True + + + True + + + False + 0 + + + + + True + True + 150 + True + + + True + True + 280 + True + + + True + True + automatic + automatic + + + True + True + + + + + False + True + + + + + True + True + automatic + automatic + + + True + True + + + + + + + True + True + + + + + False + True + + + + + True + True + + + True + True + + + + + 1 + + + + + True + 2 + + + False + 2 + + + + + + diff --git a/gui.shit/gui.py b/gui.shit/gui.py new file mode 100644 index 0000000..900b84b --- /dev/null +++ b/gui.shit/gui.py @@ -0,0 +1,92 @@ +import pygtk +pygtk.require("2.0") +import gtk +import gtk.glade + +import client + +class GUI: + """GUI class. Processed all signals and get data via dbus from daemon.""" + + def __init__(self, path): + """Set starting gui parametrs and load some data -- chans and boards.""" + + self.wTree = gtk.glade.XML(path) + self.wTree.signal_autoconnect({ 'on_main_destroy': gtk.main_quit + , 'on_chans_view_cursor_changed': self.board_selected + , 'on_chans_view_row_activated': self.board_activated + }) + + # toolbar + toolbar = self.wTree.get_widget('toolbar') + bSettings = gtk.ToolButton(label='settings') + bSettings.show() + bExit = gtk.ToolButton(label='exit') + bExit.show() + bExit.connect('clicked', gtk.main_quit) + toolbar.insert(bSettings, 0) + toolbar.insert(bExit, 1) + + # statusbar + self.statusbar = self.wTree.get_widget('statusbar') + self.statusbar.push(0, 'Stop.') + + # chans_view + chans_view = self.wTree.get_widget('chans_view') + chans_tree = gtk.TreeStore(str) + tvcolumn = gtk.TreeViewColumn('Chans') + chans_view.append_column(tvcolumn) + cell = gtk.CellRendererText() + tvcolumn.pack_start(cell, True) + tvcolumn.add_attribute(cell, 'text', 0) + chans_view.set_model(chans_tree) + + chans = client.get_chans() + for chan in chans: + parent = chans_tree.append(None, [chan]) + boards = client.get_boards(chan) + for board in boards: + chans_tree.append(parent, [board]) + + # threads_view + threads_view = self.wTree.get_widget('threads_view') + self.threads_list = gtk.ListStore(str, str) + tvcolumn1 = gtk.TreeViewColumn('id') + tvcolumn2 = gtk.TreeViewColumn('name') + threads_view.append_column(tvcolumn1) + threads_view.append_column(tvcolumn2) + tvcolumn1.pack_start(cell, True) + tvcolumn2.pack_start(cell, True) + tvcolumn1.add_attribute(cell, 'text', 0) + tvcolumn2.add_attribute(cell, 'text', 0) + threads_view.set_model(self.threads_list) + + def main(self): + """Starting gtk main loop.""" + + gtk.main() + + def board_selected(self, treeview): + """Select board.""" + + model, path = treeview.get_selection().get_selected_rows() + self.board_activated(treeview, path[0], None, False) + + def board_activated(self, treeview, path, view_column, refresh=True): + """Load threads list into threads_view then we clicked on board in chans_view.""" + + if len(path) == 2: + model = treeview.get_model() + chan_pos, board_pos = path + + iter_chan = model.get_iter((chan_pos,)) + chan = model.get_value(iter_chan, 0) + iter_board = model.get_iter(path) + board = model.get_value(iter_board, 0) + + if refresh: client.refresh_board(chan, board) + threads = client.get_threads(chan, board) + + self.threads_list.clear() + for thread in threads: + self.threads_list.append([thread, 'nyaa']) diff --git a/other/updateArchive.py b/other/updateArchive.py new file mode 100644 index 0000000..6322238 --- /dev/null +++ b/other/updateArchive.py @@ -0,0 +1,658 @@ +# coding=utf-8 +#fgsfds +from paste.script.command import Command +from fc.lib.base import * +from fc.model import * +from sqlalchemy.orm import eagerload +from sqlalchemy.orm import class_mapper +from sqlalchemy.sql import and_, or_, not_ +import sqlalchemy +import os +import cgi +import shutil +import datetime +import time +import Image +import hashlib +import re + +from fc.lib.fuser import FUser +from fc.lib.miscUtils import * +from fc.lib.constantValues import * +from fc.lib.settings import * +from fc.lib.fileHolder import AngryFileHolder +import sys +import paste.fixture +import paste.registry +import paste.deploy.config +from paste.deploy import loadapp, appconfig +from paste.script.command import Command, BadCommand +from paste.script.filemaker import FileOp +from paste.script.pluginlib import find_egg_info_dir +import urllib2 +import httplib +from lxml import etree +import StringIO +from fc.model.arch import * +import logging + +def can_import(name): + """Attempt to __import__ the specified package/module, returning True when + succeeding, otherwise False""" + try: + __import__(name) + return True + except ImportError: + return False + +def unicodify(text): + if isinstance(text, str): + text = text.decode('utf-8') + return text + +idList = {} +GFilters = {} + +class DateTimeParser: + monthes = [('Янв','Jan','января'),('Фев','Feb','февраля'),('Мар','Mar','марта'),('Апр','Apr','апреля'),('Май','May','мая'),('Июн','Jun','июня'),('Июл','Jul','июля'),('Авг','Aug','августа'),('Сен','Sep','сентября'),('Окт','Oct','октября'),('Ноя','Nov','ноября'),('Дек','Dec','декабря')] + dateRe = re.compile(r"""[^\d]+(\d+)\s+([^\d\s]+)\s+(\d+)\s+(\d+)\:(\d+)\:(\d+)""") + dateReISO = re.compile(r"""(\d+)\-(\d+)\-(\d+) (\d+)\:(\d+)\:(\d+)""") + def getDateTime(self,date): + dateP = self.dateRe.findall(date) + dateP = dateP[0] + mi = 0 + f = False + for mm in self.monthes: + mi = mi + 1 + if dateP[1] in mm: + f = True + break + if f: + return datetime.datetime(int(dateP[2]),mi,int(dateP[0]),int(dateP[3]),int(dateP[4]),int(dateP[5])) + else: + return None + def getDateTimeFromISO8601(self,date): + dateP = self.dateReISO.findall(date) + dateP = dateP[0] + return datetime.datetime(int(dateP[0]),int(dateP[1]),int(dateP[2]),int(dateP[3]),int(dateP[4]),int(dateP[5])) + +DTP = DateTimeParser() + +class IBParser: + def GetNextTag(self,el,tag,skip=0): + tag = tag.lower() + if skip: + r = el.getnext() + else: + r = el + if not r.tag or r.tag.lower() != tag: + while (r.getnext() != None) and not (r.getnext().tag and r.getnext().tag.lower() == tag): + r = r.getnext() + if r.getnext() != None: + r = r.getnext() + if r.tag and r.tag.lower() == tag: + return r + else: + return None + + def GetPreviousTag(self,el,tag,skip=0): + tag = tag.lower() + if skip: + r = el.getprevious() + else: + r = el + if not r.tag or r.tag.lower() != tag: + while (r.getprevious() != None) and not (r.getprevious().tag and r.getprevious().tag.lower() == tag): + r = r.getprevious() + if r.getprevious() != None: + r = r.getprevious() + if r.tag and r.tag.lower() == tag: + return r + else: + return None + def ResolveSecondaryId(self,thread,Ids): + id = int(Ids[1]) + if id in idList: + return idList[id][0] + + tagsf = and_(Post.tags.any(tag=thread.chanTag),Post.tags.any(tag=thread.board)) + f2 = and_(Post.parentid==-1,tagsf) + f1 = and_(Post.secondaryIndex==Ids[0],f2) + thread = meta.Session.query(Post).filter(f1).first() + if thread: + if Ids[0] == Ids[1]: + return thread.id + else: + post = meta.Session.query(Post).filter(and_(Post.secondaryIndex==int(Ids[1]),Post.parentid==thread.id)).first() + if post: + return post.id + else: + return None + else: + return None + def GetPostID(self,post): + if post.thread: + ids = self.replyIdRe.findall(post.href) + return [post.thread.tid,int(ids[0])] + else: + ids = self.postIdRe.findall(post.href) + return [int(ids[0][0]),ids[0][2] and int(ids[0][2]) or int(ids[0][0])] + +class Loader: + def parseLink(self,link): + s1 = link.split('://') + p = len(s1)>1 and s1[0] or None + p2= p and (p+'://') or '' + s2 = s1[-1].split('/') + return [p, s2[0], p2 + s2[0] + '/', p2 + '/'.join(s2[:-1]) + '/', s2[-1],'/'+'/'.join(s2[1:])] + +class LoaderLocal(Loader): + def __init__(self,link): + p = self.parseLink(link) + self.relativeUrl = p[3] + def stat(self,link): + try: + stats = os.stat(link) + return [datetime.datetime.fromtimestamp(stats[8]),stats[6]] + except OSError: + return None + def get(self,url): + return open(url,'rb').read() + def getAbsolutePath(self,url): + return self.relativeUrl + url + def getFromRelative(self,url): + return self.get(self.getAbsolutePath(url)) + +class LoaderHTTP(Loader): + def __init__(self,link): + p = self.parseLink(link) + self.proto = p[0] + self.host = p[1] + self.baseUrl = p[2] + self.relativeUrl = p[3] + def stat(self,link): + linkp = self.parseLink(link) + c = httplib.HTTPConnection(linkp[1]) + c.request('HEAD', linkp[5]) + r = c.getresponse() + if r.status == 200: + size = r.getheader('content-length',0) + date = r.getheader('last-modified',r.getheader('date',None)) + return [DTP.getDateTime(date),size] + elif r.status == 404: + return None + else: + return None + def get(self,url): + req = urllib2.Request(url) + req.add_header('Referer', self.baseUrl) + try: + f = urllib2.urlopen(req) + res = f.read() + return res + except urllib2.HTTPError: + return None + def getAbsolutePath(self,url): + if url[0] == '/': + return self.baseUrl + url + else: + return self.relativeUrl + url + def getFromRelative(self,url): + return self.get(self.getAbsolutePath(url)) +class IBFilter: + def filter(self,post): + return None +class IBFilterSage(IBFilter): + def filter(self,post): + return post.sage +class IBFilterLowres(IBFilter): + def filter(self,post): + return post.pic and post.pic.width < 50 + +class Thread: + def __init__(self,entry,parsers,directlink=None,forcetype=None): + self.parser = parsers[entry.type] + self.tid = entry.tid + self.url = entry.url + self.board = entry.board + self.chanTag= entry.chanTag + self.tags = entry.tags and entry.tags.split(',') or [] + self.type = entry.type + self.forcetype = forcetype + self.lastChanged = entry.lastChanged + self.filters = [] + filters = entry.filters and entry.filters.split(',') or [] + if filters: + for f in filters: + self.filters.append(GFilters[f]) + + self.timeDiff = entry.timeDiff + self.directlink = directlink + self.loader = Loader() + if not self.directlink: + self.directlink = self.parser.GetThreadLink(self.url,self.board,self.tid) + if self.loader.parseLink(self.directlink)[0]: + self.loader = LoaderHTTP(self.directlink) + else: + self.loader = LoaderLocal(self.directlink) + def checkState(self): + stat = self.loader.stat(self.directlink) + if not stat: + return [404] + elif stat[0] > self.lastChanged: + return [200,stat[0],stat[1]] + else: + return [304,stat[0],stat[1]] + def initialize(self): + page = self.loader.get(self.directlink) + if page: + parser = etree.HTMLParser() + if isinstance(page, str): + page = page.decode('utf-8') + self.document = etree.parse(StringIO.StringIO(page), parser) + self.posts = self.parser.GetPostsList(self) + self.threadId = self.parser.ResolveSecondaryId(self,[self.tid,self.tid]) + if self.posts: + return True + else: + return False + else: + return False + def filter(self,post): + fl = None + if self.filters: + for f in self.filters: + fl = fl or f.filter(post) + return fl + def ReplaceReference(self,m): + mgg = m.groups() + mg = [mgg[1],mgg[2]] + tid = self.parser.ResolveSecondaryId(self,[mg[0],mg[0]]) + if tid: + if mg[0] != mg[1]: + pid = self.parser.ResolveSecondaryId(self,[mg[0],mg[1]]) + else: + pid = tid + if pid: + return '>>%s' % (tid, pid, pid, mg[1]) + print "ERROR! %s/%s does not exist!" % (mg[0],mg[1]) + return '>>%s' % (mg[0], mg[1], mg[1], mg[1]) + +class WakabaParser(IBParser): + replyIdRe = re.compile(r""">>(\d+)""") + postIdRe = re.compile(r"""\/(\d+)\.x?h?t?ml?(#i?(\d+))?""") + referenceRe = re.compile("""]*href="([^"]*/)?(\d+)\.[^"]+"[^>]*>\>\;\>\;(\d+)""") + def GetThreadLink(self,url,board,thread): + return 'http://'+url+'/'+board+'/res/'+str(thread)+'.html' + def GetPostsList(self,thread): + posts = thread.document.xpath("/html/body/form//*[@class='reflink']/a") + postsList = [] + if posts: + for postA in posts: + post = Post() + post.thread = thread + post.href = postA.get('href') + post.reflink = postA.getparent() + post.Ids = self.GetPostID(post) + post.secondaryIndex = int(post.Ids[1]) + postsList.append(post) + return postsList + else: + return None + def GetImgSrc(self,post): + cont = post.l.getparent() + for t in cont: + if t.tag.lower() == 'a': + href = t.get('href') + if href and href.find('/src/') != -1: + if post.thread.forcetype: + return '../src/' + post.thread.loader.parseLink(href)[4] + else: + return href + return None + + def ParseText(self,post): + if post.bq is not None: + post.bq.tail = '' + message = etree.tostring(post.bq, pretty_print=False,encoding='utf-8') + if message[:12].lower() == '
' and message[-13:].lower() == '
': + message = message[12:-13] + else: + print "Cant parse this message : '%s'" % message + return None + message = self.referenceRe.sub(post.thread.ReplaceReference,message) + return message + else: + return u'' + def parsePost(self,post): + post.bq = self.GetNextTag(post.reflink,'blockquote') + post.l = self.GetPreviousTag(post.reflink,'label') + post.title = unicodify(post.l[1].text) + if not post.title: + post.title = u'' + post.cpn = post.l[2] + post.sage = False + if len(post.cpn)>0 and post.cpn[0].tag.lower() == 'a': + post.cpnHref = post.cpn[0].get('href') + if post.cpnHref.find('sage') > -1: + post.sage = True + post.src = self.GetImgSrc(post) + date = post.l[2].tail.encode('utf-8') + date = date.replace("\r",'').replace("\n",'') + post.date = DTP.getDateTime(date) + post.message = unicodify(self.ParseText(post)) + +class UpdateArchive(Command): + # Parser configuration + summary = "--NO SUMMARY--" + usage = "--NO USAGE--" + group_name = "fc" + parser = Command.standard_parser(verbose=False) + parser.add_option("--mode") + parser.add_option("--chan") + parser.add_option("--board") + parser.add_option("--thread") + parser.add_option("--chanTag") + parser.add_option("--type") + parser.add_option("--tags") + parser.add_option("--timeDiff") + parser.add_option("--directlink") + parser.add_option("--list") + parser.add_option("--filters") + parser.add_option("--forcetype") + parsers = {'wakaba':WakabaParser()} + def command(self): + """Main command to create a new shell""" + self.verbose = 3 + config_file = 'development.ini' + config_name = 'config:%s' % config_file + here_dir = os.getcwd() + locs = dict(__name__="pylons-admin") + conf = appconfig(config_name, relative_to=here_dir) + conf.update(dict(app_conf=conf.local_conf,global_conf=conf.global_conf)) + paste.deploy.config.CONFIG.push_thread_config(conf) + sys.path.insert(0, here_dir) + wsgiapp = loadapp(config_name, relative_to=here_dir) + test_app = paste.fixture.TestApp(wsgiapp) + tresponse = test_app.get('/_test_vars') + request_id = int(tresponse.body) + test_app.pre_request_hook = lambda self:paste.registry.restorer.restoration_end() + test_app.post_request_hook = lambda self:paste.registry.restorer.restoration_begin(request_id) + paste.registry.restorer.restoration_begin(request_id) + egg_info = find_egg_info_dir(here_dir) + f = open(os.path.join(egg_info, 'top_level.txt')) + packages = [l.strip() for l in f.readlines() if l.strip() and not l.strip().startswith('#')] + f.close() + found_base = False + for pkg_name in packages: + # Import all objects from the base module + base_module = pkg_name + '.lib.base' + found_base = can_import(base_module) + if not found_base: + # Minimal template + base_module = pkg_name + '.controllers' + found_base = can_import(base_module) + + if found_base: + break + + if not found_base: + raise ImportError("Could not import base module. Are you sure this is a Pylons app?") + + base = sys.modules[base_module] + base_public = [__name for __name in dir(base) if not \ + __name.startswith('_') or __name == '_'] + for name in base_public: + locs[name] = getattr(base, name) + locs.update(dict(wsgiapp=wsgiapp, app=test_app)) + + mapper = tresponse.config.get('routes.map') + if mapper: + locs['mapper'] = mapper + + + self.thread = self.options.thread + self.chan = self.options.chan + self.chanTag = self.options.chanTag + self.board = self.options.board + + logging.getLogger('sqlalchemy').setLevel(logging.ERROR) + GFilters['sage'] = IBFilterSage() + GFilters['lowres'] = IBFilterLowres() + #logging.getLogger( 'sqlalchemy').setLevel( logging.NONE ) + if not self.options.mode or self.options.mode == 'update': + self.UpdateArchive() + elif self.options.mode == 'add': + self.AddToArchive() + elif self.options.mode == 'thread': + if self.options.list: + f = open(self.options.list,'r') + tList = f.readlines() + else: + tList = [self.options.thread] + for t in tList: + entry = ArchiveList() + entry.tid = int(t) + entry.url = self.options.chan + entry.chanTag = self.options.chanTag + entry.board = self.options.board or 'b' + entry.tags = self.options.tags or '' + entry.type = self.options.type or 'wakaba' + entry.filters = self.options.filters or '' + entry.timeDiff = self.options.timeDiff or 0 + entry.lastChanged = datetime.datetime.fromtimestamp(0) + print "Processing %s %s %s %s" % (entry.tid,entry.url,entry.chanTag,entry.board) + thread = Thread(entry,self.parsers,self.options.directlink,self.options.forcetype) + self.processThread(thread) + + def LoadPage(self,thread,chan='2ch.ru',board='b'): + self.host = 'http://'+chan + if thread: + self.path = '/'+board+'/res/' + self.url = self.host+self.path+thread+'.html' + else: + self.path = '/'+board+'/' + self.url = self.host+self.path + print self.url + req = urllib2.Request(self.url) + req.add_header('Referer', self.host+'/'+board+'/') + f = urllib2.urlopen(req) + res = f.read() + return res + + def getTags(self,tagsList): + tags = [] + for tagName in tagsList: + tag = meta.Session.query(Tag).filter(Tag.tag==tagName).first() + if tag: + tags.append(tag) + else: + tags.append(Tag(tagName)) + return tags + + def processPost(self,post): + post.thread.parser.parsePost(post) + post.pic = False + if post.src: + post.pic = self.LoadImage(post) + if post.pic == -1: + post.pic = None + if post.pic: + post.picid = post.pic.id + print "Thread %s Post %s (Image:%s %s %sx%s) at %s, sage : %s" % (post.Ids[0],post.Ids[1],post.src,post.pic and post.pic.id or 0,post.pic and post.pic.width or 0,post.pic and post.pic.height or 0,post.date,post.sage) + if (post.thread.filter(post)): + print "Filtered out" + print "----------------------" + else: + if post.Ids[0] == post.Ids[1]: + post.parentid = -1 + post.replyCount = 0 + post.bumpDate = post.date + post.tags = self.getTags([post.thread.chanTag,post.thread.board]+post.thread.tags) + post.thread.post = post + else: + post.parentid = post.thread.post.id + if not post.sage: + post.thread.post.bumpDate = post.date + post.thread.post.replyCount += 1 + post.uidNumber = 1 + meta.Session.save(post) + meta.Session.commit() + idList[post.Ids[1]]=[post.id,post.Ids[0]] + print "Saved in DB as %s/%s" % (post.id,post.parentid) + print "----------------------" + + def processThread(self,thread): + if thread.initialize(): + if thread.threadId: + thread.post = meta.Session.query(Post).get(thread.threadId) + lastPost = meta.Session.query(Post).filter(Post.parentid==thread.post.id).filter(Post.secondaryIndex>0).order_by(Post.secondaryIndex.desc()).first() + if lastPost: + lastId = lastPost.secondaryIndex + else: + lastId = int(thread.tid) + else: + lastId = 0 + skipped = 0 + for post in thread.posts: + if int(post.Ids[1]) > lastId: + if skipped: + print "Skipped %s out of %s posts" % (skipped,len(thread.posts)) + skipped=0 + self.processPost(post) + else: + skipped += 1 + if skipped: + print "Skipped %s out of %s posts" % (skipped,len(thread.posts)) + + + def LoadImage(self,post): + url = post.thread.loader.getAbsolutePath(post.src) + fileName = post.thread.loader.parseLink(url)[4] + res = post.thread.loader.getFromRelative(post.src) + if res: + localFilePath = os.path.join(g.OPT.uploadPath, fileName) + localFile = open(localFilePath,'wb') + localFile.write(res) + localFile.close() + file = FieldStorageLike(fileName,localFilePath) + fileDescriptors = self.processFile(file, 200) + pic = False + if fileDescriptors: + pic = fileDescriptors[0] + fileHolder = fileDescriptors[1] + if pic and pic != -1 and fileHolder: + fileHolder.disableDeletion() + return pic + else: + return None + + def processFile(self, file, thumbSize=250): + if isinstance(file, cgi.FieldStorage) or isinstance(file,FieldStorageLike): + # We should check whether we got this file already or not + # If we dont have it, we add it + name = str(long(time.time() * 10**7)) + ext = file.filename.rsplit('.',1)[:0:-1] + + if ext: + ext = ext[0].lstrip(os.sep) + else: + # Panic, no extention found + ext = '' + return '' + + # Make sure its something we want to have + + extParams = meta.Session.query(Extension).filter(Extension.ext==ext).first() + + if not extParams: + return False + + localFilePath = os.path.join(g.OPT.uploadPath, name + '.' + ext) + localFile = open(localFilePath,'w+b') + shutil.copyfileobj(file.file, localFile) + localFile.seek(0) + md5 = hashlib.md5(localFile.read()).hexdigest() + file.file.close() + localFile.close() + + pic = meta.Session.query(Picture).filter(Picture.md5==md5).first() + + if pic: + os.unlink(localFilePath) + return [pic, False] + + try: + if extParams.type == 'image': + thumbFilePath = name + 's.' + ext + size = self.makeThumbnail(localFilePath, os.path.join(g.OPT.uploadPath,thumbFilePath), (thumbSize,thumbSize)) + else: + if extParams.type == 'image-jpg': + thumbFilePath = name + 's.jpg' + size = self.makeThumbnail(localFilePath, os.path.join(g.OPT.uploadPath,thumbFilePath), (thumbSize,thumbSize)) + else: + thumbFilePath = extParams.path + size = [0, 0, extParams.thwidth, extParams.thheight] + except: + return [-1, AngryFileHolder(localFilePath)] + + pic = Picture() + pic.path = name + '.' + ext + pic.thumpath = thumbFilePath + pic.width = size[0] + pic.height = size[1] + pic.thwidth = size[2] + pic.thheight = size[3] + pic.extid = extParams.id + pic.size = os.stat(localFilePath)[6] + pic.md5 = md5 + meta.Session.save(pic) + meta.Session.commit() + return [pic, AngryFileHolder(localFilePath, pic)] + else: + return False + + def makeThumbnail(self, source, dest, maxSize): + sourceImage = Image.open(source) + size = sourceImage.size + if sourceImage: + sourceImage.thumbnail(maxSize,Image.ANTIALIAS) + sourceImage.save(dest) + return size + sourceImage.size + else: + return [] + def AddToArchive(self): + if self.options.thread and self.options.chan and self.options.chanTag: + if not self.options.board: + self.options.board = 'b' + entry = meta.Session.query(ArchiveList).filter(ArchiveList.tid==self.options.thread).filter(ArchiveList.url==self.options.chan).filter(ArchiveList.board==self.options.board).first() + if entry: + print "Thread is already in the list" + else: + entry = ArchiveList() + entry.tid = self.options.thread + entry.url = self.options.chan + entry.chanTag = self.options.chanTag + entry.board = self.options.board + entry.tags = self.options.tags or '' + entry.type = self.options.type or 'wakaba' + entry.filters = self.options.filters or '' + entry.timeDiff = self.options.timeDiff or 0 + entry.lastChanged = datetime.datetime.fromtimestamp(0) + meta.Session.save(entry) + meta.Session.commit() + else: + print "Bad parameters" + def UpdateArchive(self): + archiveList = meta.Session.query(ArchiveList).all() + for entry in archiveList: + thread = Thread(entry,self.parsers) + state = thread.checkState() + print "*** Thread %s HTTP %s" % (thread.directlink,state[0]) + if state[0] == 404: + meta.Session.delete(entry) + meta.Session.commit() + elif state[0] == 200: + self.processThread(thread) + entry.lastChanged = state[1] + meta.Session.commit() diff --git a/prepare_for_commit.sh b/prepare_for_commit.sh new file mode 100755 index 0000000..821d062 --- /dev/null +++ b/prepare_for_commit.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +for file in `find . -name '*.pyc' -o -name '.*' -type f` +do + echo "Removing $file..." + rm "$file" +done diff --git a/server/chanspyd b/server/chanspyd new file mode 100755 index 0000000..5c536da --- /dev/null +++ b/server/chanspyd @@ -0,0 +1,127 @@ +#!/usr/bin/env python + +############################################### +# chanspyd - chanspy daemon +# Copyright (C) 2008 anonymous +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +############################################### + +import sys +import os +import signal + +import config +import static_generation +import server + +os.chdir(sys.path[0]) + +#---------------------------------------------- +def main(): + '''Main function. Write pid file, generate static html and run server.''' + + if os.path.exists(PID): + sys.stderr.write('chanspyd\'s already executed. If not then remove %s.\n' %PID) + sys.exit(2) + + if settings['daemon']['write_pid']: + try: + f = open(PID, 'w') + f.write(str(os.getpid())) + f.close() + except: + sys.stderr.write('Can\'t write pid to file.\n') + sys.exit(2) + signal.signal(signal.SIGTERM, sigTermCB) + + static_generation.generate(settings['chans']) + server.start_server(settings['daemon']['port']) + +#---------------------------------------------- +def help(): + '''Print help message.''' + + help = ('Usage: %s [OPTIONS]' %sys.argv[0]) +\ + '\n\nOptions:\n' +\ + '-h --help help\n' +\ + '-d --daemon run in background\n' +\ + '-k --kill kill daemon' + print(help) + +#---------------------------------------------- +def error(): + '''Invalid option.''' + + error = ('%s: invalid option.\n' %sys.argv[0]) +\ + 'Try `%s --help\' for more information.\n' %sys.argv[0] + sys.stderr.write(error) + sys.exit(1) + +#---------------------------------------------- +def kill(): + '''Kill process by pid.''' + + try: + f = open(PID) + pid = f.read() + f.close() + except: + sys.stderr.write('chanspyd is not running\n') + sys.exit(2) + else: + try: + os.kill(int(pid), signal.SIGTERM) + except: + sys.stderr.write('Cat\'t kill %s.\n' %pid) + sys.exit(2) + +#---------------------------------------------- +def sigTermCB(signum, frame): + '''Removing pid and exiting on SIGTERM.''' + + try: + os.remove(PID) + finally: + server.stop_server() + +############################################### +settings = config.load_config() +PID = settings['daemon']['PID'] + +logfile = '/dev/stdout' + +if len(sys.argv) == 2: + if sys.argv[1] == '-h' or sys.argv[1] == '--help': + help() + + elif sys.argv[1] == '-d' or sys.argv[1] == '--daemon' : + logfile = settings['daemon']['logfile'] + + pid = os.fork() + if pid == 0: + os.setsid() + pid = os.fork() + if pid == 0: + main() + + elif sys.argv[1] == '-k' or sys.argv[1] == '--kill': + kill() + + else: + error() +elif len(sys.argv) == 1: + main() +else: + error() diff --git a/server/config.py b/server/config.py new file mode 100644 index 0000000..1b31a44 --- /dev/null +++ b/server/config.py @@ -0,0 +1,19 @@ +import os +import sys +import yaml + +os.chdir(sys.path[0]) +config_path = os.path.join('configs', 'config.yaml') + +def load_config(): + '''Read yaml config.''' + + try: + file = open(config_path) + config = file.read() + except: + sys.stderr.write('Can\'t find config file at %s.\n' %os.path.join(sys.path[0], config_path)) + sys.exit(1) + else: + settings = yaml.load(config) + return settings diff --git a/server/configs/config.yaml b/server/configs/config.yaml new file mode 100644 index 0000000..ad7cb62 --- /dev/null +++ b/server/configs/config.yaml @@ -0,0 +1,28 @@ +daemon: + write_pid: False + PID: chanspyd.pid + logfile: chanspyd.log + port: 6789 + +chans: + - iichan: + base_uri: http://iichan.ru + boards: + - b: {type: wakaba, files: [jpg, png, gif]} + - a: {type: wakaba, files: [jpg, png, gif]} + # to be continued… + + - dobrochan: + base_uri: http://dobrochan.ru + boards: + - b: {type: wakaba, files: [jpg, png, gif]} + - r: {type: wakaba, files: [jpg, png, gif]} + - hr: {type: wakaba, files: [jpg, png, gif]} + - it: {type: wakaba, files: [jpg, png, gif]} + - mu: {type: wakaba, files: [jpg, png, gif]} + - oe: {type: wakaba, files: [jpg, png, gif]} + - w: {type: wakaba, files: [jpg, png, gif]} + - a: {type: wakaba, files: [jpg, png, gif]} + - sw: {type: wakaba, files: [jpg, png, gif]} + - hau: {type: wakaba, files: [jpg, png, gif]} + - d: {type: wakaba, files: [jpg, png, gif]} diff --git a/server/htdocs/blank.html b/server/htdocs/blank.html new file mode 100644 index 0000000..22587dd --- /dev/null +++ b/server/htdocs/blank.html @@ -0,0 +1,10 @@ + + + + + + blank + + + + diff --git a/server/htdocs/chans/dobrochan/index.html b/server/htdocs/chans/dobrochan/index.html new file mode 100644 index 0000000..a54e76d --- /dev/null +++ b/server/htdocs/chans/dobrochan/index.html @@ -0,0 +1,20 @@ + + + + + + + +
b
+
r
+
hr
+
it
+
mu
+
oe
+
w
+
a
+
sw
+
hau
+
d
+ + \ No newline at end of file diff --git a/server/htdocs/chans/iichan/index.html b/server/htdocs/chans/iichan/index.html new file mode 100644 index 0000000..3831ebf --- /dev/null +++ b/server/htdocs/chans/iichan/index.html @@ -0,0 +1,11 @@ + + + + + + + +
b
+
a
+ + \ No newline at end of file diff --git a/server/htdocs/chans/index.html b/server/htdocs/chans/index.html new file mode 100644 index 0000000..ecb2a77 --- /dev/null +++ b/server/htdocs/chans/index.html @@ -0,0 +1,11 @@ + + + + + + + +
iichan
+
dobrochan
+ + \ No newline at end of file diff --git a/server/htdocs/index.html b/server/htdocs/index.html new file mode 100644 index 0000000..bd24cc5 --- /dev/null +++ b/server/htdocs/index.html @@ -0,0 +1,17 @@ + + + + + + chanspy + + + + + + + + + + + diff --git a/server/htdocs/list_template.html b/server/htdocs/list_template.html new file mode 100644 index 0000000..f89875f --- /dev/null +++ b/server/htdocs/list_template.html @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/server/htdocs/settings.html b/server/htdocs/settings.html new file mode 100644 index 0000000..858b6b0 --- /dev/null +++ b/server/htdocs/settings.html @@ -0,0 +1,13 @@ + + + + + + Settings + + +
+ Неведомая хуйня: +
+ + diff --git a/server/htdocs/styles/style.css b/server/htdocs/styles/style.css new file mode 100644 index 0000000..92238ae --- /dev/null +++ b/server/htdocs/styles/style.css @@ -0,0 +1,19 @@ +html, body { + margin: 1px; + padding: 0; +} + +div { + text-align: center; +} + +div a { + display: block; + text-decoration: none; + color: #800000; + background: #F0E0D6; +} + +div a:hover { + background: #FFFFEE; +} diff --git a/server/server.py b/server/server.py new file mode 100644 index 0000000..e6e3455 --- /dev/null +++ b/server/server.py @@ -0,0 +1,42 @@ +# Based on webserver.py by Jon Berg (turtlemeat.com) + +import sys +import os +from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer +import mimetypes + +os.chdir(sys.path[0]) +server = None +# some constants +htdocs_dir = u'htdocs' + +#---------------------------------------------# +class MyHandler(BaseHTTPRequestHandler): + def do_GET(self): + path = u'%s%s' %(htdocs_dir, self.path) + if os.path.isdir(path): path += u'/index.html' + + if os.path.exists(path): + f = open(path) + + self.send_response(200) + self.send_header('Content-type', self.get_content_type(path)) + self.end_headers() + self.wfile.write(f.read()) + f.close() + else: + self.send_error(404, 'File Not Found: %s' %path) + + def get_content_type(self, filename): + return mimetypes.guess_type(filename)[0] or 'application/octet-stream' + +#---------------------------------------------# +def start_server(port): + global server + server = HTTPServer(('', port), MyHandler) + server.serve_forever() + +#---------------------------------------------# +def stop_server(): + global server + server.socket.close() diff --git a/server/static_generation.py b/server/static_generation.py new file mode 100644 index 0000000..150bfe0 --- /dev/null +++ b/server/static_generation.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- + +import sys +import shutil +import os +from BeautifulSoup import BeautifulSoup + +os.chdir(sys.path[0]) +# some constants +htdocs_dir = u'htdocs' +chans_dir = ['chans', os.path.join(htdocs_dir, u'chans')] +template = os.path.join(htdocs_dir, u'list_template.html') +chans_file = os.path.join(chans_dir[1], u'index.html') + +#---------------------------------------------# +def chans_generate(data, chans_list): + '''Generate chans list.''' + + soup = BeautifulSoup(data) + + body = u'\n' + for (chan_name, uri) in chans_list: + # TODO: do something with this shit + body += u'
%s
\n' %(uri, chan_name) + + soup.body.replaceWith(u'%s' %body) + return unicode(soup)[:-1] + +#---------------------------------------------# +def boards_generate(data, boards_list): + '''Generate boards list.''' + + soup = BeautifulSoup(data) + + body = u'\n' + for (board_name, uri) in boards_list: + # TODO: do something with this shit + body += u'
%s
\n' %(uri, board_name) + + soup.body.replaceWith(u'%s' %body) + return unicode(soup)[:-1] + +#---------------------------------------------# +def generate(chans): + '''Generate static html files.''' + + try: + f = open(template, 'r') + template_data = f.read() + f.close() + + shutil.rmtree(chans_dir[1], 'ignore_errors') + os.mkdir(chans_dir[1]) + + chans_list = [] + for chan_item in chans: + chan_name, chan_settings = chan_item.items()[0] + chans_list.append((chan_name, u'/%s/%s/index.html' %(chans_dir[0], chan_name))) + + os.mkdir(os.path.join(chans_dir[1], chan_name)) + boards_list = [] + for board_item in chan_settings['boards']: + board_name, board_settings = board_item.items()[0] + boards_list.append((board_name, u'%s/%s/' %(chan_settings['base_uri'], board_name))) + + boards_html = boards_generate(template_data, boards_list) + f = open(os.path.join(chans_dir[1], chan_name, 'index.html'), 'w') + f.write(boards_html) + f.close() + + chans_html = chans_generate(template_data, chans_list) + f = open(chans_file, 'w') + f.write(chans_html) + f.close() + except: + pass -- 2.11.4.GIT