From f6125daf5f6407f9208e623a9253298732d5b86d Mon Sep 17 00:00:00 2001 From: Gryllida A Date: Sat, 9 Mar 2013 21:58:28 +1030 Subject: [PATCH] Add support for multiple language sites --- .gitignore | 1 + bin/update_db.pl | 232 +++++++++++++++++++++++++++++-------------------------- lib/Wnstats.pm | 15 ++-- views/index.tt | 6 +- 4 files changed, 136 insertions(+), 118 deletions(-) rewrite bin/update_db.pl (90%) diff --git a/.gitignore b/.gitignore index d4f1136..681b7b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ config.yml notes +some.db diff --git a/bin/update_db.pl b/bin/update_db.pl dissimilarity index 90% index f7aaa1e..fffaa2c 100644 --- a/bin/update_db.pl +++ b/bin/update_db.pl @@ -1,108 +1,124 @@ -#/usr/bin/perl - -use Dancer ':script'; -use Dancer::Plugin::Database; -use Data::Dumper; -use MediaWiki::API; -use DateTime; -use DateTime::Format::Strptime; -use DateTime::Event::Recurrence; -# use Dancer::Plugin::NYTProf; # do not run profiler unless we need it - - - -$|++; # forces a flush after every write or print - -my $mw = MediaWiki::API->new(); -$mw->{config}->{api_url} = 'https://en.wikinews.org/w/api.php'; - -my $strp = DateTime::Format::Strptime->new( - pattern => '%Y-%m-%dT%H:%M:%SZ', - locale => 'en_AU', - time_zone => 'UTC', -); - -my $strp_db_days = DateTime::Format::Strptime->new( - pattern => '%F', # Equivalent to %Y-%m-%d. (This is the ISO style date) - locale => 'en_AU', -); - -database->do("CREATE TABLE IF NOT EXISTS stats (date VARCHAR(12) PRIMARY KEY, count INTEGER)"); - -# -# populate or update the database -# - -my %data; -my @years=(2005..2013); -my @months=(1,2,3,4,5,6,7,8,9,10,11,12); - -# retrieve last saved date -my $get_latest_date = database->prepare - ("select date from stats order by date desc limit 1"); -$get_latest_date->execute(); -my $row = $get_latest_date->fetchrow_hashref; -my $latest_date = $row - ? $strp_db_days->parse_datetime($row->{date})->add(days => 1) # don't get stats if we already have them - : DateTime->new(year=>2005,month=>01,day=>01); - -# process yearly lists of published articles -for my $y (@years){ - next if $y < $latest_date->year; - print "getting ts... "; - # earliest possible: last saved date - my $startTime = ($latest_date->year eq $y) - ? $strp->format_datetime($latest_date) - : "$y:01:01 00:00:00"; - # latest possible: today - my ($h,$m,$s) = (DateTime->now->hour,DateTime->now->minute,DateTime->now->second); - my $endTime = ($y eq DateTime->now->year) - ? $strp->format_datetime((DateTime->now)->add(hours=>-$h,minutes=>-$m,seconds=>-$s-1)) # don't get stats for today - : "$y:12:31 23:59:59"; - - print "done.\n"; - - # retrieve articles list for a year - print "from $startTime to $endTime. getting data... "; - my $articles = $mw->list ( { - action => 'query', - list => 'categorymembers', - cmtitle => 'Category:Published', - cmnamespace => 0, - cmlimit => 5000, - cmsort => 'timestamp', - cmdir => 'desc', - cmstart => $endTime, - cmend => $startTime, - cmprop => "timestamp", - } ) or die $mw->{error}->{code} . ': ' . $mw->{error}->{details}; - - # put the daily articles count into %data - print "done.\ncounting ... "; - for my $article (@{$articles}){ - my $ts = $article->{timestamp}; - my $time = $strp->parse_datetime($ts); # This is a DateTime object - $data{sprintf("%d-%02d-%02d",($time->year, $time->month, $time->day))}++; - } - print "done.\n"; -} - - - -database->begin_work; # works with AutoCommit 1 -eval { - for my $key (keys %data){ - # insert count into the database - my $add_statement = database->prepare("INSERT INTO stats VALUES (?, ?)"); - $add_statement->execute($key, $data{$key}); - } - database->commit; # commit the changes if we get this far -}; -if ($@) { - warn "Transaction aborted because $@"; - # now rollback to undo the incomplete changes - # but do it in an eval{} as it may also fail - eval { database->rollback }; - # add other application on-error-clean-up code here -} - +#/usr/bin/perl + +use Dancer ':script'; +use Dancer::Plugin::Database; +use Data::Dumper; +use MediaWiki::API; +use DateTime; +use DateTime::Format::Strptime; +use DateTime::Event::Recurrence; +use utf8; +# use Dancer::Plugin::NYTProf; # do not run profiler unless we need it + +$|++; # forces a flush after every write or print + +my %known_wikis = ( + 'ru.wikinews.org' => 'Опубликовано', + 'en.wikinews.org' => 'Published', +); + +for my $wiki (keys %known_wikis){ + print "Working on $wiki...\n"; + my $cat_name = $known_wikis{$wiki}; + print "* Category name $cat_name...\n"; + my $wiki_url = $wiki; + $wiki =~ s/\.//g; + print "* Db name $wiki...\n"; + + my $mw = MediaWiki::API->new(); + $mw->{config}->{api_url} = "https://$wiki_url/w/api.php"; + + my $strp = DateTime::Format::Strptime->new( + pattern => '%Y-%m-%dT%H:%M:%SZ', + locale => 'en_AU', + time_zone => 'UTC', + ); + + my $strp_db_days = DateTime::Format::Strptime->new( + pattern => '%F', # Equivalent to %Y-%m-%d. (This is the ISO style date) + locale => 'en_AU', + ); + print "* Created the needed variables...\n"; + print "* Creating database $wiki...\n"; + + my $createdb = database->do("CREATE TABLE IF NOT EXISTS $wiki (date VARCHAR(12) PRIMARY KEY, count INTEGER)"); + print "* Created the database if it didn't exist...\n"; + + # + # populate or update the database + # + + my %data; + my @years=(2005..2013); + my @months=(1,2,3,4,5,6,7,8,9,10,11,12); + + # retrieve last saved date + my $get_latest_date = database->prepare + ("select date from $wiki order by date desc limit 1"); + $get_latest_date->execute(); + my $row = $get_latest_date->fetchrow_hashref; + my $latest_date = $row + ? $strp_db_days->parse_datetime($row->{date})->add(days => 1) # don't get stats if we already have them + : DateTime->new(year=>2005,month=>01,day=>01); + + # process yearly lists of published articles + for my $y (@years){ + next if $y < $latest_date->year; + print "* getting ts... "; + # earliest possible: last saved date + my $startTime = ($latest_date->year eq $y) + ? $strp->format_datetime($latest_date) + : "$y:01:01 00:00:00"; + # latest possible: today + my ($h,$m,$s) = (DateTime->now->hour,DateTime->now->minute,DateTime->now->second); + my $endTime = ($y eq DateTime->now->year) + ? $strp->format_datetime((DateTime->now)->add(hours=>-$h,minutes=>-$m,seconds=>-$s-1)) # don't get stats for today + : "$y:12:31 23:59:59"; + + print "done.\n"; + + # retrieve articles list for a year + print "* from $startTime to $endTime. getting data... "; + my $articles = $mw->list ( { + action => 'query', + list => 'categorymembers', + cmtitle => "Category:$cat_name", + cmnamespace => 0, + cmlimit => 5000, + cmsort => 'timestamp', + cmdir => 'desc', + cmstart => $endTime, + cmend => $startTime, + cmprop => "timestamp", + } ) or die $mw->{error}->{code} . ': ' . $mw->{error}->{details}; + + # put the daily articles count into %data + print "done.\n* counting ... "; + for my $article (@{$articles}){ + my $ts = $article->{timestamp}; + my $time = $strp->parse_datetime($ts); # This is a DateTime object + $data{sprintf("%d-%02d-%02d",($time->year, $time->month, $time->day))}++; + } + print "done.\n"; + } + + + + database->begin_work; # works with AutoCommit 1 + eval { + for my $key (keys %data){ + # insert count into the database + my $add_statement = database->prepare("INSERT INTO $wiki VALUES (?, ?)"); + $add_statement->execute($key, $data{$key}); + } + database->commit; # commit the changes if we get this far + }; + if ($@) { + warn "Transaction aborted because $@"; + # now rollback to undo the incomplete changes + # but do it in an eval{} as it may also fail + eval { database->rollback }; + # add other application on-error-clean-up code here + } + +} diff --git a/lib/Wnstats.pm b/lib/Wnstats.pm index be75a26..ee30521 100644 --- a/lib/Wnstats.pm +++ b/lib/Wnstats.pm @@ -9,14 +9,15 @@ use DateTime::Event::Recurrence; use Data::Dumper; sub num_published_month_before{ - my ($y0,$m0,$d0,$years) = @_; + my ($y0,$m0,$d0,$home_wiki) = @_; + $home_wiki =~ s/\.//g; my $date = sprintf("%d-%02d-%02d",($y0, $m0, $d0)); my $driver_name = database->{Driver}->{Name}; my $str_query; if ($driver_name eq 'mysql') { - $str_query = 'select SUM(count) from stats where date < ? and date > ADDDATE(?,-31)'; + $str_query = 'select SUM(count) from '.$home_wiki.' where date < ? and date > ADDDATE(?,-31)'; } elsif ($driver_name eq 'SQLite') { - $str_query = 'select SUM(count) from stats where date < ? and date > DATE(?,"-1 month")'; + $str_query = 'select SUM(count) from '.$home_wiki.' where date < ? and date > DATE(?,"-1 month")'; } my ($n) = database->selectrow_array($str_query,{},$date,$date); my $count = $n ? $n : 0; @@ -25,7 +26,7 @@ sub num_published_month_before{ sub get_chart_points{ - my ($years, $interval) = @_; # interval in days, <=30 + my ($years, $interval, $home_wiki) = @_; # interval in days, <=30 my @points; # get the abscissae using start date, end date, interval @@ -44,20 +45,20 @@ sub get_chart_points{ last if $day->truncate( to => "day" ) == DateTime->today; # ... by checking the db for total last 30 days articles count my ($y, $m, $d) = ($day->year, $day->month, $day->day); - push @points, ["$y-$m-$d",num_published_month_before ($y, $m, $d)]; + push @points, ["$y-$m-$d",num_published_month_before ($y, $m, $d, $home_wiki)]; }; return \@points; } -ajax '/chart/:start_year?/:end_year?/:interval?' => sub { +ajax '/chart/:start_year?/:end_year?/:interval?/:home_wiki' => sub { # get years range input my $start_year = param('start_year') ? param('start_year') : 2005; my $end_year = param('end_year') ? param('end_year') : DateTime->now->year; # get interval input my $interval = param('interval') ? param('interval') : 10; # days # retrieve the data and return the json - my $result = to_json get_chart_points([$start_year..$end_year], $interval); + my $result = to_json get_chart_points([$start_year..$end_year], $interval, param('home_wiki')); return $result; }; diff --git a/views/index.tt b/views/index.tt index ad18127..01b7371 100644 --- a/views/index.tt +++ b/views/index.tt @@ -41,13 +41,13 @@ -

FROM - TO WITH POINTS EVERY DAYS. +

AT FROM + TO EVERY DAYS.