From 58e5a3e4b42420835066e88faf8289e9d6c86f21 Mon Sep 17 00:00:00 2001 From: Pavol Rusnak Date: Thu, 12 Mar 2009 15:40:40 +0100 Subject: [PATCH] popcorn-server - using TDB --- Makefile | 2 +- README | 78 +++++++++++++++++++++++++++++++++++++++ popcorn-client | 27 +++++--------- popcorn-rotate | 31 ++++++++++++++++ popcorn-server.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- technical.txt | 42 --------------------- 6 files changed, 227 insertions(+), 62 deletions(-) create mode 100644 README create mode 100755 popcorn-rotate delete mode 100644 technical.txt diff --git a/Makefile b/Makefile index 5b804f5..0880c56 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ CC = gcc CFLAGS = -Wall -LDFLAGS = -lsqlite3 +LDFLAGS = -ltdb all: popcorn-server diff --git a/README b/README new file mode 100644 index 0000000..6a48a1b --- /dev/null +++ b/README @@ -0,0 +1,78 @@ +POPCORN - Popularity Contest (for RPM) +-------------------------------------- + +http://repo.or.cz/w/popcorn.git + +Licensed under MIT License + +Files +----- + +popcorn-client + * client-side written in Python + * gathers info about packages and sends to server (either via e-mail or HTTP POST) + * is called from cron every week (random time for each machine) + +popcorn-server + * server-side written in C and using SQLite3 + * reads info from stdin and saves it to stats.tdb + +popcorn-rotate + * called every week on server + * move stats.tdb to stats-{year}-{week}.tdb and recreates stats.tdb + +Submission format +----------------- + +POPCORN + +... + + + +Package categories +------------------ + +(n) no-files - no watched file present in the package (see below) +(r) recent - packages has been recently installed (less than 30 days) +(v) voted - package is older than 30 days and has been used recently (less than 30 days) +(o) old - package is older than 30 days and hasn't been used recently + +note: n + r + v + o = total number of installed packages + +Watched files +------------- + +To find out whether the package has been used recently, access times of the files +in the following directories are watched: + +/bin +/boot +/lib +/lib64 +/opt/gnome/bin +/opt/gnome/sbin +/opt/kde3/bin +/opt/kde3/sbin +/sbin +/usr/bin +/usr/games +/usr/include +/usr/lib +/usr/lib64 +/usr/libexec +/usr/sbin +/usr/share + +Database format +--------------- + +Format is produced by libtdb library (http://tdb.samba.org) + +It is simply a (key,value) storage. + + key | value | meaning +-------------+----------------------+---------------------------------------------------- + arch/ | | number of reports from arch + ver/ | | number of reports from ver + | | package was reported times as n,r,v,o diff --git a/popcorn-client b/popcorn-client index 1ecab9b..14c2321 100755 --- a/popcorn-client +++ b/popcorn-client @@ -30,26 +30,18 @@ import platform import httplib import time -package_name = 'popcorn-client' +version = '0.1' popcorn_server = 'stats.opensuse.org' days_recent = 30 -days_unused = 30 +days_voted = 30 pathre = '^/(bin|boot|lib|lib64|opt/gnome/bin|opt/gnome/sbin|opt/kde3/bin|opt/kde3/sbin|sbin|usr/bin|usr/games|usr/include|usr/lib|usr/lib64|usr/libexec|usr/sbin|usr/share)/' class Statistics: - package_ver = 'unknown' arch = platform.machine() packages = [] - def __init__(self): - ts = rpm.TransactionSet() - mi = ts.dbMatch('name', package_name) - for h in mi: - self.package_ver = h['version'] - break - def fill_packages(self): pathreprg = re.compile(pathre) now = int(time.time()) @@ -69,23 +61,22 @@ class Statistics: except: pass if accessed == 0: - cat = 'nof' # no-files + cat = 'n' # no-files else: - if now - installed < 30 * 86400: - cat = 'rec' # recent + if now - installed < days_recent * 86400: + cat = 'r' # recent else: - if now - accessed < 30 * 86400: - cat = 'vot' # voted + if now - accessed < days_voted * 86400: + cat = 'v' # voted else: - cat ='old' # old + cat ='o' # old self.packages.append( (cat, name) ) def serialize(self): ret = '' - ret += 'START %s %s\n' % (self.package_ver, self.arch) + ret += 'POPCORN %s %s\n' % (version, self.arch) for pkg in self.packages: ret += '%s %s\n' % pkg - ret += 'END\n' return ret class Client: diff --git a/popcorn-rotate b/popcorn-rotate new file mode 100755 index 0000000..e2b5daf --- /dev/null +++ b/popcorn-rotate @@ -0,0 +1,31 @@ +#!/bin/sh + +# Copyright (c) 2009 Pavol Rusnak +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +STATSDIR='/var/cache/popcorn/' +STATSFILE='stats.tdb' +STATSFMT='+stats-%G-%V.tdb' + +# move current data to week-data +mv -f $STATSDIR$STATSFILE $STATSDIR`date $STATSFMT` diff --git a/popcorn-server.c b/popcorn-server.c index df4ff0e..5623589 100644 --- a/popcorn-server.c +++ b/popcorn-server.c @@ -23,9 +23,116 @@ OTHER DEALINGS IN THE SOFTWARE. */ -#include +#include +#include +#include +#include +#include + +#define STATSDIR "/var/cache/popcorn/" +#define STATSFILE "stats.tdb" + +#define QUIET 1 + +#define BUFSIZE 1024 + +#define is_sane(c) ( ((c)>='A'&&(c)<='Z') || ((c)>='a'&&(c)<='z') || ((c)>='0'&&(c)<='9') || ((c)=='.') || ((c)=='-') || ((c)=='_') ) + +typedef struct { + unsigned int n; /* no-files */ + unsigned int r; /* recent */ + unsigned int v; /* voted */ + unsigned int o; /* old */ +} TUPLE; + +void sanitize(char *str) +{ + char *c = str; + while (*c) + { + if (!is_sane(*c)) + { + str[0]='\0'; break; + } + ++c; + } + if (str[0]=='\0') strcpy(str,"unknown"); +} int main() { + int l; + unsigned int cnt; + char buf[BUFSIZE], ver[BUFSIZE], arch[BUFSIZE], cat, *pkg, *c; + TDB_CONTEXT *db; + TDB_DATA key, val; + TUPLE tuple; + + if (! fgets(buf, BUFSIZE, stdin) ) + return 1; + sscanf(buf, "POPCORN %s %s", ver, arch); + + sanitize(ver); + sanitize(arch); + + /* open database */ + db = tdb_open(STATSDIR STATSFILE, 0, 0, O_CREAT | O_RDWR, 0644); + if ( !db ) { +#ifndef QUIET + fprintf(stderr, "Can't open database: %s\n", STATSDIR STATSFILE); +#endif + return 1; + } + + // update arch + key.dsize = strlen(arch) + 5; + sprintf(buf, "arch/%s", arch); + key.dptr = (unsigned char *)buf; + val = tdb_fetch(db, key); + cnt = val.dptr ? *((unsigned int *)val.dptr) + 1 : 1; + val.dptr = (unsigned char *)&cnt; + val.dsize = sizeof(cnt); + tdb_store(db, key, val, TDB_REPLACE); + + // update ver + key.dsize = strlen(ver) + 4; + sprintf(buf, "ver/%s", ver); + key.dptr = (unsigned char *)buf; + val = tdb_fetch(db, key); + cnt = val.dptr ? *((unsigned int *)val.dptr) + 1 : 1; + val.dptr = (unsigned char *)&cnt; + val.dsize = sizeof(cnt); + tdb_store(db, key, val, TDB_REPLACE); + + // update packages + while (fgets(buf, BUFSIZE, stdin)) { + l = strlen(buf); + if (l<3) continue; + cat = buf[0]; + c = strpbrk(buf + 2, " \n\r\t"); + *c = '\0'; + pkg = buf + 2; + + key.dsize = strlen(pkg); + key.dptr = (unsigned char *)pkg; + val = tdb_fetch(db, key); + if (!val.dptr) { + tuple.n = 0; + tuple.r = 0; + tuple.v = 0; + tuple.o = 0; + } else { + tuple = *((TUPLE *)val.dptr); + } + switch (cat) { + case 'n': ++tuple.n; break; + case 'r': ++tuple.r; break; + case 'v': ++tuple.v; break; + case 'o': ++tuple.o; break; + } + tdb_store(db, key, val, TDB_REPLACE); + } + + tdb_close(db); return 0; } diff --git a/technical.txt b/technical.txt deleted file mode 100644 index 4cfebcd..0000000 --- a/technical.txt +++ /dev/null @@ -1,42 +0,0 @@ -Submission format ------------------ - -START - -... -END - - -Package categories ------------------- - -(nof) no-files - no watched file present in the package (see below) -(rec) recent - packages has been recently installed (less than 30 days) -(vot) voted - package is older than 30 days and has been used recently (less than 30 days) -(old) old - package is older than 30 days and hasn't been used recently - -note: nof + rec + vot + old = total number of installed packages - -Watched files -------------- - -To find out whether the package has been used recently, access times of the files -in the following directories are watched: - -/bin -/boot -/lib -/lib64 -/opt/gnome/bin -/opt/gnome/sbin -/opt/kde3/bin -/opt/kde3/sbin -/sbin -/usr/bin -/usr/games -/usr/include -/usr/lib -/usr/lib64 -/usr/libexec -/usr/sbin -/usr/share -- 2.11.4.GIT