updated git and svn scripts

[xrzperl.git] / urlrule_task

#!/usr/bin/perl 
# $Id$
use Cwd qw/getcwd/;
use lib $ENV{XR_PERL_MODULE_DIR};
use MyPlace::URLRule;
use MyPlace::ReEnterable;
use MyPlace::Script::Message;
use Cwd;
use strict;
use warnings;

#use MyPlace::URLRule::HostMap qw/%HOST_MAP get_task valid_hostname add_host get_hosts/;

my $phnd;
use constant {
    TASK_FILE_RESUME => '.urlrule_task_resume',
    TASK_FILE_LOG    => 'urlrule_task.log',
};

my $TASK_LOGGING    = 1;
my $TASK_DATABASE   = "DATABASE.URL";
my $HOST_DATABASE = $URLRULE_DIRECTORY . "/database/HOSTS.URL";
my $U_DOWNLOADER = 'batchget --maxtime 600 -U -c -m 3 -f';
my $S_DOWNLOADER = 'batchget --maxtime 600 -c -m 3 -f';
my $TASK_WORKDIR = getcwd();;
my $TASK_FILEHANDLE_LOG;
use MyPlace::XConfig;
my $Names = MyPlace::XConfig->new();
$Names->read_plainfile($TASK_DATABASE);

sub change_directory {
    my $path = shift;
    return 1 unless($path);
    if(! -d $path) {
        app_message("Creating directory $path...");
        if(mkdir $path) {
            print STDERR "\t[OK]\n";
        }
        else {
            print STDERR "\t[Failed]\n";
            return undef;
        }
    }
    app_message("Changing working directory to $path...");
    if(chdir $path) {
        print STDERR "\t[OK]\n";
    }
    else {
        print STDERR "\t[Failed]\n";
        return undef;
    }
    return 1;
}

### NOT FINISHED
sub urlrule_process_task {
    my $task = shift;
    return 1 unless($task);
    return 1 unless(ref $task);
    my $do_dir = shift;
    if($do_dir and $task->{work_dir}) { 
        return undef unless(change_directory($task->{work_dir}));
    }
    my $task_wd = getcwd();
    if($task->{pass_data}) {
        my $idx = 0;
        foreach my $url (@{$task->{pass_data}}) {
            if($task->{pass_name} and @{$task->{pass_name}}) {
                my $sub_wd = $task->{pass_name}->[$idx];
                next unless(change_directory($sub_wd));
            }
        }
    }
    return 1;
}


urlrule_set_callback('process_passdown',\&process_passdown_task);
sub process_passdown_task {
        my($status2,$pass_count,@passdown)
            = urlrule_process_result(@_);
        if($status2 and $pass_count and $pass_count>0) {
            my $CWD = getcwd;
            foreach(@passdown) {
                my($cwd,@args) = @{$_};
                if(@args) {
                    $phnd->push($CWD,'load_rule',$cwd,@args);
                }
            }
            chdir $CWD;
        }
}
sub load_rule {
    my $CWD = getcwd;
    my ($status1,$rule,$result) = urlrule_process_args(@_);
    if($status1) {
        process_passdown_task($rule,$result);
    }
    chdir $CWD;
}

sub convert_records {
    my ($Hosts,@records) = @_;
    my @r;
    foreach my $record (@records) {
        foreach my $path (@{$record}) {
            my($name,$id,$host) = @{$path};
            next unless($name);
            next unless($id);
            next unless($host);
            if($host =~ m/^#/) {
                push @r,[$name,$id,$host];
                next;
            }
            my $hostname = $host;
            if($hostname =~ m/^([^\|]+)\|(.*)$/) {
                $hostname = $1;
            }
            my ($url) = $Hosts->propget($hostname);
            if($url) {
                my ($level) = $Hosts->propget($hostname,$url);
                my ($id_name,@id_text) = split(/\s*:\s*/,$id);
                $url =~ s/###(?:ID|NAME])###/$id_name/g;
                my $index = 0;
                foreach(@id_text) {
                    $index++;
                    $url =~ s/###TEXT$index###/$_/g;
                }
                push @r,[$name,$id,$host,$url,$level];
            }
            else {
                app_error("HOST $host isn't valid, or not defined in :\n\t$HOST_DATABASE\n");
            }
        }
    }
    return @r;
}

sub action_ECHO {
    my $task = shift;
    my($name,$id,$host,$url,$level) = @{$task};
    $level=0 unless($level);
    if($host =~ m/^#(.*)$/) {
        app_message("$name/$1 = \'$id\'\n");
        #app_message("$name/$1 $id\n");
    }
    else {
        app_message("$name/$host [$level]$url\n");
    }
    return;
}


sub action_DUMP {
    my $task = shift;
    my($name,$id,$host,$url,$level) = @{$task};
    my($status,$rule,$result,@msg) = urlrule_process_args(undef,$url,$level);
    if($status) {
        use Data::Dumper;
        print Data::Dumper->Dump([$result],['*result']);
        return 1;
    }
    else {
        app_error("Error:",$rule,$result,@msg,"\n");
        return undef;
    }
}
sub action_DOWNLOAD {
    my $task = shift;
    my($name,$id,$host,$url,$level) = @{$task};
    my $hostname;
    if($host =~ /^[^\|]+\|(.+)$/)
    {
        $hostname = $1;
    }
    elsif($host =~ m/^#(.+)$/) {
        app_message("$name/$1 = \'$id\'\n");
        return 1;
    }
    else
    {
        $hostname = $host;
    }
    my $cwd = getcwd();
    if(! -d "$name")
    {
        app_message("Create directory: $name\n");
        mkdir $name;
    }
    if(! -d "$name/$hostname") 
    {
        app_message("Create directory: $name/$hostname\n");
        mkdir "$name/$hostname";
    }
    if(! -d "$name/$hostname") 
    {
        app_error("Can't access $name/$hostname!\n");
        return undef;
    }
    my $Names = MyPlace::XConfig->new();
    $Names->read_plainfile($TASK_DATABASE);
    $Names->propset(scalar(localtime),$name,$id,$host,"last");
    print STDERR "Saving $TASK_DATABASE...\n";
    $Names->write_plainfile($TASK_DATABASE);
    app_message("Processing $name/$host\n");
    &log("DOWNLOAD - $name/$host $url\[$level\]");
    if($host =~ /google|baidu|yahoo|sogua/) {
        load_rule("$name/$hostname",$url,$level,$U_DOWNLOADER);
    }
    else {
        load_rule("$name/$hostname",$url,$level,$S_DOWNLOADER);
    }
    return 1;
}
sub log {
    open $TASK_FILEHANDLE_LOG,">>",TASK_FILE_LOG;
    print $TASK_FILEHANDLE_LOG scalar(localtime),": ",@_,"\n" if(@_);
    close $TASK_FILEHANDLE_LOG;
}


sub sig_killed {
    app_message("saving remained tasks...\n");
    if($phnd->{lastStack}) {
        $phnd->unshift(@{$phnd->{lastStack}});
    }
    chdir($TASK_WORKDIR) if($TASK_WORKDIR);
    $phnd->saveToFile(TASK_FILE_RESUME);
    app_message($phnd->length," task(s) saved to " . TASK_FILE_RESUME . "\n");
    clean_up();
    exit 1;
}


$SIG{INT} = \&sig_killed;
sub init {
        $phnd= MyPlace::ReEnterable->new('main');
};

sub clean_up {
}

if(@ARGV) {
    die("No DATA_FILE(" . $TASK_DATABASE . ") found\n") unless(-r $TASK_DATABASE);
    my ($action,$query) = @ARGV;
    if($action eq 'DUMP') {
    }
    elsif($action eq 'DOWNLOAD') {
    }
    elsif($action eq 'ECHO') {
    }
    else {
        $query = $action;
        $action = 'ECHO';
    }
    my $Hosts = MyPlace::XConfig->new();
    $Hosts->read_plainfile($HOST_DATABASE);
    my @targets =  $Names->query($query);
    if(!@targets) {
        app_error("Nothing match query $query\n");
        exit 1;
    }
    my @records = $Names->get_records(@targets);
    my @tasks = convert_records($Hosts,\@records);
    if($action eq 'DOWNLOAD') {
        &init;
        &log($action," - ",$query);
        foreach(@tasks) {
            $phnd->push($TASK_WORKDIR,"action_$action",$_);
        }
    }
    else {
        foreach(@tasks) {
            eval("action_$action(\$_)");
        }
        exit 0;
    }
}
else {
    &init;
    $phnd->loadFromFile(TASK_FILE_RESUME);
}

unless($phnd->length) {
    app_error("Nothing to do!\n");
    exit 0;
}

while(my $tasks = $phnd->length) {
    app_error("[$tasks] tasks remained\n");
    $phnd->run();
}
app_message "All tasks completed.\n";
unlink TASK_FILE_RESUME if(-f TASK_FILE_RESUME);
exit 0;