From 85491c5800564e365f7f5bd5cb791863ce6170eb Mon Sep 17 00:00:00 2001 From: dms Date: Mon, 30 Apr 2001 14:44:13 +0000 Subject: [PATCH] - forgot to set forked{}{PID} in addForked - renamed dbGetRowInfo to dbGetColInfo - now used by Freshmeat - || factoids in /msg are now evalated - fm updated to use xml db as used by FM II - "news #DEBIAN-bots" failed - fixed. git-svn-id: https://svn.code.sf.net/p/infobot/code/trunk/blootbot@468 c11ca15a-4712-0410-83d8-924469b57eb5 --- src/Factoids/Reply.pl | 2 +- src/IRC/Irc.pl | 1 - src/Misc.pl | 17 +++-- src/Modules/Factoids.pl | 2 +- src/Modules/Freshmeat.pl | 149 +++++++++++++++++++++++---------------- src/Modules/News.pl | 2 +- src/Shm.pl | 4 +- src/core.pl | 9 ++- src/db_mysql.pl | 4 +- 9 files changed, 112 insertions(+), 78 deletions(-) diff --git a/src/Factoids/Reply.pl b/src/Factoids/Reply.pl index e7dcacd..1137e76 100644 --- a/src/Factoids/Reply.pl +++ b/src/Factoids/Reply.pl @@ -43,7 +43,7 @@ sub getReply { $poss[0] =~ s/^\s//; $poss[$#poss] =~ s/\s$//; - if ((@poss > 1) && ($msgType =~ /public/)) { + if (@poss > 1) { $result = &getRandom(@poss); $result =~ s/^\s*//; } diff --git a/src/IRC/Irc.pl b/src/IRC/Irc.pl index 9d23905..4e93b2c 100644 --- a/src/IRC/Irc.pl +++ b/src/IRC/Irc.pl @@ -308,7 +308,6 @@ sub notice { $conn->notice($target, $txt); } - sub DCCBroadcast { my ($txt,$flag) = @_; diff --git a/src/Misc.pl b/src/Misc.pl index 3db0cdc..ce98779 100644 --- a/src/Misc.pl +++ b/src/Misc.pl @@ -171,7 +171,7 @@ sub Time2String { my $prefix = ""; if ($time < 0) { - $time = - $time; + $time = - $time; $prefix = "- "; } @@ -180,12 +180,13 @@ sub Time2String { my $h = int($time / 3600) % 24; my $d = int($time / 86400); - $retval .= sprintf(" \002%d\002d", $d) if ($d != 0); - $retval .= sprintf(" \002%d\002h", $h) if ($h != 0); - $retval .= sprintf(" \002%d\002m", $m) if ($m != 0); - $retval .= sprintf(" \002%d\002s", $s) if ($s != 0); + my @data; + push(@data, sprintf("\002%d\002d", $d)) if ($d != 0); + push(@data, sprintf("\002%d\002h", $h)) if ($h != 0); + push(@data, sprintf("\002%d\002m", $m)) if ($m != 0); + push(@data, sprintf("\002%d\002s", $s)) if ($s != 0 or !@data); - return $prefix.substr($retval, 1); + return $prefix.join(' ', @data); } ### @@ -447,12 +448,16 @@ sub isStale { return 1; } + &DEBUG("!exist $file") if (! -f $file); + return 1 unless ( -f $file); if ($file =~ /idx/) { my $age2 = time() - (stat($file))[9]; &DEBUG("stale: $age2. (". &Time2String($age2) .")"); } $age *= 60*60*24 if ($age >= 0 and $age < 30); + &DEBUG("age = $age"); + &DEBUG("... = ".(stat $file)[9] ); return 1 if (time() - (stat($file))[9] > $age); return 0; diff --git a/src/Modules/Factoids.pl b/src/Modules/Factoids.pl index f223f91..b2e3646 100644 --- a/src/Modules/Factoids.pl +++ b/src/Modules/Factoids.pl @@ -56,7 +56,7 @@ sub CmdFactInfo { if (time() - $time > 60*60*24*7) { my $days = int( (time() - $time)/60/60/24 ); $string .= " at \037". scalar(localtime $time). "\037" . - " ($days days) "; + " ($days days)"; } else { $string .= " ".&Time2String(time() - $time)." ago"; } diff --git a/src/Modules/Freshmeat.pl b/src/Modules/Freshmeat.pl index 2936b9c..41c013b 100644 --- a/src/Modules/Freshmeat.pl +++ b/src/Modules/Freshmeat.pl @@ -8,12 +8,11 @@ package Freshmeat; use strict; - -### download compressed version instead? +use vars qw(@cols @data $string %pkg $i $locktime); my %urls = ( - 'public' => 'http://www.freshmeat.net/backend/appindex.txt', - 'private' => 'http://feed.freshmeat.net/appindex/appindex.txt', + 'public' => 'http://www.freshmeat.net/backend/fm-projects.rdf.bz2', +# 'private' => 'http://feed.freshmeat.net/appindex/appindex.txt', ); #### @@ -21,24 +20,29 @@ my %urls = ( sub Freshmeat { my $sstr = lc($_[0]); my $refresh = &::getChanConfDefault("freshmeatRefreshInterval", - "", 24) * 60 * 60; + "", 24) * 60 * 60 * 7; - my $last_refresh = &::dbGet("freshmeat", "name","_","stable"); + my $last_refresh = &::dbGet("freshmeat", "projectname_short", "_", "latest_version"); my $renewtable = 0; - if (defined $last_refresh) { + if (defined $last_refresh and $last_refresh =~ /^\d+$/) { $renewtable++ if (time() - $last_refresh > $refresh); } else { $renewtable++; } - $renewtable++ if (&::countKeys("freshmeat") < 10); + $renewtable++ if (&::countKeys("freshmeat") < 1000); - if ($renewtable and $$ == $::bot_pid) { - &::Forker("freshmeat", sub { + if ($renewtable) { + if ($$ == $::bot_pid) { + &::Forker("freshmeat", sub { &downloadIndex(); &Freshmeat($sstr); - } ); - # both parent/fork runs here, in case the following looks weird. + } ); + # both parent/fork runs here, in case the following looks weird. + } else { + &downloadIndex(); + } + return if ($$ == $::bot_pid); } @@ -47,12 +51,12 @@ sub Freshmeat { my %hash; # search by key/NAME first. - foreach (&::searchTable("freshmeat", "name","name",$sstr)) { + foreach (&::searchTable("freshmeat", "projectname_short", "projectname_short",$sstr)) { $hash{$_} = 1 unless exists $hash{$_}; } # search by description line. - foreach (&::searchTable("freshmeat", "name","oneliner", $sstr)) { + foreach (&::searchTable("freshmeat", "projectname_short", "desc_short", $sstr)) { $hash{$_} = 1 unless exists $hash{$_}; last if (scalar keys %hash > 15); } @@ -80,17 +84,18 @@ sub Freshmeat { sub showPackage { my ($pkg) = @_; - my @fm = &::dbGet("freshmeat", "name",$pkg,"*"); + my @fm = &::dbGet("freshmeat", "projectname_short", $pkg, "*"); if (scalar @fm) { #1: perfect match of name. my $retval; - $retval = "$fm[0] \002(\002$fm[11]\002)\002, "; - $retval .= "section $fm[3], "; - $retval .= "is $fm[4]. "; - $retval .= "Stable: \002$fm[1]\002, "; - $retval .= "Development: \002$fm[2]\002. "; - $retval .= $fm[5] || $fm[6]; # fallback to 'download'. - $retval .= " deb: ".$fm[8] if ($fm[8] ne ""); # 'deb'. + $retval = "$fm[0] \002(\002$fm[5]\002)\002, "; +# $retval .= "section $fm[3], "; + $retval .= "is $fm[2]. "; + $retval .= "Version: \002$fm[1]\002, "; +# $retval .= "Development: \002$fm[2]\002. "; + $retval .= $fm[4]; +### ??? +# $retval .= " deb: ".$fm[3] if ($fm[3] ne ""); # 'deb'. &::performStrictReply($retval); return 1; } else { @@ -119,14 +124,30 @@ sub randPackage { sub downloadIndex { my $start_time = &::timeget(); # set the start time. - my $idx = "$::param{tempDir}/fm_index.txt"; + my $idx = "$::param{tempDir}/fm-projects.rdf.bz2"; + + if (!&::loadPerlModule("XML::Parser")) { + &::WARN("don't have xml::parser..."); + return; + } + my $p = new XML::Parser(Style => 'Objects'); + my %pkg; + my $string; + + $p->setHandlers( + Char => \&xml_text, + End => \&xml_end, + ); &::msg($::who, "Updating freshmeat index... please wait"); if (&::isStale($idx, 1)) { &::status("Freshmeat: fetching data."); + foreach (keys %urls) { - my $retval = &::getURLAsFile($urls{$_}, $idx); + $urls{$_} =~ /^.*\/(.*)$/; + $idx = "$::param{tempDir}/$1"; + my $retval = &::getURLAsFile($urls{$_}, $idx); next if ($retval =~ /^(403|500)$/); &::DEBUG("FM: last! retval => '$retval'."); @@ -161,48 +182,16 @@ sub downloadIndex { ### lets get on with business. # set the last refresh time. fixes multiple spawn bug. - &::dbSet("freshmeat", "name","_","stable",time()); - - my $i = 0; - while (my $line = ) { - chop $line; - $i++ if ($line eq "%%"); - last if ($i == 2); - } + &::dbSet("freshmeat", "projectname_short", "_", "latest_version", time()); &::dbRaw("LOCK", "LOCK TABLES freshmeat WRITE"); - my @data; - my @done; - while (my $line = ) { - chop $line; - if ($line ne "%%") { - push(@data,$line); - next; - } + @cols = &::dbGetColInfo("freshmeat"); - if ($i % 200 == 0 and $i != 0) { - &::DEBUG("FM: unlocking and locking."); - &::dbRaw("UNLOCK", "UNLOCK TABLES"); - ### another lame hack to "prevent" errors. - select(undef, undef, undef, 0.2); - &::dbRaw("LOCK", "LOCK TABLES freshmeat WRITE"); - } - - if (grep /^\Q$data[0]\E$/, @done) { - &::DEBUG("dupe? $data[0]"); - @data = (); - next; - } - - $i++; - pop @data; - $data[1] ||= "none"; - $data[2] ||= "none"; - &::dbSetRow("freshmeat", @data); - push(@done,$data[0]); - @data = (); - } + $locktime = time(); + # todo: prevent severe memory usage whilst opening this file!!! + $p->parse(*IN, ProtocolEncoding => 'ISO-8859-1'); close IN; + &::DEBUG("FM: data ".scalar(@data) ); &::dbRaw("UNLOCK", "UNLOCK TABLES"); @@ -267,4 +256,40 @@ sub freshmeatAnnounce { return "Freshmeat update: ".join(" \002::\002 ", @new); } +sub xml_text { + my($expat,$text) = @_; + return if ($text =~ /^\s+$/); + + $string = $text; +} + +sub xml_end { + my($expat,$text) = @_; + + $pkg{$text} = $string; + + if ($expat->depth == 1) { + for (my $j=0; $j $forked{$name}{PID}"); - if (-d "/proc/$forked{$name}{PID}") { + if ( -d "/proc/$forked{$name}{PID}") { &status("fork: still running; good. BAIL OUT."); + return 0; } else { &WARN("Found dead fork; removing and resetting."); $continue = 1; @@ -146,6 +147,7 @@ sub addForked { } $forked{$name}{Time} = time(); + $forked{$name}{PID} = $$; $forkedtime = time(); $count{'Fork'}++; return 1; diff --git a/src/core.pl b/src/core.pl index 5d3a2b2..20f75b3 100644 --- a/src/core.pl +++ b/src/core.pl @@ -271,11 +271,14 @@ sub getChanConf { $chan ||= "_default"; my @c = grep /^$chan$/i, keys %chanconf; - if (@c and $c[0] ne $chan) { - &WARN("c ne chan ($c[0] ne $chan)"); + if (@c) { + if ($c[0] ne $chan) { + &WARN("c ne chan ($c[0] ne $chan)"); + } + return $chanconf{$c[0]}{$param}; } - return $chanconf{$c[0]}{$param} || $chanconf{"_default"}{$param}; + return $chanconf{"_default"}{$param}; } sub showProc { diff --git a/src/db_mysql.pl b/src/db_mysql.pl index 754bcda..9bb777f 100644 --- a/src/db_mysql.pl +++ b/src/db_mysql.pl @@ -105,8 +105,8 @@ sub dbGetCol { } #### -# Usage: &dbGetRowInfo($table); -sub dbGetRowInfo { +# Usage: &dbGetColInfo($table); +sub dbGetColInfo { my ($table) = @_; my $query = "SHOW COLUMNS from $table"; -- 2.39.5