- new time delta function

[infobot.git] / src / Modules / Freshmeat.pl

diff --git a/src/Modules/Freshmeat.pl b/src/Modules/Freshmeat.pl
index f7c60db35c815b2fe6f01087e0662d29fd3ab288..a09e810c929017c918bb94c8ff3056ed0d6c34fc 100644 (file)
--- a/src/Modules/Freshmeat.pl
+++ b/src/Modules/Freshmeat.pl
@@ -12,7 +12,7 @@ use strict;
 ### download compressed version instead?
 
 my %urls = (
-       'public'  => 'http://core.freshmeat.net/backend/appindex.txt',
+       'public'  => 'http://www.freshmeat.net/backend/appindex.txt',
        'private' => 'http://feed.freshmeat.net/appindex/appindex.txt',
 );
 
@@ -42,7 +42,7 @@ sub Freshmeat {
     }
 
     if (!&showPackage($sstr)) {                # no exact match.
-       my $start_time = &main::gettimeofday();
+       my $start_time = &main::timeget();
        my %hash;
 
        # search by key/NAME first.
@@ -65,7 +65,7 @@ sub Freshmeat {
        }
 
        # show how long it took.
-       my $delta_time = &main::gettimeofday() - $start_time;
+       my $delta_time = &main::timedelta($start_time);
        &main::status(sprintf("freshmeat: %.02f sec to complete query.", $delta_time)) if ($delta_time > 0);
 
        for (@list) {
@@ -98,8 +98,8 @@ sub showPackage {
 }
 
 sub downloadIndex {
-    my $start_time     = &main::gettimeofday(); # set the start time.
-    my $idx            = "$main::bot_base_dir/Temp/fm_index.txt";
+    my $start_time     = &main::timeget(); # set the start time.
+    my $idx            = "$main::param{tempDir}/fm_index.txt";
 
     &main::msg($main::who, "Updating freshmeat index... please wait");
 
@@ -128,7 +128,13 @@ sub downloadIndex {
        return;
     }
 
-    open(IN, $idx);
+    if ($idx =~ /bz2$/) {
+       open(IN, "bzcat $idx |");
+    } elsif ($idx =~ /gz$/) {
+       open(IN, "gzcat $idx |");
+    } else {
+       open(IN, $idx);
+    }
 
     # delete the table before we redo it.
     &main::deleteTable("freshmeat");
@@ -146,6 +152,7 @@ sub downloadIndex {
 
     &main::dbRaw("LOCK", "LOCK TABLES freshmeat WRITE");
     my @data;
+    my @done;
     while (my $line = <IN>) {
        chop $line;
        if ($line ne "%%") {
@@ -153,7 +160,7 @@ sub downloadIndex {
            next;
        }
 
-       if ($i % 100 == 0 and $i != 0) {
+       if ($i % 200 == 0 and $i != 0) {
            &main::DEBUG("FM: unlocking and locking.");
            &main::dbRaw("UNLOCK", "UNLOCK TABLES");
            ### another lame hack to "prevent" errors.
@@ -161,18 +168,25 @@ sub downloadIndex {
            &main::dbRaw("LOCK", "LOCK TABLES freshmeat WRITE");
        }
 
+       if (grep /^\Q$data[0]\E$/, @done) {
+           &main::DEBUG("dupe? $data[0]");
+           @data = ();
+           next;
+       }
+
        $i++;
        pop @data;
        $data[1] ||= "none";
        $data[2] ||= "none";
        &main::dbSetRow("freshmeat", @data);
+       push(@done,$data[0]);
        @data = ();
     }
     close IN;
     &main::DEBUG("FM: data ".scalar(@data) );
     &main::dbRaw("UNLOCK", "UNLOCK TABLES");
 
-    my $delta_time = &main::gettimeofday() - $start_time;
+    my $delta_time = &main::timedelta($start_time);
     &main::status(sprintf("Freshmeat: %.02f sec to complete.", $delta_time)) if ($delta_time > 0);
 
     my $count = &main::countKeys("freshmeat");
@@ -180,7 +194,7 @@ sub downloadIndex {
 }
 
 sub freshmeatAnnounce {
-    my $file = "$main::bot_base_dir/Temp/fm_recent.txt";
+    my $file = "$main::param{tempDir}/fm_recent.txt";
     my @old;
 
     ### if file exists, lets read it.