From a14fb89b6c0279acee78340b6f7a547f8fe1a3cf Mon Sep 17 00:00:00 2001
From: dms <dms@c11ca15a-4712-0410-83d8-924469b57eb5>
Date: Fri, 24 Nov 2000 12:17:22 +0000
Subject: [PATCH] - prevent dupe errors. - support bz2/gz for appindex.

git-svn-id: https://svn.code.sf.net/p/infobot/code/trunk/blootbot@171 c11ca15a-4712-0410-83d8-924469b57eb5
---
 src/Modules/Freshmeat.pl | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/Modules/Freshmeat.pl b/src/Modules/Freshmeat.pl
index 78eb6a5..df78405 100644
--- a/src/Modules/Freshmeat.pl
+++ b/src/Modules/Freshmeat.pl
@@ -128,7 +128,13 @@ sub downloadIndex {
 	return;
     }
 
-    open(IN, $idx);
+    if ($idx =~ /bz2$/) {
+	open(IN, "bzcat $idx |");
+    } elsif ($idx =~ /gz$/) {
+	open(IN, "gzcat $idx |");
+    } else {
+	open(IN, $idx);
+    }
 
     # delete the table before we redo it.
     &main::deleteTable("freshmeat");
@@ -146,6 +152,7 @@ sub downloadIndex {
 
     &main::dbRaw("LOCK", "LOCK TABLES freshmeat WRITE");
     my @data;
+    my @done;
     while (my $line = <IN>) {
 	chop $line;
 	if ($line ne "%%") {
@@ -161,11 +168,18 @@ sub downloadIndex {
 	    &main::dbRaw("LOCK", "LOCK TABLES freshmeat WRITE");
 	}
 
+	if (grep /^\Q$data[0]\E$/, @done) {
+	    &main::DEBUG("dupe? $data[0]");
+	    @data = ();
+	    next;
+	}
+
 	$i++;
 	pop @data;
 	$data[1] ||= "none";
 	$data[2] ||= "none";
 	&main::dbSetRow("freshmeat", @data);
+	push(@done,$data[0]);
 	@data = ();
     }
     close IN;
-- 
2.39.5