2 # Debian.pl: Frontend to debian contents and packages files
4 # Version: v0.7b (20000527)
12 # format: "alias=real".
13 my $defaultdist = "woody";
15 "unstable" => "woody",
18 "incoming" => "incoming",
22 "debian/Contents-##DIST-i386.gz" =>
23 "ftp://ftp.us.debian.org".
24 "/debian/dists/##DIST/Contents-i386.gz",
26 "debian/Contents-##DIST-i386-non-US.gz" =>
27 "ftp://non-us.debian.org".
28 "/debian-non-US/dists/##DIST/non-US/Contents-i386.gz",
32 "debian/Packages-##DIST-main-i386.gz" =>
33 "ftp://ftp.us.debian.org".
34 "/debian/dists/##DIST/main/binary-i386/Packages.gz",
35 "debian/Packages-##DIST-contrib-i386.gz" =>
36 "ftp://ftp.us.debian.org".
37 "/debian/dists/##DIST/contrib/binary-i386/Packages.gz",
38 "debian/Packages-##DIST-non-free-i386.gz" =>
39 "ftp://ftp.us.debian.org".
40 "/debian/dists/##DIST/non-free/binary-i386/Packages.gz",
41 "debian/Packages-##DIST-non-US-i386.gz" =>
42 "ftp://non-us.debian.org".
43 "/debian-non-US/dists/##DIST/non-US/binary-i386/Packages.gz",
47 ### COMMON FUNCTION....
48 #######################
51 # Usage: &DebianDownload(%hash);
53 my ($dist, %urls) = @_;
54 my $refresh = $main::param{'debianRefreshInterval'} * 60 * 60 * 24;
58 &main::status("Debian: Downloading files for '$dist'.");
61 &main::status("Debian: creating debian dir.");
62 mkdir("debian/",0755);
65 %urls = &fixNonUS($dist, %urls);
70 foreach $file (keys %urls) {
71 my $url = $urls{$file};
72 $url =~ s/##DIST/$dist/g;
73 $file =~ s/##DIST/$dist/g;
77 my $last_refresh = (stat($file))[9];
78 $update++ if (time() - $last_refresh > $refresh);
80 &main::DEBUG("Debian: local '$file' does not exist.");
84 next unless ($update);
86 if ($good + $bad == 0) {
87 &main::msg($main::who, "Updating debian files... please wait.");
90 if (exists $main::debian{$url}) {
91 &main::DEBUG("2: ".(time - $main::debian{$url})." <= $refresh");
92 next if (time() - $main::debian{$url} <= $refresh);
93 &main::DEBUG("stale for url $url; updating!");
96 if ($url =~ /^ftp:\/\/(.*?)\/(\S+)\/(\S+)$/) {
97 my ($host,$path,$thisfile) = ($1,$2,$3);
99 # error internally to ftp.
100 # hope it doesn't do anything bad.
101 if (!&main::ftpGet($host,$path,$thisfile,$file)) {
102 &main::DEBUG("deb: down: ftpGet: bad.");
108 &main::DEBUG("deb: down: ftpGet: !file");
113 &main::DEBUG("deb: download: good.");
116 &main::ERROR("Debian: invalid format of url => ($url).");
123 &generateIndex($dist);
126 return -1 unless ($bad); # no download.
127 &main::DEBUG("DD: !good and bad($bad). :(");
132 ###########################
133 # DEBIAN CONTENTS SEARCH FUNCTIONS.
137 # Usage: &searchContents($query);
139 my ($dist, $query) = &getDistroFromStr($_[0]);
140 &main::status("Debian: Contents search for '$query' on $dist.");
143 $dccsend++ if ($query =~ s/^dcc\s+//i);
145 # $query = $query.'(\.so\.)?([.[[:digit:]]+\.]+)?$';
147 $query =~ s/\\([\^\$])/$1/g;
148 $query =~ s/^\s+|\s+$//g;
149 $query =~ s/\*/\\S*/g; # does it even work?
151 if (!&main::validExec($query)) {
152 &main::msg($main::who, "search string looks fuzzy.");
156 if ($dist eq "incoming") { # nothing yet.
157 &main::DEBUG("sC: dist = 'incoming'. no contents yet.");
160 my %urls = &fixDist($dist, %urlcontents);
161 # download contents file.
162 &main::DEBUG("deb: download 1.");
163 if (!&DebianDownload($dist, %urls)) {
164 &main::ERROR("Debian: could not download files.");
170 my $start_time = &main::gettimeofday();
174 my $search = "$query.*\[ \t]";
175 my $files = join(' ', keys %urlcontents);
176 $files =~ s/##DIST/$dist/g;
178 open(IN,"zegrep -h '$search' $files |");
180 if (/^\.?\/?(.*?)[\t\s]+(\S+)\n$/) {
181 my ($file,$package) = ("/".$1,$2);
182 if ($query =~ /\//) {
183 next unless ($file =~ /\Q$query\E/);
185 my ($basename) = $file =~ /^.*\/(.*)$/;
186 next unless ($basename =~ /\Q$query\E/);
188 next if ($query !~ /\.\d\.gz/ and $file =~ /\/man\//);
190 $contents{$package}{$file} = 1;
198 ### send results with dcc.
200 if (exists $main::dcc{'SEND'}{$main::who}) {
201 &main::msg($main::who, "DCC already active!");
205 if (!scalar %contents) {
206 &main::msg($main::who,"search returned no results.");
214 my $file = "temp/$main::who.txt";
215 if (!open(OUT,">$file")) {
216 &main::ERROR("Debian: cannot write file for dcc send.");
220 foreach $pkg (keys %contents) {
221 foreach (keys %{$contents{$pkg}}) {
222 # TODO: correct padding.
223 print OUT "$_\t\t\t$pkg\n";
228 &main::shmWrite($main::shm, "DCC SEND $main::who $file");
233 &main::status("Debian: $found results.");
236 foreach $pkg (keys %contents) {
237 my @tmplist = &main::fixFileList(keys %{$contents{$pkg}});
238 my @sublist = sort { length $a <=> length $b } @tmplist;
240 pop @sublist while (scalar @sublist > 3);
242 $pkg =~ s/\,/\037\,\037/g; # underline ','.
243 push(@list, "(". join(', ',@sublist) .") in $pkg");
245 # sort the total list from shortest to longest...
246 @list = sort { length $a <=> length $b } @list;
248 # show how long it took.
249 my $delta_time = &main::gettimeofday() - $start_time;
250 &main::status(sprintf("Debian: %.02f sec to complete query.", $delta_time)) if ($delta_time > 0);
252 my $prefix = "Debian Search of '$query' ";
253 &main::performStrictReply( &main::formListReply(0, $prefix, @list) );
257 # Usage: &searchAuthor($query);
259 my ($dist, $query) = &getDistroFromStr($_[0]);
260 &main::DEBUG("searchAuthor: dist => '$dist', query => '$query'.");
261 $query =~ s/^\s+|\s+$//g;
264 my $start_time = &main::gettimeofday();
265 &main::status("Debian: starting author search.");
268 my ($bad,$good) = (0,0);
269 my %urls = %urlpackages;
270 ### potato now has the "new" non-US tree like woody does.
271 if ($dist =~ /^(woody|potato)$/) {
272 %urls = &fixNonUS($dist, %urlpackages);
275 foreach (keys %urlpackages) {
287 &main::DEBUG("good = $good, bad = $bad...");
289 if ($good == 0 and $bad != 0) {
290 my %urls = &fixDist($dist, %urlpackages);
291 &main::DEBUG("deb: download 2.");
292 if (!&DebianDownload($dist, %urls)) {
293 &main::ERROR("Debian(sA): could not download files.");
298 my (%maint, %pkg, $package);
299 open(IN,"zegrep -h '^Package|^Maintainer' $files |");
301 if (/^Package: (\S+)$/) {
303 } elsif (/^Maintainer: (.*) \<(\S+)\>$/) {
305 $pkg{$1}{$package} = 1;
307 &main::WARN("invalid line: '$_'.");
313 # TODO: can we use 'map' here?
314 foreach (grep /\Q$query\E/i, keys %maint) {
318 # TODO: should we only search email if '@' is used?
319 if (scalar keys %hash < 15) {
321 foreach $name (keys %maint) {
323 foreach $email (keys %{$maint{$name}}) {
324 next unless ($email =~ /\Q$query\E/i);
325 next if (exists $hash{$name});
331 my @list = keys %hash;
332 if (scalar @list != 1) {
333 my $prefix = "Debian Author Search of '$query' ";
334 &main::performStrictReply( &main::formListReply(0, $prefix, @list) );
338 &main::DEBUG("showing all packages by '$list[0]'...");
340 my @pkg = sort keys %{$pkg{$list[0]}};
342 # show how long it took.
343 my $delta_time = &main::gettimeofday() - $start_time;
344 &main::status(sprintf("Debian: %.02f sec to complete query.", $delta_time)) if ($delta_time > 0);
346 my $email = join(', ', keys %{$maint{$list[0]}});
347 my $prefix = "Debian Packages by $list[0] \002<\002$email\002>\002 ";
348 &main::performStrictReply( &main::formListReply(0, $prefix, @pkg) );
352 # Usage: &generateIncoming();
353 sub generateIncoming {
354 my $interval = $main::param{'debianRefreshInterval'};
355 my $pkgfile = "debian/Packages-incoming";
356 my $idxfile = $pkgfile.".idx";
358 $stale++ if (&main::isStale($pkgfile.".gz", $interval));
359 $stale++ if (&main::isStale($idxfile.".gz", $interval));
360 &main::DEBUG("gI: stale => '$stale'.");
361 return 0 unless ($stale);
364 my %ftp = &main::ftpList("llug.sep.bnl.gov", "/pub/debian/Incoming/");
366 if (!open(PKG,">$pkgfile")) {
367 &main::ERROR("cannot write to pkg $pkgfile.");
370 if (!open(IDX,">$idxfile")) {
371 &main::ERROR("cannot write to idx $idxfile.");
375 print IDX "*$pkgfile.gz\n";
377 foreach $file (sort keys %ftp) {
378 next unless ($file =~ /deb$/);
380 if ($file =~ /^(\S+)\_(\S+)\_(\S+)\.deb$/) {
382 print PKG "Package: $1\n";
383 print PKG "Version: $2\n";
384 print PKG "Architecture: ", (defined $4) ? $4 : "all", "\n";
386 print PKG "Filename: $file\n";
387 print PKG "Size: $ftp{$file}\n";
393 system("gzip -9fv $pkgfile"); # lame fix.
395 &main::status("Debian: generateIncoming() complete.");
399 ##############################
400 # DEBIAN PACKAGE INFO FUNCTIONS.
403 # Usage: &getPackageInfo($query,$file);
405 my ($package, $file) = @_;
408 &main::status("gPI: file $file does not exist?");
415 open(IN, "zcat $file 2>&1 |");
421 next if (/^ \S+/); # package long description.
424 if (/^Package: (.*)\n$/) {
426 if ($pkg =~ /^$package$/i) {
427 $found++; # we can use pkg{'package'} instead.
428 $pkg{'package'} = $pkg;
437 if (/^Version: (.*)$/) {
438 $pkg{'version'} = $1;
439 } elsif (/^Priority: (.*)$/) {
440 $pkg{'priority'} = $1;
441 } elsif (/^Section: (.*)$/) {
442 $pkg{'section'} = $1;
443 } elsif (/^Size: (.*)$/) {
445 } elsif (/^i.*size: (.*)$/) {
446 $pkg{'installed'} = $1;
447 } elsif (/^Description: (.*)$/) {
449 } elsif (/^Filename: (.*)$/) {
451 } elsif (/^Pre-Depends: (.*)$/) {
452 $pkg{'depends'} = "pre-depends on $1";
453 } elsif (/^Depends: (.*)$/) {
454 if (exists $pkg{'depends'}) {
455 $pkg{'depends'} .= "; depends on $1";
457 $pkg{'depends'} = "depends on $1";
459 } elsif (/^Maintainer: (.*)$/) {
461 } elsif (/^Provides: (.*)$/) {
462 $pkg{'provides'} = $1;
463 } elsif (/^Suggests: (.*)$/) {
464 $pkg{'suggests'} = $1;
465 } elsif (/^Conflicts: (.*)$/) {
466 $pkg{'conflicts'} = $1;
469 ### &main::DEBUG("=> '$_'.");
479 next if (defined $pkg);
487 # Usage: &infoPackages($query,$package);
489 my ($query,$dist,$package) = ($_[0], &getDistroFromStr($_[1]));
490 my $interval = $main::param{'debianRefreshInterval'} || 7;
492 &main::status("Debian: Searching for package '$package' in '$dist'.");
494 # download packages file.
496 my %urls = &fixDist($dist, %urlpackages);
497 if ($dist ne "incoming") {
498 &main::DEBUG("deb: download 3.");
499 if (!&DebianDownload($dist, %urls)) { # no good download.
500 &main::WARN("Debian(iP): could not download ANY files.");
504 # check if the package is valid.
506 my @files = &validPackage($package, $dist);
507 if (!scalar @files) {
508 &main::status("Debian: no valid package found; checking incoming.");
509 @files = &validPackage($package, "incoming");
511 &main::status("Debian: cool, it exists in incoming.");
514 &main::msg($main::who, "Package '$package' does not exist.");
519 if (scalar @files > 1) {
520 &main::WARN("same package in more than one file; random.");
521 &main::DEBUG("THIS SHOULD BE FIXED SOMEHOW!!!");
522 $files[0] = &main::getRandom(@files);
525 if (! -f $files[0]) {
526 &main::WARN("files[0] ($files[0]) doesn't exist.");
527 &main::msg($main::who, "WARNING: $files[0] does not exist? FIXME");
531 ### TODO: if specific package is requested, note down that a version
532 ### exists in incoming.
535 my $file = $files[0];
538 ### TODO: use fe, dump to a hash. if only one version of the package
539 ### exists. do as normal otherwise list all versions.
541 &main::ERROR("D:iP: file '$file' DOES NOT EXIST!!! should never happen.");
544 my %pkg = &getPackageInfo($package, $file);
547 if ($query eq "info") {
548 if (scalar keys %pkg > 5) {
549 $pkg{'info'} = "\002(\002". $pkg{'desc'} ."\002)\002";
550 $pkg{'info'} .= ", section ".$pkg{'section'};
551 $pkg{'info'} .= ", is ".$pkg{'priority'};
552 $pkg{'info'} .= ". Version: \002$pkg{'version'}\002";
553 $pkg{'info'} .= ", Packaged size: \002". int($pkg{'size'}/1024) ."\002 kB";
554 $pkg{'info'} .= ", Installed size: \002$pkg{'installed'}\002 kB";
557 &main::status("iP: info requested and pkg is in incoming, too.");
558 my %incpkg = &getPackageInfo($query, "debian/Packages-incoming");
560 if (scalar keys %incpkg) {
561 $pkg{'info'} .= ". Is in incoming ($incpkg{'file'}).";
563 &main::ERROR("iP: pkg $query is in incoming but we couldn't get any info?");
567 &main::DEBUG("running debianCheck() due to problems (".scalar(keys %pkg).").");
569 &main::DEBUG("end of debianCheck()");
571 &main::msg($main::who,"Debian: Package appears to exist but I could not retrieve info about it...");
576 if ($dist eq "incoming") {
577 $pkg{'info'} .= "Version: \002$pkg{'version'}\002";
578 $pkg{'info'} .= ", Packaged size: \002". int($pkg{'size'}/1024) ."\002 kB";
579 $pkg{'info'} .= ", is in incoming!!!";
582 if (!exists $pkg{$query}) {
583 if ($query eq "suggests") {
584 $pkg{$query} = "has no suggestions";
585 } elsif ($query eq "conflicts") {
586 $pkg{$query} = "does not conflict with any other package";
587 } elsif ($query eq "depends") {
588 $pkg{$query} = "does not depend on anything";
589 } elsif ($query eq "maint") {
590 $pkg{$query} = "has no maintainer";
592 $pkg{$query} = "has nothing about $query";
596 &main::performStrictReply("$package: $pkg{$query}");
599 # Usage: &infoStats($dist);
602 $dist = &getDistro($dist);
603 return unless (defined $dist);
605 &main::DEBUG("infoS: dist => '$dist'.");
606 my $interval = $main::param{'debianRefreshInterval'} || 7;
608 # download packages file if needed.
609 my %urls = &fixDist($dist, %urlpackages);
610 &main::DEBUG("deb: download 4.");
611 if (!&DebianDownload($dist, %urls)) {
612 &main::WARN("Debian(iS): could not download ANY files.");
613 &main::msg($main::who, "Debian(iS): internal error.");
620 foreach $file (keys %urlpackages) {
621 $file =~ s/##DIST/$dist/g; # won't work for incoming.
622 &main::DEBUG("file => '$file'.");
623 if (exists $stats{$file}{'count'}) {
624 &main::DEBUG("hrm... duplicate open with $file???");
628 open(IN,"zcat $file 2>&1 |");
631 &main::DEBUG("iS: $file does not exist.");
638 next if (/^ \S+/); # package long description.
640 if (/^Package: (.*)\n$/) { # counter.
641 $stats{$file}{'count'}++;
643 } elsif (/^Maintainer: .* <(\S+)>$/) {
644 $stats{$file}{'maint'}{$1}++;
645 $total{'maint'}{$1}++;
646 } elsif (/^Size: (.*)$/) { # compressed size.
647 $stats{$file}{'csize'} += $1;
648 $total{'csize'} += $1;
649 } elsif (/^i.*size: (.*)$/) { # installed size.
650 $stats{$file}{'isize'} += $1;
651 $total{'isize'} += $1;
654 ### &main::DEBUG("=> '$_'.");
659 &main::performStrictReply(
660 "Debian Distro Stats on $dist... ".
661 "\002$total{'count'}\002 packages, ".
662 "\002".scalar(keys %{$total{'maint'}})."\002 maintainers, ".
663 "\002". int($total{'isize'}/1024)."\002 MB installed size, ".
664 "\002". int($total{'csize'}/1024/1024)."\002 MB compressed size."
667 ### TODO: do individual stats? if so, we need _another_ arg.
668 # foreach $file (keys %stats) {
669 # foreach (keys %{$stats{$file}}) {
670 # &main::DEBUG(" '$file' '$_' '$stats{$file}{$_}'.");
680 # HELPER FUNCTIONS FOR INFOPACKAGES...
683 # Usage: &generateIndex();
686 &main::DEBUG("Debian: generateIndex() called.");
687 if (!scalar @dists) {
688 &main::ERROR("gI: no dists to generate index.");
693 my $dist = &getDistro($_); # incase the alias is returned, possible?
694 my $idx = "debian/Packages-$dist.idx";
696 # TODO: check if any of the Packages file have been updated then
697 # regenerate it, even if it's not stale.
698 # TODO: also, regenerate the index if the packages file is newer
700 next unless (&main::isStale($idx, $main::param{'debianRefreshInterval'}));
702 &main::DEBUG("gIndex: calling generateIncoming()!");
707 &main::DEBUG("gIndeX: calling DebianDownload($dist, ...).");
708 &DebianDownload($dist, %urlpackages);
710 &main::status("Debian: generating index for '$_'.");
711 if (!open(OUT,">$idx")) {
712 &main::ERROR("cannot write to $idx.");
717 foreach $packages (keys %urlpackages) {
718 $packages =~ s/##DIST/$dist/;
720 if (! -e $packages) {
721 &main::ERROR("gIndex: '$packages' does not exist?");
725 print OUT "*$packages\n";
726 open(IN,"zcat $packages |");
729 if (/^Package: (.*)\n$/) {
741 # Usage: &validPackage($package, $dist);
743 my ($package,$dist) = @_;
747 &main::DEBUG("D: validPackage($package, $dist) called.");
750 while (!open(IN, "debian/Packages-$dist.idx")) {
752 &main::ERROR("Packages-$dist.idx does not exist (#1).");
756 &generateIndex($dist);
768 if (/^$package\n$/) {
775 &main::DEBUG("vP: scanned $count items in index.");
781 my ($dist, $query) = &getDistroFromStr($_[0]);
782 my $file = "debian/Packages-$dist.idx";
786 &main::status("Debian: Search package matching '$query' in '$dist'.");
788 &main::DEBUG("sP: $file == NULL; removing, redoing.");
792 while (!open(IN, $file)) {
793 &main::ERROR("$file does not exist (#2).");
794 if ($dist eq "incoming") {
795 &main::DEBUG("sP: dist == incoming; calling gI().");
800 &main::ERROR("could not generate index!!!");
804 &generateIndex(($dist));
811 &main::DEBUG("sP: hrm => '$1'.");
813 if (&main::isStale($file, $main::param{'debianRefreshInterval'})) {
814 &main::DEBUG("STALE $file! regen.");
815 &generateIndex(($dist));
816 @files = searchPackage("$query $dist");
817 &main::DEBUG("EVIL HACK HACK HACK.");
837 if (!defined $dist or $dist eq "") {
838 &main::DEBUG("gD: dist == NULL; dist = defaultdist.");
839 $dist = $defaultdist;
842 if (exists $dists{$dist}) {
843 return $dists{$dist};
845 if (!grep /^\Q$dist\E$/i, %dists) {
846 &main::msg($main::who, "invalid dist '$dist'.");
854 sub getDistroFromStr {
856 my $dists = join '|', %dists;
857 my $dist = $defaultdist;
859 if ($str =~ s/\s+($dists)$//i) {
860 $dist = &getDistro(lc $1);
863 $str =~ s/\\([\$\^])/$1/g;
869 my ($dist, %urls) = @_;
873 while (($key,$val) = each %urls) {
874 $key =~ s/##DIST/$dist/;
875 $val =~ s/##DIST/$dist/;
876 ### TODO: what should we do if the sar wasn't done.
883 ### H-H-H-HACK HACK HACK :)
885 my ($dist, $query) = &getDistroFromStr($str);
886 my @results = sort &searchPackage($str);
888 if (!scalar @results) {
889 &main::Forker("debian", sub { &searchContents($str); } );
890 } elsif (scalar @results == 1) {
891 &main::status("searchPackage returned one result; getting info of package instead!");
892 &main::Forker("debian", sub { &infoPackages("info", "$results[0] $dist"); } );
894 my $prefix = "Debian Package Listing of '$str' ";
895 &main::performStrictReply( &main::formListReply(0, $prefix, @results) );
899 ### TODO: move DWH to &fixDist() or leave it being called by DD?
901 my ($dist, %urls) = @_;
903 foreach (keys %urls) {
904 last unless ($dist =~ /^(woody|potato)$/);
905 next unless (/non-US/);
906 &main::DEBUG("DD: Enabling hack(??) for $dist non-US.");
910 delete $urls{$file}; # heh.
912 foreach ("main","contrib","non-free") {
913 my ($newfile,$newurl) = ($file,$url);
914 # only needed for Packages for now, not Contents; good.
915 $newfile =~ s/non-US/non-US_$_/;
916 $newurl =~ s#non-US/bin#non-US/$_/bin#;
917 $urls{$newfile} = $newurl;
920 &main::DEBUG("DD: Files: ".scalar(keys %urls));
931 &main::status("debianCheck() called.");
933 ### TODO: remove the following loop (check if dir exists before)
935 last if (opendir(DEBIAN, $dir));
937 &main::ERROR("dC: cannot opendir debian.");
946 while (defined($file = readdir DEBIAN)) {
947 next unless ($file =~ /(gz|bz2)$/);
949 my $exit = system("gzip -t '$dir/$file'");
951 &main::DEBUG("hmr... => ".(time() - (stat($file))[8])."'.");
952 next unless (time() - (stat($file))[8] > 3600);
954 &main::DEBUG("dC: exit => '$exit'.");
955 &main::WARN("dC: '$dir/$file' corrupted? deleting!");
956 unlink $dir."/".$file;