1 # This module is part of debbugs, and is released
2 # under the terms of the GPL version 2, or any later
3 # version at your option.
4 # See the file README and COPYING for more information.
6 # [Other people have contributed to this file; their copyrights should
8 # Copyright 2007 by Don Armstrong <don@donarmstrong.com>.
10 package Debbugs::Common;
14 Debbugs::Common -- Common routines for all of Debbugs
18 use Debbugs::Common qw(:url :html);
23 This module is a replacement for the general parts of errorlib.pl.
24 subroutines in errorlib.pl will be gradually phased out and replaced
25 with equivalent (or better) functionality here.
33 use vars qw($VERSION $DEBUG %EXPORT_TAGS @EXPORT_OK @EXPORT);
34 use base qw(Exporter);
38 $DEBUG = 0 unless defined $DEBUG;
41 %EXPORT_TAGS = (util => [qw(getbugcomponent getbuglocation getlocationpath get_hashname),
42 qw(appendfile overwritefile buglog getparsedaddrs getmaintainers),
44 qw(getmaintainers_reverse),
46 qw(package_maintainer),
49 misc => [qw(make_list globify_scalar english_join checkpid),
50 qw(cleanup_eval_fail),
53 utf8 => [qw(encode_utf8_structure encode_utf8_safely),
55 date => [qw(secs_to_english)],
57 lock => [qw(filelock unfilelock lockpid)],
60 Exporter::export_ok_tags(keys %EXPORT_TAGS);
61 $EXPORT_TAGS{all} = [@EXPORT_OK];
64 #use Debbugs::Config qw(:globals);
69 use Debbugs::Config qw(:config);
72 use Debbugs::MIME qw(decode_rfc1522);
75 use Encode qw(encode_utf8 is_utf8 decode);
77 use Storable qw(dclone);
79 use Params::Validate qw(validate_with :types);
81 use Fcntl qw(:DEFAULT :flock);
83 our $DEBUG_FH = \*STDERR if not defined $DEBUG_FH;
87 The following functions are exported by the C<:util> tag
89 =head2 getbugcomponent
91 my $file = getbugcomponent($bug_number,$extension,$location)
93 Returns the path to the bug file in location C<$location>, bug number
94 C<$bugnumber> and extension C<$extension>
99 my ($bugnum, $ext, $location) = @_;
101 if (not defined $location) {
102 $location = getbuglocation($bugnum, $ext);
103 # Default to non-archived bugs only for now; CGI scripts want
104 # archived bugs but most of the backend scripts don't. For now,
105 # anything that is prepared to accept archived bugs should call
106 # getbuglocation() directly first.
107 return undef if defined $location and
108 ($location ne 'db' and $location ne 'db-h');
110 my $dir = getlocationpath($location);
111 return undef if not defined $dir;
112 if (defined $location and $location eq 'db') {
113 return "$dir/$bugnum.$ext";
115 my $hash = get_hashname($bugnum);
116 return "$dir/$hash/$bugnum.$ext";
120 =head2 getbuglocation
122 getbuglocation($bug_number,$extension)
124 Returns the the location in which a particular bug exists; valid
125 locations returned currently are archive, db-h, or db. If the bug does
126 not exist, returns undef.
131 my ($bugnum, $ext) = @_;
132 my $archdir = get_hashname($bugnum);
133 return 'archive' if -r getlocationpath('archive')."/$archdir/$bugnum.$ext";
134 return 'db-h' if -r getlocationpath('db-h')."/$archdir/$bugnum.$ext";
135 return 'db' if -r getlocationpath('db')."/$bugnum.$ext";
140 =head2 getlocationpath
142 getlocationpath($location)
144 Returns the path to a specific location
148 sub getlocationpath {
150 if (defined $location and $location eq 'archive') {
151 return "$config{spool_dir}/archive";
152 } elsif (defined $location and $location eq 'db') {
153 return "$config{spool_dir}/db";
155 return "$config{spool_dir}/db-h";
164 Returns the hash of the bug which is the location within the archive
169 return "" if ( $_[ 0 ] < 0 );
170 return sprintf "%02d", $_[ 0 ] % 100;
177 Returns the path to the logfile corresponding to the bug.
179 Returns undef if the bug does not exist.
185 my $location = getbuglocation($bugnum, 'log');
186 return getbugcomponent($bugnum, 'log', $location) if ($location);
187 $location = getbuglocation($bugnum, 'log.gz');
188 return getbugcomponent($bugnum, 'log.gz', $location) if ($location);
197 Returns the path to the summary file corresponding to the bug.
199 Returns undef if the bug does not exist.
205 my $location = getbuglocation($bugnum, 'summary');
206 return getbugcomponent($bugnum, 'summary', $location) if ($location);
212 appendfile($file,'data','to','append');
214 Opens a file for appending and writes data to it.
219 my ($file,@data) = @_;
220 my $fh = IO::File->new($file,'a') or
221 die "Unable top open $file for appending: $!";
222 print {$fh} @data or die "Unable to write to $file: $!";
223 close $fh or die "Unable to close $file: $!";
228 ovewritefile($file,'data','to','append');
230 Opens file.new, writes data to it, then moves file.new to file.
235 my ($file,@data) = @_;
236 my $fh = IO::File->new("${file}.new",'w') or
237 die "Unable top open ${file}.new for writing: $!";
238 print {$fh} @data or die "Unable to write to ${file}.new: $!";
239 close $fh or die "Unable to close ${file}.new: $!";
240 rename("${file}.new",$file) or
241 die "Unable to rename ${file}.new to $file: $!";
248 =head2 getparsedaddrs
250 my $address = getparsedaddrs($address);
251 my @address = getparsedaddrs($address);
253 Returns the output from Mail::Address->parse, or the cached output if
254 this address has been parsed before. In SCALAR context returns the
255 first address parsed.
263 return () unless defined $addr;
264 return wantarray?@{$_parsedaddrs{$addr}}:$_parsedaddrs{$addr}[0]
265 if exists $_parsedaddrs{$addr};
267 # don't display the warnings from Mail::Address->parse
268 local $SIG{__WARN__} = sub { };
269 @{$_parsedaddrs{$addr}} = Mail::Address->parse($addr);
271 return wantarray?@{$_parsedaddrs{$addr}}:$_parsedaddrs{$addr}[0];
274 =head2 getmaintainers
276 my $maintainer = getmaintainers()->{debbugs}
278 Returns a hashref of package => maintainer pairs.
282 our $_maintainer = undef;
283 our $_maintainer_rev = undef;
285 return $_maintainer if defined $_maintainer;
286 package_maintainer(rehash => 1);
290 =head2 getmaintainers_reverse
292 my @packages = @{getmaintainers_reverse->{'don@debian.org'}||[]};
294 Returns a hashref of maintainer => [qw(list of packages)] pairs.
298 sub getmaintainers_reverse{
299 return $_maintainer_rev if defined $_maintainer_rev;
300 package_maintainer(rehash => 1);
301 return $_maintainer_rev;
304 =head2 package_maintainer
306 my @s = package_maintainer(source => [qw(foo bar baz)],
307 binary => [qw(bleh blah)],
312 =item source -- scalar or arrayref of source package names to return
313 maintainers for, defaults to the empty arrayref.
315 =item binary -- scalar or arrayref of binary package names to return
316 maintainers for; automatically returns source package maintainer if
317 the package name starts with 'src:', defaults to the empty arrayref.
319 =item reverse -- whether to return the source/binary packages a
320 maintainer maintains instead
322 =item rehash -- whether to reread the maintainer and source maintainer
329 our $_source_maintainer = undef;
330 our $_source_maintainer_rev = undef;
331 sub package_maintainer {
332 my %param = validate_with(params => \@_,
333 spec => {source => {type => SCALAR|ARRAYREF,
336 binary => {type => SCALAR|ARRAYREF,
339 maintainer => {type => SCALAR|ARRAYREF,
342 rehash => {type => BOOLEAN,
345 reverse => {type => BOOLEAN,
350 my @binary = make_list($param{binary});
351 my @source = make_list($param{source});
352 my @maintainers = make_list($param{maintainer});
353 if ((@binary or @source) and @maintainers) {
354 croak "It is nonsensical to pass both maintainers and source or binary";
356 if ($param{rehash}) {
357 $_source_maintainer = undef;
358 $_source_maintainer_rev = undef;
359 $_maintainer = undef;
360 $_maintainer_rev = undef;
362 if (not defined $_source_maintainer or
363 not defined $_source_maintainer_rev) {
364 $_source_maintainer = {};
365 $_source_maintainer_rev = {};
366 for my $fn (@config{('source_maintainer_file',
367 'source_maintainer_file_override',
368 'pseudo_maint_file')}) {
369 next unless defined $fn;
371 warn "Missing source maintainer file '$fn'";
374 __add_to_hash($fn,$_source_maintainer,
375 $_source_maintainer_rev);
378 if (not defined $_maintainer or
379 not defined $_maintainer_rev) {
381 $_maintainer_rev = {};
382 for my $fn (@config{('maintainer_file',
383 'maintainer_file_override',
384 'pseudo_maint_file')}) {
385 next unless defined $fn;
387 warn "Missing maintainer file '$fn'";
390 __add_to_hash($fn,$_maintainer,
395 for my $binary (@binary) {
396 if (not $param{reverse} and $binary =~ /^src:/) {
397 push @source,$binary;
400 push @return,grep {defined $_} make_list($_maintainer->{$binary});
402 for my $source (@source) {
403 $source =~ s/^src://;
404 push @return,grep {defined $_} make_list($_source_maintainer->{$source});
406 for my $maintainer (grep {defined $_} @maintainers) {
407 push @return,grep {defined $_}
408 make_list($_maintainer_rev->{$maintainer});
409 push @return,map {$_ !~ /^src:/?'src:'.$_:$_}
411 make_list($_source_maintainer_rev->{$maintainer});
416 #=head2 __add_to_hash
418 # __add_to_hash($file,$forward_hash,$reverse_hash,'address');
420 # Reads a maintainer/source maintainer/pseudo desc file and adds the
421 # maintainers from it to the forward and reverse hashref; assumes that
422 # the forward is unique; makes no assumptions of the reverse.
427 my ($fn,$forward,$reverse,$type) = @_;
428 if (ref($forward) ne 'HASH') {
429 croak "__add_to_hash must be passed a hashref for the forward";
431 if (defined $reverse and not ref($reverse) eq 'HASH') {
432 croak "if reverse is passed to __add_to_hash, it must be a hashref";
435 my $fh = IO::File->new($fn,'r') or
436 die "Unable to open $fn for reading: $!";
437 binmode($fh,':encoding(UTF-8)');
440 next unless m/^(\S+)\s+(\S.*\S)\s*$/;
441 my ($key,$value)=($1,$2);
443 $forward->{$key}= $value;
444 if (defined $reverse) {
445 if ($type eq 'address') {
446 for my $m (map {lc($_->address)} (getparsedaddrs($value))) {
447 push @{$reverse->{$m}},$key;
451 push @{$reverse->{$value}}, $key;
460 my $pseudopkgdesc = getpseudodesc(...);
462 Returns the entry for a pseudo package from the
463 $config{pseudo_desc_file}. In cases where pseudo_desc_file is not
464 defined, returns an empty arrayref.
466 This function can be used to see if a particular package is a
467 pseudopackage or not.
471 our $_pseudodesc = undef;
473 return $_pseudodesc if defined $_pseudodesc;
475 __add_to_hash($config{pseudo_desc_file},$_pseudodesc) if
476 defined $config{pseudo_desc_file};
482 sort_versions('1.0-2','1.1-2');
484 Sorts versions using AptPkg::Versions::compare if it is available, or
485 Debbugs::Versions::Dpkg::vercmp if it isn't.
491 use Debbugs::Versions::Dpkg;
492 $vercmp=\&Debbugs::Versions::Dpkg::vercmp;
494 # eventually we'll use AptPkg:::Version or similar, but the current
495 # implementation makes this *super* difficult.
498 # use AptPkg::Version;
499 # $vercmp=\&AptPkg::Version::compare;
504 return sort {$vercmp->($a,$b)} @_;
510 my $english = secs_to_english($seconds);
511 my ($days,$english) = secs_to_english($seconds);
513 XXX This should probably be changed to use Date::Calc
520 my $days = int($seconds / 86400);
521 my $years = int($days / 365);
525 push @age, "1 year" if ($years == 1);
526 push @age, "$years years" if ($years > 1);
527 push @age, "1 day" if ($days == 1);
528 push @age, "$days days" if ($days > 1);
529 $result .= join(" and ", @age);
531 return wantarray?(int($seconds/86400),$result):$result;
537 These functions are exported with the :lock tag
542 filelock($lockfile,$locks);
544 FLOCKs the passed file. Use unfilelock to unlock it.
546 Can be passed an optional $locks hashref, which is used to track which
547 files are locked (and how many times they have been locked) to allow
548 for cooperative locking.
557 # NB - NOT COMPATIBLE WITH `with-lock'
558 my ($lockfile,$locks) = @_;
559 if ($lockfile !~ m{^/}) {
560 $lockfile = cwd().'/'.$lockfile;
562 # This is only here to allow for relocking bugs inside of
563 # Debbugs::Control. Nothing else should be using it.
564 if (defined $locks and exists $locks->{locks}{$lockfile} and
565 $locks->{locks}{$lockfile} >= 1) {
566 if (exists $locks->{relockable} and
567 exists $locks->{relockable}{$lockfile}) {
568 $locks->{locks}{$lockfile}++;
569 # indicate that the bug for this lockfile needs to be reread
570 $locks->{relockable}{$lockfile} = 1;
571 push @{$locks->{lockorder}},$lockfile;
576 confess "Locking already locked file: $lockfile\n".Data::Dumper->Dump([$lockfile,$locks],[qw(lockfile locks)]);
580 $count= 10; $errors= '';
583 my $fh2 = IO::File->new($lockfile,'w')
584 or die "Unable to open $lockfile for writing: $!";
585 flock($fh2,LOCK_EX|LOCK_NB)
586 or die "Unable to lock $lockfile $!";
593 push @filelocks, {fh => $fh, file => $lockfile};
594 if (defined $locks) {
595 $locks->{locks}{$lockfile}++;
596 push @{$locks->{lockorder}},$lockfile;
603 croak "failed to get lock on $lockfile -- $errors".
604 (defined $locks?Data::Dumper->Dump([$locks],[qw(locks)]):'');
610 # clean up all outstanding locks at end time
623 Unlocks the file most recently locked.
625 Note that it is not currently possible to unlock a specific file
626 locked with filelock.
632 if (@filelocks == 0) {
633 carp "unfilelock called with no active filelocks!\n";
636 if (defined $locks and ref($locks) ne 'HASH') {
637 croak "hash not passsed to unfilelock";
639 if (defined $locks and exists $locks->{lockorder} and
640 @{$locks->{lockorder}} and
641 exists $locks->{locks}{$locks->{lockorder}[-1]}) {
642 my $lockfile = pop @{$locks->{lockorder}};
643 $locks->{locks}{$lockfile}--;
644 if ($locks->{locks}{$lockfile} > 0) {
647 delete $locks->{locks}{$lockfile};
649 my %fl = %{pop(@filelocks)};
650 flock($fl{fh},LOCK_UN)
651 or warn "Unable to unlock lockfile $fl{file}: $!";
653 or warn "Unable to close lockfile $fl{file}: $!";
655 or warn "Unable to unlink lockfile $fl{file}: $!";
661 lockpid('/path/to/pidfile');
663 Creates a pidfile '/path/to/pidfile' if one doesn't exist or if the
664 pid in the file does not respond to kill 0.
666 Returns 1 on success, false on failure; dies on unusual errors.
673 my $pid = checkpid($pidfile);
674 die "Unable to read pidfile $pidfile: $!" if not defined $pid;
675 return 0 if $pid != 0;
677 die "Unable to unlink stale pidfile $pidfile $!";
679 my $pidfh = IO::File->new($pidfile,O_CREAT|O_EXCL|O_WRONLY) or
680 die "Unable to open $pidfile for writing: $!";
681 print {$pidfh} $$ or die "Unable to write to $pidfile $!";
682 close $pidfh or die "Unable to close $pidfile $!";
688 checkpid('/path/to/pidfile');
690 Checks a pid file and determines if the process listed in the pidfile
691 is still running. Returns the pid if it is, 0 if it isn't running, and
692 undef if the pidfile doesn't exist or cannot be read.
699 my $pidfh = IO::File->new($pidfile, 'r') or
704 ($pid) = $pid =~ /(\d+)/;
705 if (defined $pid and kill(0,$pid)) {
718 These functions are exported with the :quit tag.
724 Exits the program by calling die.
726 Usage of quit is deprecated; just call die instead.
731 print {$DEBUG_FH} "quitting >$_[0]<\n" if $DEBUG;
732 carp "quit() is deprecated; call die directly instead";
738 These functions are exported with the :misc tag
742 LIST = make_list(@_);
744 Turns a scalar or an arrayref into a list; expands a list of arrayrefs
747 That is, make_list([qw(a b c)]); returns qw(a b c); make_list([qw(a
748 b)],[qw(c d)] returns qw(a b c d);
753 return map {(ref($_) eq 'ARRAY')?@{$_}:$_} @_;
759 print english_join(list => \@list);
760 print english_join(\@list);
762 Joins list properly to make an english phrase.
766 =item normal -- how to separate most values; defaults to ', '
768 =item last -- how to separate the last two values; defaults to ', and '
770 =item only_two -- how to separate only two values; defaults to ' and '
772 =item list -- ARRAYREF values to join; if the first argument is an
773 ARRAYREF, it's assumed to be the list of values to join
777 In cases where C<list> is empty, returns ''; when there is only one
778 element, returns that element.
783 if (ref $_[0] eq 'ARRAY') {
784 return english_join(list=>$_[0]);
786 my %param = validate_with(params => \@_,
787 spec => {normal => {type => SCALAR,
790 last => {type => SCALAR,
793 only_two => {type => SCALAR,
796 list => {type => ARRAYREF,
800 my @list = @{$param{list}};
802 return @list?$list[0]:'';
805 return join($param{only_two},@list);
807 my $ret = $param{last} . pop(@list);
808 return join($param{normal},@list) . $ret;
812 =head2 globify_scalar
814 my $handle = globify_scalar(\$foo);
816 if $foo isn't already a glob or a globref, turn it into one using
817 IO::Scalar. Gives a new handle to /dev/null if $foo isn't defined.
819 Will carp if given a scalar which isn't a scalarref or a glob (or
820 globref), and return /dev/null. May return undef if IO::Scalar or
821 IO::File fails. (Check $!)
828 if (defined $scalar) {
829 if (defined ref($scalar)) {
830 if (ref($scalar) eq 'SCALAR' and
831 not UNIVERSAL::isa($scalar,'GLOB')) {
832 open $handle, '>:scalar:utf8', $scalar;
839 elsif (UNIVERSAL::isa(\$scalar,'GLOB')) {
843 carp "Given a non-scalar reference, non-glob to globify_scalar; returning /dev/null handle";
846 return IO::File->new('/dev/null','>:utf8');
849 =head2 cleanup_eval_fail()
851 print "Something failed with: ".cleanup_eval_fail($@);
853 Does various bits of cleanup on the failure message from an eval (or
854 any other die message)
856 Takes at most two options; the first is the actual failure message
857 (usually $@ and defaults to $@), the second is the debug level
858 (defaults to $DEBUG).
860 If debug is non-zero, the code at which the failure occured is output.
864 sub cleanup_eval_fail {
865 my ($error,$debug) = @_;
866 if (not defined $error or not @_) {
867 $error = $@ // 'unknown reason';
870 $debug = $DEBUG // 0;
872 $debug = 0 if not defined $debug;
877 # ditch the "at foo/bar/baz.pm line 5"
878 $error =~ s/\sat\s\S+\sline\s\d+//;
879 # ditch croak messages
880 $error =~ s/^\t+.+\n?//g;
881 # ditch trailing multiple periods in case there was a cascade of
883 $error =~ s/\.+$/\./;
889 hash_slice(%hash,qw(key1 key2 key3))
891 For each key, returns matching values and keys of the hash if they exist
896 # NB: We use prototypes here SPECIFICALLY so that we can be passed a
897 # hash without uselessly making a reference to first. DO NOT USE
898 # PROTOTYPES USELESSLY ELSEWHERE.
899 sub hash_slice(\%@) {
900 my ($hashref,@keys) = @_;
901 return map {exists $hashref->{$_}?($_,$hashref->{$_}):()} @keys;
907 These functions are exported with the :utf8 tag
909 =head2 encode_utf8_structure
911 %newdata = encode_utf8_structure(%newdata);
913 Takes a complex data structure and encodes any strings with is_utf8
914 set into their constituent octets.
919 sub encode_utf8_structure {
923 if (ref($_) eq 'HASH') {
924 push @ret, {encode_utf8_structure(%{$depth == 1 ? dclone($_):$_})};
926 elsif (ref($_) eq 'ARRAY') {
927 push @ret, [encode_utf8_structure(@{$depth == 1 ? dclone($_):$_})];
930 # we don't know how to handle non hash or non arrays
934 push @ret,encode_utf8_safely($_);
941 =head2 encode_utf8_safely
943 $octets = encode_utf8_safely($string);
945 Given a $string, returns the octet equivalent of $string if $string is
946 in perl's internal encoding; otherwise returns $string.
948 Silently returns REFs without encoding them. [If you want to deeply
949 encode REFs, see encode_utf8_structure.]
954 sub encode_utf8_safely{
957 if (not ref($r) and is_utf8($r)) {
958 $r = encode_utf8($r);
962 return wantarray ? @ret : (length @_ > 1 ? @ret : $_[0]);
965 =head2 convert_to_utf8
967 $utf8 = convert_to_utf8("text","charset");
971 our %iconv_converters;
973 sub convert_to_utf8 {
974 my ($data,$charset) = @_;
975 if (is_utf8($data)) {
976 return encode_utf8($data);
978 $charset = uc($charset);
979 if (not defined $iconv_converters{$charset}) {
981 $iconv_converters{$charset} = Text::Iconv->new($charset,"UTF-8") or
982 die "Unable to create converter for '$charset'";
986 # We weren't able to create the converter, so use Encode
988 return __fallback_convert_to_utf8($data,$charset);
990 # It shouldn't be necessary when converting to UTF8, but lets
991 # allow for transliteration and silent discarding of broken
994 $iconv_converters{$charset}->set_attr("transliterate");
995 $iconv_converters{$charset}->set_attr("discard_ilseq");
997 # This shouldn't fail on Debian systems; we're warning here
998 # just in case we've made a mistake above. This warning should
999 # probably be disabled on non-GNU libc systems.
1002 if (not defined $iconv_converters{$charset}) {
1003 warn "The converter for $charset wasn't created properly somehow!";
1004 return __fallback_convert_to_utf8($data,$charset);
1006 my $converted_data = $iconv_converters{$charset}->convert($data);
1007 # if the conversion failed, retval will be undefined or perhaps
1009 if (not defined $iconv_converters{$charset}->retval() or
1010 $iconv_converters{$charset}->retval() < 0
1012 # Fallback to encode, which will probably also fail.
1013 return __fallback_convert_to_utf8($data,$charset);
1015 return $converted_data;
1019 # we're switching this to return UTF8 octets instead of perl's internal
1021 sub __Fallback_convert_to_utf8 {
1022 my ($data, $charset) = @_;
1023 # raw data just gets returned (that's the charset WordDecorder
1024 # uses when it doesn't know what to do)
1025 return $data if $charset eq 'raw';
1026 if (not defined $charset and not is_utf8($data)) {
1027 warn ("Undefined charset, and string '$data' is not in perl's internal encoding");
1030 # lets assume everything that doesn't have a charset is utf8
1031 $charset //= 'utf8';
1034 $result = decode($charset,$data) unless is_utf8($data);
1035 $result = encode_utf8($result);
1038 warn "Unable to decode charset; '$charset' and '$data': $@";