X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=Debbugs%2FCommon.pm;h=abc98dc6efe6f5d65191ae3f012004c4ff8891f1;hb=593feb86562b711efd71e4d03239e188ddeea051;hp=0917c6570b7860e0098a1309617c5dc3846788c9;hpb=02e70ed5c05c0ed436e2f6e51ea2122e8fc0d9d8;p=debbugs.git diff --git a/Debbugs/Common.pm b/Debbugs/Common.pm index 0917c65..abc98dc 100644 --- a/Debbugs/Common.pm +++ b/Debbugs/Common.pm @@ -39,27 +39,30 @@ BEGIN{ @EXPORT = (); %EXPORT_TAGS = (util => [qw(getbugcomponent getbuglocation getlocationpath get_hashname), - qw(appendfile buglog getparsedaddrs getmaintainers), + qw(appendfile overwritefile buglog getparsedaddrs getmaintainers), qw(bug_status), qw(getmaintainers_reverse), qw(getpseudodesc), qw(package_maintainer), + qw(sort_versions), ], misc => [qw(make_list globify_scalar english_join checkpid), qw(cleanup_eval_fail), + qw(hash_slice), ], date => [qw(secs_to_english)], quit => [qw(quit)], - lock => [qw(filelock unfilelock lockpid)], + lock => [qw(filelock unfilelock lockpid simple_filelock simple_unlockfile)], ); @EXPORT_OK = (); - Exporter::export_ok_tags(qw(lock quit date util misc)); + Exporter::export_ok_tags(keys %EXPORT_TAGS); $EXPORT_TAGS{all} = [@EXPORT_OK]; } #use Debbugs::Config qw(:globals); use Carp; +$Carp::Verbose = 1; use Debbugs::Config qw(:config); use IO::File; @@ -67,10 +70,13 @@ use IO::Scalar; use Debbugs::MIME qw(decode_rfc1522); use Mail::Address; use Cwd qw(cwd); +use Storable qw(dclone); +use Time::HiRes qw(usleep); use Params::Validate qw(validate_with :types); -use Fcntl qw(:flock); +use Fcntl qw(:DEFAULT :flock); +use Encode qw(is_utf8 decode_utf8); our $DEBUG_FH = \*STDERR if not defined $DEBUG_FH; @@ -215,6 +221,28 @@ sub appendfile { close $fh or die "Unable to close $file: $!"; } +=head2 overwritefile + + ovewritefile($file,'data','to','append'); + +Opens file.new, writes data to it, then moves file.new to file. + +=cut + +sub overwritefile { + my ($file,@data) = @_; + my $fh = IO::File->new("${file}.new",'w') or + die "Unable top open ${file}.new for writing: $!"; + print {$fh} @data or die "Unable to write to ${file}.new: $!"; + close $fh or die "Unable to close ${file}.new: $!"; + rename("${file}.new",$file) or + die "Unable to rename ${file}.new to $file: $!"; +} + + + + + =head2 getparsedaddrs my $address = getparsedaddrs($address); @@ -306,6 +334,9 @@ sub package_maintainer { binary => {type => SCALAR|ARRAYREF, default => [], }, + maintainer => {type => SCALAR|ARRAYREF, + default => [], + }, rehash => {type => BOOLEAN, default => 0, }, @@ -314,6 +345,12 @@ sub package_maintainer { }, }, ); + my @binary = make_list($param{binary}); + my @source = make_list($param{source}); + my @maintainers = make_list($param{maintainer}); + if ((@binary or @source) and @maintainers) { + croak "It is nonsensical to pass both maintainers and source or binary"; + } if ($param{rehash}) { $_source_maintainer = undef; $_source_maintainer_rev = undef; @@ -326,8 +363,12 @@ sub package_maintainer { $_source_maintainer_rev = {}; for my $fn (@config{('source_maintainer_file', 'source_maintainer_file_override', - 'pseduo_maint_file')}) { + 'pseudo_maint_file')}) { next unless defined $fn; + if (not -e $fn) { + warn "Missing source maintainer file '$fn'"; + next; + } __add_to_hash($fn,$_source_maintainer, $_source_maintainer_rev); } @@ -338,25 +379,34 @@ sub package_maintainer { $_maintainer_rev = {}; for my $fn (@config{('maintainer_file', 'maintainer_file_override', - 'pseduo_maint_file')}) { + 'pseudo_maint_file')}) { next unless defined $fn; + if (not -e $fn) { + warn "Missing maintainer file '$fn'"; + next; + } __add_to_hash($fn,$_maintainer, $_maintainer_rev); } } my @return; - my @extra_source; - my $b = $param{reverse}?$_maintainer_rev:$_maintainer; - for my $binary (make_list($param{binary})) { + for my $binary (@binary) { if (not $param{reverse} and $binary =~ /^src:/) { - push @extra_source,$binary; + push @source,$binary; next; } - push @return,grep {defined $_} make_list($b->{$binary}); + push @return,grep {defined $_} make_list($_maintainer->{$binary}); + } + for my $source (@source) { + $source =~ s/^src://; + push @return,grep {defined $_} make_list($_source_maintainer->{$source}); } - my $s = $param{reverse}?$_source_maintainer_rev:$_source_maintainer; - for my $source (make_list($param{source},@extra_source)) { - push @return,grep {defined $_} make_list($s->{$source}); + for my $maintainer (grep {defined $_} @maintainers) { + push @return,grep {defined $_} + make_list($_maintainer_rev->{$maintainer}); + push @return,map {$_ !~ /^src:/?'src:'.$_:$_} + grep {defined $_} + make_list($_source_maintainer_rev->{$maintainer}); } return @return; } @@ -382,10 +432,14 @@ sub __add_to_hash { $type //= 'address'; my $fh = IO::File->new($fn,'r') or die "Unable to open $fn for reading: $!"; + binmode($fh,':encoding(UTF-8)'); while (<$fh>) { chomp; - next unless m/^(\S+)\s+(\S.*\S)\s*$/; - my ($key,$value)=($1,$2); + my @elements = split /\t/; + next unless @elements >=2; + # we do this because the source maintainer file contains the + # archive location, which we don't care about + my ($key,$value)=($elements[0],$elements[-1]); $key = lc $key; $forward->{$key}= $value; if (defined $reverse) { @@ -424,6 +478,33 @@ sub getpseudodesc { return $_pseudodesc; } +=head2 sort_versions + + sort_versions('1.0-2','1.1-2'); + +Sorts versions using AptPkg::Versions::compare if it is available, or +Debbugs::Versions::Dpkg::vercmp if it isn't. + +=cut + +our $vercmp; +BEGIN{ + use Debbugs::Versions::Dpkg; + $vercmp=\&Debbugs::Versions::Dpkg::vercmp; + +# eventually we'll use AptPkg:::Version or similar, but the current +# implementation makes this *super* difficult. + +# eval { +# use AptPkg::Version; +# $vercmp=\&AptPkg::Version::compare; +# }; +} + +sub sort_versions{ + return sort {$vercmp->($a,$b)} @_; +} + =head1 DATE @@ -458,43 +539,115 @@ These functions are exported with the :lock tag =head2 filelock - filelock + filelock($lockfile); + filelock($lockfile,$locks); FLOCKs the passed file. Use unfilelock to unlock it. +Can be passed an optional $locks hashref, which is used to track which +files are locked (and how many times they have been locked) to allow +for cooperative locking. + =cut our @filelocks; +use Carp qw(cluck); + sub filelock { # NB - NOT COMPATIBLE WITH `with-lock' - my ($lockfile) = @_; + my ($lockfile,$locks) = @_; if ($lockfile !~ m{^/}) { $lockfile = cwd().'/'.$lockfile; } - my ($count,$errors); - $count= 10; $errors= ''; - for (;;) { - my $fh = eval { + # This is only here to allow for relocking bugs inside of + # Debbugs::Control. Nothing else should be using it. + if (defined $locks and exists $locks->{locks}{$lockfile} and + $locks->{locks}{$lockfile} >= 1) { + if (exists $locks->{relockable} and + exists $locks->{relockable}{$lockfile}) { + $locks->{locks}{$lockfile}++; + # indicate that the bug for this lockfile needs to be reread + $locks->{relockable}{$lockfile} = 1; + push @{$locks->{lockorder}},$lockfile; + return; + } + else { + use Data::Dumper; + confess "Locking already locked file: $lockfile\n".Data::Dumper->Dump([$lockfile,$locks],[qw(lockfile locks)]); + } + } + my ($fh,$t_lockfile,$errors) = + simple_filelock($lockfile,10,1); + if ($fh) { + push @filelocks, {fh => $fh, file => $lockfile}; + if (defined $locks) { + $locks->{locks}{$lockfile}++; + push @{$locks->{lockorder}},$lockfile; + } + } else { + use Data::Dumper; + croak "failed to get lock on $lockfile -- $errors". + (defined $locks?Data::Dumper->Dump([$locks],[qw(locks)]):''); + } +} + +=head2 simple_filelock + + my ($fh,$t_lockfile,$errors) = + simple_filelock($lockfile,$count,$wait); + +Does a flock of lockfile. If C<$count> is zero, does a blocking lock. +Otherwise, does a non-blocking lock C<$count> times, waiting C<$wait> +seconds in between. + +In list context, returns the lockfile filehandle, lockfile name, and +any errors which occured. + +When the lockfile filehandle is undef, locking failed. + +These lockfiles must be unlocked manually at process end. + + +=cut + +sub simple_filelock { + my ($lockfile,$count,$wait) = @_; + if (not defined $count) { + $count = 10; + } + if ($count < 0) { + $count = 0; + } + if (not defined $wait) { + $wait = 1; + } + my $errors= ''; + my $fh; + while (1) { + $fh = eval { my $fh2 = IO::File->new($lockfile,'w') or die "Unable to open $lockfile for writing: $!"; - flock($fh2,LOCK_EX|LOCK_NB) + # Do a blocking lock if count is zero + flock($fh2,LOCK_EX|($count == 0?0:LOCK_NB)) or die "Unable to lock $lockfile $!"; return $fh2; }; if ($@) { $errors .= $@; } - if ($fh) { - push @filelocks, {fh => $fh, file => $lockfile}; - last; - } - if (--$count <=0) { - $errors =~ s/\n+$//; - die "failed to get lock on $lockfile -- $errors"; + if ($fh) { + last; } - sleep 10; + # use usleep for fractional wait seconds + usleep($wait * 1_000_000); + } continue { + last unless (--$count > 0); + } + if ($fh) { + return wantarray?($fh,$lockfile,$errors):$fh } + return wantarray?(undef,$lockfile,$errors):undef; } # clean up all outstanding locks at end time @@ -504,10 +657,28 @@ END { } } +=head2 simple_unlockfile + + simple_unlockfile($fh,$lockfile); + + +=cut + +sub simple_unlockfile { + my ($fh,$lockfile) = @_; + flock($fh,LOCK_UN) + or warn "Unable to unlock lockfile $lockfile: $!"; + close($fh) + or warn "Unable to close lockfile $lockfile: $!"; + unlink($lockfile) + or warn "Unable to unlink lockfile $lockfile: $!"; +} + =head2 unfilelock unfilelock() + unfilelock($locks); Unlocks the file most recently locked. @@ -517,17 +688,26 @@ locked with filelock. =cut sub unfilelock { + my ($locks) = @_; if (@filelocks == 0) { - warn "unfilelock called with no active filelocks!\n"; + carp "unfilelock called with no active filelocks!\n"; return; } + if (defined $locks and ref($locks) ne 'HASH') { + croak "hash not passsed to unfilelock"; + } + if (defined $locks and exists $locks->{lockorder} and + @{$locks->{lockorder}} and + exists $locks->{locks}{$locks->{lockorder}[-1]}) { + my $lockfile = pop @{$locks->{lockorder}}; + $locks->{locks}{$lockfile}--; + if ($locks->{locks}{$lockfile} > 0) { + return + } + delete $locks->{locks}{$lockfile}; + } my %fl = %{pop(@filelocks)}; - flock($fl{fh},LOCK_UN) - or warn "Unable to unlock lockfile $fl{file}: $!"; - close($fl{fh}) - or warn "Unable to close lockfile $fl{file}: $!"; - unlink($fl{file}) - or warn "Unable to unlink lockfile $fl{file}: $!"; + simple_unlockfile($fl{fh},$fl{file}); } @@ -551,7 +731,7 @@ sub lockpid { unlink $pidfile or die "Unable to unlink stale pidfile $pidfile $!"; } - my $pidfh = IO::File->new($pidfile,'w') or + my $pidfh = IO::File->new($pidfile,O_CREAT|O_EXCL|O_WRONLY) or die "Unable to open $pidfile for writing: $!"; print {$pidfh} $$ or die "Unable to write to $pidfile $!"; close $pidfh or die "Unable to close $pidfile $!"; @@ -695,6 +875,10 @@ Will carp if given a scalar which isn't a scalarref or a glob (or globref), and return /dev/null. May return undef if IO::Scalar or IO::File fails. (Check $!) +The scalar will fill with octets, not perl's internal encoding, so you +must use decode_utf8() after on the scalar, and encode_utf8() on it +before. This appears to be a bug in the underlying modules. + =cut sub globify_scalar { @@ -704,7 +888,12 @@ sub globify_scalar { if (defined ref($scalar)) { if (ref($scalar) eq 'SCALAR' and not UNIVERSAL::isa($scalar,'GLOB')) { - return IO::Scalar->new($scalar); + if (is_utf8(${$scalar})) { + ${$scalar} = decode_utf8(${$scalar}); + carp(q(\$scalar must not be in perl's internal encoding)); + } + open $handle, '>:scalar:utf8', $scalar; + return $handle; } else { return $scalar; @@ -717,7 +906,7 @@ sub globify_scalar { carp "Given a non-scalar reference, non-glob to globify_scalar; returning /dev/null handle"; } } - return IO::File->new('/dev/null','w'); + return IO::File->new('/dev/null','>:encoding(UTF-8)'); } =head2 cleanup_eval_fail() @@ -750,12 +939,31 @@ sub cleanup_eval_fail { } # ditch the "at foo/bar/baz.pm line 5" $error =~ s/\sat\s\S+\sline\s\d+//; + # ditch croak messages + $error =~ s/^\t+.+\n?//g; # ditch trailing multiple periods in case there was a cascade of # die messages. $error =~ s/\.+$/\./; return $error; } +=head2 hash_slice + + hash_slice(%hash,qw(key1 key2 key3)) + +For each key, returns matching values and keys of the hash if they exist + +=cut + + +# NB: We use prototypes here SPECIFICALLY so that we can be passed a +# hash without uselessly making a reference to first. DO NOT USE +# PROTOTYPES USELESSLY ELSEWHERE. +sub hash_slice(\%@) { + my ($hashref,@keys) = @_; + return map {exists $hashref->{$_}?($_,$hashref->{$_}):()} @keys; +} + 1;