X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=Debbugs%2FPackages.pm;h=2cdef213fce3b89abec6349e5e909d1166fb97a7;hb=cd0b9752cb6b5cec796baf1c07951cee5971b42e;hp=88dc8b47a859dd081b62487e787cea7b580814c6;hpb=8e65acb57543b68313d883d47f85167192f4d9e7;p=debbugs.git diff --git a/Debbugs/Packages.pm b/Debbugs/Packages.pm index 88dc8b4..2cdef21 100644 --- a/Debbugs/Packages.pm +++ b/Debbugs/Packages.pm @@ -12,7 +12,7 @@ package Debbugs::Packages; use warnings; use strict; -use base qw(Exporter); +use Exporter qw(import); use vars qw($VERSION @EXPORT_OK %EXPORT_TAGS @EXPORT); use Carp; @@ -25,7 +25,8 @@ BEGIN { @EXPORT = (); %EXPORT_TAGS = (versions => [qw(getversions get_versions make_source_versions)], mapping => [qw(getpkgsrc getpkgcomponent getsrcpkgs), - qw(binary_to_source sourcetobinary makesourceversions) + qw(binary_to_source sourcetobinary makesourceversions), + qw(source_to_binary), ], ); @EXPORT_OK = (); @@ -37,9 +38,9 @@ use Fcntl qw(O_RDONLY); use MLDBM qw(DB_File Storable); use Storable qw(dclone); use Params::Validate qw(validate_with :types); -use Debbugs::Common qw(make_list globify_scalar); - -use List::Util qw(min max); +use Debbugs::Common qw(make_list globify_scalar sort_versions); +use DateTime::Format::Pg; +use List::AllUtils qw(min max uniq); use IO::File; @@ -72,13 +73,14 @@ our $_pkgcomponent; our $_srcpkg; sub getpkgsrc { return $_pkgsrc if $_pkgsrc; - return {} unless defined $Debbugs::Packages::gPackageSource; + return {} unless defined $config{package_source} and + length $config{package_source}; my %pkgsrc; my %pkgcomponent; my %srcpkg; my $fh = IO::File->new($config{package_source},'r') - or die("Unable to open $config{package_source} for reading: $!"); + or croak("Unable to open $config{package_source} for reading: $!"); while(<$fh>) { next unless m/^(\S+)\s+(\S+)\s+(\S.*\S)\s*$/; my ($bin,$cmp,$src)=($1,$2,$3); @@ -168,7 +170,19 @@ binary_to_source. # the two global variables below are used to tie the source maps; we # probably should be retying them in long lived processes. our %_binarytosource; +sub _tie_binarytosource { + if (not tied %_binarytosource) { + tie %_binarytosource, MLDBM => $config{binary_source_map}, O_RDONLY or + die "Unable to open $config{binary_source_map} for reading"; + } +} our %_sourcetobinary; +sub _tie_sourcetobinary { + if (not tied %_sourcetobinary) { + tie %_sourcetobinary, MLDBM => $config{source_binary_map}, O_RDONLY or + die "Unable to open $config{source_binary_map} for reading"; + } +} sub binary_to_source{ my %param = validate_with(params => \@_, spec => {binary => {type => SCALAR|ARRAYREF, @@ -186,12 +200,15 @@ sub binary_to_source{ cache => {type => HASHREF, default => {}, }, + schema => {type => OBJECT, + optional => 1, + }, }, ); # TODO: This gets hit a lot, especially from buggyversion() - probably # need an extra cache for speed here. - return () unless defined $gBinarySourceMap; + return () unless defined $gBinarySourceMap or defined $param{schema}; if ($param{scalar_only} or not wantarray) { $param{source_only} = 1; @@ -203,6 +220,7 @@ sub binary_to_source{ my @versions = grep {defined $_} make_list(exists $param{version}?$param{version}:[]); my @archs = grep {defined $_} make_list(exists $param{arch}?$param{arch}:[]); return () unless @binaries; + my $cache_key = join("\1", join("\0",@binaries), join("\0",@versions), @@ -212,19 +230,75 @@ sub binary_to_source{ return $param{scalar_only} ? $param{cache}{$cache_key}[0]: @{$param{cache}{$cache_key}}; } - for my $binary (@binaries) { - if ($binary =~ m/^src:(.+)$/) { - push @source,[$1,'']; - next; + # any src:foo is source package foo with unspecified version + @source = map {/^src:(.+)$/? + [$1,'']:()} @binaries; + @binaries = grep {$_ !~ /^src:/} @binaries; + if ($param{schema}) { + if ($param{source_only}) { + @source = map {$_->[0]} @source; + my $src_rs = $param{schema}->resultset('SrcPkg')-> + search_rs({'bin_pkg.pkg' => [@binaries], + @versions?('bin_vers.ver' => [@versions]):(), + @archs?('arch.arch' => [@archs]):(), + }, + {join => {'src_vers'=> + {'bin_vers'=> ['arch','bin_pkg']} + }, + columns => [qw(pkg)], + order_by => [qw(pkg)], + result_class => 'DBIx::Class::ResultClass::HashRefInflator', + distinct => 1, + }, + ); + push @source, + map {$_->{pkg}} $src_rs->all; + if ($param{scalar_only}) { + @source = join(',',@source); + } + $param{cache}{$cache_key} = \@source; + return $param{scalar_only}?$source[0]:@source; } - if (not tied %_binarytosource) { - tie %_binarytosource, MLDBM => $config{binary_source_map}, O_RDONLY or - die "Unable to open $config{binary_source_map} for reading"; + my $src_rs = $param{schema}->resultset('SrcVer')-> + search_rs({'bin_pkg.pkg' => [@binaries], + @versions?('bin_vers.ver' => [@versions]):(), + @archs?('arch.arch' => [@archs]):(), + }, + {join => ['src_pkg', + {'bin_vers' => ['arch','binpkg']}, + ], + columns => ['src_pkg.pkg','src_ver.ver'], + result_class => 'DBIx::Class::ResultClass::HashRefInflator', + order_by => ['src_pkg.pkg','src_ver.ver'], + distinct => 1, + }, + ); + push @source, + map {[$_->{src_pkg}{pkg}, + $_->{src_ver}{ver}, + ]} $src_rs->all; + if (not @source and not @versions and not @archs) { + $src_rs = $param{schema}->resultset('SrcPkg')-> + search_rs({pkg => [@binaries]}, + {join => ['src_vers'], + columns => ['src_pkg.pkg','src_vers.ver'], + distinct => 1, + }, + ); + push @source, + map {[$_->{src_pkg}{pkg}, + $_->{src_vers}{ver}, + ]} $src_rs->all; } + $param{cache}{$cache_key} = \@source; + return $param{scalar_only}?$source[0]:@source; + } + for my $binary (@binaries) { + _tie_binarytosource; # avoid autovivification my $bin = $_binarytosource{$binary}; + next unless defined $bin; if (not @versions) { - next unless defined $bin; for my $ver (keys %{$bin}) { for my $ar (keys %{$bin->{$ver}}) { my $src = $bin->{$ver}{$ar}; @@ -234,7 +308,6 @@ sub binary_to_source{ } } else { - my $found_one_version = 0; for my $version (@versions) { next unless exists $bin->{$version}; if (exists $bin->{$version}{all}) { @@ -264,11 +337,7 @@ sub binary_to_source{ # if any the packages we've been given are a valid source # package name, and there's no binary of the same name (we got # here, so there isn't), return it. - - if (not tied %_sourcetobinary) { - tie %_sourcetobinary, MLDBM => $config{source_binary_map}, O_RDONLY or - die "Unable top open $gSourceBinaryMap for reading"; - } + _tie_sourcetobinary(); for my $maybe_sourcepkg (@binaries) { if (exists $_sourcetobinary{$maybe_sourcepkg}) { push @source,[$maybe_sourcepkg,$_] for keys %{$_sourcetobinary{$maybe_sourcepkg}}; @@ -283,6 +352,9 @@ sub binary_to_source{ if ($param{source_only}) { my %uniq; for my $s (@source) { + # we shouldn't need to do this, but do this temporarily to + # stop the warning. + next unless defined $s->[0]; $uniq{$s->[0]} = 1; } @result = sort keys %uniq; @@ -306,6 +378,247 @@ sub binary_to_source{ return $param{scalar_only} ? $result[0] : @result; } +=head2 source_to_binary + + source_to_binary(package => 'foo', + version => '1.2.3', + arch => 'i386'); + + +Turn a source package (at optional version) into a single (or set) of all binary +packages (optionally) with associated versions. + +By default, in LIST context, returns a LIST of array refs of binary package, +binary version, architecture triples corresponding to the source package(s) and +verion(s) passed. + +In SCALAR context, only the corresponding binary packages are returned, +concatenated with ', ' if necessary. + +If no binaries can be found, returns undef in scalar context, or the +empty list in list context. + +=over + +=item source -- source package name(s) as a SCALAR or ARRAYREF + +=item version -- binary package version(s) as a SCALAR or ARRAYREF; +optional, defaults to all versions. + +=item dist -- list of distributions to return corresponding binary packages for +as a SCALAR or ARRAYREF. + +=item binary_only -- return only the source name (forced on if in SCALAR +context), defaults to false. [If in LIST context, returns a list of binary +names.] + +=item scalar_only -- return a scalar only (forced true if in SCALAR +context, also causes binary_only to be true), defaults to false. + +=item cache -- optional HASHREF to be used to cache results of +binary_to_source. + +=back + +=cut + +# the two global variables below are used to tie the source maps; we +# probably should be retying them in long lived processes. +sub source_to_binary{ + my %param = validate_with(params => \@_, + spec => {source => {type => SCALAR|ARRAYREF, + }, + version => {type => SCALAR|ARRAYREF, + optional => 1, + }, + dist => {type => SCALAR|ARRAYREF, + optional => 1, + }, + binary_only => {default => 0, + }, + scalar_only => {default => 0, + }, + cache => {type => HASHREF, + default => {}, + }, + schema => {type => OBJECT, + optional => 1, + }, + }, + ); + if (not defined $config{source_binary_map} and + not defined $param{schema} + ) { + return (); + } + + if ($param{scalar_only} or not wantarray) { + $param{binary_only} = 1; + $param{scalar_only} = 1; + } + + my @binaries; + my @sources = sort grep {defined $_} + make_list(exists $param{source}?$param{source}:[]); + my @versions = sort grep {defined $_} + make_list(exists $param{version}?$param{version}:[]); + return () unless @sources; + + # any src:foo is source package foo with unspecified version + @sources = map {s/^src://; $_} @sources; + if ($param{schema}) { + if ($param{binary_only}) { + my $bin_rs = $param{schema}->resultset('BinPkg')-> + search_rs({'src_pkg.pkg' => [@sources], + @versions?('src_ver.ver' => [@versions]):(), + }, + {join => {'bin_vers'=> + {'src_ver'=> 'src_pkg'} + }, + columns => [qw(pkg)], + order_by => [qw(pkg)], + result_class => 'DBIx::Class::ResultClass::HashRefInflator', + distinct => 1, + }, + ); + if (exists $param{dist}) { + $bin_rs = $bin_rs-> + search({-or => + {'suite.codename' => [make_list($param{dist})], + 'suite.suite_name' => [make_list($param{dist})], + }}, + {join => {'bin_vers' => + {'bin_associations' => + 'suite' + }}, + }); + } + push @binaries, + map {$_->{pkg}} $bin_rs->all; + if ($param{scalar_only}) { + return join(', ',@binaries); + } + return @binaries; + + } + my $src_rs = $param{schema}->resultset('BinVer')-> + search_rs({'src_pkg.pkg' => [@sources], + @versions?('src_ver.ver' => [@versions]):(), + }, + {join => ['bin_pkg', + 'arch', + {'src_ver' => ['src_pkg']}, + ], + columns => ['src_pkg.pkg','src_ver.ver','arch.arch'], + order_by => ['src_pkg.pkg','src_ver.ver','arch.arch'], + result_class => 'DBIx::Class::ResultClass::HashRefInflator', + distinct => 1, + }, + ); + push @binaries, + map {[$_->{src_pkg}{pkg}, + $_->{src_ver}{ver}, + $_->{arch}{arch}, + ]} + $src_rs->all; + if (not @binaries and not @versions) { + $src_rs = $param{schema}->resultset('BinPkg')-> + search_rs({pkg => [@sources]}, + {join => {'bin_vers' => + ['arch', + {'src_ver'=>'src_pkg'}], + }, + distinct => 1, + result_class => 'DBIx::Class::ResultClass::HashRefInflator', + columns => ['src_pkg.pkg','src_ver.ver','arch.arch'], + order_by => ['src_pkg.pkg','src_ver.ver','arch.arch'], + }, + ); + push @binaries, + map {[$_->{src_pkg}{pkg}, + $_->{src_ver}{ver}, + $_->{arch}{arch}, + ]} $src_rs->all; + } + return @binaries; + } + my $cache_key = join("\1", + join("\0",@sources), + join("\0",@versions), + join("\0",@param{qw(binary_only scalar_only)})); + if (exists $param{cache}{$cache_key}) { + return $param{scalar_only} ? $param{cache}{$cache_key}[0]: + @{$param{cache}{$cache_key}}; + } + my @return; + my %binaries; + if ($param{binary_only}) { + for my $source (@sources) { + _tie_sourcetobinary; + # avoid autovivification + my $src = $_sourcetobinary{$source}; + if (not defined $src) { + next if @versions; + _tie_binarytosource; + if (exists $_binarytosource{$source}) { + $binaries{$source} = 1; + } + next; + } + my @src_vers = @versions; + if (not @versions) { + @src_vers = keys %{$src}; + } + for my $ver (@src_vers) { + $binaries{$_->[0]} = 1 + foreach @{$src->{$ver}//[]}; + } + } + # return if we have any results. + @return = sort keys %binaries; + if ($param{scalar_only}) { + @return = join(', ',@return); + } + goto RETURN_RESULT; + } + for my $source (@sources) { + _tie_sourcetobinary; + my $src = $_sourcetobinary{$source}; + # there isn't a source package, so return this as a binary packages if a + # version hasn't been specified + if (not defined $src) { + next if @versions; + _tie_binarytosource; + if (exists $_binarytosource{$source}) { + my $bin = $_binarytosource{$source}; + for my $ver (keys %{$bin}) { + for my $arch (keys %{$bin->{$ver}}) { + $binaries{$bin}{$ver}{$arch} = 1; + } + } + } + next; + } + for my $bin_ver_archs (values %{$src}) { + for my $bva (@{$bin_ver_archs}) { + $binaries{$bva->[0]}{$bva->[1]}{$bva->[2]} = 1; + } + } + } + for my $bin (sort keys %binaries) { + for my $ver (sort keys %{$binaries{$bin}}) { + for my $arch (sort keys %{$binaries{$bin}{$ver}}) { + push @return, + [$bin,$ver,$arch]; + } + } + } +RETURN_RESULT: + $param{cache}{$cache_key} = \@return; + return $param{scalar_only} ? $return[0] : @return; +} + + =head2 sourcetobinary Returns a list of references to triplets of binary package names, versions, @@ -319,14 +632,7 @@ returned, without the architecture. sub sourcetobinary { my ($srcname, $srcver) = @_; - - if (not tied %_sourcetobinary) { - tie %_sourcetobinary, MLDBM => $config{source_binary_map}, O_RDONLY or - die "Unable top open $config{source_binary_map} for reading"; - } - - - + _tie_sourcetobinary; # avoid autovivification my $source = $_sourcetobinary{$srcname}; return () unless defined $source; @@ -390,6 +696,11 @@ may change in the future, so if you care, please code accordingly.) =item return_archs -- returns a version=>[archs] hash indicating which architectures are at which versions. +=item largest_source_version_only -- if there is more than one source +version in a particular distribution, discards all versions but the +largest in that distribution. Defaults to 1, as this used to be the +way that the Debian archive worked. + =back When called in scalar context, this function will return hashrefs or @@ -423,8 +734,132 @@ sub get_versions{ return_archs => {type => BOOLEAN, default => 0, }, + largest_source_version_only => {type => BOOLEAN, + default => 1, + }, + schema => {type => OBJECT, + optional => 1, + }, }, - ); + ); + if (defined $param{schema}) { + my @src_packages; + my @bin_packages; + for my $pkg (make_list($param{package})) { + if ($pkg =~ /^src:(.+)/) { + push @src_packages, + $1; + } else { + push @bin_packages,$pkg; + } + } + + my $s = $param{schema}; + use Data::Printer; + p @src_packages; + my %return; + if (@src_packages) { + my $src_rs = $s->resultset('SrcVer')-> + search({'src_pkg.pkg'=>[@src_packages], + -or => {'suite.codename' => [make_list($param{dist})], + 'suite.suite_name' => [make_list($param{dist})], + } + }, + {join => ['src_pkg', + { + src_associations=>'suite'}, + ], + '+select' => [qw(src_pkg.pkg), + qw(suite.codename), + qw(src_associations.modified), + q(CONCAT(src_pkg.pkg,'/',me.ver))], + '+as' => ['src_pkg_name','codename', + 'modified_time', + qw(src_pkg_ver)], + result_class => 'DBIx::Class::ResultClass::HashRefInflator', + order_by => {-desc => 'me.ver'}, + }, + ); + my %completed_dists; + for my $src ($src_rs->all()) { + my $val = 'source'; + if ($param{time}) { + $val = DateTime::Format::Pg-> + parse_datetime($src->{modified_time})-> + epoch(); + } + if ($param{largest_source_version_only}) { + next if $completed_dists{$src->{codename}}; + $completed_dists{$src->{codename}} = 1; + } + if ($param{source}) { + $return{$src->{src_pkg_ver}} = $val; + } else { + $return{$src->{ver}} = $val; + } + } + } + if (@bin_packages) { + my $bin_rs = $s->resultset('BinVer')-> + search({'bin_pkg.pkg' => [@bin_packages], + -or => {'suite.codename' => [make_list($param{dist})], + 'suite.suite_name' => [make_list($param{dist})], + }, + }, + {join => ['bin_pkg', + { + 'src_ver'=>'src_pkg'}, + { + bin_associations => 'suite'}, + 'arch', + ], + '+select' => [qw(bin_pkg.pkg arch.arch suite.codename), + qw(bin_associations.modified), + qw(src_pkg.pkg),q(CONCAT(src_pkg.pkg,'/',me.ver)), + ], + '+as' => ['bin_pkg','arch','codename', + 'modified_time', + 'src_pkg_name','src_pkg_ver'], + result_class => 'DBIx::Class::ResultClass::HashRefInflator', + order_by => {-desc => 'src_ver.ver'}, + }); + if (exists $param{arch}) { + $bin_rs = + $bin_rs->search({'arch.arch' => [make_list($param{arch})]}, + { + join => 'arch'} + ); + } + my %completed_dists; + for my $bin ($bin_rs->all()) { + my $key = $bin->{ver}; + if ($param{source}) { + $key = $bin->{src_pkg_ver}; + } + my $val = $bin->{arch}; + if ($param{time}) { + $val = DateTime::Format::Pg-> + parse_datetime($bin->{modified_time})-> + epoch(); + } + if ($param{largest_source_version_only}) { + if ($completed_dists{$bin->{codename}} and not + exists $return{$key}) { + next; + } + $completed_dists{$bin->{codename}} = 1; + } + push @{$return{$key}}, + $val; + } + } + if ($param{return_archs}) { + if ($param{time} or $param{return_archs}) { + return wantarray?%return :\%return; + } + return wantarray?keys %return :[keys %return]; + } + } my $versions; if ($param{time}) { return () if not defined $gVersionTimeIndex; @@ -444,6 +879,10 @@ sub get_versions{ } my %versions; for my $package (make_list($param{package})) { + my $source_only = 0; + if ($package =~ s/^src://) { + $source_only = 1; + } my $version = $versions->{$package}; next unless defined $version; for my $dist (make_list($param{dist})) { @@ -451,12 +890,18 @@ sub get_versions{ make_list($param{arch}): (grep {not $param{no_source_arch} or $_ ne 'source' - } keys %{$version->{$dist}})) { + } $source_only?'source':keys %{$version->{$dist}})) { next unless defined $version->{$dist}{$arch}; - for my $ver (ref $version->{$dist}{$arch} ? - keys %{$version->{$dist}{$arch}} : - $version->{$dist}{$arch} - ) { + my @vers = ref $version->{$dist}{$arch} eq 'HASH' ? + keys %{$version->{$dist}{$arch}} : + make_list($version->{$dist}{$arch}); + if ($param{largest_source_version_only} and + $arch eq 'source' and @vers > 1) { + # order the versions, then pick the biggest version number + @vers = sort_versions(@vers); + @vers = $vers[-1]; + } + for my $ver (@vers) { my $f_ver = $ver; if ($param{source}) { ($f_ver) = make_source_versions(package => $package, @@ -509,7 +954,6 @@ sub makesourceversions { arch => 'source', versions => '0.1.1', guess_source => 1, - debug => \$debug, warnings => \$warnings, ); @@ -549,10 +993,12 @@ sub make_source_versions { warnings => {type => SCALARREF|HANDLE, optional => 1, }, + schema => {type => OBJECT, + optional => 1, + }, }, ); my ($warnings) = globify_scalar(exists $param{warnings}?$param{warnings}:undef); - my ($debug) = globify_scalar(exists $param{debug} ?$param{debug} :undef); my @packages = grep {defined $_ and length $_ } make_list($param{package}); my @archs = grep {defined $_ } make_list ($param{arch}); @@ -589,7 +1035,7 @@ sub make_source_versions { my @bin_versions = sourcetobinary($1,$version); if (not @bin_versions or @{$bin_versions[0]} != 3) { - print {$warnings} "The source $1 and version $2 do not appear to match any binary packages\n"; + print {$warnings} "The source '$1' and version '$version' do not appear to match any binary packages\n"; } next; } @@ -611,7 +1057,7 @@ sub make_source_versions { } my @srcinfo = binary_to_source(binary => $pkg, version => $version, - arch => $arch); + length($arch)?(arch => $arch):()); if (not @srcinfo) { # We don't have explicit information about the # binary-to-source mapping for this version