X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bin%2Fwanna-build;h=be16dbe886dab82a8bb9f8ed7150939608c8edcd;hb=87ad762e32834250d7d6d28b3d074afa79425442;hp=552f07cd863b757d237b2730c6caa87eb7dd5234;hpb=6f4a16d6d5684ba68e88c89c8e60626498d728b7;p=wannabuild.git diff --git a/bin/wanna-build b/bin/wanna-build index 552f07c..be16dbe 100755 --- a/bin/wanna-build +++ b/bin/wanna-build @@ -32,7 +32,6 @@ die "dbbase is empty\n" if ! $dbbase; die "transactlog is empty\n" if ! $transactlog; die "mailprog binary $conf::mailprog does not exist or isn't executable\n" if !-x $conf::mailprog; -die "no distributions defined\n" if ! %distributions; package main; use strict; @@ -40,7 +39,6 @@ use POSIX; use FileHandle; use File::Copy; use DBI; -use lib '/org/wanna-build/bin'; use lib '/org/wanna-build/lib'; #use lib 'lib'; use WannaBuild; @@ -51,6 +49,7 @@ use String::Format; use Date::Parse; use List::Util qw[max]; use Dpkg::Version qw(vercmp); # TODO: change this for running with squeeze dpkg +use Dpkg::Deps; # TODO: same our ($verbose, $mail_logs, $list_order, $list_state, $curr_date, $op_mode, $user, $real_user, $distribution, @@ -60,12 +59,14 @@ our ($verbose, $mail_logs, $list_order, $list_state, $category, %catval, %short_category, $short_date, $list_min_age, $dbbase, @curr_time, $build_priority, %new_vers, $binNMUver, %merge_srcvers, %merge_binsrc, - $printformat, $ownprintformat, $privmode, $extra_depends, $extra_conflicts + $printformat, $ownprintformat, $privmode, $extra_depends, $extra_conflicts, + %distributions, %distribution_aliases ); our $Pas = '/org/buildd.debian.org/etc/packages-arch-specific/Packages-arch-specific'; our $simulate = 0; our $simulate_edos = 0; -our $api = 0; # allow buildds to specify an different api +our $api = undef; # allow buildds to specify an different api +our $recorduser = undef; # global vars $ENV{'PATH'} = "/bin:/usr/bin:/usr/local/bin:/org/wanna-build/bin/"; @@ -180,7 +181,7 @@ my %options = { short => "O", arg => \$list_order, code => sub { die "Bad ordering character\n" - if $list_order !~ /^[PSpsncbCW]+$/; + if $list_order !~ /^[PSpsncbCWT]+$/; } }, message => { short => "m", arg => \$fail_reason }, # database is deprecated, use arch instead. @@ -214,6 +215,8 @@ my %options = export => { arg => \$export_to, mode => "export" }, import => { arg => \$import_from, mode => "import" }, "manual-edit" => { mode => "manual-edit" }, + "distribution-architectures" => { mode => "distribution-architectures" }, + "distribution-aliases" => { mode => "distribution-aliases" }, ); while( @ARGV && $ARGV[0] =~ /^-/ ) { @@ -257,9 +260,15 @@ while( @ARGV && $ARGV[0] =~ /^-/ ) { } } -$op_mode = $category ? "set-failed" : "set-building" - if !$op_mode; # default operation -$distribution ||= "unstable"; +my $dbh; + +END { + if (defined $dbh) + { + $dbh->disconnect or warn $dbh->errstr; + } +} + if ($distribution eq 'any-priv') { $privmode = 'yes'; $distribution = 'any'; @@ -268,12 +277,51 @@ if ($distribution eq 'any-unpriv') { $privmode = 'no'; $distribution = 'any'; } + +my $schema_suffix = ''; +$recorduser //= (not -t and $user =~ /^buildd_/); +if (isin( $op_mode, qw(list info)) && $distribution !~ /security/ && !$recorduser && !($privmode eq 'yes')) { + $dbh = DBI->connect("DBI:Pg:service=wanna-build") || + die "FATAL: Cannot open database: $DBI::errstr\n"; + $schema_suffix = '_public'; +} +else +{ + $dbh = DBI->connect("DBI:Pg:service=wanna-build-privileged") || + die "FATAL: Cannot open database: $DBI::errstr\n"; +} + +# TODO: This shouldn't be needed, file a bug. +$dbh->{pg_server_prepare} = 0; + +$dbh->begin_work or die $dbh->errstr; + +my $q = 'SELECT distribution, public, auto_dep_wait, build_dep_resolver, archive FROM distributions'; +my $rows = $dbh->selectall_hashref($q, 'distribution'); +foreach my $name (keys %$rows) { + $distributions{$name} = {}; + $distributions{$name}->{'noadw'} = 1 if !($rows->{$name}->{'auto_dep_wait'}); + $distributions{$name}->{'hidden'} = 1 if !($rows->{$name}->{'public'}); + $distributions{$name}->{'build_dep_resolver'} = $rows->{$name}->{'build_dep_resolver'} if $rows->{$name}->{'build_dep_resolver'}; + $distributions{$name}->{'archive'} = $rows->{$name}->{'archive'} if $rows->{$name}->{'archive'}; +} + +$q = 'SELECT alias, distribution FROM distribution_aliases'; +$rows = $dbh->selectall_hashref($q, 'alias'); +foreach my $name (keys %$rows) { + $distribution_aliases{$name} = $rows->{$name}->{'distribution'}; +} +$distribution = $distribution_aliases{$distribution} if (isin($distribution, keys %distribution_aliases)); + +$op_mode = $category ? "set-failed" : "set-building" + if !$op_mode; # default operation +$distribution ||= "sid"; undef $distribution if $distribution eq 'any'; if ($distribution) { my @dists = split(/[, ]+/, $distribution); foreach my $dist (@dists) { die "Bad distribution '$distribution'\n" - if !isin($dist, keys %conf::distributions); + if !isin($dist, keys %distributions); } } if (!isin ( $op_mode, qw(list) ) && ( !$distribution || $distribution =~ /[ ,]/)) { @@ -298,7 +346,8 @@ if ($verbose) { if (!@ARGV && !isin( $op_mode, qw(list merge-quinn merge-partial-quinn import export merge-packages manual-edit - merge-sources))) { + merge-sources distribution-architectures + distribution-aliases))) { warn "No packages given.\n"; usage(); } @@ -357,32 +406,8 @@ if (not $yamlmap) { } $list_order = $yamlmap->{"list-order"}{$list_state} if !$list_order and $list_state; $list_order ||= $yamlmap->{"list-order"}{'default'}; - -my $dbh; - -END { - if (defined $dbh) - { - $dbh->disconnect or warn $dbh->errstr; - } -} - -my $schema_suffix = ''; -if (isin( $op_mode, qw(list info)) && $distribution !~ /security/ && !(not -t and $user =~ /buildd-/) && !($privmode eq 'yes')) { - $dbh = DBI->connect("DBI:Pg:service=wanna-build") || - die "FATAL: Cannot open database: $DBI::errstr\n"; - $schema_suffix = '_public'; -} -else -{ - $dbh = DBI->connect("DBI:Pg:service=wanna-build-privileged") || - die "FATAL: Cannot open database: $DBI::errstr\n"; -} - -# TODO: This shouldn't be needed, file a bug. -$dbh->{pg_server_prepare} = 0; - -$dbh->begin_work or die $dbh->errstr; +$api //= $yamlmap->{"api"}; +$api //= 0; process(); @@ -543,10 +568,18 @@ sub process { export_db( $export_to ); last SWITCH; }; + /^distribution-architectures/ && do { + show_distribution_architectures(); + last SWITCH; + }; + /^distribution-aliases/ && do { + show_distribution_aliases(); + last SWITCH; + }; die "Unexpected operation mode $op_mode\n"; } - if (not -t and $user =~ /buildd-/) { + if ($recorduser) { my $userinfo = get_user_info($user); if (!defined $userinfo) { @@ -747,15 +780,15 @@ sub add_one_building { } print "$name: $ok\n" if $verbose; } else { - print "- $name:\n"; - print " - status: ok\n"; - print " - extra-depends: $pkg->{'extra_depends'}\n" if $pkg->{'extra_depends'}; - print " - extra-conflicts: $pkg->{'extra_conflicts'}\n" if $pkg->{'extra_conflicts'}; - if ($pkg->{'binary_nmu_version'}) { - print " - binNMU:\n"; - print " - version: $pkg->{'binary_nmu_version'}\n"; - print " - changelog: $pkg->{'binary_nmu_changelog'}\n"; - } + print "- $name:\n"; + print " - status: ok\n"; + printf " - pkg-ver: %s_%s\n", $name, $version; + print " - binNMU: $pkg->{'binary_nmu_version'}\n" if $pkg->{'binary_nmu_version'}; + print " - extra-changelog: $pkg->{'binary_nmu_changelog'}\n" if $pkg->{'binary_nmu_changelog'} && $pkg->{'binary_nmu_version'}; + print " - extra-depends: $pkg->{'extra_depends'}\n" if $pkg->{'extra_depends'}; + print " - extra-conflicts: $pkg->{'extra_conflicts'}\n" if $pkg->{'extra_conflicts'}; + print " - archive: $distributions{$distribution}->{'archive'}\n" if $distributions{$distribution}->{'archive'}; + print " - build_dep_resolver: $distributions{$distribution}->{'build_dep_resolver'}\n" if $distributions{$distribution}->{'build_dep_resolver'}; } change_state( \$pkg, 'Building' ); $pkg->{'package'} = $name; @@ -1092,7 +1125,7 @@ sub add_one_needsbuild { "Skipping.\n"; return; } - if ($distribution eq "unstable") { + if (!defined $distributions{$distribution}{noadw}) { change_state( \$pkg, 'BD-Uninstallable' ); $pkg->{'bd_problem'} = "Installability of build dependencies not tested yet"; } else { @@ -1140,6 +1173,11 @@ sub set_one_binnmu { $pkg->{'notes'} = 'out-of-date'; $pkg->{'buildpri'} = $pkg->{'permbuildpri'} if (defined $pkg->{'permbuildpri'}); + if (defined $distributions{$distribution}{noadw}) { + change_state( \$pkg, 'Installed' ); + } else { + change_state( \$pkg, 'BD-Uninstallable' ); + } } log_ta( $pkg, "--binNMU" ); update_source_info($pkg); @@ -1161,7 +1199,7 @@ sub set_one_binnmu { return; } - if ($distribution eq "unstable") { + if (!defined $distributions{$distribution}{noadw}) { change_state( \$pkg, 'BD-Uninstallable' ); $pkg->{'bd_problem'} = "Installability of build dependencies not tested yet"; } @@ -1867,9 +1905,9 @@ BEGIN { sub sort_list_func { my $map_funcs = { - 'C' => ['<=>', sub { return (-1) * $_[0]->{'calprio'}; }], - 'W' => ['<=>', sub { return (-1) * $_[0]->{'state_days'}; }], - 'P' => ['<=>', sub { return (-1) * $_[0]->{'buildpri'}; }], + 'C' => ['<->', sub { return $_[0]->{'calprio'}; }], + 'W' => ['<->', sub { return $_[0]->{'state_days'}; }], + 'P' => ['<->', sub { return $_[0]->{'buildpri'}; }], 'p' => ['<=>', sub { return $prioval{$_[0]->{'priority'}}; }], 's' => ['<=>', sub { return $sectval{$_[0]->{'section'}}; }], 'n' => ['cmp', sub { return $_[0]->{'package'}; }], @@ -1877,10 +1915,12 @@ sub sort_list_func { 'c' => ['<=>', sub { return ($_[0]->{'notes'} =~ /^(out-of-date|partial)/) ? 0: ($_[0]->{'notes'} =~ /^uncompiled/) ? 2 : 1; }], 'f' => ['<=>', sub { return $catval{ $_[0]->{'failed_category'} ? $_[0]->{'failed_category'}: "none" }; }], 'S' => ['<=>', sub { return $prioval{$_[0]->{'priority'}} > $prioval{'standard'}; }], + 'T' => ['<->', sub { return $_[0]->{'state_time'} % 86400;} ], # Fractions of a day }; foreach my $letter (split( //, $list_order )) { my $r; + $r = &{$map_funcs->{$letter}[1]}($b) <=> &{$map_funcs->{$letter}[1]}($a) if $map_funcs->{$letter}[0] eq '<->'; $r = &{$map_funcs->{$letter}[1]}($a) <=> &{$map_funcs->{$letter}[1]}($b) if $map_funcs->{$letter}[0] eq '<=>'; $r = &{$map_funcs->{$letter}[1]}($a) cmp &{$map_funcs->{$letter}[1]}($b) if $map_funcs->{$letter}[0] eq 'cmp'; return $r if $r != 0; @@ -1905,6 +1945,13 @@ sub calculate_prio { my $scale = $priomap->{'waitingdays'}->{'scale'} || 1; $pkg->{'calprio'} += $days * $scale; + my $btime = max($pkg->{'anytime'}, $pkg->{'successtime'}); + my $bhours = defined($btime) ? int($btime/3600) : ($priomap->{'buildhours'}->{'default'} || 2); + $bhours = $priomap->{'buildhours'}->{'min'} if $priomap->{'buildhours'}->{'min'} and $bhours < $priomap->{'buildhours'}->{'min'}; + $bhours = $priomap->{'buildhours'}->{'max'} if $priomap->{'buildhours'}->{'max'} and $bhours > $priomap->{'buildhours'}->{'max'}; + $scale = $priomap->{'buildhours'}->{'scale'} || 1; + $pkg->{'calprio'} -= $bhours * $scale; + $pkg->{'calprio'} += $pkg->{'permbuildpri'} if $pkg->{'permbuildpri'}; $pkg->{'calprio'} += $pkg->{'buildpri'} if $pkg->{'buildpri'}; @@ -2109,7 +2156,7 @@ sub info_packages { my( $name, $pkg, $key, $dist ); my @firstkeys = qw(package version builder state section priority installed_version previous_state state_change); - my @dists = $info_all_dists ? keys %conf::distributions : ($distribution); + my @dists = $info_all_dists ? keys %distributions : ($distribution); my %beautykeys = ( 'package' => 'Package', 'version' => 'Version', 'builder' => 'Builder', 'state' => 'State', 'section' => 'Section', 'priority' => 'Priority', 'installed_version' => 'Installed-Version', 'previous_state' => 'Previous-State', @@ -2120,7 +2167,7 @@ sub info_packages { 'failed_category' => 'Failed-Category', 'notes' => 'Notes', 'distribution' => 'Distribution', 'old_failed' => 'Old-Failed', 'permbuildpri' => 'PermBuildPri', 'rel' => 'Rel', - 'calprio' => 'CalculatedPri', 'state_days' => 'State-Days', + 'calprio' => 'CalculatedPri', 'state_days' => 'State-Days', 'state_time' => 'State-Time', 'successtime' => 'Success-build-time', 'anytime' => 'Build-time', 'extra_depends' => 'Extra-Dependencies', @@ -2430,24 +2477,9 @@ sub build_deplist { } -sub greparch { - my ($gapkglong, $gaarch) = @_; - my ($gapkg, $gaarchs) = split(/ \[/, $gapkglong); - if ($gaarchs) { - $_ = $gapkg; - chop($gaarchs); # take away the ] - my @gaarches = split(/ /, $gaarchs); - if (substr($gaarches[0], 0, 1) eq '!') { - return 0 if grep /^!$gaarch$/, @gaarches; - } else { # positive case - return 0 unless grep /^$gaarch$/, @gaarches; - } - }; - return 1; -} sub filterarch { - my $faarch = $_[1]; - return join(', ', grep { &greparch($_, $faarch) } split(/, ?/, $_[0])); + return "" unless $_[0]; + return Dpkg::Deps::parse($_[0], ("reduce_arch" => 1, "host_arch" => $_[1]))->dump(); } sub wb_edos_builddebcheck { @@ -2544,7 +2576,7 @@ sub call_edos_depcheck { my $srcs = $args->{'srcs'}; my $key; - return if defined ($conf::distributions{$distribution}{noadw}) && not defined $args->{'depwait'}; + return if defined ($distributions{$distribution}{noadw}) && not defined $args->{'depwait'}; # We need to check all of needs-build, as any new upload could make # something in needs-build have uninstallable deps @@ -2554,13 +2586,13 @@ sub call_edos_depcheck { my $db = get_all_source_info(); foreach $key (keys %$db) { my $pkg = $db->{$key}; - if (defined $pkg and isin($pkg->{'state'}, qw/Needs-Build BD-Uninstallable/) and not defined ($conf::distributions{$distribution}{noadw})) { + if (defined $pkg and isin($pkg->{'state'}, qw/Needs-Build BD-Uninstallable/) and not defined ($distributions{$distribution}{noadw})) { $interesting_packages{$key} = undef; } if (defined $pkg and isin($pkg->{'state'}, qw/Dep-Wait/) and defined $args->{'depwait'}) { $interesting_packages_depwait{$key} = undef; # we always check for BD-Uninstallability in depwait - could be that depwait is satisfied but package is uninstallable - $interesting_packages{$key} = undef unless defined ($conf::distributions{$distribution}{noadw}); + $interesting_packages{$key} = undef unless defined ($distributions{$distribution}{noadw}); } } @@ -2761,7 +2793,7 @@ sub pkg_history_table_name { sub get_readonly_source_info { my $name = shift; # SELECT FLOOR(EXTRACT('epoch' FROM age(localtimestamp, '2010-01-22 23:45')) / 86400) -- change to that? - my $q = "SELECT rel, priority, state_change, permbuildpri, section, buildpri, failed, state, binary_nmu_changelog, bd_problem, version, package, distribution, installed_version, notes, failed_category, builder, old_failed, previous_state, binary_nmu_version, depends, extract(days from date_trunc('days', now() - state_change)) as state_days" + my $q = "SELECT rel, priority, state_change, permbuildpri, section, buildpri, failed, state, binary_nmu_changelog, bd_problem, version, package, distribution, installed_version, notes, failed_category, builder, old_failed, previous_state, binary_nmu_version, depends, extract(days from date_trunc('days', now() - state_change)) as state_days, floor(extract(epoch from now()) - extract(epoch from state_change)) as state_time" . ", (SELECT max(build_time) FROM ".pkg_history_table_name()." WHERE pkg_history.package = packages.package AND pkg_history.distribution = packages.distribution AND result = 'successful') AS successtime" . ", (SELECT max(build_time) FROM ".pkg_history_table_name()." WHERE pkg_history.package = packages.package AND pkg_history.distribution = packages.distribution ) AS anytime" . ", extra_depends, extra_conflicts" @@ -2775,7 +2807,7 @@ sub get_readonly_source_info { sub get_source_info { my $name = shift; return get_readonly_source_info($name) if $simulate; - my $pkg = $dbh->selectrow_hashref('SELECT *, extract(days from date_trunc(\'days\', now() - state_change)) as state_days FROM ' . + my $pkg = $dbh->selectrow_hashref('SELECT *, extract(days from date_trunc(\'days\', now() - state_change)) as state_days, floor(extract(epoch from now()) - extract(epoch from state_change)) as state_time FROM ' . table_name() . ' WHERE package = ? AND distribution = ?' . ' FOR UPDATE', undef, $name, $distribution); @@ -2785,7 +2817,7 @@ sub get_source_info { sub get_all_source_info { my %options = @_; - my $q = "SELECT rel, priority, state_change, permbuildpri, section, buildpri, failed, state, binary_nmu_changelog, bd_problem, version, package, distribution, installed_version, notes, failed_category, builder, old_failed, previous_state, binary_nmu_version, depends, extract(days from date_trunc('days', now() - state_change)) as state_days" + my $q = "SELECT rel, priority, state_change, permbuildpri, section, buildpri, failed, state, binary_nmu_changelog, bd_problem, version, package, distribution, installed_version, notes, failed_category, builder, old_failed, previous_state, binary_nmu_version, depends, extract(days from date_trunc('days', now() - state_change)) as state_days, floor(extract(epoch from now()) - extract(epoch from state_change)) as state_time" # . ", (SELECT max(build_time) FROM ".pkg_history_table_name()." WHERE pkg_history.package = packages.package AND pkg_history.distribution = packages.distribution AND result = 'successful') AS successtime" # . ", (SELECT max(build_time) FROM ".pkg_history_table_name()." WHERE pkg_history.package = packages.package AND pkg_history.distribution = packages.distribution ) AS anytime" . ", successtime.build_time as successtime, anytime.build_time as anytime, extra_depends, extra_conflicts" @@ -2836,6 +2868,22 @@ sub get_all_source_info { return $db; } +sub show_distribution_architectures { + my $q = 'SELECT distribution, spacecat_all(architecture) AS architectures '. + 'FROM distribution_architectures '. + 'GROUP BY distribution'; + my $rows = $dbh->selectall_hashref($q, 'distribution'); + foreach my $name (keys %$rows) { + print $name.': '.$rows->{$name}->{'architectures'}."\n"; + } +} + +sub show_distribution_aliases { + foreach my $alias (keys %distribution_aliases) { + print $alias.': '.$distribution_aliases{$alias}."\n"; + } +} + sub update_source_info { my $pkg = shift; $pkg->{'extra_depends'} = $extra_depends if defined $extra_depends; @@ -2996,7 +3044,7 @@ sub parse_all_v3() { ($pkg->{'binary_nmu_version'} ? ";b".$pkg->{'binary_nmu_version'} : ""). ", $pkg->{'state'}):"; - if (isin($pkgs->{'status'}, qw (installed related)) && $pkg->{'binary_nmu_version'} && $pkgs->{'binnmu'} < int($pkg->{'binary_nmu_version'})) { + if (isin($pkgs->{'status'}, qw (installed related)) && $pkgs->{'version'} eq $pkg->{'version'} && $pkg->{'binary_nmu_version'} && $pkgs->{'binnmu'} < int($pkg->{'binary_nmu_version'})) { $pkgs->{'status'} = 'out-of-date'; } if (isin($pkgs->{'status'}, qw (installed related))) {