X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=bin%2Fwanna-build;h=91d05df91f8c7c346941d69e1303a6aa0caba1f3;hb=55fedfd5508b0805df3127a71afc0b40509511cb;hp=5b51462bcc91e63d422b944480ba86a6e653d521;hpb=5bfd763bbea3d99fe06d40bcc02521ec1b42a4f6;p=wannabuild.git diff --git a/bin/wanna-build b/bin/wanna-build index 5b51462..91d05df 100755 --- a/bin/wanna-build +++ b/bin/wanna-build @@ -32,7 +32,6 @@ die "dbbase is empty\n" if ! $dbbase; die "transactlog is empty\n" if ! $transactlog; die "mailprog binary $conf::mailprog does not exist or isn't executable\n" if !-x $conf::mailprog; -die "no distributions defined\n" if ! %distributions; package main; use strict; @@ -40,7 +39,6 @@ use POSIX; use FileHandle; use File::Copy; use DBI; -use lib '/org/wanna-build/bin'; use lib '/org/wanna-build/lib'; #use lib 'lib'; use WannaBuild; @@ -51,6 +49,7 @@ use String::Format; use Date::Parse; use List::Util qw[max]; use Dpkg::Version qw(vercmp); # TODO: change this for running with squeeze dpkg +use Dpkg::Deps; # TODO: same our ($verbose, $mail_logs, $list_order, $list_state, $curr_date, $op_mode, $user, $real_user, $distribution, @@ -60,12 +59,14 @@ our ($verbose, $mail_logs, $list_order, $list_state, $category, %catval, %short_category, $short_date, $list_min_age, $dbbase, @curr_time, $build_priority, %new_vers, $binNMUver, %merge_srcvers, %merge_binsrc, - $printformat, $ownprintformat, $privmode + $printformat, $ownprintformat, $privmode, $extra_depends, $extra_conflicts, + %distributions, %distribution_aliases ); our $Pas = '/org/buildd.debian.org/etc/packages-arch-specific/Packages-arch-specific'; our $simulate = 0; our $simulate_edos = 0; -our $api = 0; # allow buildds to specify an different api +our $api = undef; # allow buildds to specify an different api +our $recorduser = undef; # global vars $ENV{'PATH'} = "/bin:/usr/bin:/usr/local/bin:/org/wanna-build/bin/"; @@ -84,9 +85,6 @@ my %prognames = ( "uploaded-build" => "set-uploaded", "give-back-build" => "set-needs-build", "dep-wait-build" => "set-dep-wait", "forget-build" => "forget", - "merge-quinn" => "merge-quinn", - "merge-packages" => "merge-packages", - "merge-sources" => "merge-sources", "build-info" => "info" ); %short_category = ( u => "uploaded-fixed-pkg", @@ -137,13 +135,14 @@ my %options = "dep-wait" => { mode => "set-dep-wait" }, forget => { mode => "forget" }, 'forget-user' => { mode => 'forget-user' }, - "merge-quinn" => { mode => "merge-quinn" }, - "merge-partial-quinn" => { mode => "merge-partial-quinn" }, - "merge-packages" => { mode => "merge-packages" }, - "merge-sources" => { mode => "merge-sources" }, + update => { mode => "set-update" }, + #"merge-quinn" => { mode => "merge-quinn" }, + #"merge-partial-quinn" => { mode => "merge-partial-quinn" }, + #"merge-packages" => { mode => "merge-packages" }, + #"merge-sources" => { mode => "merge-sources" }, "pretend-avail" => { short => "p", mode => "pretend-avail" }, - "merge-all" => { mode => "merge-all" }, - "merge-all-secondary" => { mode => "merge-all-secondary" }, + #"merge-all" => { mode => "merge-all" }, + #"merge-all-secondary" => { mode => "merge-all-secondary" }, "merge-v3" => { mode => "merge-v3" }, info => { short => "i", mode => "info" }, 'binNMU' => { mode => 'set-binary-nmu', arg => \$binNMUver, @@ -210,10 +209,14 @@ my %options = "format" => { arg => \$printformat }, "own-format" => { arg => \$ownprintformat }, "Pas" => { arg => \$Pas }, + "extra-depends"=> { arg => \$extra_depends }, + "extra-conflicts"=> { arg => \$extra_conflicts }, # special actions export => { arg => \$export_to, mode => "export" }, import => { arg => \$import_from, mode => "import" }, "manual-edit" => { mode => "manual-edit" }, + "distribution-architectures" => { mode => "distribution-architectures" }, + "distribution-aliases" => { mode => "distribution-aliases" }, ); while( @ARGV && $ARGV[0] =~ /^-/ ) { @@ -257,9 +260,15 @@ while( @ARGV && $ARGV[0] =~ /^-/ ) { } } -$op_mode = $category ? "set-failed" : "set-building" - if !$op_mode; # default operation -$distribution ||= "unstable"; +my $dbh; + +END { + if (defined $dbh) + { + $dbh->disconnect or warn $dbh->errstr; + } +} + if ($distribution eq 'any-priv') { $privmode = 'yes'; $distribution = 'any'; @@ -268,12 +277,49 @@ if ($distribution eq 'any-unpriv') { $privmode = 'no'; $distribution = 'any'; } + +my $schema_suffix = ''; +$recorduser //= (not -t and $user =~ /^buildd_/); +if (isin( $op_mode, qw(list info)) && $distribution !~ /security/ && !$recorduser && !($privmode eq 'yes')) { + $dbh = DBI->connect("DBI:Pg:service=wanna-build") || + die "FATAL: Cannot open database: $DBI::errstr\n"; + $schema_suffix = '_public'; +} +else +{ + $dbh = DBI->connect("DBI:Pg:service=wanna-build-privileged") || + die "FATAL: Cannot open database: $DBI::errstr\n"; +} + +# TODO: This shouldn't be needed, file a bug. +$dbh->{pg_server_prepare} = 0; + +$dbh->begin_work or die $dbh->errstr; + +my $q = 'SELECT distribution, public, auto_dep_wait FROM distributions'; +my $rows = $dbh->selectall_hashref($q, 'distribution'); +foreach my $name (keys %$rows) { + $distributions{$name} = {}; + $distributions{$name}->{'noadw'} = 1 if !($rows->{$name}->{'auto_dep_wait'}); + $distributions{$name}->{'hidden'} = 1 if !($rows->{$name}->{'public'}); +} + +$q = 'SELECT alias, distribution FROM distribution_aliases'; +$rows = $dbh->selectall_hashref($q, 'alias'); +foreach my $name (keys %$rows) { + $distribution_aliases{$name} = $rows->{$name}->{'distribution'}; +} +$distribution = $distribution_aliases{$distribution} if (isin($distribution, keys %distribution_aliases)); + +$op_mode = $category ? "set-failed" : "set-building" + if !$op_mode; # default operation +$distribution ||= "sid"; undef $distribution if $distribution eq 'any'; if ($distribution) { my @dists = split(/[, ]+/, $distribution); foreach my $dist (@dists) { die "Bad distribution '$distribution'\n" - if !isin($dist, keys %conf::distributions); + if !isin($dist, keys %distributions); } } if (!isin ( $op_mode, qw(list) ) && ( !$distribution || $distribution =~ /[ ,]/)) { @@ -298,7 +344,8 @@ if ($verbose) { if (!@ARGV && !isin( $op_mode, qw(list merge-quinn merge-partial-quinn import export merge-packages manual-edit - merge-sources))) { + merge-sources distribution-architectures + distribution-aliases))) { warn "No packages given.\n"; usage(); } @@ -357,32 +404,8 @@ if (not $yamlmap) { } $list_order = $yamlmap->{"list-order"}{$list_state} if !$list_order and $list_state; $list_order ||= $yamlmap->{"list-order"}{'default'}; - -my $dbh; - -END { - if (defined $dbh) - { - $dbh->disconnect or warn $dbh->errstr; - } -} - -my $schema_suffix = ''; -if (isin( $op_mode, qw(list info)) && $distribution !~ /security/ && !(not -t and $user =~ /-/) && !($privmode eq 'yes')) { - $dbh = DBI->connect("DBI:Pg:service=wanna-build") || - die "FATAL: Cannot open database: $DBI::errstr\n"; - $schema_suffix = '_public'; -} -else -{ - $dbh = DBI->connect("DBI:Pg:service=wanna-build-privileged") || - die "FATAL: Cannot open database: $DBI::errstr\n"; -} - -# TODO: This shouldn't be needed, file a bug. -$dbh->{pg_server_prepare} = 0; - -$dbh->begin_work or die $dbh->errstr; +$api //= $yamlmap->{"api"}; +$api //= 0; process(); @@ -543,10 +566,18 @@ sub process { export_db( $export_to ); last SWITCH; }; + /^distribution-architectures/ && do { + show_distribution_architectures(); + last SWITCH; + }; + /^distribution-aliases/ && do { + show_distribution_aliases(); + last SWITCH; + }; die "Unexpected operation mode $op_mode\n"; } - if (not -t and $user =~ /-/) { + if ($recorduser) { my $userinfo = get_user_info($user); if (!defined $userinfo) { @@ -609,6 +640,9 @@ sub add_packages { elsif ($op_mode eq "set-binary-nmu") { set_one_binnmu( $name, $version ); } + elsif ($op_mode eq "set-update") { + set_one_update( $name, $version ); + } } } @@ -744,13 +778,13 @@ sub add_one_building { } print "$name: $ok\n" if $verbose; } else { - print "- $name:\n"; - print " - status: ok\n"; - if ($pkg->{'binary_nmu_version'}) { - print " - binNMU:\n"; - print " - version: $pkg->{'binary_nmu_version'}\n"; - print " - changelog: $pkg->{'binary_nmu_changelog'}\n"; - } + print "- $name:\n"; + print " - status: ok\n"; + printf " - pkg-ver: %s_%s\n", $name, $version; + print " - binNMU: $pkg->{'binary_nmu_version'}\n" if $pkg->{'binary_nmu_version'}; + print " - extra-changelog: $pkg->{'binary_nmu_changelog'}\n" if $pkg->{'binary_nmu_changelog'} && $pkg->{'binary_nmu_version'}; + print " - extra-depends: $pkg->{'extra_depends'}\n" if $pkg->{'extra_depends'}; + print " - extra-conflicts: $pkg->{'extra_conflicts'}\n" if $pkg->{'extra_conflicts'}; } change_state( \$pkg, 'Building' ); $pkg->{'package'} = $name; @@ -1135,6 +1169,11 @@ sub set_one_binnmu { $pkg->{'notes'} = 'out-of-date'; $pkg->{'buildpri'} = $pkg->{'permbuildpri'} if (defined $pkg->{'permbuildpri'}); + if (defined $distributions{$distribution}{noadw}) { + change_state( \$pkg, 'Installed' ); + } else { + change_state( \$pkg, 'BD-Uninstallable' ); + } } log_ta( $pkg, "--binNMU" ); update_source_info($pkg); @@ -1156,7 +1195,7 @@ sub set_one_binnmu { return; } - if ($distribution eq "unstable") { + if (!defined $distributions{$distribution}{noadw}) { change_state( \$pkg, 'BD-Uninstallable' ); $pkg->{'bd_problem'} = "Installability of build dependencies not tested yet"; } @@ -1285,6 +1324,21 @@ sub add_one_depwait { print "$name: registered as waiting for dependencies\n" if $verbose; } +sub set_one_update { + my $name = shift; + my $version = shift; + my $pkg = get_source_info($name); + + if (!defined($pkg)) { + print "$name: not registered yet.\n"; + return; + } + $pkg->{'version'} =~ s/\+b[0-9]+$//; + + log_ta( $pkg, "--update" ); + update_source_info($pkg); +} + sub parse_sources { my %pkgs; @@ -1885,6 +1939,13 @@ sub calculate_prio { my $scale = $priomap->{'waitingdays'}->{'scale'} || 1; $pkg->{'calprio'} += $days * $scale; + my $btime = max($pkg->{'anytime'}, $pkg->{'successtime'}); + my $bhours = defined($btime) ? int($btime/3600) : ($priomap->{'buildhours'}->{'default'} || 2); + $bhours = $priomap->{'buildhours'}->{'min'} if $priomap->{'buildhours'}->{'min'} and $bhours < $priomap->{'buildhours'}->{'min'}; + $bhours = $priomap->{'buildhours'}->{'max'} if $priomap->{'buildhours'}->{'max'} and $bhours > $priomap->{'buildhours'}->{'max'}; + $scale = $priomap->{'buildhours'}->{'scale'} || 1; + $pkg->{'calprio'} -= $bhours * $scale; + $pkg->{'calprio'} += $pkg->{'permbuildpri'} if $pkg->{'permbuildpri'}; $pkg->{'calprio'} += $pkg->{'buildpri'} if $pkg->{'buildpri'}; @@ -2029,6 +2090,8 @@ sub list_packages { # filter components @list = grep { my $i = $_->{'component'}; grep { $i eq $_ } split /[, ]+/, $yamlmap->{"restrict"}{'component'} } @list; + # extra depends / conflicts only from api 1 on + @list = grep { !$_->{'extra_depends'} and !$_->{'extra_conflicts'} } @list if $api < 1 ; # first adjust ownprintformat, then set printformat accordingly $printformat ||= $yamlmap->{"format"}{$ownprintformat}; @@ -2087,7 +2150,7 @@ sub info_packages { my( $name, $pkg, $key, $dist ); my @firstkeys = qw(package version builder state section priority installed_version previous_state state_change); - my @dists = $info_all_dists ? keys %conf::distributions : ($distribution); + my @dists = $info_all_dists ? keys %distributions : ($distribution); my %beautykeys = ( 'package' => 'Package', 'version' => 'Version', 'builder' => 'Builder', 'state' => 'State', 'section' => 'Section', 'priority' => 'Priority', 'installed_version' => 'Installed-Version', 'previous_state' => 'Previous-State', @@ -2100,7 +2163,9 @@ sub info_packages { 'permbuildpri' => 'PermBuildPri', 'rel' => 'Rel', 'calprio' => 'CalculatedPri', 'state_days' => 'State-Days', 'successtime' => 'Success-build-time', - 'anytime' => 'Build-time' + 'anytime' => 'Build-time', + 'extra_depends' => 'Extra-Dependencies', + 'extra_conflicts' => 'Extra-Conflicts', ); foreach $name (@_) { @@ -2406,24 +2471,9 @@ sub build_deplist { } -sub greparch { - my ($gapkglong, $gaarch) = @_; - my ($gapkg, $gaarchs) = split(/ \[/, $gapkglong); - if ($gaarchs) { - $_ = $gapkg; - chop($gaarchs); # take away the ] - my @gaarches = split(/ /, $gaarchs); - if (substr($gaarches[0], 0, 1) eq '!') { - return 0 if grep /^!$gaarch$/, @gaarches; - } else { # positive case - return 0 unless grep /^$gaarch$/, @gaarches; - } - }; - return 1; -} sub filterarch { - my $faarch = $_[1]; - return join(', ', grep { &greparch($_, $faarch) } split(/, ?/, $_[0])); + return "" unless $_[0]; + return Dpkg::Deps::parse($_[0], ("reduce_arch" => 1, "host_arch" => $_[1]))->dump(); } sub wb_edos_builddebcheck { @@ -2520,7 +2570,7 @@ sub call_edos_depcheck { my $srcs = $args->{'srcs'}; my $key; - return if defined ($conf::distributions{$distribution}{noadw}) && not defined $args->{'depwait'}; + return if defined ($distributions{$distribution}{noadw}) && not defined $args->{'depwait'}; # We need to check all of needs-build, as any new upload could make # something in needs-build have uninstallable deps @@ -2530,13 +2580,13 @@ sub call_edos_depcheck { my $db = get_all_source_info(); foreach $key (keys %$db) { my $pkg = $db->{$key}; - if (defined $pkg and isin($pkg->{'state'}, qw/Needs-Build BD-Uninstallable/) and not defined ($conf::distributions{$distribution}{noadw})) { + if (defined $pkg and isin($pkg->{'state'}, qw/Needs-Build BD-Uninstallable/) and not defined ($distributions{$distribution}{noadw})) { $interesting_packages{$key} = undef; } if (defined $pkg and isin($pkg->{'state'}, qw/Dep-Wait/) and defined $args->{'depwait'}) { $interesting_packages_depwait{$key} = undef; # we always check for BD-Uninstallability in depwait - could be that depwait is satisfied but package is uninstallable - $interesting_packages{$key} = undef unless defined ($conf::distributions{$distribution}{noadw}); + $interesting_packages{$key} = undef unless defined ($distributions{$distribution}{noadw}); } } @@ -2555,8 +2605,12 @@ sub call_edos_depcheck { print $SOURCES "Package: source---$key\n"; print $SOURCES "Version: $pkg->{'version'}\n"; my $t = &filterarch($srcs->{$key}{'dep'} || $srcs->{$key}{'depends'}, $arch); + my $tt = &filterarch($pkg->{'extra_depends'}, $arch); + $t = $t ? ($tt ? "$t, $tt" : $t) : $tt; print $SOURCES "Depends: $t\n" if $t; my $u = &filterarch($srcs->{$key}{'conf'} || $srcs->{$key}{'conflicts'}, $arch); + my $uu = &filterarch($pkg->{'extra_conflicts'}, $arch); + $u = $u ? ($uu ? "$u, $uu" : $u) : $uu; print $SOURCES "Conflicts: $u\n" if $u; print $SOURCES "Architecture: all\n"; print $SOURCES "\n"; @@ -2736,6 +2790,7 @@ sub get_readonly_source_info { my $q = "SELECT rel, priority, state_change, permbuildpri, section, buildpri, failed, state, binary_nmu_changelog, bd_problem, version, package, distribution, installed_version, notes, failed_category, builder, old_failed, previous_state, binary_nmu_version, depends, extract(days from date_trunc('days', now() - state_change)) as state_days" . ", (SELECT max(build_time) FROM ".pkg_history_table_name()." WHERE pkg_history.package = packages.package AND pkg_history.distribution = packages.distribution AND result = 'successful') AS successtime" . ", (SELECT max(build_time) FROM ".pkg_history_table_name()." WHERE pkg_history.package = packages.package AND pkg_history.distribution = packages.distribution ) AS anytime" + . ", extra_depends, extra_conflicts" . " FROM " . table_name() . ' WHERE package = ? AND distribution = ?'; my $pkg = $dbh->selectrow_hashref( $q, @@ -2759,7 +2814,7 @@ sub get_all_source_info { my $q = "SELECT rel, priority, state_change, permbuildpri, section, buildpri, failed, state, binary_nmu_changelog, bd_problem, version, package, distribution, installed_version, notes, failed_category, builder, old_failed, previous_state, binary_nmu_version, depends, extract(days from date_trunc('days', now() - state_change)) as state_days" # . ", (SELECT max(build_time) FROM ".pkg_history_table_name()." WHERE pkg_history.package = packages.package AND pkg_history.distribution = packages.distribution AND result = 'successful') AS successtime" # . ", (SELECT max(build_time) FROM ".pkg_history_table_name()." WHERE pkg_history.package = packages.package AND pkg_history.distribution = packages.distribution ) AS anytime" - . ", successtime.build_time as successtime, anytime.build_time as anytime" + . ", successtime.build_time as successtime, anytime.build_time as anytime, extra_depends, extra_conflicts" . " FROM " . table_name() . " left join ( " . "select distinct on (package, distribution) build_time, package, distribution from ".pkg_history_table_name()." where result = 'successful' order by package, distribution, timestamp " @@ -2807,8 +2862,29 @@ sub get_all_source_info { return $db; } +sub show_distribution_architectures { + my $q = 'SELECT distribution, spacecat_all(architecture) AS architectures '. + 'FROM distribution_architectures '. + 'GROUP BY distribution'; + my $rows = $dbh->selectall_hashref($q, 'distribution'); + foreach my $name (keys %$rows) { + print $name.': '.$rows->{$name}->{'architectures'}."\n"; + } +} + +sub show_distribution_aliases { + foreach my $alias (keys %distribution_aliases) { + print $alias.': '.$distribution_aliases{$alias}."\n"; + } +} + sub update_source_info { my $pkg = shift; + $pkg->{'extra_depends'} = $extra_depends if defined $extra_depends; + undef $pkg->{'extra_depends'} unless $pkg->{'extra_depends'}; + $pkg->{'extra_conflicts'} = $extra_conflicts if defined $extra_conflicts; + undef $pkg->{'extra_conflicts'} unless $pkg->{'extra_conflicts'}; + print Dumper $pkg if $verbose and $simulate; return if $simulate; my $pkg2 = get_source_info($pkg->{'package'}); @@ -2836,6 +2912,8 @@ sub update_source_info { 'buildpri = ?, ' . 'depends = ?, ' . 'rel = ?, ' . + 'extra_depends = ?, ' . + 'extra_conflicts = ?, ' . 'bd_problem = ? ' . 'WHERE package = ? AND distribution = ?', undef, @@ -2856,6 +2934,8 @@ sub update_source_info { $pkg->{'buildpri'}, $pkg->{'depends'}, $pkg->{'rel'}, + $pkg->{'extra_depends'}, + $pkg->{'extra_conflicts'}, $pkg->{'bd_problem'}, $pkg->{'package'}, $distribution) or die $dbh->errstr; @@ -2958,7 +3038,7 @@ sub parse_all_v3() { ($pkg->{'binary_nmu_version'} ? ";b".$pkg->{'binary_nmu_version'} : ""). ", $pkg->{'state'}):"; - if (isin($pkgs->{'status'}, qw (installed related)) && $pkg->{'binary_nmu_version'} && $pkgs->{'binnmu'} < int($pkg->{'binary_nmu_version'})) { + if (isin($pkgs->{'status'}, qw (installed related)) && $pkgs->{'version'} eq $pkg->{'version'} && $pkg->{'binary_nmu_version'} && $pkgs->{'binnmu'} < int($pkg->{'binary_nmu_version'})) { $pkgs->{'status'} = 'out-of-date'; } if (isin($pkgs->{'status'}, qw (installed related))) { @@ -2966,6 +3046,8 @@ sub parse_all_v3() { if ($pkg->{'state'} ne 'Installed') { change_state( \$pkg, 'Installed'); delete $pkg->{'depends'}; + delete $pkg->{'extra_depends'}; + delete $pkg->{'extra_conflicts'}; $change++; } my $attrs = { 'version' => 'version', 'installed_version' => 'version', 'binary_nmu_version' => 'binnmu', 'section' => 'section', 'priority' => 'priority' };