use DBI;
use lib '/org/wanna-build/bin';
use WannaBuild;
+use YAML::Tiny;
+use Data::Dumper;
+use Hash::Merge qw ( merge );
our ($verbose, $mail_logs, $list_order, $list_state,
$curr_date, $op_mode, $user, $real_user, $distribution,
# global vars
$ENV{'PATH'} = "/bin:/usr/bin:/usr/local/bin:/org/wanna-build/bin/";
+$ENV{'LC_ALL'} = 'C';
$verbose = 0;
$mail_logs = "";
@curr_time = gmtime;
"merge-sources" => { mode => "merge-sources" },
"pretend-avail" => { short => "p", mode => "pretend-avail" },
"merge-all" => { mode => "merge-all" },
- "merge-all-overlay" => { mode => "merge-all-overlay" },
+ "merge-all-secondary" => { mode => "merge-all-secondary" },
info => { short => "i", mode => "info" },
'binNMU' => { mode => 'set-binary-nmu', arg => \$binNMUver,
code => sub { die "Invalid binNMU version: $binNMUver\n"
{ short => "O", arg => \$list_order,
code => sub {
die "Bad ordering character\n"
- if $list_order !~ /^[PSpsncb]+$/;
+ if $list_order !~ /^[PSpsncbCW]+$/;
} },
message => { short => "m", arg => \$fail_reason },
# database is deprecated, use arch instead.
}
}
+my $yamlmap = ();
+my $yamldir = "/org/wanna-build/etc/yaml";
+my @files = ('wanna-build.yaml');
+if ($user =~ /(buildd.*)-/) { push (@files, "$1.yaml") };
+push ( @files, "$user.yaml");
+foreach my $file (@files) {
+ if ($verbose >= 2) { print "Trying to read $file ...\n"; }
+ next unless -f $yamldir."/".$file;
+ if ($verbose >= 2) { print "Read $file ...\n"; }
+ my $m = YAML::Tiny->read( $yamldir."/".$file )->[0];
+ $yamlmap = merge($m, $yamlmap);
+}
+if (not $yamlmap) {
+ die "FATAL: no configuration found\n";
+}
+
my $dbh;
END {
if (defined @conf::admin_users and
!isin( $real_user, @conf::admin_users));
lock_table();
- parse_packages();
+ parse_packages(0);
last SWITCH;
};
/^merge-sources/ && do {
lock_table();
my @ARGS = @ARGV;
@ARGV = ( $ARGS[0] );
- my $pkgs = parse_packages();
+ my $pkgs = parse_packages(0);
@ARGV = ( $ARGS[1] );
parse_quinn_diff(0);
@ARGV = ( $ARGS[2] );
call_edos_depcheck( $ARGS[0], $srcs );
last SWITCH;
};
- /^merge-all-overlay/ && do {
+ /^merge-all-secondary/ && do {
die "This operation is restricted to admin users\n"
if (defined @conf::admin_users and
!isin( $real_user, @conf::admin_users));
+ # This is in case the chroot has multiple unrelated
+ # dist, for instance unstable and experimental.
+ # This is not for stable and proposed-updates.
+ # The second packages file contains a combination
+ # of all Packages files known to the buildd, the
+ # first only for the current dist.
lock_table();
my @ARGS = @ARGV;
@ARGV = ( $ARGS[0] );
- my $pkgs = parse_packages();
+ my $pkgs = parse_packages(0);
+ @ARGV = ( $ARGS[3] );
+ my $pkgs = parse_packages(1);
@ARGV = ( $ARGS[1] );
parse_quinn_diff(0);
@ARGV = ( $ARGS[2] );
return;
}
- if ($pkg->{'state'} ne "Building" ) {
+ if (($pkg->{'state'} ne "Building") && ($pkg->{'state'} ne "Build-Attempted")) {
print "$name: not taken for building (state is $pkg->{'state'}). ",
"Skipping.\n";
return;
return;
}
- if ($pkg->{'state'} ne "Building" ) {
+ if (($pkg->{'state'} ne "Building") && ($pkg->{'state'} ne "Build-Attempted")) {
print "$name: not taken for building (state is $pkg->{'state'}). ",
"Skipping.\n";
return;
# This function looks through a Packages file and sets the state of
# packages to 'Installed'
sub parse_packages {
+ my $depwait_only = shift;
my $installed;
my $pkgs = get_all_source_info();
next if (defined ($installed->{$name}) and $installed->{$name}{'version'} ne "" and
version_lesseq( $version, $installed->{$name}{'version'} ));
$installed->{$name}{'version'} = $version;
+ next if $depwait_only;
$installed->{$name}{'depends'} = $depends;
$installed->{$name}{'all'} = 1 if $architecture eq "all";
undef $installed->{$name}{'Provider'};
foreach $letter (split( "", $list_order )) {
SWITCH: foreach ($letter) {
+ /C/ && do {
+ $x = $b->{'calprio'} <=> $a->{'calprio'};
+ return $x if $x != 0;
+ last SWITCH;
+ };
+ /W/ && do {
+ $x = $b->{'state_days'} <=> $a->{'state_days'};
+ return $x if $x != 0;
+ last SWITCH;
+ };
/P/ && do {
$x = $b->{'buildpri'} <=> $a->{'buildpri'};
return $x if $x != 0;
return 0;
}
+sub calculate_prio {
+ my $priomap = $yamlmap->{priority};
+ my $pkg = shift;
+ $pkg->{'calprio'} = 0;
+ foreach my $k (keys %{$priomap->{keys}}) {
+ $pkg->{'calprio'} += $priomap->{keys}->{$k}{$pkg->{$k}} if $pkg->{$k} and $priomap->{keys}->{$k}{$pkg->{$k}};
+ }
+
+ my $days = $pkg->{'state_days'};
+ $days = $priomap->{'waitingdays'}->{'min'} if $priomap->{'waitingdays'}->{'min'} and $days < $priomap->{'waitingdays'}->{'min'};
+ $days = $priomap->{'waitingdays'}->{'max'} if $priomap->{'waitingdays'}->{'max'} and $days > $priomap->{'waitingdays'}->{'max'};
+ my $scale = $priomap->{'waitingdays'}->{'scale'} || 1;
+ $pkg->{'calprio'} += $days * $scale;
+
+ $pkg->{'calprio'} += $pkg->{'permbuildpri'} if $pkg->{'permbuildpri'};
+ $pkg->{'calprio'} += $pkg->{'buildpri'} if $pkg->{'buildpri'};
+
+ return $pkg;
+}
+
+
sub list_packages {
my $state = shift;
my( $name, $pkg, @list );
my $db = get_all_source_info(state => $state, user => $user, category => $category, list_min_age => $list_min_age);
foreach $name (keys %$db) {
next if $name =~ /^_/;
- push @list, $db->{$name};
+ push @list, calculate_prio($db->{$name});
}
foreach $pkg (sort sort_list_func @list) {
if defined $pkg->{'buildpri'};
print ":binNMU{" . $pkg->{'binary_nmu_version'} . "}"
if defined $pkg->{'binary_nmu_version'};
+ print ":calprio{". $pkg->{'calprio'}."}";
+ print ":days{". $pkg->{'state_days'}."}";
print "]\n";
print " Reasons for failing:\n",
" [Category: ",
print " Previous state was $pkg->{'previous_state'} until ",
"$pkg->{'state_change'}\n"
if $verbose && $pkg->{'previous_state'};
+ print " Previous state $pkg->{'previous_state'} left $pkg->{'state_time'} ago\n"
+ if $verbose && $pkg->{'previous_state'};
print " No previous state recorded\n"
if $verbose && !$pkg->{'previous_state'};
print " Previous failing reasons:\n ",
my @firstkeys = qw(package version builder state section priority
installed_version previous_state state_change);
my @dists = $info_all_dists ? keys %conf::distributions : ($distribution);
+ my %beautykeys = ( 'package' => 'Package', 'version' => 'Version', 'builder' => 'Builder',
+ 'state' => 'State', 'section' => 'Section', 'priority' => 'Priority',
+ 'installed_version' => 'Installed-Version', 'previous_state' => 'Previous-State',
+ 'state_change' => 'State-Change',
+ 'bd_problem' => 'BD-Problem',
+ 'binary_nmu_changelog' => 'Binary-NMU-Changelog', 'binary_nmu_version' => 'Binary-NMU-Version',
+ 'buildpri' => 'BuildPri', 'depends' => 'Depends', 'failed' => 'Failed',
+ 'failed_category' => 'Failed-Category', 'notes' => 'Notes',
+ 'distribution' => 'Distribution', 'old_failed' => 'Old-Failed',
+ 'permbuildpri' => 'PermBuildPri', 'rel' => 'Rel',
+ 'calprio' => 'CalculatedPri', 'state_days' => 'State-Days'
+ );
foreach $name (@_) {
$name =~ s/_.*$//; # strip version
print "$pname: not registered\n";
next;
}
+ $pkg = calculate_prio($pkg);
print "$pname:\n";
foreach $key (@firstkeys) {
$val = "\n$val" if isin( $key, qw(Failed Old-Failed));
$val =~ s/\n/\n /g;
my $print_key = $key;
- $print_key = 'Package' if ($key eq 'package');
- $print_key = 'Version' if ($key eq 'version');
- $print_key = 'Builder' if ($key eq 'builder');
- $print_key = 'State' if ($key eq 'state');
- $print_key = 'Section' if ($key eq 'section');
- $print_key = 'Priority' if ($key eq 'priority');
- $print_key = 'Installed-Version' if ($key eq 'installed_version');
- $print_key = 'Previous-State' if ($key eq 'previous_state');
- $print_key = 'State-Change' if ($key eq 'state_change');
+ $print_key = $beautykeys{$print_key} if $beautykeys{$print_key};
printf " %-20s: %s\n", $print_key, $val;
}
foreach $key (sort keys %$pkg) {
$val = "\n$val" if isin( $key, qw(Failed Old-Failed));
$val =~ s/\n/\n /g;
my $print_key = $key;
- $print_key = 'BD-Problem' if ($key eq 'bd_problem');
- $print_key = 'Binary-NMU-Changelog' if ($key eq 'binary_nmu_changelog');
- $print_key = 'Binary-NMU-Version' if ($key eq 'binary_nmu_version');
- $print_key = 'BuildPri' if ($key eq 'buildpri');
- $print_key = 'Depends' if ($key eq 'depends');
- $print_key = 'Failed' if ($key eq 'failed');
- $print_key = 'Failed-Category' if ($key eq 'failed_category');
- $print_key = 'Notes' if ($key eq 'notes');
- $print_key = 'Distribution' if ($key eq 'distribution');
- $print_key = 'Old-Failed' if ($key eq 'old_failed');
- $print_key = 'PermBuildPri' if ($key eq 'permbuildpri');
- $print_key = 'Rel' if ($key eq 'rel');
+ $print_key = $beautykeys{$print_key} if $beautykeys{$print_key};
printf " %-20s: %s\n", $print_key, $val;
}
}
return if defined($$state) and $$state eq $newstate;
$pkg->{'previous_state'} = $$state if defined($$state);
$pkg->{'state_change'} = $curr_date;
+ $pkg->{'do_state_change'} = 1;
if (defined($$state) and $$state eq 'Failed') {
$pkg->{'old_failed'} =
return \%result;
}
-# for parsing Build-Depends from Sources
-sub parse_srcdeplist {
- my $pkg = shift;
- my $deps = shift;
- my $arch = shift;
- my $dep;
- my @results;
-
- foreach $dep (split( /\s*,\s*/, $deps )) {
- my @alts = split( /\s*\|\s*/, $dep );
- # Anything with an | is ignored, as it can be configured on a
- # per-buildd basis what will be installed
- next if $#alts != 0;
- $_ = shift @alts;
- if (!/^([^\s([]+)\s*(\(\s*([<=>]+)\s*(\S+)\s*\))?(\s*\[([^]]+)\])?/) {
- warn( "parse_srcdeplist: bad dependency $_\n" );
- next;
- }
- my($dep, $rel, $relv, $archlist) = ($1, $3, $4, $6);
- if ($archlist) {
- $archlist =~ s/^\s*(.*)\s*$/$1/;
- my @archs = split( /\s+/, $archlist );
- my ($use_it, $ignore_it, $include) = (0, 0, 0);
- foreach (@archs) {
- if (/^!/) {
- $ignore_it = 1 if substr($_, 1) eq $arch;
- } else {
- $use_it = 1 if $_ eq $arch;
- $include = 1;
- }
- }
- warn "Warning: inconsistent arch restriction on ",
- "$pkg: $dep depedency\n"
- if $ignore_it && $use_it;
- next if $ignore_it || ($include && !$use_it);
- }
- my $neg = 0;
- if ($dep =~ /^!/) {
- $dep =~ s/^!\s*//;
- $neg = 1;
- }
- my $result;
- $result->{'package'} = $dep;
- $result->{'Neg'} = $neg;
- if ($rel && $relv) {
- $result->{'rel'} = $rel;
- $result->{'version'} = $relv;
- }
- push @results, $result;
- }
- return \@results;
-}
-
sub build_deplist {
my $list = shift;
my($key, $result);
return $result;
}
-sub get_unsatisfied_dep {
- my $bd = shift;
- my $pkgs = shift;
- my $dep = shift;
- my $savedep = shift;
-
- my $pkgname = $dep->{'package'};
-
- if (defined $pkgs->{$pkgname}{'Provider'}) {
- # provides. leave them for buildd/sbuild.
- return "";
- }
-
- # check cache
- return $pkgs->{$pkgname}{'Unsatisfied'} if $savedep and defined($pkgs->{$pkgname}{'Unsatisfied'});
-
- # Return unsatisfied deps to a higher caller to process
- if ((!defined($pkgs->{$pkgname})) or
- (defined($dep->{'rel'}) and !version_compare( $pkgs->{$pkgname}{'version'}, $dep->{'rel'}, $dep->{'Version'} ) ) ) {
- my %deplist;
- $deplist{$pkgname} = $dep;
- my $deps = build_deplist(\%deplist);
- $pkgs->{$pkgname}{'Unsatisfied'} = $deps if $savedep;
- return $deps;
- }
-
- # set cache to "" to avoid infinite recursion
- $pkgs->{$pkgname}{'Unsatisfied'} = "" if $savedep;
-
- if (defined $pkgs->{$dep->{'package'}}{'depends'}) {
- my $deps = parse_deplist( $pkgs->{$dep->{'package'}}{'depends'} );
- foreach (keys %$deps) {
- $dep = $$deps{$_};
- # recur on dep.
- my $ret = get_unsatisfied_dep($bd,$pkgs,$dep,1);
- if ($ret ne "") {
- my $retdep = parse_deplist( $ret );
- foreach (keys %$retdep) {
- $dep = $$retdep{$_};
-
- $dep->{'rel'} = '>=' if defined($dep->{'rel'}) and $dep->{'rel'} =~ '^=';
-
- if (defined($dep->{'rel'}) and $dep->{'rel'} =~ '^>' and defined ($pkgs->{$dep->{'package'}}) and
- version_compare($bd->{$pkgs->{$dep->{'package'}}{'Source'}}{'ver'},'>>',$pkgs->{$dep->{'package'}}{'Sourcev'})) {
- if (not defined($merge_binsrc{$dep->{'package'}})) {
- # the uninstallable package doesn't exist in the new source; look for something else that does.
- delete $$retdep{$dep->{'package'}};
- foreach (sort (split( /\s*,\s*/, $bd->{$pkgs->{$dep->{'package'}}{'Source'}}{'bin'}))) {
- next if ($pkgs->{$_}{'all'} or not defined $pkgs->{$_}{'version'});
- $dep->{'package'} = $_;
- $dep->{'rel'} = '>>';
- $dep->{'version'} = $pkgs->{$_}{'Version'};
- $$retdep{$_} = $dep;
- last;
- }
- }
- } else {
- # sanity check to make sure the depending binary still exists, and the depended binary exists and dep-wait on a new version of it
- if ( defined($merge_binsrc{$pkgname}) and defined($pkgs->{$dep->{'package'}}{'version'}) ) {
- delete $$retdep{$dep->{'package'}};
- $dep->{'package'} = $pkgname;
- $dep->{'rel'} = '>>';
- $dep->{'version'} = $pkgs->{$pkgname}{'Version'};
- $$retdep{$pkgname} = $dep;
- }
- delete $$retdep{$dep->{'package'}} if (defined ($dep->{'rel'}) and $dep->{'rel'} =~ '^>');
- }
- }
- $ret = build_deplist($retdep);
- $pkgs->{$pkgname}{'Unsatisfied'} = $ret if $savedep;
- return $ret;
- }
- }
- }
- return "";
-}
-
sub call_edos_depcheck {
my $packagesfile = shift;
my $srcs = shift;
sub get_readonly_source_info {
my $name = shift;
- my $pkg = $dbh->selectrow_hashref('SELECT * FROM ' .
+ # SELECT FLOOR(EXTRACT('epoch' FROM age(localtimestamp, '2010-01-22 23:45')) / 86400) -- change to that?
+ my $pkg = $dbh->selectrow_hashref('SELECT *, extract(days from date_trunc(\'days\', now() - state_change::timestamp)) as state_days FROM ' .
table_name() . ' WHERE package = ? AND distribution = ?',
undef, $name, $distribution);
return $pkg;
sub get_source_info {
my $name = shift;
- my $pkg = $dbh->selectrow_hashref('SELECT * FROM ' .
+ my $pkg = $dbh->selectrow_hashref('SELECT *, extract(days from date_trunc(\'days\', now() - state_change::timestamp)) as state_days FROM ' .
table_name() . ' WHERE package = ? AND distribution = ?' .
' FOR UPDATE',
undef, $name, $distribution);
sub get_all_source_info {
my %options = @_;
- my $q = 'SELECT * FROM ' . table_name()
+ my $q = 'SELECT *, '.
+ 'extract(days from date_trunc(\'days\', now() - state_change::timestamp)) as state_days, '.
+ 'date_trunc(\'seconds\', now() - state_change::timestamp) as state_time'.
+ ' FROM ' . table_name()
. ' WHERE distribution = ? ';
my @args = ($distribution);
if ($options{state} && uc($options{state}) ne "ALL") {
'priority = ?, ' .
'installed_version = ?, ' .
'previous_state = ?, ' .
- 'state_change = ?, ' .
+ (($pkg->{'do_state_change'}) ? "state_change = now()," : "").
'notes = ?, ' .
'builder = ?, ' .
'failed = ?, ' .
$pkg->{'priority'},
$pkg->{'installed_version'},
$pkg->{'previous_state'},
- $pkg->{'state_change'},
$pkg->{'notes'},
$pkg->{'builder'},
$pkg->{'failed'},