+# -*- mode:sh -*-
# log something (basically echo it together with a timestamp)
#
# Set $PROGRAM to a string to have it added to the output.
}
function wbtrigger() {
- MODE=${1:-"often"}
SSHOPT="-o BatchMode=yes -o ConnectTimeout=30 -o SetupTimeout=240"
if lockfile -r 3 -l 3600 "${LOCK_BUILDD}"; then
- if [ "x${MODE}x" = "xdailyx" ]; then
- ssh ${SSHOPT} wbadm@buildd /org/wanna-build/trigger.daily || echo "W-B trigger.daily failed" | mail -s "W-B Daily trigger failed" ftpmaster@ftp-master.debian.org
- elif [ "x${MODE}x" = "xoftenx" ]; then
- ssh -q -q ${SSHOPT} wbadm@buildd /org/wanna-build/trigger.often
- else
- log_error "Unknown wb trigger mode called"
- fi
+ ssh -q -q ${SSHOPT} wbadm@buildd /org/wanna-build/trigger.often
fi
rm -f "${LOCK_BUILDD}"
}
function sync_debbugs () {
# sync with debbugs
echo "--" >> $report
+ timestamp=$(date "+%Y-%m-%d-%H:%M")
+ mkdir -p $queuedir/${timestamp}
+ rsync -aq $queuedir/bts_version_track/ $queuedir/bts_version_track_archive/${timestamp}
+ rmdir --ignore-fail-on-non-empty $queuedir/${timestamp} # remove if empty.
rsync -aq -e "ssh -o Batchmode=yes -o ConnectTimeout=30 -o SetupTimeout=30" --remove-source-files $queuedir/bts_version_track/ bugs-sync:/org/bugs.debian.org/versions/queue/ftp-master/ 2>/dev/null && touch $lockdir/synced_bts_version || true
NOW=$(date +%s)
TSTAMP=$(stat -c %Y $lockdir/synced_bts_version)
touch "${DINSTALLSTART}"
ts "startup"
+DINSTALLBEGIN="$(date -u +"%a %b %d %T %Z %Y (%s)")"
+state "Startup"
lockfile -l 3600 "${LOCK_DAILY}"
trap onerror ERR
)
stage $GO &
-GO=(
- FUNC="punew"
- TIME="p-u-new"
- ARGS="p-u-new"
- ERR=""
-)
-### TODO: policy-new
-#stage $GO
-
-GO=(
- FUNC="opunew"
- TIME="o-p-u-new"
- ARGS="o-p-u-new"
- ERR=""
-)
-### TODO: policy-new
-#stage $GO
-
GO=(
FUNC="i18n1"
TIME="i18n 1"
lockfile "$LOCK_NEW"
GO=(
- FUNC="process_unchecked"
- TIME="unchecked"
+ FUNC="punew"
+ TIME="p-u-new"
+ ARGS="proposedupdates"
+ ERR="false"
+)
+stage $GO
+
+GO=(
+ FUNC="opunew"
+ TIME="o-p-u-new"
+ ARGS="oldproposedupdates"
+ ERR="false"
+)
+stage $GO
+
+GO=(
+ FUNC="newstage"
+ TIME="newstage"
ARGS=""
ERR=""
)
-# disabled until p-u is faster than now. it runs often enough, so wont hurt to save
-# the time here.
-#stage $GO
+stage $GO
GO=(
FUNC="cruft"
rm -f "$LOCK_ACCEPTED"
rm -f "$LOCK_NEW"
+state "indices"
+
GO=(
FUNC="dominate"
TIME="dominate"
)
stage $GO &
+state "packages/contents"
GO=(
FUNC="packages"
TIME="apt-ftparchive"
# functions before it. We no longer have a 1.5hour sync point then.
stage $GO
+state "dists/"
GO=(
FUNC="pdiff"
TIME="pdiff"
)
stage $GO
+state "scripts"
GO=(
FUNC="mkmaintainers"
TIME="mkmaintainers"
)
stage $GO
-GO=(
- FUNC="wb"
- TIME="w-b"
- ARGS=""
- ERR=""
-)
-stage $GO &
-
rm -f "${LOCK_DAILY}"
ts "locked part finished"
+state "postlock"
GO=(
FUNC="pgdump_post"
ERR="false"
)
stage $GO
+state "all done"
# Now, at the very (successful) end of dinstall, make sure we remove
export SCRIPTVARS=/srv/ftp.debian.org/dak/config/debian/vars
. $SCRIPTVARS
-date -u > $ftpdir/project/trace/ftp-master.debian.org
-echo "Using dak v1" >> $ftpdir/project/trace/ftp-master.debian.org
-echo "Running on host: $(hostname -f)" >> $ftpdir/project/trace/ftp-master.debian.org
dak import-users-from-passwd
dak queue-report -n > $webdir/new.html
# We used to have accepted in here, but it doesn't exist in that form any more
fi
done
make_buildd_dir
- wbtrigger "often"
+ wbtrigger
fi
}
+# -*- mode:sh -*-
# Timestamp. Used for dinstall stat graphs
function ts() {
echo "Archive maintenance timestamp ($1): $(date +%H:%M:%S)"
# Process (oldstable)-proposed-updates "NEW" queue
function punew_do() {
- cd "${queuedir}/${1}"
date -u -R >> REPORT
- dak process-new -a -C COMMENTS >> REPORT || true
+ dak process-policy $1 | tee -a REPORT | mail -e -s "NEW changes in $1" debian-release@lists.debian.org
echo >> REPORT
}
function punew() {
log "Doing automated p-u-new processing"
+ cd "${queuedir}/p-u-new"
punew_do "$1"
}
function opunew() {
log "Doing automated o-p-u-new processing"
+ cd "${queuedir}/o-p-u-new"
punew_do "$1"
}
function mirror() {
log "Regenerating \"public\" mirror/ hardlink fun"
+ DATE_SERIAL=$(date +"%Y%m%d01")
+ FILESOAPLUS1=$(awk '/serial/ { print $3+1 }' ${TRACEFILE} )
+ if [ "$DATE_SERIAL" -gt "$FILESOAPLUS1" ]; then
+ SERIAL="$DATE_SERIAL"
+ else
+ SERIAL="$FILESOAPLUS1"
+ fi
+ date -u > ${TRACEFILE}
+ echo "Using dak v1" >> ${TRACEFILE}
+ echo "Running on host: $(hostname -f)" >> ${TRACEFILE}
+ echo "Archive serial: ${SERIAL}" >> ${TRACEFILE}
cd ${mirrordir}
rsync -aH --link-dest ${ftpdir} --delete --delete-after --ignore-errors ${ftpdir}/. .
}
-function wb() {
- log "Trigger daily wanna-build run"
- wbtrigger "daily"
-}
-
function expire() {
log "Expiring old database dumps..."
cd $base/backup
do_unchecked
sync_debbugs
}
+
+# do a run of newstage only before dinstall is on.
+function newstage() {
+ log "Processing the newstage queue"
+ UNCHECKED_WITHOUT_LOCK="-p"
+ do_newstage
+}
+
+# Function to update a "statefile" telling people what we are doing
+# (more or less).
+#
+# This should be called with the argument(s)
+# - Status name we want to show.
+#
+function state() {
+ RIGHTNOW="$(date -u +"%a %b %d %T %Z %Y (%s)")"
+ cat >"${DINSTALLSTATE}" <<EOF
+Dinstall start: ${DINSTALLBEGIN}
+Current action: ${1}
+Action start: ${RIGHTNOW}
+EOF
+}
+# -*- mode:sh -*-
# usually we are not using debug logs. Set to 1 if you want them.
DEBUG=0
# Lock buildd updates
LOCK_BUILDD="$lockdir/buildd.lock"
+
+# Statefile for the users
+DINSTALLSTATE="${webdir}/dinstall.status"
+
+# The mirror trace file
+TRACEFILE="${ftpdir}/project/trace/ftp-master.debian.org"
#!/usr/bin/env python
-""" Check for obsolete binary packages """
-# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
+"""
+Check for obsolete binary packages
+
+@contact: Debian FTP Master <ftpmaster@debian.org>
+@copyright: 2000-2006 James Troup <james@nocrew.org>
+@copyright: 2009 Torsten Werner <twerner@debian.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
################################################################################
-def do_nbs(real_nbs):
- output = "Not Built from Source\n"
- output += "---------------------\n\n"
+def queryWithoutSource(suite_id, session):
+ """searches for arch: all packages from suite that do no longer
+ reference a source package in the same suite
- cmd_output = ""
- nbs_keys = real_nbs.keys()
- nbs_keys.sort()
- for source in nbs_keys:
- output += " * %s_%s builds: %s\n" % (source,
- source_versions.get(source, "??"),
- source_binaries.get(source, "(source does not exist)"))
- output += " but no longer builds:\n"
- versions = real_nbs[source].keys()
- versions.sort(apt_pkg.VersionCompare)
- all_packages = []
- for version in versions:
- packages = real_nbs[source][version].keys()
- packages.sort()
- all_packages.extend(packages)
- output += " o %s: %s\n" % (version, ", ".join(packages))
- if all_packages:
- all_packages.sort()
- cmd_output += " dak rm -m \"[auto-cruft] NBS (was built by %s)\" -s %s -b %s -R\n\n" % (source, suite.suite_name, " ".join(all_packages))
+ subquery unique_binaries: selects all packages with only 1 version
+ in suite since 'dak rm' does not allow to specify version numbers"""
- output += "\n"
+ query = """
+ with unique_binaries as
+ (select package, max(version) as version, max(source) as source
+ from bin_associations_binaries
+ where architecture = 2 and suite = :suite_id
+ group by package having count(*) = 1)
+ select ub.package, ub.version
+ from unique_binaries ub
+ left join src_associations_src sas
+ on ub.source = sas.src and sas.suite = :suite_id
+ where sas.id is null
+ order by ub.package"""
+ return session.execute(query, { 'suite_id': suite_id })
+
+def reportWithoutSource(suite_name, suite_id, session):
+ rows = queryWithoutSource(suite_id, session)
+ title = 'packages without source in suite %s' % suite_name
+ if rows.rowcount > 0:
+ print '%s\n%s\n' % (title, '-' * len(title))
+ message = '"[auto-cruft] no longer built from source"'
+ for row in rows:
+ (package, version) = row
+ print "* package %s in version %s is no longer built from source" % \
+ (package, version)
+ print " - suggested command:"
+ print " dak rm -m %s -s %s -a all -p -R -b %s\n" % \
+ (message, suite_name, package)
+
+def queryNewerAll(suite_name, session):
+ """searches for arch != all packages that have an arch == all
+ package with a higher version in the same suite"""
- if len(cmd_output):
- print output
- print "Suggested commands:\n"
- print cmd_output
+ query = """
+select bab1.package, bab1.version as oldver,
+ array_to_string(array_agg(a.arch_string), ',') as oldarch,
+ bab2.version as newver
+ from bin_associations_binaries bab1
+ join bin_associations_binaries bab2
+ on bab1.package = bab2.package and bab1.version < bab2.version and
+ bab1.suite = bab2.suite and bab1.architecture > 2 and
+ bab2.architecture = 2
+ join architecture a on bab1.architecture = a.id
+ join suite s on bab1.suite = s.id
+ where s.suite_name = :suite_name
+ group by bab1.package, oldver, bab1.suite, newver"""
+ return session.execute(query, { 'suite_name': suite_name })
+
+def reportNewerAll(suite_name, session):
+ rows = queryNewerAll(suite_name, session)
+ title = 'obsolete arch any packages in suite %s' % suite_name
+ if rows.rowcount > 0:
+ print '%s\n%s\n' % (title, '-' * len(title))
+ message = '"[auto-cruft] obsolete arch any package"'
+ for row in rows:
+ (package, oldver, oldarch, newver) = row
+ print "* package %s is arch any in version %s but arch all in version %s" % \
+ (package, oldver, newver)
+ print " - suggested command:"
+ print " dak rm -m %s -s %s -a %s -p -b %s\n" % \
+ (message, suite_name, oldarch, package)
+
+def queryNBS(suite_id, session):
+ """This one is really complex. It searches arch != all packages that
+ are no longer built from current source packages in suite.
+
+ temp table unique_binaries: will be populated with packages that
+ have only one version in suite because 'dak rm' does not allow
+ specifying version numbers
+
+ temp table newest_binaries: will be populated with packages that are
+ built from current sources
+
+ subquery uptodate_arch: returns all architectures built from current
+ sources
+
+ subquery unique_binaries_uptodate_arch: returns all packages in
+ architectures from uptodate_arch
+
+ subquery unique_binaries_uptodate_arch_agg: same as
+ unique_binaries_uptodate_arch but with column architecture
+ aggregated to array
+
+ subquery uptodate_packages: similar to uptodate_arch but returns all
+ packages built from current sources
+
+ subquery outdated_packages: returns all packages with architectures
+ no longer built from current source
+ """
+
+ query = """
+create temp table unique_binaries (
+ package text not null,
+ architecture integer not null,
+ source integer not null);
+
+insert into unique_binaries
+ select bab.package, bab.architecture, max(bab.source)
+ from bin_associations_binaries bab
+ where bab.suite = :suite_id and bab.architecture > 2
+ group by package, architecture having count(*) = 1;
+
+create temp table newest_binaries (
+ package text not null,
+ architecture integer not null,
+ source text not null,
+ version debversion not null);
+
+insert into newest_binaries
+ select ub.package, ub.architecture, nsa.source, nsa.version
+ from unique_binaries ub
+ join newest_src_association nsa
+ on ub.source = nsa.src and nsa.suite = :suite_id;
+
+with uptodate_arch as
+ (select architecture, source, version
+ from newest_binaries
+ group by architecture, source, version),
+ unique_binaries_uptodate_arch as
+ (select ub.package, ub.architecture, ua.source, ua.version
+ from unique_binaries ub
+ join source s
+ on ub.source = s.id
+ join uptodate_arch ua
+ on ub.architecture = ua.architecture and s.source = ua.source),
+ unique_binaries_uptodate_arch_agg as
+ (select ubua.package,
+ array(select unnest(array_agg(a.arch_string)) order by 1) as arch_list,
+ ubua.source, ubua.version
+ from unique_binaries_uptodate_arch ubua
+ join architecture a
+ on ubua.architecture = a.id
+ group by ubua.source, ubua.version, ubua.package),
+ uptodate_packages as
+ (select package, source, version
+ from newest_binaries
+ group by package, source, version),
+ outdated_packages as
+ (select array(select unnest(array_agg(package)) order by 1) as pkg_list,
+ arch_list, source, version
+ from unique_binaries_uptodate_arch_agg
+ where package not in
+ (select package from uptodate_packages)
+ group by arch_list, source, version)
+ select * from outdated_packages order by source"""
+ return session.execute(query, { 'suite_id': suite_id })
+
+def reportNBS(suite_name, suite_id):
+ session = DBConn().session()
+ nbsRows = queryNBS(suite_id, session)
+ title = 'NBS packages in suite %s' % suite_name
+ if nbsRows.rowcount > 0:
+ print '%s\n%s\n' % (title, '-' * len(title))
+ for row in nbsRows:
+ (pkg_list, arch_list, source, version) = row
+ pkg_string = ' '.join(pkg_list)
+ arch_string = ','.join(arch_list)
+ print "* source package %s version %s no longer builds" % \
+ (source, version)
+ print " binary package(s): %s" % pkg_string
+ print " on %s" % arch_string
+ print " - suggested command:"
+ message = '"[auto-cruft] NBS (no longer built by %s)"' % source
+ print " dak rm -m %s -s %s -a %s -p -R -b %s\n" % \
+ (message, suite_name, arch_string, pkg_string)
+ session.close()
+
+def reportAllNBS(suite_name, suite_id, session):
+ reportWithoutSource(suite_name, suite_id, session)
+ reportNewerAll(suite_name, session)
+ reportNBS(suite_name, suite_id)
################################################################################
if "obsolete source" in checks:
report_obsolete_source(suite_name, session)
+ if "nbs" in checks:
+ reportAllNBS(suite_name, suite_id, session)
+
bin_not_built = {}
if "bnb" in checks:
# Distinguish dubious (version numbers match) and 'real' NBS (they don't)
dubious_nbs = {}
- real_nbs = {}
for source in nbs.keys():
for package in nbs[source].keys():
versions = nbs[source][package].keys()
source_version = source_versions.get(source,"0")
if apt_pkg.VersionCompare(latest_version, source_version) == 0:
add_nbs(dubious_nbs, source, latest_version, package, suite_id, session)
- else:
- add_nbs(real_nbs, source, latest_version, package, suite_id, session)
if "nviu" in checks:
do_newer_version('unstable', 'experimental', 'NVIU', session)
if "nvit" in checks:
do_newer_version('testing', 'testing-proposed-updates', 'NVIT', session)
- if "nbs" in checks:
- do_nbs(real_nbs)
-
###
if Options["Mode"] == "full":
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+################################################################################
+
+# Ganneff> Please go and try to lock mhy now. After than try to lock NEW.
+# twerner> !lock mhy
+# dak> twerner: You suck, this is already locked by Ganneff
+# Ganneff> now try with NEW
+# twerner> !lock NEW
+# dak> twerner: also locked NEW
+# mhy> Ganneff: oy, stop using me for locks and highlighting me you tall muppet
+# Ganneff> hehe :)
+
+################################################################################
+
from daklib.dbconn import *
from daklib.config import Config
from daklib import utils
def getBinaries(suite, component, architecture, type, session):
query = """
- SELECT path, filename
- FROM binfiles_suite_component_arch
- WHERE suite = :suite AND component = :component AND type = :type AND
- (architecture = :architecture OR architecture = 2)
- ORDER BY filename
+CREATE TEMP TABLE gf_candidates (
+ filename text,
+ path text,
+ architecture integer,
+ src integer,
+ source text);
+
+INSERT INTO gf_candidates (filename, path, architecture, src, source)
+ SELECT f.filename, l.path, b.architecture, b.source as src, s.source
+ FROM binaries b
+ JOIN bin_associations ba ON b.id = ba.bin
+ JOIN source s ON b.source = s.id
+ JOIN files f ON b.file = f.id
+ JOIN location l ON f.location = l.id
+ WHERE ba.suite = :suite AND b.type = :type AND
+ l.component = :component AND b.architecture IN (2, :architecture);
+
+WITH arch_any AS
+
+ (SELECT path, filename FROM gf_candidates
+ WHERE architecture > 2),
+
+ arch_all_with_any AS
+ (SELECT path, filename FROM gf_candidates
+ WHERE architecture = 2 AND
+ src IN (SELECT src FROM gf_candidates WHERE architecture > 2)),
+
+ arch_all_without_any AS
+ (SELECT path, filename FROM gf_candidates
+ WHERE architecture = 2 AND
+ source NOT IN (SELECT DISTINCT source FROM gf_candidates WHERE architecture > 2)),
+
+ filelist AS
+ (SELECT * FROM arch_any
+ UNION
+ SELECT * FROM arch_all_with_any
+ UNION
+ SELECT * FROM arch_all_without_any)
+
+ SELECT * FROM filelist ORDER BY filename
"""
args = { 'suite': suite.suite_id,
'component': component.component_id,
file.write(filename + '\n')
file.close()
-def writeBinaryList(suite, component, architecture, type, session):
+def writeBinaryList(suite, component, architecture, type):
file = listPath(suite, component, architecture, type)
+ session = DBConn().session()
for filename in getBinaries(suite, component, architecture, type, session):
file.write(filename + '\n')
+ session.close()
file.close()
def usage():
print """Usage: dak generate_filelist [OPTIONS]
Create filename lists for apt-ftparchive.
- -s, --suite=SUITE act on this suite
+ -s, --suite=SUITE act on this suite
-c, --component=COMPONENT act on this component
- -a, --architecture=ARCH act on this architecture
- -h, --help show this help and exit
+ -a, --architecture=ARCH act on this architecture
+ -h, --help show this help and exit
ARCH, COMPONENT and SUITE can be comma (or space) separated list, e.g.
--suite=testing,unstable"""
if architecture_name == 'source':
writeSourceList(suite, component, session)
elif architecture_name != 'all':
- writeBinaryList(suite, component, architecture, 'deb', session)
- writeBinaryList(suite, component, architecture, 'udeb', session)
+ writeBinaryList(suite, component, architecture, 'deb')
+ writeBinaryList(suite, component, architecture, 'udeb')
except:
pass
# this script doesn't change the database
- session.rollback()
+ session.close()
if __name__ == '__main__':
main()
except CantGetLockError:
print "Hello? Operator! Give me the number for 911!"
print "Dinstall in the locked area, cant process packages, come back later"
-# (new, byhand) = check_status(files)
-# if new or byhand:
-# if new:
-# do_new(u, session)
-# if byhand:
-# do_byhand(u, session)
-# (new, byhand) = check_status(files)
-
-# if not new and not byhand:
-# try:
-# check_daily_lock()
-# do_accept(u)
-# except CantGetLockError:
-# print "Hello? Operator! Give me the number for 911!"
-# print "Dinstall in the locked area, cant process packages, come back later"
+
except AlreadyLockedError, e:
print "Seems to be locked by %s already, skipping..." % (e)
if not Options["No-Action"]:
destqueue = get_policy_queue('newstage', session)
- if changes_to_queue(u, srcqueue, destqueue, session):
+ if changes_to_queue(u, srcqueue, destqueue, session):
+ print " ACCEPT"
Logger.log(["Policy Queue ACCEPT: %s: %s" % (srcqueue.queue_name, u.pkg.changes_file)])
- else:
+ else:
print "E: Failed to migrate %s" % u.pkg.changes_file
################################################################################
u.pkg.remove_known_changes(session=session)
session.commit()
+ print " REJECT"
Logger.log(["Policy Queue REJECT: %s: %s" % (srcqueue.queue_name, u.pkg.changes_file)])
from daklib.summarystats import SummaryStats
from daklib.holding import Holding
from daklib.config import Config
+from daklib.regexes import re_match_expired
###############################################################################
if u.pkg.changes["fingerprint"]:
valid_changes_p = u.load_changes(changespath)
else:
+ for reason in rejects:
+ if re_match_expired.match(reason):
+ # Hrm, key expired. Lets see if we can still parse the .changes before
+ # we reject. Then we would be able to mail the maintainer, instead of
+ # just silently dropping the upload.
+ u.load_changes(changespath)
valid_changes_p = False
u.rejects.extend(rejects)
age = Cnf["Queue-Report::Options::Age"]
if Cnf.has_key("Queue-Report::Options::New"):
# If we produce html we always have oldest first.
- direction.append([4,-1,"ao"])
+ direction.append([5,-1,"ao"])
else:
if Cnf.has_key("Queue-Report::Options::Sort"):
for i in Cnf["Queue-Report::Options::Sort"].split(","):
if i == "ao":
# Age, oldest first.
- direction.append([4,-1,age])
+ direction.append([5,-1,age])
elif i == "an":
# Age, newest first.
- direction.append([4,1,age])
+ direction.append([5,1,age])
elif i == "na":
# Name, Ascending.
direction.append([0,1,0])
direction.append([0,-1,0])
elif i == "nl":
# Notes last.
- direction.append([3,1,0])
+ direction.append([4,1,0])
elif i == "nf":
# Notes first.
- direction.append([3,-1,0])
+ direction.append([4,-1,0])
entries.sort(lambda x, y: sortfunc(x, y))
# Yes, in theory you can add several sort options at the commandline with. But my mind is to small
# at the moment to come up with a real good sorting function that considers all the sidesteps you
log.write("\n")
if Cnf.has_key("Queue-Report::Options::New"):
- direction.append([4,1,"ao"])
+ direction.append([5,1,"ao"])
entries.sort(lambda x, y: sortfunc(x, y))
# Output for a html file. First table header. then table_footer.
# Any line between them is then a <tr> printed from subroutine table_row.
@contact: Debian FTPMaster <ftpmaster@debian.org>
@copyright: 2000, 2001, 2002, 2003, 2004, 2006 James Troup <james@nocrew.org>
@copyright: 2008-2009 Mark Hymers <mhy@debian.org>
-@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
@copyright: 2009 Mike O'Connor <stew@debian.org>
@license: GNU General Public License version 2 or later
"""
# reflection
class DebVersion(sqltypes.Text):
+ """
+ Support the debversion type
+ """
+
def get_col_spec(self):
return "DEBVERSION"
"""
Returns list of Suite objects for given C{architecture} name
- @type source: str
- @param source: Architecture name to search for
+ @type architecture: str
+ @param architecture: Architecture name to search for
@type session: Session
@param session: Optional SQL session object (a temporary one will be
"""
Returns list of Suite objects which given C{package} name is in
- @type source: str
- @param source: DBBinary package name to search for
+ @type package: str
+ @param package: DBBinary package name to search for
@rtype: list
@return: list of Suite objects for the given package
@type version: str or None
@param version: Version to search for (or None)
- @type package: str, list or None
- @param package: Architectures to limit to (or None if no limit)
+ @type architecture: str, list or None
+ @param architecture: Architectures to limit to (or None if no limit)
@type session: Session
@param session: Optional SQL session object (a temporary one will be
If no matching file is found, a row is inserted.
- @type filename: string
- @param filename: The filepath
+ @type filepath: string
+ @param filepath: The filepath
+
@type session: SQLAlchemy
@param session: Optional SQL session object (a temporary one will be
generated if not supplied). If not passed, a commit will be performed at
def check_poolfile(filename, filesize, md5sum, location_id, session=None):
"""
Returns a tuple:
- (ValidFileFound [boolean or None], PoolFile object or None)
+ (ValidFileFound [boolean or None], PoolFile object or None)
@type filename: string
@param filename: the filename of the file to check against the DB
@rtype: tuple
@return: Tuple of length 2.
- If more than one file found with that name:
- (None, None)
- If valid pool file found: (True, PoolFile object)
- If valid pool file not found:
- (False, None) if no file found
- (False, PoolFile object) if file found with size/md5sum mismatch
+ - If more than one file found with that name: (C{None}, C{None})
+ - If valid pool file found: (C{True}, C{PoolFile object})
+ - If valid pool file not found:
+ - (C{False}, C{None}) if no file found
+ - (C{False}, C{PoolFile object}) if file found with size/md5sum mismatch
"""
q = session.query(PoolFile).filter_by(filename=filename)
"""
returns DBChange object for given C{filename}.
- @type archive: string
- @param archive: the name of the arhive
+ @type filename: string
+ @param filename: the name of the file
@type session: Session
@param session: Optional SQLA session object (a temporary one will be
generated if not supplied)
- @rtype: Archive
- @return: Archive object for the given name (None if not present)
+ @rtype: DBChange
+ @return: DBChange object for the given filename (C{None} if not present)
"""
q = session.query(DBChange).filter_by(changesname=filename)
@param component: the component name (if None, no restriction applied)
@type archive: string
- @param archive_id: the archive name (if None, no restriction applied)
+ @param archive: the archive name (if None, no restriction applied)
@rtype: Location / None
@return: Either a Location object or None if one can't be found
1. exact match => 1.0-3
2. bin-only NMU => 1.0-3+b1 , 1.0-3.1+b1
- @type package: string
- @param package: package source name
+ @type source: string
+ @param source: source name
@type source_version: string
@param source_version: expected source version
@type source: str
@param source: DBSource package name to search for
- @type source: str or None
- @param source: DBSource version name to search for or None if not applicable
+ @type version: str or None
+ @param version: DBSource version name to search for or None if not applicable
@type dm_upload_allowed: bool
@param dm_upload_allowed: If None, no effect. If True or False, only
uploader_ids.append(get_or_set_maintainer(up, session).maintainer_id)
added_ids = {}
- for up in uploader_ids:
- if added_ids.has_key(up):
- utils.warn("Already saw uploader %s for source %s" % (up, source.source))
+ for up_id in uploader_ids:
+ if added_ids.has_key(up_id):
+ import utils
+ utils.warn("Already saw uploader %s for source %s" % (up_id, source.source))
continue
- added_ids[u]=1
+ added_ids[up_id]=1
su = SrcUploader()
- su.maintainer_id = up
+ su.maintainer_id = up_id
su.source_id = source.source_id
session.add(su)
"""
Returns list of Architecture objects for given C{suite} name
- @type source: str
- @param source: Suite name to search for
+ @type suite: str
+ @param suite: Suite name to search for
@type skipsrc: boolean
@param skipsrc: Whether to skip returning the 'source' architecture entry
+#!/usr/bin/python
+
+""" Helper functions for the various changes formats
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Chris Lamb <lamby@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# <mhy> !!!!11111iiiiiioneoneoneone
+# <dak> mhy: Error: "!!!11111iiiiiioneoneoneone" is not a valid command.
+# <mhy> dak: oh shut up
+# <dak> mhy: Error: "oh" is not a valid command.
+
+################################################################################
+
from regexes import re_verwithext
from dak_exceptions import UnknownFormatError
(8, 4, 'hardy')
If the format doesn't match these forms, raises UnknownFormatError.
+
+ @type txt: string
+ @param txt: Format string to parse
+
+ @rtype: tuple
+ @return: Parsed format
+
+ @raise UnknownFormatError: Unknown Format: line
"""
format = re_verwithext.search(txt)
+#!/usr/bin/python
+
+""" Utility functions for lintian checks in dak
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Chris Lamb <lamby@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# <mhy> I often wonder if we should use NSA bot or something instead and get dinstall
+# to send emails telling us about its progress :-)
+# <mhy> dinstall: I'm processing openoffice
+# <mhy> dinstall: I'm choking, please help me
+# <Ganneff> yeah. get floods in here, for 600 accepted packages.
+# <mhy> hehe
+# <Ganneff> im not sure the other opers will like it if i oper up the bot, just so it
+# can flood faster
+# <mhy> flood all debian related channels
+# <mhy> just to be safe
+# <Ganneff> /msg #debian-* dinstall: starting
+# <Ganneff> more interesting would be the first message in #debian, the next in
+# #d-devel, then #d-qa
+# <Ganneff> and expect people to monitor all.
+# <Ganneff> i bet we have enough debian channels to at least put the timestamps in
+# seperate channels each
+# <Ganneff> and if not - we can make it go multi-network
+# <Ganneff> first oftc, then opn, then ircnet, then - we will find some. quakenet anyone?
+# <mhy> I should know better than to give you ideas
+
+################################################################################
+
from regexes import re_parse_lintian
def parse_lintian_output(output):
>>> list(parse_lintian_output('W: pkgname: some-tag path/to/file'))
[('W', 'pkgname', 'some-tag', 'path/to/file')]
+
+ @type output: string
+ @param output: The output from lintian
"""
for line in output.split('\n'):
"""
Generates package reject messages by comparing parsed lintian output with
tag definitions. Returns a generator containing the reject messages.
+
+ @param parsed_tags: Parsed lintian tags as returned by L{parse_lintian_output}
+
+ @param tag_definitions: YAML.load lintian tag definitions to reject on
+
+ @return: Reject message(s), if any
"""
tags = set()
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2001 - 2006 James Troup <james@nocrew.org>
-@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
@license: GNU General Public License version 2 or later
"""
###############################################################################
-def check_status(files):
- new = byhand = 0
- for f in files.keys():
- if files[f].has_key("byhand"):
- byhand = 1
- elif files[f].has_key("new"):
- new = 1
- return (new, byhand)
-
-###############################################################################
-
# Used by Upload.check_timestamps
class TarTime(object):
def __init__(self, future_cutoff, past_cutoff):
###########################################################################
def load_changes(self, filename):
"""
+ Load a changes file and setup a dictionary around it. Also checks for mandantory
+ fields within.
+
+ @type filename: string
+ @param filename: Changes filename, full path.
+
@rtype: boolean
- @rvalue: whether the changes file was valid or not. We may want to
+ @return: whether the changes file was valid or not. We may want to
reject even if this is True (see what gets put in self.rejects).
This is simply to prevent us even trying things later which will
fail because we couldn't properly parse the file.
def check_source(self):
# Bail out if:
# a) there's no source
- # or c) the orig files are MIA
- if not self.pkg.changes["architecture"].has_key("source") \
- or len(self.pkg.orig_files) == 0:
+ if not self.pkg.changes["architecture"].has_key("source"):
return
tmpdir = utils.temp_dirname()
found = False
# Look in the pool
- for poolfile in get_poolfile_like_name('/%s' % filename, session_):
+ for poolfile in get_poolfile_like_name('%s' % filename, session_):
poolfile_path = os.path.join(
poolfile.location.path, poolfile.filename
)
## Helper stuff for DebBugs Version Tracking
if cnf.Find("Dir::Queue::BTSVersionTrack"):
- # ??? once queue/* is cleared on *.d.o and/or reprocessed
- # the conditionalization on dsc["bts changelog"] should be
- # dropped.
-
- # Write out the version history from the changelog
- if self.pkg.changes["architecture"].has_key("source") and \
- self.pkg.dsc.has_key("bts changelog"):
-
+ if self.pkg.changes["architecture"].has_key("source"):
(fd, temp_filename) = utils.temp_filename(cnf["Dir::Queue::BTSVersionTrack"], prefix=".")
version_history = os.fdopen(fd, 'w')
version_history.write(self.pkg.dsc["bts changelog"])
directory it will be moved to the morgue to make way for
the new file.
- @type files: dict
- @param files: file dictionary
+ @type reject_files: dict
+ @param reject_files: file dictionary
"""
try:
dest_fd = os.open(dest_file, os.O_RDWR | os.O_CREAT | os.O_EXCL, 0644)
except OSError, e:
- # File exists? Let's try and move it to the morgue
+ # File exists? Let's find a new name by adding a number
if e.errno == errno.EEXIST:
- morgue_file = os.path.join(cnf["Dir::Morgue"], cnf["Dir::MorgueReject"], file_entry)
try:
- morgue_file = utils.find_next_free(morgue_file)
+ dest_file = utils.find_next_free(dest_file, 255)
except NoFreeFilenameError:
# Something's either gone badly Pete Tong, or
# someone is trying to exploit us.
- utils.warn("**WARNING** failed to move %s from the reject directory to the morgue." % (file_entry))
+ utils.warn("**WARNING** failed to find a free filename for %s in %s." % (file_entry, cnf["Dir::Queue::Reject"]))
return
- utils.move(dest_file, morgue_file, perms=0660)
+
+ # Make sure we really got it
try:
dest_fd = os.open(dest_file, os.O_RDWR|os.O_CREAT|os.O_EXCL, 0644)
except OSError, e:
@contact: Debian FTP Master <ftpmaster@debian.org>
@copyright: 2001, 2002, 2003, 2004, 2005, 2006 James Troup <james@nocrew.org>
@copyright: 2009 Mark Hymers <mhy@debian.org>
-@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
@license: GNU General Public License version 2 or later
"""
import re
+#: Is it a number?
re_isanum = re.compile (r"^\d+$")
+
+#: Looking for the default reply
re_default_answer = re.compile(r"\[(.*)\]")
+#: Used in build_summaries to make changes output look better
re_fdnic = re.compile(r"\n\n")
+#: Detect a binnmu
re_bin_only_nmu = re.compile(r"\+b\d+$")
+#: To sort out comment lines
re_comments = re.compile(r"\#.*")
+#: To ignore comment and whitespace lines.
re_whitespace_comment = re.compile(r"^\s*(#|$)")
re_no_epoch = re.compile(r"^\d+\:")
re_no_revision = re.compile(r"-[^-]+$")
re_source_ext = re.compile("(" + orig_source_ext_re + r"|debian\.tar\.(?:gz|bz2)|diff\.gz|tar\.(?:gz|bz2)|dsc)$")
re_issource = re.compile(r"(.+)_(.+?)\." + re_source_ext.pattern)
re_is_orig_source = re.compile (r"(.+)_(.+?)\.orig(?:-.+)?\.tar\.(?:gz|bz2)$")
+#re_is_orig_source = re.compile (r"(.+)_(.+?)\.(?:orig\.)?tar\.(?:gz|bz2)$")
re_single_line_field = re.compile(r"^(\S*?)\s*:\s*(.*)")
re_multi_line_field = re.compile(r"^\s(.*)")
re_re_mark = re.compile(r'^RE:')
re_parse_lintian = re.compile(r"^(?P<level>W|E|O): (?P<package>.*?): (?P<tag>[^ ]*) ?(?P<description>.*)$")
+
+# in process-upload
+re_match_expired = re.compile(r"^The key used to sign .+ has expired on .+$")
+#!/usr/bin/python
+
+""" Helper functions for the various source formats
+
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@copyright: 2009, 2010 Joerg Jaspert <joerg@debian.org>
+@copyright: 2009 Chris Lamb <lamby@debian.org>
+@license: GNU General Public License version 2 or later
+"""
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+
+################################################################################
+
+# <sgran> hey, I think something's wrong with your git repo
+# <sgran> when I git pulled this last time, I got something that looked almost
+# like python instead of dak
+# <mhy> sgran: slander
+# <sgran> sorry, I take it back, I've had a better look now
+
+################################################################################
import re
from dak_exceptions import UnknownFormatError
<helix> elmo: I can't believe people pay you to fix computers
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-<mhy> I often wonder if we should use NSA bot or something instead and get dinstall to send emails telling us about its progress :-)
-<mhy> dinstall: I'm processing openoffice
-<mhy> dinstall: I'm choking, please help me
-<Ganneff> yeah. get floods in here, for 600 accepted packages.
-<mhy> hehe
-<Ganneff> im not sure the other opers will like it if i oper up the bot, just so it can flood faster
-<mhy> flood all debian related channels
-<mhy> just to be safe
-<Ganneff> /msg #debian-* dinstall: starting
-<Ganneff> more interesting would be the first message in #debian, the next in #d-devel, then #d-qa
-<Ganneff> and expect people to monitor all.
-<Ganneff> i bet we have enough debian channels to at least put the timestamps in seperate channels each
-<Ganneff> and if not - we can make it go multi-network
-<Ganneff> first oftc, then opn, then ircnet, then - we will find some. quakenet anyone?
-<mhy> I should know better than to give you ideas
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-<mhy> !!!!11111iiiiiioneoneoneone
-<dak> mhy: Error: "!!!11111iiiiiioneoneoneone" is not a valid command.
-<mhy> dak: oh shut up
-<dak> mhy: Error: "oh" is not a valid command.
-
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
-<sgran> hey, I think something's wrong with your git repo
-<sgran> when I git pulled this last time, I got something that looked almost like python instead of dak
-<mhy> sgran: slander
-<sgran> sorry, I take it back, I've had a better look now
<li>Barry deFreese</li>
<li>Torsten Werner</li>
</ul>
-
+
<p>This information (and more like it) is available from
<a href="http://www.debian.org/intro/organization">Debian's Organizational Structure</a>.</p>
</div>
<div id="dak">
- <h1>'dak'</h1>
+ <h1>'dak'</h1>
<p>dak (Debian Archive Kit) is the collection of scripts that have replaced dinstall and friends.</p>
-
+
<p>The source is managed in git and is available
from: <a href="http://ftp-master.debian.org/git/">http://ftp-master.debian.org/git/</a></p>
</div>
- <div id="new">
+ <div id="new">
<h1>New Packages</h1>
<p>New Packages uploaded to the archive, but not yet accepted,
can be <a href="http://ftp-master.debian.org/new.html">seen here</a>.
until the daily cron run moves them into the pool.</p>
</div>
+ <div id="deferred">
+ <h1>Deferred Packages</h1>
+ <p>Deferred Packages which were uploaded to a DELAYED queue and have not reached their specified delay can be <a href="http://ftp-master.debian.org/deferred.html">seen here</a>.</p> </div>
+
<div id="pending">
<h1>Pending removals</h1>
<p>Packages which are scheduled to be removed from Debian
projects / dak.git / commitdiff