# Stolen from userdir-ldap
# Compute a random password using /dev/urandom.
def GenPass():
- # Generate a 10 character random string
- SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/."
- Rand = open("/dev/urandom")
- Password = ""
- for i in range(0,15):
- Password = Password + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)]
- return Password
+ # Generate a 10 character random string
+ SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/."
+ Rand = open("/dev/urandom")
+ Password = ""
+ for i in range(0,15):
+ Password = Password + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)]
+ return Password
# Compute the MD5 crypted version of the given password
def HashPass(Password):
- import crypt
- # Hash it telling glibc to use the MD5 algorithm - if you dont have
- # glibc then just change Salt = "$1$" to Salt = ""
- SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/."
- Salt = "$1$"
- Rand = open("/dev/urandom")
- for x in range(0,10):
- Salt = Salt + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)]
- Pass = crypt.crypt(Password,Salt)
- if len(Pass) < 14:
- raise "Password Error", "MD5 password hashing failed, not changing the password!"
- return Pass
+ import crypt
+ # Hash it telling glibc to use the MD5 algorithm - if you dont have
+ # glibc then just change Salt = "$1$" to Salt = ""
+ SaltVals = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789/."
+ Salt = "$1$"
+ Rand = open("/dev/urandom")
+ for x in range(0,10):
+ Salt = Salt + SaltVals[ord(Rand.read(1)[0]) % len(SaltVals)]
+ Pass = crypt.crypt(Password,Salt)
+ if len(Pass) < 14:
+ raise "Password Error", "MD5 password hashing failed, not changing the password!"
+ return Pass
################################################################################
]
for i in [ "help", "create" ]:
- if not Cnf.has_key("Add-User::Options::%s" % (i)):
- Cnf["Add-User::Options::%s" % (i)] = ""
+ if not Cnf.has_key("Add-User::Options::%s" % (i)):
+ Cnf["Add-User::Options::%s" % (i)] = ""
apt_pkg.ParseCommandLine(Cnf, Arguments, sys.argv)
(result, output) = commands.getstatusoutput(cmd)
m = re_gpg_fingerprint.search(output)
if not m:
- print output
+ print output
utils.fubar("0x%s: (1) No fingerprint found in gpg output but it returned 0?\n%s" \
- % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, \
- " [GPG output:] ")))
+ % (Cnf["Add-User::Options::Key"], utils.prefix_multi_line_string(output, \
+ " [GPG output:] ")))
primary_key = m.group(1)
primary_key = primary_key.replace(" ","")
yn = utils.our_raw_input(prompt).lower()
if yn == "y":
-# Create an account for the user?
- summary = ""
- if Cnf.FindB("Add-User::CreateAccount") or Cnf["Add-User::Options::Create"]:
- password = GenPass()
- pwcrypt = HashPass(password)
- if Cnf.has_key("Add-User::GID"):
- cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' -G %s %s" \
- % (pwcrypt, name, Cnf["Add-User::GID"], uid)
- else:
- cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' %s" \
- % (pwcrypt, name, uid)
- (result, output) = commands.getstatusoutput(cmd)
- if (result != 0):
- utils.fubar("Invocation of '%s' failed:\n%s\n" % (cmd, output), result)
- try:
- summary+=createMail(uid, password, Cnf["Add-User::Options::Key"], Cnf["Dinstall::GPGKeyring"])
- except:
- summary=""
- utils.warn("Could not prepare password information for mail, not sending password.")
-
-# Now add user to the database.
- # Note that we provide a session, so we're responsible for committing
- uidobj = get_or_set_uid(uid, session=session)
- uid_id = uidobj.uid_id
- add_database_user(uid)
- session.commit()
-# The following two are kicked out in rhona, so we don't set them. kelly adds
-# them as soon as she installs a package with unknown ones, so no problems to expect here.
-# Just leave the comment in, to not think about "Why the hell aren't they added" in
-# a year, if we ever touch uma again.
-# maint_id = database.get_or_set_maintainer_id(name)
-# session.execute("INSERT INTO fingerprint (fingerprint, uid) VALUES (:fingerprint, uid)",
-# {'fingerprint': primary_key, 'uid': uid_id})
-
-# Lets add user to the email-whitelist file if its configured.
- if Cnf.has_key("Dinstall::MailWhiteList") and Cnf["Dinstall::MailWhiteList"] != "":
- file = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
- for mail in emails:
- file.write(mail+'\n')
- file.close()
-
- print "Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id, \
- name, primary_key)
-
-# Should we send mail to the newly added user?
- if Cnf.FindB("Add-User::SendEmail"):
- mail = name + "<" + emails[0] +">"
- Subst = {}
- Subst["__NEW_MAINTAINER__"] = mail
- Subst["__UID__"] = uid
- Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
- Subst["__PRIMARY_KEY__"] = primary_key
- Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
- Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"]
- Subst["__SUMMARY__"] = summary
- new_add_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/add-user.added")
- utils.send_mail(new_add_message)
+ # Create an account for the user?
+ summary = ""
+ if Cnf.FindB("Add-User::CreateAccount") or Cnf["Add-User::Options::Create"]:
+ password = GenPass()
+ pwcrypt = HashPass(password)
+ if Cnf.has_key("Add-User::GID"):
+ cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' -G %s %s" \
+ % (pwcrypt, name, Cnf["Add-User::GID"], uid)
+ else:
+ cmd = "sudo /usr/sbin/useradd -g users -m -p '%s' -c '%s' %s" \
+ % (pwcrypt, name, uid)
+ (result, output) = commands.getstatusoutput(cmd)
+ if (result != 0):
+ utils.fubar("Invocation of '%s' failed:\n%s\n" % (cmd, output), result)
+ try:
+ summary+=createMail(uid, password, Cnf["Add-User::Options::Key"], Cnf["Dinstall::GPGKeyring"])
+ except:
+ summary=""
+ utils.warn("Could not prepare password information for mail, not sending password.")
+
+ # Now add user to the database.
+ # Note that we provide a session, so we're responsible for committing
+ uidobj = get_or_set_uid(uid, session=session)
+ uid_id = uidobj.uid_id
+ add_database_user(uid)
+ session.commit()
+
+ # The following two are kicked out in rhona, so we don't set them. kelly adds
+ # them as soon as she installs a package with unknown ones, so no problems to expect here.
+ # Just leave the comment in, to not think about "Why the hell aren't they added" in
+ # a year, if we ever touch uma again.
+ # maint_id = database.get_or_set_maintainer_id(name)
+ # session.execute("INSERT INTO fingerprint (fingerprint, uid) VALUES (:fingerprint, uid)",
+ # {'fingerprint': primary_key, 'uid': uid_id})
+
+ # Lets add user to the email-whitelist file if its configured.
+ if Cnf.has_key("Dinstall::MailWhiteList") and Cnf["Dinstall::MailWhiteList"] != "":
+ f = utils.open_file(Cnf["Dinstall::MailWhiteList"], "a")
+ for mail in emails:
+ f.write(mail+'\n')
+ f.close()
+
+ print "Added:\nUid:\t %s (ID: %s)\nMaint:\t %s\nFP:\t %s" % (uid, uid_id, \
+ name, primary_key)
+
+ # Should we send mail to the newly added user?
+ if Cnf.FindB("Add-User::SendEmail"):
+ mail = name + "<" + emails[0] +">"
+ Subst = {}
+ Subst["__NEW_MAINTAINER__"] = mail
+ Subst["__UID__"] = uid
+ Subst["__KEYID__"] = Cnf["Add-User::Options::Key"]
+ Subst["__PRIMARY_KEY__"] = primary_key
+ Subst["__FROM_ADDRESS__"] = Cnf["Dinstall::MyEmailAddress"]
+ Subst["__HOSTNAME__"] = Cnf["Dinstall::MyHost"]
+ Subst["__SUMMARY__"] = summary
+ new_add_message = utils.TemplateSubst(Subst,Cnf["Dir::Templates"]+"/add-user.added")
+ utils.send_mail(new_add_message)
else:
- uid = None
-
+ uid = None
#######################################################################################
if __name__ == '__main__':
main()
-
print suite_name
print "-" * len(suite_name)
print
- suite = get_suite(suite)
- q = s.execute("""
+ suite = get_suite(suite_name)
+ q = session.execute("""
SELECT DISTINCT b.package FROM binaries b, bin_associations ba
WHERE b.id = ba.bin AND ba.suite = :suiteid AND NOT EXISTS
(SELECT 1 FROM override o WHERE o.suite = :suiteid AND o.package = b.package)"""
for j in q.fetchall():
print j[0]
- q = s.execute("""
+ q = session.execute("""
SELECT DISTINCT s.source FROM source s, src_associations sa
WHERE s.id = sa.source AND sa.suite = :suiteid AND NOT EXISTS
(SELECT 1 FROM override o WHERE o.suite = :suiteid and o.package = s.source)"""
"""
for suite in [ "stable", "testing", "unstable" ]:
for component in Cnf.ValueList("Suite::%s::Components" % (suite)):
- architectures = database.get_suite_architectures(suite)
- for arch in [ i.lower() for i in architectures ]:
+ architectures = get_suite_architectures(suite)
+ for arch in [ i.arch_string.lower() for i in architectures ]:
if arch == "source":
validate_sources(suite, component)
elif arch == "all":
def check_build_depends():
""" Validate build-dependencies of .dsc files in the archive """
+ cnf = Config()
os.path.walk(cnf["Dir::Root"], chk_bd_process_dir, None)
################################################################################
import logging
import gzip
import threading
+import traceback
import Queue
import apt_pkg
from daklib import utils
WHERE ba.suite = :suiteid AND ba.bin = b.id
AND b.architecture = a.id AND b.package = :package""",
{'suiteid': suite_id, 'package': binary})
+ ql = q.fetchall()
versions = []
- for i in q.fetchall():
+ for i in ql:
arch = i[0]
version = i[1]
if architectures.has_key(arch):
#!/usr/bin/env python
-""" Create all the Release files """
-
-# Copyright (C) 2001, 2002, 2006 Anthony Towns <ajt@debian.org>
+""" Create all the Release files
+@contact: Debian FTPMaster <ftpmaster@debian.org>
+@Copyright: 2001, 2002, 2006 Anthony Towns <ajt@debian.org>
+@copyright: 2009 Joerg Jaspert <joerg@debian.org>
+@license: GNU General Public License version 2 or later
+"""
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
################################################################################
-import sys, os, stat, time
-import gzip, bz2
+import sys
+import os
+import stat
+import time
+import gzip
+import bz2
import apt_pkg
from daklib import utils
dest = Cnf["Dir::Root"] + tree + "/Release.gpg"
if os.path.exists(dest):
os.unlink(dest)
+ inlinedest = Cnf["Dir::Root"] + tree + "/InRelease"
+ if os.path.exists(inlinedest):
+ os.unlink(inlinedest)
for keyid in signkeyids:
- if keyid != "": defkeyid = "--default-key %s" % keyid
- else: defkeyid = ""
+ if keyid != "":
+ defkeyid = "--default-key %s" % keyid
+ else:
+ defkeyid = ""
os.system("gpg %s %s %s --detach-sign <%s >>%s" %
(keyring, defkeyid, arguments,
Cnf["Dir::Root"] + tree + "/Release", dest))
+ os.system("gpg %s %s %s --clearsign <%s >>%s" %
+ (keyring, defkeyid, arguments,
+ Cnf["Dir::Root"] + tree + "/Release", inlinedest))
#######################################################################################
sys.exit(exit_code)
def check_signature (sig_filename, data_filename=""):
+ fingerprint = None
+
keyrings = [
"/home/joerg/keyring/keyrings/debian-keyring.gpg",
"/home/joerg/keyring/keyrings/debian-keyring.pgp",
def __init__(self):
self.next_in_line = None
self.next_lock = threading.Condition()
+ self.die = False
+
+ def plsDie(self):
+ self.die = True
+ self.next_lock.notify()
def enqueue(self, next):
self.next_lock.acquire()
while self.next_in_line:
+ if self.die:
+ return
self.next_lock.wait()
assert( not self.next_in_line )
def dequeue(self):
self.next_lock.acquire()
while not self.next_in_line:
+ if self.die:
+ return
self.next_lock.wait()
- result = self.next_in_line
- if isinstance(result, EndOfChanges):
- return None
+ result = self.next_in_line
self.next_in_line = None
self.next_lock.notify()
self.next_lock.release()
+
+ if isinstance(result, EndOfChanges):
+ return None
+
return result
class ChangesToImport(object):
class ChangesGenerator(threading.Thread):
"""enqueues changes files to be imported"""
- def __init__(self, queue):
+ def __init__(self, parent, queue):
threading.Thread.__init__(self)
self.queue = queue
self.session = DBConn().session()
+ self.parent = parent
+ self.die = False
+
+ def plsDie(self):
+ self.die = True
def run(self):
cnf = Config()
if not filenames:
# Empty directory (or only subdirectories), next
continue
- for changesfile in filenames:
- if not changesfile.endswith(".changes"):
- # Only interested in changes files.
- continue
- count += 1
- if not get_knownchange(changesfile, self.session):
- to_import = ChangesToImport(dirpath, changesfile, count)
- print("enqueue: %s" % to_import)
- self.queue.enqueue(to_import)
+ for changesfile in filenames:
+ try:
+ if not changesfile.endswith(".changes"):
+ # Only interested in changes files.
+ continue
+ count += 1
+
+ if not get_knownchange(changesfile, self.session):
+ to_import = ChangesToImport(dirpath, changesfile, count)
+ if self.die:
+ return
+ self.queue.enqueue(to_import)
+ except KeyboardInterrupt:
+ print("got Ctrl-c in enqueue thread. terminating")
+ self.parent.plsDie()
+ sys.exit(1)
self.queue.enqueue(EndOfChanges())
class ImportThread(threading.Thread):
- def __init__(self, queue):
+ def __init__(self, parent, queue):
threading.Thread.__init__(self)
self.queue = queue
self.session = DBConn().session()
+ self.parent = parent
+ self.die = False
+
+ def plsDie(self):
+ self.die = True
def run(self):
while True:
try:
+ if self.die:
+ return
to_import = self.queue.dequeue()
if not to_import:
return
changes = Changes()
changes.changes_file = to_import.changesfile
changesfile = os.path.join(to_import.dirpath, to_import.changesfile)
- print( "STU: %s / %s" % (to_import.dirpath, to_import.changesfile))
changes.changes = parse_changes(changesfile, signing_rules=-1)
changes.changes["fingerprint"] = check_signature(changesfile)
changes.add_known_changes(to_import.dirpath, self.session)
except InvalidDscError, line:
warn("syntax error in .dsc file '%s', line %s." % (f, line))
-# failure += 1
except ChangesUnicodeError:
warn("found invalid changes file, not properly utf-8 encoded")
-# failure += 1
-
- print "Directory %s, file %7d, failures %3d. (%s)" % (dirpath[-10:], count, failure, changesfile)
+ except KeyboardInterrupt:
+ print("Caught C-c; on ImportThread. terminating.")
+ self.parent.plsDie()
+ sys.exit(1)
except:
traceback.print_exc()
+ self.parent.plsDie()
+ sys.exit(1)
+
+class ImportKnownChanges(object):
+ def __init__(self,num_threads):
+ self.queue = OneAtATime()
+ self.threads = [ ChangesGenerator(self,self.queue) ]
+
+ for i in range(num_threads):
+ self.threads.append( ImportThread(self,self.queue) )
+
+ try:
+ for thread in self.threads:
+ thread.start()
+
+ except KeyboardInterrupt:
+ print("Caught C-c; terminating.")
+ utils.warn("Caught C-c; terminating.")
+ self.plsDie()
+
+ def plsDie(self):
+ traceback.print_stack90
+ for thread in self.threads:
+ print( "STU: before ask %s to die" % thread )
+ thread.plsDie()
+ print( "STU: after ask %s to die" % thread )
+
+ self.threads=[]
+ sys.exit(1)
+
def main():
cnf = Config()
if Config().has_key( "%s::%s" %(options_prefix,"Concurrency")):
num_threads = int(Config()[ "%s::%s" %(options_prefix,"Concurrency")])
+ ImportKnownChanges(num_threads)
- queue = OneAtATime()
- ChangesGenerator(queue).start()
- for i in range(num_threads):
- ImportThread(queue).start()
if __name__ == '__main__':
################################################################################
def do_da_do_da():
+ cnf = Config()
+
# If we're only doing a subset of suites, ensure we do enough to
# be able to do arch: all mapping.
if Options["Suite"]:
<th align="center">Closes</th>
</tr>
"""
- return res
def table_footer():
return '</table><br/><p>non-NEW uploads are <a href="/deferred/">available</a>, see the <a href="ftp://ftp-master.debian.org/pub/UploadQueue/README">UploadQueue-README</a> for more information.</p></center><br/>\n'
def mark_missing_fields(self):
"""add "missing" in fields which we will require for the known_changes table"""
- for key in ['urgency', 'maintainer', 'fingerprint', 'changedby' ]:
+ for key in ['urgency', 'maintainer', 'fingerprint', 'changed-by' ]:
if (not self.changes.has_key(key)) or (not self.changes[key]):
self.changes[key]='missing'
distribution, urgency, maintainer, fingerprint, changedby, date)
VALUES (:changesfile,:filetime,:source,:binary, :architecture,
:version,:distribution,:urgency,:maintainer,:fingerprint,:changedby,:date)""",
- { 'changesfile':changesfile,
+ { 'changesfile':self.changes_file,
'filetime':filetime,
'source':self.changes["source"],
'binary':self.changes["binary"],
projects / dak.git / commitdiff