From 0bd6cf6bbad8203a4dbef219359f71e3d15e2e1f Mon Sep 17 00:00:00 2001 From: blundellc Date: Sat, 13 Sep 2008 13:21:48 +0000 Subject: [PATCH] depends: add forced dependencies, separate run/build depends. changes sufficient to make failures in rJava and RBGL external: openjdk is not ready yet (latest crashes during configure), and libboost-graph-dev 1.34.1-11 is not compatible with gcc 4.3 (see BioC list archives). also limit the frequency of pbuilder update and cache update. note: database must be deleted and rebuilt with this patch. git-svn-id: svn://svn.r-forge.r-project.org/svnroot/cran2deb@86 edb9625f-4e0d-4859-8d74-9fd3b1da38cb --- pkg/trunk/R/db.R | 110 ++++++++++++++++++++----- pkg/trunk/R/debcontrol.R | 42 +++++++--- pkg/trunk/data/populate_depend_aliases | 37 +++++++++ pkg/trunk/data/populate_forcedep | 4 + pkg/trunk/data/populate_sysreq | 66 +++++++-------- pkg/trunk/data/quit | 1 + pkg/trunk/exec/depend | 80 ++++++++++++++++++ pkg/trunk/exec/sysreq | 40 --------- pkg/trunk/exec/update | 14 +++- 9 files changed, 285 insertions(+), 109 deletions(-) create mode 100644 pkg/trunk/data/populate_depend_aliases create mode 100644 pkg/trunk/data/populate_forcedep create mode 100644 pkg/trunk/data/quit create mode 100755 pkg/trunk/exec/depend delete mode 100755 pkg/trunk/exec/sysreq diff --git a/pkg/trunk/R/db.R b/pkg/trunk/R/db.R index db35927..57fd7ae 100644 --- a/pkg/trunk/R/db.R +++ b/pkg/trunk/R/db.R @@ -5,10 +5,26 @@ db_start <- function() { tables <- dbListTables(con) if (!dbExistsTable(con,'sysreq_override')) { dbGetQuery(con,paste('CREATE TABLE sysreq_override (' - ,' debian_name TEXT NOT NULL' + ,' depend_alias TEXT NOT NULL' ,',r_pattern TEXT PRIMARY KEY NOT NULL' ,')')) } + if (!dbExistsTable(con,'debian_dependency')) { + dbGetQuery(con,paste('CREATE TABLE debian_dependency (' + ,' id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL' + ,',alias TEXT NOT NULL' + ,',build INTEGER NOT NULL' + ,',debian_pkg TEXT NOT NULL' + ,',UNIQUE (alias,build,debian_pkg)' + ,')')) + } + if (!dbExistsTable(con,'forced_depends')) { + dbGetQuery(con,paste('CREATE TABLE forced_depends (' + ,' r_name TEXT NOT NULL' + ,',depend_alias TEXT NOT NULL' + ,',PRIMARY KEY (r_name,depend_alias)' + ,')')) + } if (!dbExistsTable(con,'license_override')) { dbGetQuery(con,paste('CREATE TABLE license_override (' ,' name TEXT PRIMARY KEY NOT NULL' @@ -33,27 +49,24 @@ db_quote <- function(text) { } db_sysreq_override <- function(sysreq_text) { - sysreq_text <- tolower(sysreq_text) con <- db_start() results <- dbGetQuery(con,paste( - 'SELECT DISTINCT debian_name FROM sysreq_override WHERE' - ,db_quote(sysreq_text),'LIKE r_pattern')) + 'SELECT DISTINCT depend_alias FROM sysreq_override WHERE' + ,db_quote(tolower(sysreq_text)),'LIKE r_pattern')) db_stop(con) if (length(results) == 0) { return(NA) } - return(results$debian_name) + return(results$depend_alias) } -db_add_sysreq_override <- function(pattern,debian_name) { - pattern <- tolower(pattern) - debian_name <- tolower(debian_name) +db_add_sysreq_override <- function(pattern,depend_alias) { con <- db_start() results <- dbGetQuery(con,paste( 'INSERT OR REPLACE INTO sysreq_override' - ,'(debian_name, r_pattern) VALUES (' - ,' ',db_quote(debian_name) - ,',',db_quote(pattern) + ,'(depend_alias, r_pattern) VALUES (' + ,' ',db_quote(tolower(depend_alias)) + ,',',db_quote(tolower(pattern)) ,')')) db_stop(con) } @@ -65,13 +78,69 @@ db_sysreq_overrides <- function() { return(overrides) } +db_get_depends <- function(depend_alias,build=F) { + con <- db_start() + results <- dbGetQuery(con,paste( + 'SELECT DISTINCT debian_pkg FROM debian_dependency WHERE' + ,db_quote(tolower(depend_alias)),'= alias' + ,'AND',as.integer(build),'= build')) + db_stop(con) + return(results$debian_pkg) +} + +db_add_depends <- function(depend_alias,debian_pkg,build=F) { + con <- db_start() + results <- dbGetQuery(con,paste( + 'INSERT OR REPLACE INTO debian_dependency' + ,'(alias, build, debian_pkg) VALUES (' + ,' ',db_quote(tolower(depend_alias)) + ,',',as.integer(build) + ,',',db_quote(tolower(debian_pkg)) + ,')')) + db_stop(con) +} + +db_depends <- function() { + con <- db_start() + depends <- dbGetQuery(con,paste('SELECT * FROM debian_dependency')) + db_stop(con) + return(depends) +} + +db_get_forced_depends <- function(r_name) { + con <- db_start() + forced_depends <- dbGetQuery(con, + paste('SELECT depend_alias FROM forced_depends WHERE' + ,db_quote(r_name),'= r_name')) + db_stop(con) + return(forced_depends$depend_alias) +} + +db_add_forced_depends <- function(r_name, depend_alias) { + if (!length(db_get_depends(depend_alias,build=F)) && + !length(db_get_depends(depend_alias,build=T))) { + stop(paste('Debian dependency alias',depend_alias,'is not know,' + ,'yet trying to force a dependency on it?')) + } + con <- db_start() + dbGetQuery(con, + paste('INSERT OR REPLACE INTO forced_depends (r_name, depend_alias)' + ,'VALUES (',db_quote(r_name),',',db_quote(depend_alias),')')) + db_stop(con) +} + +db_forced_depends <- function() { + con <- db_start() + depends <- dbGetQuery(con,paste('SELECT * FROM forced_depends')) + db_stop(con) + return(depends) +} db_license_override_name <- function(name) { - name <- tolower(name) con <- db_start() results <- dbGetQuery(con,paste( 'SELECT DISTINCT accept FROM license_override WHERE' - ,db_quote(name),'= name')) + ,db_quote(tolower(name)),'= name')) db_stop(con) if (length(results) == 0) { return(NA) @@ -80,7 +149,6 @@ db_license_override_name <- function(name) { } db_add_license_override <- function(name,accept) { - name <- tolower(name) message(paste('adding',name,'accept?',accept)) if (accept != TRUE && accept != FALSE) { stop('accept must be TRUE or FALSE') @@ -89,20 +157,19 @@ db_add_license_override <- function(name,accept) { results <- dbGetQuery(con,paste( 'INSERT OR REPLACE INTO license_override' ,'(name, accept) VALUES (' - ,' ',db_quote(name) + ,' ',db_quote(tolower(name)) ,',',as.integer(accept) ,')')) db_stop(con) } db_license_override_hash <- function(license_sha1) { - license_sha1 <- tolower(license_sha1) con <- db_start() results <- dbGetQuery(con,paste( 'SELECT DISTINCT accept FROM license_override' ,'INNER JOIN license_hashes' ,'ON license_hashes.name = license_override.name WHERE' - ,db_quote(license_sha1),'= license_hashes.sha1')) + ,db_quote(tolower(license_sha1)),'= license_hashes.sha1')) db_stop(con) if (length(results) == 0) { return(NA) @@ -120,15 +187,16 @@ db_license_overrides <- function() { } db_add_license_hash <- function(name,license_sha1) { - name <- tolower(name) - license_sha1 <- tolower(license_sha1) + if (is.na(db_license_override_name(name))) { + stop(paste('license',name,'is not know, yet trying to add a hash for it?')) + } message(paste('adding hash',license_sha1,'for',name)) con <- db_start() dbGetQuery(con,paste( 'INSERT OR REPLACE INTO license_hashes' ,'(name, sha1) VALUES (' - ,' ',db_quote(name) - ,',',db_quote(license_sha1) + ,' ',db_quote(tolower(name)) + ,',',db_quote(tolower(license_sha1)) ,')')) db_stop(con) } diff --git a/pkg/trunk/R/debcontrol.R b/pkg/trunk/R/debcontrol.R index 50bf742..6030783 100644 --- a/pkg/trunk/R/debcontrol.R +++ b/pkg/trunk/R/debcontrol.R @@ -17,8 +17,14 @@ get_dependencies <- function(pkg,extra_deps) { # add the system requirements if ('SystemRequirements' %in% colnames(pkg$description)) { sysreq <- sysreqs_as_debian(pkg$description[1,'SystemRequirements']) - depends$bin = c(sysreq,depends$bin) - depends$build = c(sysreq,depends$build) + depends$bin = c(sysreq$bin,depends$bin) + depends$build = c(sysreq$build,depends$build) + } + + forced <- forced_deps_as_debian(pkg$name) + if (length(forced)) { + depends$bin = c(forced$bin,depends$bin) + depends$build = c(forced$build,depends$build) } # make sure we depend upon R in some way... @@ -49,7 +55,7 @@ get_dependencies <- function(pkg,extra_deps) { sysreqs_as_debian <- function(sysreq_text) { # form of this field is unspecified (ugh) but most people seem to stick # with this - debs <- c() + aliases <- c() sysreq_text <- gsub('[[:space:]]and[[:space:]]',' , ',tolower(sysreq_text)) for (sysreq in strsplit(sysreq_text,'[[:space:]]*,[[:space:]]*')[[1]]) { startreq = sysreq @@ -66,20 +72,32 @@ sysreqs_as_debian <- function(sysreq_text) { sysreq = gsub('(ht|f)tps?://[[:alnum:]!?*"\'(),%$_@.&+/=-]*','',sysreq) # squish out space sysreq = chomp(gsub('[[:space:]]+',' ',sysreq)) - deb <- db_sysreq_override(sysreq) - if (is.na(deb)) { + alias <- db_sysreq_override(sysreq) + if (is.na(alias)) { message(paste('E: do not know what to do with SystemRequirement:',sysreq)) message(paste('E: original SystemRequirement:',startreq)) stop('unmet system requirement') } - message(paste('N: mapped SystemRequirement',startreq,'onto',deb,'via',sysreq)) - if (deb == 'build-essential') { - # already in any build environment so no explicit depend. - message(paste('N: SystemRequirement',startreq,'dropped')) - } else { - debs = c(debs,deb) - } + message(paste('N: mapped SystemRequirement',startreq,'onto',alias,'via',sysreq)) + aliases = c(aliases,alias) + } + return(map_aliases_to_debian(aliases)) +} + +forced_deps_as_debian <- function(r_name) { + aliases <- db_get_forced_depends(r_name) + return(map_aliases_to_debian(aliases)) +} + +map_aliases_to_debian <- function(aliases) { + if (!length(aliases)) { + return(aliases) } + debs <- list() + debs$bin = unlist(sapply(aliases, db_get_depends)) + debs$build = unlist(sapply(aliases, db_get_depends, build=T)) + debs$bin = debs$bin[debs$bin != 'build-essential'] + debs$build = debs$build[debs$build != 'build-essential'] return(debs) } diff --git a/pkg/trunk/data/populate_depend_aliases b/pkg/trunk/data/populate_depend_aliases new file mode 100644 index 0000000..0d5e3f6 --- /dev/null +++ b/pkg/trunk/data/populate_depend_aliases @@ -0,0 +1,37 @@ +alias_build boost libboost-dev +alias_build boost libboost-graph-dev +alias ggobi ggobbi +alias glade libglade2-dev +alias glib libglib2.0-dev +alias glu libglu1-mesa-dev +alias gmp libgmp3-dev +alias gsl libgsl0-dev +alias_build ignore build-essential +alias_build java openjdk-6-jdk +alias_run java openjdk-6-jre +alias libatk libatk1.0-dev +alias libcairo libcairo2-dev +alias_run libcurl libcurl3 +alias_build libcurl libcurl4-openssl-dev +alias libdieharder libdieharder-dev +alias libfontconfig libfontconfig1-dev +alias libfreetype libfreetype6-dev +alias libgdal libgdal1-dev +alias libgd libgd2-noxpm-dev +alias libgraphviz libgraphviz-dev +alias libgtk libgtk2.0-dev +alias libjpeg libjpeg62-dev +alias libmagick libmagick9-dev +alias libpango libpango1.0-dev +alias libpng libpng12-dev +alias libxml libxml2-dev +alias msttcorefonts msttcorefonts +alias_run netcdf libnetcdf4 +alias_build netcdf libnetcdf-dev +alias opengl libgl1-mesa-dev +alias pari-gp pari/gp +alias proj proj +alias quantlib libquantlib0-dev +alias_run sqlite libsqlite3-0 +alias_build sqlite libsqlite3-dev +alias zlib zlib1g-dev diff --git a/pkg/trunk/data/populate_forcedep b/pkg/trunk/data/populate_forcedep new file mode 100644 index 0000000..3f54dc3 --- /dev/null +++ b/pkg/trunk/data/populate_forcedep @@ -0,0 +1,4 @@ +force java rJava +force sqlite RSQLite +force boost RBGL +force netcdf ncdf diff --git a/pkg/trunk/data/populate_sysreq b/pkg/trunk/data/populate_sysreq index 17a9d49..a674312 100644 --- a/pkg/trunk/data/populate_sysreq +++ b/pkg/trunk/data/populate_sysreq @@ -1,33 +1,33 @@ -add build-essential gcc -add build-essential gnu make -add build-essential % if present -add build-essential none -add curl libcurl4-openssl-dev -add ggobi ggobi -add libatk1.0-dev atk -add libcairo2-dev cairo -add libdieharder-dev dieharder% -add libfontconfig1-dev fontconfig -add libfreetype6-dev freetype -add libfreetype6-dev %freetype -add libgd2-noxpm-dev libgd -add libgdal1-dev %gdal% -add libgl1-mesa-dev opengl -add libglade2-dev %glade -add libglib2.0-dev glib -add libglu1-mesa-dev glu library -add libgmp3-dev gmp -add libgraphviz-dev graphviz -add libgsl0-dev gnu gsl% -add libgsl0-dev gnu scientific library -add libgtk2.0-dev gtk% -add libjpeg62-dev libjpeg% -add libmagick9-dev imagemagick -add libpango1.0-dev pango -add libpng12-dev libpng -add libquantlib0-dev quantlib% -add libxml2-dev libxml% -add msttcorefonts msttcorefonts -add pari-gp pari/gp -add proj proj% -add zlib1g-dev zlib +sysreq ignore gcc +sysreq ignore gnu make +sysreq ignore % if present +sysreq ignore none +sysreq libcurl curl +sysreq ggobi ggobi +sysreq libatk atk +sysreq libcairo cairo +sysreq libdieharder dieharder% +sysreq libfontconfig fontconfig +sysreq libfreetype freetype +sysreq libfreetype %freetype +sysreq libgd libgd +sysreq libgdal %gdal% +sysreq opengl opengl +sysreq glade %glade +sysreq glib glib +sysreq glu glu library +sysreq gmp gmp +sysreq libgraphviz graphviz +sysreq gsl gnu gsl% +sysreq gsl gnu scientific library +sysreq libgtk gtk% +sysreq libjpeg libjpeg% +sysreq libmagick imagemagick +sysreq libpango pango +sysreq libpng libpng +sysreq quantlib quantlib% +sysreq libxml libxml% +sysreq msttcorefonts msttcorefonts +sysreq pari-gp pari/gp +sysreq proj proj% +sysreq zlib zlib diff --git a/pkg/trunk/data/quit b/pkg/trunk/data/quit new file mode 100644 index 0000000..ff60466 --- /dev/null +++ b/pkg/trunk/data/quit @@ -0,0 +1 @@ +quit diff --git a/pkg/trunk/exec/depend b/pkg/trunk/exec/depend new file mode 100755 index 0000000..ee32af3 --- /dev/null +++ b/pkg/trunk/exec/depend @@ -0,0 +1,80 @@ +#!/usr/bin/env r + +suppressPackageStartupMessages(library(cran2deb)) +suppressPackageStartupMessages(library(digest)) + +exec_cmd <- function(argc, argv) { + usage <- function() + message(paste('usage: alias ' + ,' alias_run ' + ,' alias_build ' + ,' sysreq ' + ,' force ' + ,' ls [aliases|force|sysreq]' + ,' quit' + ,sep='\n')) + + message(paste('N: command',paste(argv,collapse=' '))) + if (argc < 1) { + return() + } + cmd = argv[1] + + if (cmd == 'alias') { + if (argc < 3) { + usage() + return() + } + alias = argv[2] + pkg = argv[3] + db_add_depends(alias, pkg, build=T) + pkg = gsub('-dev$','',pkg) + db_add_depends(alias, pkg, build=F) + } else if (cmd == 'alias_run' || cmd == 'alias_build') { + if (argc < 3) { + usage() + return() + } + db_add_depends(argv[2], argv[3], cmd == 'alias_build') + } else if (cmd == 'sysreq') { + if (argc < 3) { + usage() + return() + } + sysreq = paste(argv[3:argc],collapse=' ') + db_add_sysreq_override(sysreq,argv[2]) + } else if (cmd == 'force') { + if (argc < 3) { + usage() + return() + } + db_add_forced_depends(argv[3],argv[2]) + } else if (cmd == 'ls') { + if (argc < 2 || argv[2] == 'aliases') { + print(db_depends()) + } else if (argv[2] == 'sysreq') { + print(db_sysreq_overrides()) + } else if (argv[2] == 'force') { + print(db_forced_depends()) + } else { + usage() + return() + } + } else if (cmd == 'quit') { + exit() + } else if (cmd == '#') { + } else { + usage() + return() + } +} + +argc <- length(argv) +if (argc > 1) { + exec_cmd(argc-1,argv[c(2:argc)]) +} else { + while(T) { + argv <- strsplit(readline('depend> '),'[[:space:]]+')[[1]] + exec_cmd(length(argv),argv) + } +} diff --git a/pkg/trunk/exec/sysreq b/pkg/trunk/exec/sysreq deleted file mode 100755 index c4d76b1..0000000 --- a/pkg/trunk/exec/sysreq +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env r - -suppressPackageStartupMessages(library(cran2deb)) -suppressPackageStartupMessages(library(digest)) - -exec_cmd <- function(argc, argv) { - usage <- function() - message('usage: add |ls|quit|help') - - if (argc < 1) { - exit() - } - cmd = argv[1] - - if (cmd == 'add') { - if (argc < 3) { - usage() - return() - } - sysreq = paste(argv[3:argc],collapse=' ') - db_add_sysreq_override(sysreq,argv[2]) - } else if (cmd == 'ls') { - print(db_sysreq_overrides()) - } else if (cmd == 'help') { - usage() - return() - } else if (cmd == 'quit') { - exit() - } -} - -argc <- length(argv) -if (argc > 1) { - exec_cmd(argc-1,argv[c(2:argc)]) -} else { - while(T) { - argv <- strsplit(readline('sysreq> '),'[[:space:]]+')[[1]] - exec_cmd(length(argv),argv) - } -} diff --git a/pkg/trunk/exec/update b/pkg/trunk/exec/update index 7659205..f013396 100755 --- a/pkg/trunk/exec/update +++ b/pkg/trunk/exec/update @@ -12,13 +12,21 @@ if ([ ! -e $root/var/archive ]) { mkdir $root/var/archive || exit 1 } mini-dinstall --batch -c $root/etc/mini-dinstall.conf || exit 1 -if (! ~ $1 quick) { +delta=`{hoc -e `{date +%s}^-^`{stat -c '%Y' /var/cache/cran2deb/cache.rda}} +update_period=10800 +echo Cache is $delta seconds out of date. +if (~ $1 full) { + delta=`{hoc -e $update_period^'+1'} +} +if (![ -e /var/cache/cran2deb/cache.rda ] || [ $delta -gt $update_period ]) { mode=create if ([ -e /var/cache/pbuilder/base-cran2deb.tgz ]) { mode=update } sudo pbuilder $mode --override-config --configfile $root/etc/pbuilderrc $root/exec/update_cache $root - $root/exec/license <$root/data/populate_licenses - $root/exec/sysreq <$root/data/populate_sysreq } +if (![ -e /var/cache/cran2deb/cran2deb.db ] || [ $delta -gt $update_period ]) { + cat $root/data/^(populate_licenses quit) | $root/exec/license $root +} +cat $root/data/^(populate_depend_aliases populate_sysreq populate_forcedep quit) | $root/exec/depend $root -- 2.39.5