X-Git-Url: https://git.donarmstrong.com/?p=uiuc_igb_scripts.git;a=blobdiff_plain;f=dqsub;h=571bc281cd88c45827ae35fa7672844b66a98875;hp=98e494780e6883c2d46c298fcd4268487743df92;hb=9d5820c033136980f80cccad98a499ba58c0cde5;hpb=bc41e78bdea398679782ff3a53aa33d80ae6e097 diff --git a/dqsub b/dqsub index 98e4947..571bc28 100755 --- a/dqsub +++ b/dqsub @@ -26,9 +26,14 @@ dqsub [options] --nodes nodes to use --array array mode (one of 'chdir' or 'xargs' or '') --array-from file to read arrays from (default STDIN) + --array-per-job number of array items to handle in each job (default 1) + --array-all-in-one-job Run all of the array items in one job --ppn processors per node to use --mem memory to request --dir Directory to run the script in (default current directory) + --account, -A Account name to use + --join, -J join error and output streams (default) + --name, -N Name of the job --debug, -d debugging level (Default 0) --help, -h display this help --man, -m display manual @@ -57,6 +62,20 @@ command run in the current directory. File to read array arguments from. If not provided, and B<--array> is given, arguments will be read from STDIN. +=item B<--account, -A> + +Account name to use + +=item B<--join, J> + +Whether to join STDOUT and STDERR. On by default; disable with +C<--nojoin>. + +=item B<--batch> + +Which batch system to use. If sbatch exists, assume it's slurm, +otherwise, PBS. + =item B<--debug, -d> Debug verbosity. (Default 0) @@ -79,6 +98,8 @@ dqsub use IO::File; use Cwd qw(getcwd abs_path); +use POSIX qw(ceil); +use List::Util qw(min); use vars qw($DEBUG); my %options = (nodes => 1, @@ -88,17 +109,30 @@ my %options = (nodes => 1, help => 0, man => 0, interactive => 0, + array_per_job => 1, + join => 1, ); GetOptions(\%options, 'queue|q=s', + 'batch=s', 'interactive|I!', 'nodes=i', 'array=s', 'array_from|array-from=s', - 'ppn|processors-per-node=i', + 'array_per_job|array-per-job=i', + 'array_slot_limit|array-slot-limit=i', + 'array_all_in_one_job|array-all-in-one-job!', + 'ppn|cpus|processors-per-node=i', + 'account|A=s', + 'join|J!', 'mem|memory=s', + 'time|walltime=s','cputime|cput=s','host=s', + 'pmem|process_mem|process-mem=s', + 'pvmem|process_virtual_mem|process-virtiual-mem=s', + 'max_file|max-file|file=s', 'dir=s', + 'name=s', 'debug|d+','help|h|?','man|m'); # pod2usage() if $options{help}; @@ -121,49 +155,153 @@ if ($options{interactive} and @ARGV) { push @USAGE_ERRORS,"Don't provide commands when you're asking for an interactive shell"; } +if (not defined $options{batch}) { + qx/which sbatch/; + if ($?) { + $options{batch} = 'slurm' + } else { + $options{batch} = 'pbs' + } +} + +if ($options{batch} !~ /^pbs|slurm$/) { + push @USAGE_ERRORS,"Unsupported batch system '$options{batch}'; ". + "supported systems are pbs or slurm"; +} + # pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS; -print STDERR join("\n",@USAGE_ERRORS) and exit 1 if @USAGE_ERRORS; +if (@USAGE_ERRORS) { + print STDERR map {"$_\n"} @USAGE_ERRORS; + exit 1; +} + +my $JOB_SUBMITTER = 'qsub'; # OK. Generate the options to qsub which we'll be using -my @qsub_options = generate_qsub_options(\%options); +my @qsub_options; +if ($options{batch} eq 'pbs') { + @qsub_options = generate_qsub_options(\%options,\@ARGV); + $JOB_SUBMITTER = 'qsub'; +} elsif ($options{batch} eq 'slurm') { + @qsub_options = generate_slurm_options(\%options,\@ARGV); + $JOB_SUBMITTER = 'sbatch'; +} else { + die "Unsupported batch system '$options{batch}'"; +} + if ($options{interactive}) { print STDERR 'running: qsub '.join(' ',@qsub_options) if $DEBUG; - exec('qsub',@qsub_options); + exec($JOB_SUBMITTER,@qsub_options); } else { my @array = (); if ($options{array}) { @array = read_array_options(\%options) if $options{array}; # the -t option gives the range of elements for an array job - push @qsub_options,'-t','1-'. scalar @array; + if ($options{array_all_in_one_job}) { + $options{array_per_job} = scalar @array; + } else { + push @qsub_options,'-t','1-'. ceil(scalar @array / $options{array_per_job}); + if ($options{array_slot_limit}) { + $qsub_options[$#qsub_options] .= '%'.$options{array_slot_limit}; + } + } + } + if ($options{batch} eq 'pbs') { + push @qsub_options,'-'; } call_qsub(\@qsub_options,write_qsub_script(\%options,\@ARGV,\@array)); } sub generate_qsub_options{ - my ($options) = @_; + my ($options,$args) = @_; my @qo; if (defined $options->{queue} and length $options->{queue}) { push @qo,'-q',$options->{queue}; } - if (defined $options->{dir}) { - push @qo,'-d',abs_path($options->{dir}); - } else { - push @qo,'-d',getcwd; - } ## handle the -l options my @l; push @l, 'nodes='.$options->{nodes}; if (defined $options->{ppn}) { $l[$#l] .= ':ppn='.$options->{ppn}; } - if ($options->{mem}) { - push @l,'mem='.$options->{mem}; + if (defined $options->{account}) { + push @qo,'-A',$options->{account}; + } + my %l_options = + (mem => 'vmem', + time => 'walltime', + cputime => 'cput', + host => 'host', + pmem => 'pmem', + pvmem => 'pvmem', + max_file => 'file', + ); + for my $k (keys %l_options) { + if ($options->{$k}) { + push @l,$l_options{$k}.'='.$options{$k}; + } } push @qo,'-l',join(',',@l) if @l; if ($options->{interactive}) { push @qo,'-I'; } + if ($options->{name}) { + push @qo,'-N',$options->{name}; + } else { + push @qo,'-N',join('_', + map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;} + @{$args}[0..min($#{$args},2)]); + } + # join error and output streams + if ($options->{join}) { + push @qo,'-j','oe'; + } + return @qo; +} + +sub generate_slurm_options{ + my ($options,$args) = @_; + my @qo; + if (defined $options->{queue} and length $options->{queue}) { + push @qo,'-p',$options->{queue}; + } + ## handle the -l options + if (defined $options->{account}) { + push @qo,'-A',$options->{account}; + } + my %options_map = + (mem => 'mem', + ppn => 'cpus-per-task', + time => 'time', + cputime => 'cput', + host => 'host', + pmem => 'pmem', + pvmem => 'pvmem', + max_file => 'file', + ); + for my $k (keys %options_map) { + if ($options->{$k}) { + push @qo,'--'.$options_map{$k}.'=',$options{$k}; + } + } + if ($options{mem}) { + push @qo,'--mem=',$options{mem}; + } + if ($options->{interactive}) { + push @qo,'-I'; + } + if ($options->{name}) { + push @qo,'-J',$options->{name}; + } else { + push @qo,'-J',join('_', + map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;} + @{$args}[0..min($#{$args},2)]); + } + # join error and output streams + if ($options->{join}) { + push @qo,'-j','oe'; + } return @qo; } @@ -185,41 +323,84 @@ sub read_array_options{ sub call_qsub { my ($qsub_options,$script) = @_; my $qsub_fh; - open $qsub_fh,'|-','qsub',@{$qsub_options},'-' or - die "Unable to start qsub: $!"; + open $qsub_fh,'|-',$JOB_SUBMITTER,@{$qsub_options} or + die "Unable to start $JOB_SUBMITTER: $!"; print {$qsub_fh} $script or - die "Unable to print to qsub: $!"; + die "Unable to print to $JOB_SUBMITTER: $!"; close($qsub_fh) or - die "Unable to close qsub filehandle: $!"; + die "Unable to close $JOB_SUBMITTER filehandle: $!"; } sub write_qsub_script { my ($opt,$arg,$array) = @_; my $script = "#!/bin/bash\n"; - my $command = join(' ',map {qq('$_')} @{$arg}); + my $command = join(' ',map {$_ =~ /\s/?qq('$_'):$_} @{$arg}); $script .= <{dir}) { + $directory = abs_path($opt->{dir}); + } + # we really should be quoting this instead + $script .=<{array}) { + my @subshell = ('',''); my $array_opt = join("\n",@{$array}); + my $max_array = scalar @{$array}; + my $apjm1 = $opt->{array_per_job} - 1; + $script .= <{array_per_job} > 1) { + # we will use subshells if there are more than one array + # items per job + @subshell = ('(',')'); + $script .= <{array_per_job}); do +# in some cases, the jobs aren't going to come out evenly. Handle that. +JOBNUM=\$(( \${MYARRAYID:=1} * $opt->{array_per_job} + \$i - $opt->{array_per_job} )) +if [ \$JOBNUM -le $max_array ]; then +OPT=\$(sed -n -e "\$JOBNUM p"<<'_HERE_DOC_END_' +EOF + } else { + $script .= <{array} eq 'chdir') { $script .= <{array_per_job} > 1) { + $script .= <