2 # dqsub submits jobs using qsub with better options
3 # and is released under the terms of the GNU GPL version 3, or any
4 # later version, at your option. See the file README and COPYING for
6 # Copyright 2014 by Don Armstrong <don@donarmstrong.com>.
17 dqsub - submits jobs using qsub with better options
24 --queue, -q Queue to use
25 --interactive, -I call qsub interactively
27 --array array mode (one of 'chdir' or 'xargs' or '')
28 --array-from file to read arrays from (default STDIN)
29 --array-per-job number of array items to handle in each job (default 1)
30 --array-all-in-one-job Run all of the array items in one job
31 --ppn processors per node to use
32 --mem memory to request
33 --dir Directory to run the script in (default current directory)
34 --account, -A Account name to use
35 --join, -J join error and output streams (default)
36 --name, -N Name of the job
37 --precommand Optional command to run before each command
38 --debug, -d debugging level (Default 0)
39 --help, -h display this help
40 --man, -m display manual
48 This describes how dqsub will generate array jobs.
50 If no B<--array> is given, then the command and any additional
51 arguments given will be run using qsub.
53 If B<--array> is C<chdir>, then each line of the input given in
54 B<--array-from> will be used as a directory and the command and any
55 additional arguments given will run in each directory.
57 IF B<--array> is C<xargs>, then each line of the input given will be
58 considered to be an additional argument which will be given to the
59 command run in the current directory.
63 File to read array arguments from. If not provided, and B<--array> is
64 given, arguments will be read from STDIN.
66 =item B<--account, -A>
72 Whether to join STDOUT and STDERR. On by default; disable with
77 Which batch system to use. If sbatch exists, assume it's slurm,
82 Debug verbosity. (Default 0)
86 Display brief usage information.
101 use Cwd qw(getcwd abs_path);
103 use List::Util qw(min);
106 my %options = (nodes => 1,
117 GetOptions(\%options,
123 'array_from|array-from=s',
124 'array_per_job|array-per-job=i',
125 'array_slot_limit|array-slot-limit=i',
126 'array_all_in_one_job|array-all-in-one-job!',
127 'ppn|cpus|processors-per-node=i',
131 'time|walltime=s','cputime|cput=s','host=s',
132 'pmem|process_mem|process-mem=s',
133 'pvmem|process_virtual_mem|process-virtiual-mem=s',
134 'max_file|max-file|file=s',
135 'precommand|pre-command|pre_command=s',
138 'debug|d+','help|h|?','man|m');
140 # pod2usage() if $options{help};
141 # pod2usage({verbose=>2}) if $options{man};
143 $DEBUG = $options{debug};
146 if (not @ARGV and not $options{interactive}) {
147 push @USAGE_ERRORS,"You must provide a command to run";
149 if (defined $options{array} and $options{array} !~ /^(?:|chdir|xargs)$/i) {
150 push @USAGE_ERRORS,"--array must be one of chdir, xargs or '' if provided";
151 $options{array} = lc($options{array});
152 if ($options{array} eq '') {
153 $options{array} = undef;
156 if ($options{interactive} and @ARGV) {
157 push @USAGE_ERRORS,"Don't provide commands when you're asking for an interactive shell";
160 if (not defined $options{batch}) {
161 qx{which sbatch >/dev/null 2>&1};
163 $options{batch} = 'slurm'
165 $options{batch} = 'pbs'
169 if ($options{batch} !~ /^pbs|slurm$/) {
170 push @USAGE_ERRORS,"Unsupported batch system '$options{batch}'; ".
171 "supported systems are pbs or slurm";
174 # pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
176 print STDERR map {"$_\n"} @USAGE_ERRORS;
181 my $JOB_SUBMITTER = 'qsub';
182 # OK. Generate the options to qsub which we'll be using
184 if ($options{batch} eq 'pbs') {
185 @qsub_options = generate_qsub_options(\%options,\@ARGV);
186 $JOB_SUBMITTER = 'qsub';
187 } elsif ($options{batch} eq 'slurm') {
188 @qsub_options = generate_slurm_options(\%options,\@ARGV);
189 $JOB_SUBMITTER = 'sbatch';
191 die "Unsupported batch system '$options{batch}'";
195 if ($options{interactive}) {
196 print STDERR 'running: '.$JOB_SUBMITTER.' '.join(' ',@qsub_options) if $DEBUG;
197 exec($JOB_SUBMITTER,@qsub_options);
200 if ($options{array}) {
201 @array = read_array_options(\%options) if $options{array};
202 # the -t option gives the range of elements for an array job
203 if ($options{array_all_in_one_job}) {
204 $options{array_per_job} = scalar @array;
206 if ($options{batch} eq 'pbs') {
207 push @qsub_options,'-t';
209 push @qsub_options,'-a';
211 push @qsub_options,'1-'. ceil(scalar @array / $options{array_per_job});
212 if ($options{array_slot_limit}) {
213 $qsub_options[$#qsub_options] .= '%'.$options{array_slot_limit};
217 if ($options{batch} eq 'pbs') {
218 push @qsub_options,'-';
220 call_qsub(\@qsub_options,write_qsub_script(\%options,\@ARGV,\@array));
223 sub generate_qsub_options{
224 my ($options,$args) = @_;
226 if (defined $options->{queue} and length $options->{queue}) {
227 push @qo,'-q',$options->{queue};
229 ## handle the -l options
231 push @l, 'nodes='.$options->{nodes};
232 if (defined $options->{ppn}) {
233 $l[$#l] .= ':ppn='.$options->{ppn};
235 if (defined $options->{account}) {
236 push @qo,'-A',$options->{account};
247 for my $k (keys %l_options) {
248 if ($options->{$k}) {
249 push @l,$l_options{$k}.'='.$options{$k};
252 push @qo,'-l',join(',',@l) if @l;
253 if ($options->{interactive}) {
256 if ($options->{name}) {
257 push @qo,'-N',$options->{name};
259 push @qo,'-N',join('_',
260 map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;}
261 @{$args}[0..min($#{$args},2)]);
263 # join error and output streams
264 if ($options->{join}) {
270 sub generate_slurm_options{
271 my ($options,$args) = @_;
273 if (defined $options->{queue} and length $options->{queue}) {
274 push @qo,'-p',$options->{queue};
276 ## handle the -l options
277 if (defined $options->{account}) {
278 push @qo,'-A',$options->{account};
282 ppn => 'cpus-per-task',
290 for my $k (keys %options_map) {
291 if ($options->{$k}) {
292 push @qo,'--'.$options_map{$k}.'='.$options{$k};
296 push @qo,'--mem='.$options{mem};
298 if ($options->{interactive}) {
301 if ($options->{name}) {
302 push @qo,'-J',$options->{name};
304 push @qo,'-J',join('_',
305 map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;}
306 @{$args}[0..min($#{$args},2)]);
311 sub read_array_options{
314 if (defined $options->{array_from}) {
315 $fh = IO::File->new(defined $options->{array_from}) or
316 die "Unable to open $options->{array_from} for reading: $!";
327 my ($qsub_options,$script) = @_;
329 open $qsub_fh,'|-',$JOB_SUBMITTER,@{$qsub_options} or
330 die "Unable to start $JOB_SUBMITTER: $!";
331 print {$qsub_fh} $script or
332 die "Unable to print to $JOB_SUBMITTER: $!";
334 die "Unable to close $JOB_SUBMITTER filehandle: $!";
337 sub write_qsub_script {
338 my ($opt,$arg,$array) = @_;
340 my $script = "#!/bin/bash\n";
341 my $command = join(' ',map {$_ =~ /\s/?qq('$_'):$_} @{$arg});
343 # this script was written by dqsub
345 # if there is a precommand, write it out
346 if ($opt->{precommand}) {
348 # this is the precommand _BEGIN_
353 my $directory = getcwd;
354 if (defined $opt->{dir}) {
355 $directory = abs_path($opt->{dir});
357 # we really should be quoting this instead
359 # change to the working directory
362 if (defined $opt->{array}) {
363 my @subshell = ('','');
364 my $array_opt = join("\n",@{$array});
365 my $max_array = scalar @{$array};
366 my $apjm1 = $opt->{array_per_job} - 1;
368 if [ -n "\$PBS_ARRAYID" ]; then
369 MYARRAYID="\${PBS_ARRAYID:=1}"
371 MYARRAYID="\${SLURM_ARRAY_TASK_ID:=1}"
374 if ($opt->{array_per_job} > 1) {
375 # we will use subshells if there are more than one array
377 @subshell = ('(',')');
379 for i in \$(seq 1 $opt->{array_per_job}); do
380 # in some cases, the jobs aren't going to come out evenly. Handle that.
381 JOBNUM=\$(( \${MYARRAYID:=1} * $opt->{array_per_job} + \$i - $opt->{array_per_job} ))
382 if [ \$JOBNUM -le $max_array ]; then
383 OPT=\$(sed -n -e "\$JOBNUM p"<<'_HERE_DOC_END_'
387 OPT=\$(sed -n -e "\${MYARRAYID:=1} p"<<'_HERE_DOC_END_'
395 if ($opt->{array} eq 'chdir') {
405 exec ${command} "\$OPT";
409 if ($opt->{array_per_job} > 1) {
417 # there's no array, so just executing the command with arguments