2 # dqsub submits jobs using qsub with better options
3 # and is released under the terms of the GNU GPL version 3, or any
4 # later version, at your option. See the file README and COPYING for
6 # Copyright 2014 by Don Armstrong <don@donarmstrong.com>.
17 dqsub - submits jobs using qsub with better options
24 --queue, -q Queue to use
25 --interactive, -I call qsub interactively
27 --array array mode (one of 'chdir' or 'xargs' or '')
28 --array-from file to read arrays from (default STDIN)
29 --array-per-job number of array items to handle in each job (default 1)
30 --array-all-in-one-job Run all of the array items in one job
31 --ppn processors per node to use
32 --mem memory to request
33 --dir Directory to run the script in (default current directory)
34 --account, -A Account name to use
35 --join, -J join error and output streams (default)
36 --name, -N Name of the job
37 --precommand Optional command to run before each command
38 --debug, -d debugging level (Default 0)
39 --help, -h display this help
40 --man, -m display manual
48 This describes how dqsub will generate array jobs.
50 If no B<--array> is given, then the command and any additional
51 arguments given will be run using qsub.
53 If B<--array> is C<chdir>, then each line of the input given in
54 B<--array-from> will be used as a directory and the command and any
55 additional arguments given will run in each directory.
57 IF B<--array> is C<xargs>, then each line of the input given will be
58 considered to be an additional argument which will be given to the
59 command run in the current directory.
63 File to read array arguments from. If not provided, and B<--array> is
64 given, arguments will be read from STDIN.
66 =item B<--account, -A>
72 Whether to join STDOUT and STDERR. On by default; disable with
77 Which batch system to use. If sbatch exists, assume it's slurm,
82 Debug verbosity. (Default 0)
86 Display brief usage information.
101 use Cwd qw(getcwd abs_path);
103 use List::Util qw(min);
106 my %options = (nodes => 1,
117 GetOptions(\%options,
123 'array_from|array-from=s',
124 'array_per_job|array-per-job=i',
125 'array_slot_limit|array-slot-limit=i',
126 'array_all_in_one_job|array-all-in-one-job!',
127 'ppn|cpus|processors-per-node=i',
131 'time|walltime=s','cputime|cput=s','host=s',
132 'pmem|process_mem|process-mem=s',
133 'pvmem|process_virtual_mem|process-virtiual-mem=s',
134 'max_file|max-file|file=s',
135 'precommand|pre-command|pre_command=s',
138 'debug|d+','help|h|?','man|m');
140 # pod2usage() if $options{help};
141 # pod2usage({verbose=>2}) if $options{man};
143 $DEBUG = $options{debug};
146 if (not @ARGV and not $options{interactive}) {
147 push @USAGE_ERRORS,"You must provide a command to run";
149 if (defined $options{array} and $options{array} !~ /^(?:|chdir|xargs)$/i) {
150 push @USAGE_ERRORS,"--array must be one of chdir, xargs or '' if provided";
151 $options{array} = lc($options{array});
152 if ($options{array} eq '') {
153 $options{array} = undef;
156 if ($options{interactive} and @ARGV) {
157 push @USAGE_ERRORS,"Don't provide commands when you're asking for an interactive shell";
160 if (not defined $options{batch}) {
161 qx{which sbatch >/dev/null 2>&1};
163 $options{batch} = 'slurm'
165 $options{batch} = 'pbs'
169 if ($options{batch} !~ /^pbs|slurm$/) {
170 push @USAGE_ERRORS,"Unsupported batch system '$options{batch}'; ".
171 "supported systems are pbs or slurm";
174 # pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
176 print STDERR map {"$_\n"} @USAGE_ERRORS;
181 my $JOB_SUBMITTER = 'qsub';
182 # OK. Generate the options to qsub which we'll be using
184 if ($options{batch} eq 'pbs') {
185 @qsub_options = generate_qsub_options(\%options,\@ARGV);
186 $JOB_SUBMITTER = 'qsub';
187 } elsif ($options{batch} eq 'slurm') {
188 @qsub_options = generate_slurm_options(\%options,\@ARGV);
189 $JOB_SUBMITTER = 'sbatch';
191 die "Unsupported batch system '$options{batch}'";
195 if ($options{interactive}) {
196 print STDERR 'running: '.$JOB_SUBMITTER.' '.join(' ',@qsub_options) if $DEBUG;
197 if ($options{batch} eq 'pbs') {
198 exec($JOB_SUBMITTER,@qsub_options);
200 exec('srun',@qsub_options);
205 if ($options{array}) {
206 @array = read_array_options(\%options) if $options{array};
207 # the -t option gives the range of elements for an array job
208 if ($options{array_all_in_one_job}) {
209 $options{array_per_job} = scalar @array;
211 if ($options{batch} eq 'pbs') {
212 push @qsub_options,'-t';
214 push @qsub_options,'-a';
216 push @qsub_options,'1-'. ceil(scalar @array / $options{array_per_job});
217 if ($options{array_slot_limit}) {
218 $qsub_options[$#qsub_options] .= '%'.$options{array_slot_limit};
222 if ($options{batch} eq 'pbs') {
223 push @qsub_options,'-';
225 call_qsub(\@qsub_options,write_qsub_script(\%options,\@ARGV,\@array));
228 sub generate_qsub_options{
229 my ($options,$args) = @_;
231 if (defined $options->{queue} and length $options->{queue}) {
232 push @qo,'-q',$options->{queue};
234 ## handle the -l options
236 push @l, 'nodes='.$options->{nodes};
237 if (defined $options->{ppn}) {
238 $l[$#l] .= ':ppn='.$options->{ppn};
240 if (defined $options->{account}) {
241 push @qo,'-A',$options->{account};
252 for my $k (keys %l_options) {
253 if ($options->{$k}) {
254 push @l,$l_options{$k}.'='.$options{$k};
257 push @qo,'-l',join(',',@l) if @l;
258 if ($options->{interactive}) {
261 if ($options->{name}) {
262 push @qo,'-N',$options->{name};
264 push @qo,'-N',join('_',
265 map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;}
266 @{$args}[0..min($#{$args},2)]);
268 # join error and output streams
269 if ($options->{join}) {
275 sub generate_slurm_options{
276 my ($options,$args) = @_;
278 if (defined $options->{queue} and length $options->{queue}) {
279 push @qo,'-p',$options->{queue};
281 ## handle the -l options
282 if (defined $options->{account}) {
283 push @qo,'-A',$options->{account};
287 ppn => 'cpus-per-task',
295 for my $k (keys %options_map) {
296 if ($options->{$k}) {
297 push @qo,'--'.$options_map{$k}.'='.$options{$k};
301 push @qo,'--mem='.$options{mem};
303 if ($options->{interactive}) {
304 push @qo,'--pty',$ENV{SHELL}//'bash';
306 if ($options->{name}) {
307 push @qo,'-J',$options->{name};
309 push @qo,'-J',join('_',
310 map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;}
311 @{$args}[0..min($#{$args},2)]);
316 sub read_array_options{
319 if (defined $options->{array_from}) {
320 $fh = IO::File->new(defined $options->{array_from}) or
321 die "Unable to open $options->{array_from} for reading: $!";
332 my ($qsub_options,$script) = @_;
334 open $qsub_fh,'|-',$JOB_SUBMITTER,@{$qsub_options} or
335 die "Unable to start $JOB_SUBMITTER: $!";
336 print {$qsub_fh} $script or
337 die "Unable to print to $JOB_SUBMITTER: $!";
339 die "Unable to close $JOB_SUBMITTER filehandle: $!";
342 sub write_qsub_script {
343 my ($opt,$arg,$array) = @_;
345 my $script = "#!/bin/bash\n";
346 my $command = join(' ',map {$_ =~ /\s/?qq('$_'):$_} @{$arg});
348 # this script was written by dqsub
350 # if there is a precommand, write it out
351 if ($opt->{precommand}) {
353 # this is the precommand _BEGIN_
358 my $directory = getcwd;
359 if (defined $opt->{dir}) {
360 $directory = abs_path($opt->{dir});
362 # we really should be quoting this instead
364 # change to the working directory
367 if (defined $opt->{array}) {
368 my @subshell = ('','');
369 my $array_opt = join("\n",@{$array});
370 my $max_array = scalar @{$array};
371 my $apjm1 = $opt->{array_per_job} - 1;
373 if [ -n "\$PBS_ARRAYID" ]; then
374 MYARRAYID="\${PBS_ARRAYID:=1}"
376 MYARRAYID="\${SLURM_ARRAY_TASK_ID:=1}"
379 if ($opt->{array_per_job} > 1) {
380 # we will use subshells if there are more than one array
382 @subshell = ('(',')');
384 for i in \$(seq 1 $opt->{array_per_job}); do
385 # in some cases, the jobs aren't going to come out evenly. Handle that.
386 JOBNUM=\$(( \${MYARRAYID:=1} * $opt->{array_per_job} + \$i - $opt->{array_per_job} ))
387 if [ \$JOBNUM -le $max_array ]; then
388 OPT=\$(sed -n -e "\$JOBNUM p"<<'_HERE_DOC_END_'
392 OPT=\$(sed -n -e "\${MYARRAYID:=1} p"<<'_HERE_DOC_END_'
400 if ($opt->{array} eq 'chdir') {
410 exec ${command} "\$OPT";
414 if ($opt->{array_per_job} > 1) {
422 # there's no array, so just executing the command with arguments