2 # dqsub submits jobs using qsub with better options
3 # and is released under the terms of the GNU GPL version 3, or any
4 # later version, at your option. See the file README and COPYING for
6 # Copyright 2014 by Don Armstrong <don@donarmstrong.com>.
17 dqsub - submits jobs using qsub with better options
24 --queue, -q Queue to use
25 --interactive, -I call qsub interactively
27 --array array mode (one of 'chdir' or 'xargs' or '')
28 --array-from file to read arrays from (default STDIN)
29 --array-per-job number of array items to handle in each job (default 1)
30 --array-all-in-one-job Run all of the array items in one job
31 --ppn processors per node to use
32 --mem memory to request
33 --dir Directory to run the script in (default current directory)
34 --account, -A Account name to use
35 --join, -J join error and output streams (default)
36 --name, -N Name of the job
37 --precommand Optional command to run before each command
38 --debug, -d debugging level (Default 0)
39 --help, -h display this help
40 --man, -m display manual
48 This describes how dqsub will generate array jobs.
50 If no B<--array> is given, then the command and any additional
51 arguments given will be run using qsub.
53 If B<--array> is C<chdir>, then each line of the input given in
54 B<--array-from> will be used as a directory and the command and any
55 additional arguments given will run in each directory.
57 IF B<--array> is C<xargs>, then each line of the input given will be
58 considered to be an additional argument which will be given to the
59 command run in the current directory.
63 File to read array arguments from. If not provided, and B<--array> is
64 given, arguments will be read from STDIN.
66 =item B<--account, -A>
72 Whether to join STDOUT and STDERR. On by default; disable with
77 Which batch system to use. If sbatch exists, assume it's slurm,
82 Debug verbosity. (Default 0)
86 Display brief usage information.
101 use Cwd qw(getcwd abs_path);
103 use List::Util qw(min);
106 my %options = (nodes => 1,
117 GetOptions(\%options,
123 'array_from|array-from=s',
124 'array_per_job|array-per-job=i',
125 'array_slot_limit|array-slot-limit=i',
126 'array_all_in_one_job|array-all-in-one-job!',
127 'ppn|cpus|processors-per-node=i',
131 'time|walltime=s','cputime|cput=s','host=s',
132 'pmem|process_mem|process-mem=s',
133 'pvmem|process_virtual_mem|process-virtiual-mem=s',
134 'max_file|max-file|file=s',
135 'precommand|pre-command|pre_command=s',
138 'debug|d+','help|h|?','man|m');
140 # pod2usage() if $options{help};
141 # pod2usage({verbose=>2}) if $options{man};
143 $DEBUG = $options{debug};
146 if (not @ARGV and not $options{interactive}) {
147 push @USAGE_ERRORS,"You must provide a command to run";
149 if (defined $options{array} and $options{array} !~ /^(?:|chdir|xargs)$/i) {
150 push @USAGE_ERRORS,"--array must be one of chdir, xargs or '' if provided";
151 $options{array} = lc($options{array});
152 if ($options{array} eq '') {
153 $options{array} = undef;
156 if ($options{interactive} and @ARGV) {
157 push @USAGE_ERRORS,"Don't provide commands when you're asking for an interactive shell";
160 if (not defined $options{batch}) {
163 $options{batch} = 'slurm'
165 $options{batch} = 'pbs'
169 if ($options{batch} !~ /^pbs|slurm$/) {
170 push @USAGE_ERRORS,"Unsupported batch system '$options{batch}'; ".
171 "supported systems are pbs or slurm";
174 # pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
176 print STDERR map {"$_\n"} @USAGE_ERRORS;
181 my $JOB_SUBMITTER = 'qsub';
182 # OK. Generate the options to qsub which we'll be using
184 if ($options{batch} eq 'pbs') {
185 @qsub_options = generate_qsub_options(\%options,\@ARGV);
186 $JOB_SUBMITTER = 'qsub';
187 } elsif ($options{batch} eq 'slurm') {
188 @qsub_options = generate_slurm_options(\%options,\@ARGV);
189 $JOB_SUBMITTER = 'sbatch';
191 die "Unsupported batch system '$options{batch}'";
195 if ($options{interactive}) {
196 print STDERR 'running: '.$JOB_SUBMITTER.' '.join(' ',@qsub_options) if $DEBUG;
197 exec($JOB_SUBMITTER,@qsub_options);
200 if ($options{array}) {
201 @array = read_array_options(\%options) if $options{array};
202 # the -t option gives the range of elements for an array job
203 if ($options{array_all_in_one_job}) {
204 $options{array_per_job} = scalar @array;
206 push @qsub_options,'-t','1-'. ceil(scalar @array / $options{array_per_job});
207 if ($options{array_slot_limit}) {
208 $qsub_options[$#qsub_options] .= '%'.$options{array_slot_limit};
212 if ($options{batch} eq 'pbs') {
213 push @qsub_options,'-';
215 call_qsub(\@qsub_options,write_qsub_script(\%options,\@ARGV,\@array));
218 sub generate_qsub_options{
219 my ($options,$args) = @_;
221 if (defined $options->{queue} and length $options->{queue}) {
222 push @qo,'-q',$options->{queue};
224 ## handle the -l options
226 push @l, 'nodes='.$options->{nodes};
227 if (defined $options->{ppn}) {
228 $l[$#l] .= ':ppn='.$options->{ppn};
230 if (defined $options->{account}) {
231 push @qo,'-A',$options->{account};
242 for my $k (keys %l_options) {
243 if ($options->{$k}) {
244 push @l,$l_options{$k}.'='.$options{$k};
247 push @qo,'-l',join(',',@l) if @l;
248 if ($options->{interactive}) {
251 if ($options->{name}) {
252 push @qo,'-N',$options->{name};
254 push @qo,'-N',join('_',
255 map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;}
256 @{$args}[0..min($#{$args},2)]);
258 # join error and output streams
259 if ($options->{join}) {
265 sub generate_slurm_options{
266 my ($options,$args) = @_;
268 if (defined $options->{queue} and length $options->{queue}) {
269 push @qo,'-p',$options->{queue};
271 ## handle the -l options
272 if (defined $options->{account}) {
273 push @qo,'-A',$options->{account};
277 ppn => 'cpus-per-task',
285 for my $k (keys %options_map) {
286 if ($options->{$k}) {
287 push @qo,'--'.$options_map{$k}.'='.$options{$k};
291 push @qo,'--mem='.$options{mem};
293 if ($options->{interactive}) {
296 if ($options->{name}) {
297 push @qo,'-J',$options->{name};
299 push @qo,'-J',join('_',
300 map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;}
301 @{$args}[0..min($#{$args},2)]);
306 sub read_array_options{
309 if (defined $options->{array_from}) {
310 $fh = IO::File->new(defined $options->{array_from}) or
311 die "Unable to open $options->{array_from} for reading: $!";
322 my ($qsub_options,$script) = @_;
324 open $qsub_fh,'|-',$JOB_SUBMITTER,@{$qsub_options} or
325 die "Unable to start $JOB_SUBMITTER: $!";
326 print {$qsub_fh} $script or
327 die "Unable to print to $JOB_SUBMITTER: $!";
329 die "Unable to close $JOB_SUBMITTER filehandle: $!";
332 sub write_qsub_script {
333 my ($opt,$arg,$array) = @_;
335 my $script = "#!/bin/bash\n";
336 my $command = join(' ',map {$_ =~ /\s/?qq('$_'):$_} @{$arg});
338 # this script was written by dqsub
340 # if there is a precommand, write it out
341 if ($opt->{precommand}) {
343 # this is the precommand _BEGIN_
348 my $directory = getcwd;
349 if (defined $opt->{dir}) {
350 $directory = abs_path($opt->{dir});
352 # we really should be quoting this instead
354 # change to the working directory
357 if (defined $opt->{array}) {
358 my @subshell = ('','');
359 my $array_opt = join("\n",@{$array});
360 my $max_array = scalar @{$array};
361 my $apjm1 = $opt->{array_per_job} - 1;
363 if [ -n "\$PBS_ARRAYID" ]; then
364 MYARRAYID="\${PBS_ARRAYID:=1}"
366 MYARRAYID="\${SLURM_ARRAY_TASK_ID:=1}"
369 if ($opt->{array_per_job} > 1) {
370 # we will use subshells if there are more than one array
372 @subshell = ('(',')');
374 for i in \$(seq 1 $opt->{array_per_job}); do
375 # in some cases, the jobs aren't going to come out evenly. Handle that.
376 JOBNUM=\$(( \${MYARRAYID:=1} * $opt->{array_per_job} + \$i - $opt->{array_per_job} ))
377 if [ \$JOBNUM -le $max_array ]; then
378 OPT=\$(sed -n -e "\$JOBNUM p"<<'_HERE_DOC_END_'
382 OPT=\$(sed -n -e "\${MYARRAYID:=1} p"<<'_HERE_DOC_END_'
390 if ($opt->{array} eq 'chdir') {
400 exec ${command} "\$OPT";
404 if ($opt->{array_per_job} > 1) {
412 # there's no array, so just executing the command with arguments