2 # dqsub submits jobs using qsub with better options
3 # and is released under the terms of the GNU GPL version 3, or any
4 # later version, at your option. See the file README and COPYING for
6 # Copyright 2014 by Don Armstrong <don@donarmstrong.com>.
17 dqsub - submits jobs using qsub with better options
24 --queue, -q Queue to use
25 --interactive, -I call qsub interactively
27 --array array mode (one of 'chdir' or 'xargs' or '')
28 --array-from file to read arrays from (default STDIN)
29 --array-per-job number of array items to handle in each job (default 1)
30 --array-all-in-one-job Run all of the array items in one job
31 --ppn processors per node to use
32 --mem memory to request
33 --dir Directory to run the script in (default current directory)
34 --account, -A Account name to use
35 --join, -J join error and output streams (default)
36 --name, -N Name of the job
37 --debug, -d debugging level (Default 0)
38 --help, -h display this help
39 --man, -m display manual
47 This describes how dqsub will generate array jobs.
49 If no B<--array> is given, then the command and any additional
50 arguments given will be run using qsub.
52 If B<--array> is C<chdir>, then each line of the input given in
53 B<--array-from> will be used as a directory and the command and any
54 additional arguments given will run in each directory.
56 IF B<--array> is C<xargs>, then each line of the input given will be
57 considered to be an additional argument which will be given to the
58 command run in the current directory.
62 File to read array arguments from. If not provided, and B<--array> is
63 given, arguments will be read from STDIN.
65 =item B<--account, -A>
71 Whether to join STDOUT and STDERR. On by default; disable with
76 Which batch system to use. If sbatch exists, assume it's slurm,
81 Debug verbosity. (Default 0)
85 Display brief usage information.
100 use Cwd qw(getcwd abs_path);
102 use List::Util qw(min);
105 my %options = (nodes => 1,
116 GetOptions(\%options,
122 'array_from|array-from=s',
123 'array_per_job|array-per-job=i',
124 'array_slot_limit|array-slot-limit=i',
125 'array_all_in_one_job|array-all-in-one-job!',
126 'ppn|cpus|processors-per-node=i',
130 'time|walltime=s','cputime|cput=s','host=s',
131 'pmem|process_mem|process-mem=s',
132 'pvmem|process_virtual_mem|process-virtiual-mem=s',
133 'max_file|max-file|file=s',
136 'debug|d+','help|h|?','man|m');
138 # pod2usage() if $options{help};
139 # pod2usage({verbose=>2}) if $options{man};
141 $DEBUG = $options{debug};
144 if (not @ARGV and not $options{interactive}) {
145 push @USAGE_ERRORS,"You must provide a command to run";
147 if (defined $options{array} and $options{array} !~ /^(?:|chdir|xargs)$/i) {
148 push @USAGE_ERRORS,"--array must be one of chdir, xargs or '' if provided";
149 $options{array} = lc($options{array});
150 if ($options{array} eq '') {
151 $options{array} = undef;
154 if ($options{interactive} and @ARGV) {
155 push @USAGE_ERRORS,"Don't provide commands when you're asking for an interactive shell";
158 if (not defined $options{batch}) {
161 $options{batch} = 'slurm'
163 $options{batch} = 'pbs'
167 if ($options{batch} !~ /^pbs|slurm$/) {
168 push @USAGE_ERRORS,"Unsupported batch system '$options{batch}'; ".
169 "supported systems are pbs or slurm";
172 # pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
174 print STDERR map {"$_\n"} @USAGE_ERRORS;
179 my $JOB_SUBMITTER = 'qsub';
180 # OK. Generate the options to qsub which we'll be using
182 if ($options{batch} eq 'pbs') {
183 @qsub_options = generate_qsub_options(\%options,\@ARGV);
184 $JOB_SUBMITTER = 'qsub';
185 } elsif ($options{batch} eq 'slurm') {
186 @qsub_options = generate_slurm_options(\%options,\@ARGV);
187 $JOB_SUBMITTER = 'sbatch';
189 die "Unsupported batch system '$options{batch}'";
193 if ($options{interactive}) {
194 print STDERR 'running: qsub '.join(' ',@qsub_options) if $DEBUG;
195 exec($JOB_SUBMITTER,@qsub_options);
198 if ($options{array}) {
199 @array = read_array_options(\%options) if $options{array};
200 # the -t option gives the range of elements for an array job
201 if ($options{array_all_in_one_job}) {
202 $options{array_per_job} = scalar @array;
204 push @qsub_options,'-t','1-'. ceil(scalar @array / $options{array_per_job});
205 if ($options{array_slot_limit}) {
206 $qsub_options[$#qsub_options] .= '%'.$options{array_slot_limit};
210 if ($options{batch} eq 'pbs') {
211 push @qsub_options,'-';
213 call_qsub(\@qsub_options,write_qsub_script(\%options,\@ARGV,\@array));
216 sub generate_qsub_options{
217 my ($options,$args) = @_;
219 if (defined $options->{queue} and length $options->{queue}) {
220 push @qo,'-q',$options->{queue};
222 ## handle the -l options
224 push @l, 'nodes='.$options->{nodes};
225 if (defined $options->{ppn}) {
226 $l[$#l] .= ':ppn='.$options->{ppn};
228 if (defined $options->{account}) {
229 push @qo,'-A',$options->{account};
240 for my $k (keys %l_options) {
241 if ($options->{$k}) {
242 push @l,$l_options{$k}.'='.$options{$k};
245 push @qo,'-l',join(',',@l) if @l;
246 if ($options->{interactive}) {
249 if ($options->{name}) {
250 push @qo,'-N',$options->{name};
252 push @qo,'-N',join('_',
253 map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;}
254 @{$args}[0..min($#{$args},2)]);
256 # join error and output streams
257 if ($options->{join}) {
263 sub generate_slurm_options{
264 my ($options,$args) = @_;
266 if (defined $options->{queue} and length $options->{queue}) {
267 push @qo,'-p',$options->{queue};
269 ## handle the -l options
270 if (defined $options->{account}) {
271 push @qo,'-A',$options->{account};
275 ppn => 'cpus-per-task',
283 for my $k (keys %options_map) {
284 if ($options->{$k}) {
285 push @qo,'--'.$options_map{$k},$options{$k};
289 push @qo,'--mem=',$options{mem};
291 if ($options->{interactive}) {
294 if ($options->{name}) {
295 push @qo,'-J',$options->{name};
297 push @qo,'-J',join('_',
298 map {my $a = $_; $a =~ s/[^a-zA-Z0-9]*//g; $a;}
299 @{$args}[0..min($#{$args},2)]);
301 # join error and output streams
302 if ($options->{join}) {
308 sub read_array_options{
311 if (defined $options->{array_from}) {
312 $fh = IO::File->new(defined $options->{array_from}) or
313 die "Unable to open $options->{array_from} for reading: $!";
324 my ($qsub_options,$script) = @_;
326 open $qsub_fh,'|-',$JOB_SUBMITTER,@{$qsub_options} or
327 die "Unable to start $JOB_SUBMITTER: $!";
328 print {$qsub_fh} $script or
329 die "Unable to print to $JOB_SUBMITTER: $!";
331 die "Unable to close $JOB_SUBMITTER filehandle: $!";
334 sub write_qsub_script {
335 my ($opt,$arg,$array) = @_;
337 my $script = "#!/bin/bash\n";
338 my $command = join(' ',map {$_ =~ /\s/?qq('$_'):$_} @{$arg});
340 # this script was written by dqsub
342 my $directory = getcwd;
343 if (defined $opt->{dir}) {
344 $directory = abs_path($opt->{dir});
346 # we really should be quoting this instead
348 # change to the working directory
351 if (defined $opt->{array}) {
352 my @subshell = ('','');
353 my $array_opt = join("\n",@{$array});
354 my $max_array = scalar @{$array};
355 my $apjm1 = $opt->{array_per_job} - 1;
357 if [ -n "\$PBS_ARRAYID" ]; then
358 MYARRAYID="\${PBS_ARRAYID:=1}"
360 MYARRAYID="\${SLURM_ARRAY_TASK_ID:=1}"
363 if ($opt->{array_per_job} > 1) {
364 # we will use subshells if there are more than one array
366 @subshell = ('(',')');
368 for i in \$(seq 1 $opt->{array_per_job}); do
369 # in some cases, the jobs aren't going to come out evenly. Handle that.
370 JOBNUM=\$(( \${MYARRAYID:=1} * $opt->{array_per_job} + \$i - $opt->{array_per_job} ))
371 if [ \$JOBNUM -le $max_array ]; then
372 OPT=\$(sed -n -e "\$JOBNUM p"<<'_HERE_DOC_END_'
376 OPT=\$(sed -n -e "\${MYARRAYID:=1} p"<<'_HERE_DOC_END_'
384 if ($opt->{array} eq 'chdir') {
394 exec ${command} "\$OPT";
398 if ($opt->{array_per_job} > 1) {
406 # there's no array, so just executing the command with arguments