jobs_to_org

   1 #!/usr/bin/perl
   2 # jobs_to_org finds jobs in different job sites and turns them into org things
   3 # and is released under the terms of the GNU GPL version 3, or any
   4 # later version, at your option. See the file README and COPYING for
   5 # more information.
   6 # Copyright 2016 by Don Armstrong <don@donarmstrong.com>.
   7
   8
   9 use warnings;
  10 use strict;
  11
  12 use Getopt::Long;
  13 use Pod::Usage;
  14
  15 =head1 NAME
  16
  17 jobs_to_org - finds jobs in different job sites and turns them into org things
  18
  19 =head1 SYNOPSIS
  20
  21 jobs_to_org [options]
  22
  23  Options:
  24    --debug, -d debugging level (Default 0)
  25    --help, -h display this help
  26    --man, -m display manual
  27
  28 =head1 OPTIONS
  29
  30 =over
  31
  32 =item B<--debug, -d>
  33
  34 Debug verbosity. (Default 0)
  35
  36 =item B<--help, -h>
  37
  38 Display brief usage information.
  39
  40 =item B<--man, -m>
  41
  42 Display this manual.
  43
  44 =back
  45
  46 =head1 EXAMPLES
  47
  48 jobs_to_org
  49
  50 =cut
  51
  52 use OSSP::uuid;
  53 use WWW::Mechanize;
  54 use WWW::Mechanize::TreeBuilder;
  55 use vars qw($DEBUG);
  56 use Data::Printer;
  57 use Text::Wrap;
  58 use POSIX qw(strftime);
  59 use DateTime;
  60
  61 tie my $uuid, 'OSSP::uuid::tie';
  62 $uuid= ["v1"];
  63
  64 my %options = (debug           => 0,
  65                help            => 0,
  66                man             => 0,
  67                pages           => 1,
  68                site            => 'herc',
  69               );
  70
  71 GetOptions(\%options,
  72            'site|s=s',
  73            'pages|p=i',
  74            'debug|d+','help|h|?','man|m');
  75
  76 pod2usage() if $options{help};
  77 pod2usage({verbose=>2}) if $options{man};
  78
  79 $DEBUG = $options{debug};
  80
  81 my @USAGE_ERRORS;
  82 # if (1) {
  83 #     push @USAGE_ERRORS,"You must pass something";
  84 # }
  85
  86 pod2usage(join("\n",@USAGE_ERRORS)) if @USAGE_ERRORS;
  87
  88 my %sites =
  89     (herc => {url => 'http://main.hercjobs.org/jobs/search?keywords=professor+AND+%28genomics+OR+bioinformatics+OR+biology+OR+informatics%29&discipline=academic-faculty&category=academic-faculty&category=allied-health&category=biological-biomedical-sciences&category=computer-information-sciences&category=education&category=interdisciplinary&category=mathematics-statistics&category=medical-research&category=physical-sciences&sort=DATE_POSTED+DESC',
  90               next_selector => [class => "bti-pagination-previous-link bti-pagination-prev-next",text => '>'],
  91               job_selector => [url_regex => qr{^\/jobs\/\d+/.+}],
  92               university => [itemprop=>"hiringOrganization",itemtype=> qr{https?://schema.org/Organization}],
  93               description => [class=>"bti-jd-description",itemprop=>"description"],
  94               date => [class=>"bti-jd-detail-text",
  95                        sub {scalar $_[0]->parent()->attr('class') eq 'bti-jd-details-action'}],
  96               position => [class=>"bti-jd-title",itemprop=>"title"],
  97              },
  98      nature => {url => 'http://www.nature.com/naturejobs/science/jobs?utf8=%E2%9C%93&q%5B%5D=professor&job_type%5B%5D=Assistant+Professor&job_type%5B%5D=Professor&order_by=created_on',
  99                 next_selector => [class=>"next_page",url_regex=>qr{^/naturejobs/science/jobs},],
 100                 job_selector => [url_regex => qr{^/naturejobs/science/jobs/\d+-.+}],
 101                 university => [href => qr{^/naturejobs/science/employer-directory/\d+$}],
 102                 description => [class=>"job-description"],
 103                 date => [_tag => 'dd', sub {$_[0]->as_text() =~ qr/\d+\s+days\s+ago$/},],
 104                 position => [class=>'job-title heading'],
 105                },
 106      vitae => {url => 'https://chroniclevitae.com/job_search?job_search%5Bdistance_from_zip%5D=10&job_search%5Bemployment_type%5D=Full-time&job_search%5Bposition_type%5D=63',
 107                next_selector => [text_regex => qr{Next}, url_regex=>qr{^/job_search}],
 108                job_selector => [url_regex => qr{/jobs/\d+-\d+$}],
 109                university => [href => qr{/institutions/\d+$}],
 110                description => [class => 'job-listing__content__description'],
 111                date => [_tag => 'td', sub {$_[0]->as_text() =~ qr/\,\s+20\d{2}$/}],
 112                position => [_tag => 'h1',
 113                             sub {defined $_[0]->parent()->attr('class') and
 114                                      $_[0]->parent()->attr('class') eq
 115                                      'page-title page-title--two-col'},
 116                            ],
 117               },
 118      higheredjobs => {url => 'https://www.higheredjobs.com/search/advanced_action.cfm?JobCat=113&JobCat=259&JobCat=99&JobCat=100&JobCat=108&JobCat=107&PosType=1&InstType=1&InstType=2&InstType=3&Keyword=&Remote=1&Remote=2&Region=&Submit=Search+Jobs',
 119                       next_selector_tree => [class => 'js-click-submit',
 120                                              href => qr{advanced_action\.cfm},
 121                                              sub {return 1
 122                                                       if defined
 123                                                       $_[0]->look_down(src =>
 124                                                                        qr/active-right/);
 125                                                   return 0;
 126                                               },
 127                                             ],
 128                       job_selector => [url_regex => qr{^details.cfm\?JobCode=\d+},
 129                                       ],
 130                       university => [class => qr/field-value/,
 131                                      sub {my $p = $_[0]->parent();
 132                                     my $c = $p->look_down(class => qr/field-label/,
 133                                                          );
 134                                     defined $c and $c->as_text() =~ qr/institution/i;
 135                                 }
 136                               ],
 137                       date => [class => qr/field-value/,
 138                                sub {my $p = $_[0]->parent();
 139                                     my $c = $p->look_down(class => qr/field-label/,
 140                                                          );
 141                                     defined $c and $c->as_text() =~ qr/posted/i;
 142                                 }
 143                               ],
 144                       position => [id => 'jobtitle-header',
 145                                    _tag => 'h1',
 146                                   ],
 147                       description => [id => 'jobDesc'],
 148                      },
 149     );
 150
 151 binmode STDOUT,":utf8";
 152 get_jobs($options{site},$options{pages});
 153
 154 sub get_jobs {
 155     my ($site,$pages) = @_;
 156
 157     my $todays_date = strftime('%Y-%m-%d %H:%M:%S',localtime());
 158
 159     my $m = WWW::Mechanize->new();
 160     WWW::Mechanize::TreeBuilder->meta->apply($m);
 161     if (not defined $sites{$site}) {
 162         die "Unknown site $site";
 163     }
 164     print "* Jobs from $site\n";
 165     my $s = $sites{$site};
 166     $m->get($s->{url});
 167     for (1..$pages) {
 168         my %seen;
 169         my @job_urls = grep { ! $seen{ $_->URI()->abs() }++ }
 170             $m->find_all_links(@{$s->{job_selector}});
 171         my $link;
 172         if (exists $s->{next_selector}) {
 173             ($link) = map {$_->URI()->abs()}
 174                 $m->find_all_links(@{$s->{next_selector}});
 175         } elsif (exists $s->{next_selector_tree}) {
 176             $link = $m->tree->look_down(@{$s->{next_selector_tree}});
 177             if (not defined $link) {
 178                 $m->tree->dump;
 179             }
 180             die "Unable to find next link" unless defined $link;
 181             $link = $link->attr('href');
 182         }
 183         for my $j_u (sort @job_urls) {
 184             $m->get($j_u) or next;
 185             my $university = 'No university';
 186             eval {
 187                 $university = $m->tree->look_down(@{$s->{university}})->as_text();
 188             };
 189             my $date = $todays_date;
 190             eval {
 191                 $date = $m->tree->look_down(@{$s->{date}})->as_text() // $todays_date if
 192                     @{$s->{date}};
 193                 if ($date =~ /^\s*(\d+)\s*days\s*ago\s*$/) {
 194                     $date = strftime('%Y-%m-%d %H:%M:%S',
 195                                      localtime((DateTime->now() -
 196                                                 DateTime::Duration->new(days=>$1))->epoch));
 197                 }
 198             };
 199             my $description = 'unknown';
 200             eval {
 201                 $description = $m->tree->look_down(@{$s->{description}})->as_text();
 202             };
 203             my $position = 'Unknown';
 204             eval {
 205                 $position = $m->tree->look_down(@{$s->{position}})->as_text();
 206             };
 207             print format_job($university,$position,$j_u->URI->abs(),$description,$date);
 208             $m->back() or die "Unable to go back";
 209         }
 210         $m->get($link);
 211     }
 212 }
 213
 214 sub format_job {
 215     my ($university,$position,$url,$text,$date) = @_;
 216     $text = wrap('   ','   ',$text);
 217     my $ret = <<"EOF";
 218 ** TODO $university -- $position
 219    :PROPERTIES:
 220    :ID:       $uuid
 221    :END:
 222 [[$url]]
 223 [$date]
 224
 225 $text
 226
 227 EOF
 228     return $ret;
 229 }
 230
 231
 232
 233 __END__