X-Git-Url: https://git.donarmstrong.com/?a=blobdiff_plain;f=jobs_to_org;h=d76392fa12acf57d5e45a50ba93f8c388025ccb5;hb=3d5241a316e3ff729b19b878b0841558120f75e9;hp=5d5d6036ea34e046b7111cc0e6925007852b24c4;hpb=1cefceeb6e04db3ebbdab8af36d53fb2f7d775bb;p=bin.git

diff --git a/jobs_to_org b/jobs_to_org
index 5d5d603..d76392f 100755
--- a/jobs_to_org
+++ b/jobs_to_org
@@ -56,6 +56,7 @@ use vars qw($DEBUG);
 use Data::Printer;
 use Text::Wrap;
 use POSIX qw(strftime);
+use DateTime;
 
 tie my $uuid, 'OSSP::uuid::tie';
 $uuid= ["v1"];
@@ -99,7 +100,7 @@ my %sites =
                 job_selector => [url_regex => qr{^/naturejobs/science/jobs/\d+-.+}],
                 university => [href => qr{^/naturejobs/science/employer-directory/\d+$}],
                 description => [class=>"job-description"],
-                date => [],
+                date => [_tag => 'dd', sub {$_[0]->as_text() =~ qr/\d+\s+days\s+ago$/},],
                 position => [class=>'job-title heading'],
                },
      vitae => {url => 'https://chroniclevitae.com/job_search?job_search%5Bdistance_from_zip%5D=10&job_search%5Bemployment_type%5D=Full-time&job_search%5Bposition_type%5D=63',
@@ -107,13 +108,44 @@ my %sites =
                job_selector => [url_regex => qr{/jobs/\d+-\d+$}],
                university => [href => qr{/institutions/\d+$}],
                description => [class => 'job-listing__content__description'],
-               date => [_tag => 'td', content => qr/\,\s+20\d{2}$/],
+               date => [_tag => 'td', sub {$_[0]->as_text() =~ qr/\,\s+20\d{2}$/}],
                position => [_tag => 'h1',
                             sub {defined $_[0]->parent()->attr('class') and
                                      $_[0]->parent()->attr('class') eq
                                      'page-title page-title--two-col'},
                            ],
               },
+     higheredjobs => {url => 'https://www.higheredjobs.com/search/advanced_action.cfm?JobCat=113&JobCat=259&JobCat=99&JobCat=100&JobCat=108&JobCat=107&PosType=1&InstType=1&InstType=2&InstType=3&Keyword=&Remote=1&Remote=2&Region=&Submit=Search+Jobs',
+                      next_selector_tree => [class => 'js-click-submit',
+                                             href => qr{advanced_action\.cfm},
+                                             sub {return 1
+                                                      if defined
+                                                      $_[0]->look_down(src =>
+                                                                       qr/active-right/);
+                                                  return 0;
+                                              },
+                                            ],
+                      job_selector => [url_regex => qr{^details.cfm\?JobCode=\d+},
+                                      ],
+                      university => [class => qr/field-value/,
+                                     sub {my $p = $_[0]->parent();
+                                    my $c = $p->look_down(class => qr/field-label/,
+                                                         );
+                                    defined $c and $c->as_text() =~ qr/institution/i;
+                                }
+                              ],
+                      date => [class => qr/field-value/,
+                               sub {my $p = $_[0]->parent();
+                                    my $c = $p->look_down(class => qr/field-label/,
+                                                         );
+                                    defined $c and $c->as_text() =~ qr/posted/i;
+                                }
+                              ],
+                      position => [id => 'jobtitle-header',
+                                   _tag => 'h1',
+                                  ],
+                      description => [id => 'jobDesc'],
+                     },
     );
 
 binmode STDOUT,":utf8";
@@ -129,13 +161,26 @@ sub get_jobs {
     if (not defined $sites{$site}) {
         die "Unknown site $site";
     }
+    print "* Jobs from $site\n";
     my $s = $sites{$site};
     $m->get($s->{url});
     for (1..$pages) {
         my %seen;
         my @job_urls = grep { ! $seen{ $_->URI()->abs() }++ }
             $m->find_all_links(@{$s->{job_selector}});
-        for my $j_u (@job_urls) {
+        my $link;
+        if (exists $s->{next_selector}) {
+            ($link) = map {$_->URI()->abs()}
+                $m->find_all_links(@{$s->{next_selector}});
+        } elsif (exists $s->{next_selector_tree}) {
+            $link = $m->tree->look_down(@{$s->{next_selector_tree}});
+            if (not defined $link) {
+                $m->tree->dump;
+            }
+            die "Unable to find next link" unless defined $link;
+            $link = $link->attr('href');
+        }
+        for my $j_u (sort @job_urls) {
             $m->get($j_u) or next;
             my $university = 'No university';
             eval {
@@ -145,6 +190,11 @@ sub get_jobs {
             eval {
                 $date = $m->tree->look_down(@{$s->{date}})->as_text() // $todays_date if
                     @{$s->{date}};
+                if ($date =~ /^\s*(\d+)\s*days\s*ago\s*$/) {
+                    $date = strftime('%Y-%m-%d %H:%M:%S',
+                                     localtime((DateTime->now() -
+                                                DateTime::Duration->new(days=>$1))->epoch));
+                }
             };
             my $description = 'unknown';
             eval {
@@ -155,9 +205,9 @@ sub get_jobs {
                 $position = $m->tree->look_down(@{$s->{position}})->as_text();
             };
             print format_job($university,$position,$j_u->URI->abs(),$description,$date);
-            $m->back();
+            $m->back() or die "Unable to go back";
         }
-        $m->follow_link(@{$s->{next_selector}}) or die "Unable to find next link";
+        $m->get($link);
     }
 }