File Coverage

blib/lib/NNexus/Index/Template.pm

Criterion	Covered	Total	%
statement	69	83	83.1
branch	14	24	58.3
condition	5	9	55.5
subroutine	14	21	66.6
pod	11	14	78.5
total	113	151	74.8

# /=====================================================================\ #
# |  NNexus Autolinker                                                  | #
# | Template for Indexing Plug-ins, PULL API                            | #
# |=====================================================================| #
# | Part of the Planetary project: http://trac.mathweb.org/planetary    | #
# |  Research software, produced as part of work done by:               | #
# |  the KWARC group at Jacobs University                               | #
# | Copyright (c) 2012                                                  | #
# | Released under the MIT License (MIT)                                | #
# |---------------------------------------------------------------------| #
# | Adapted from the original NNexus code by                            | #
# |                                  James Gardner and Aaron Krowne     | #
# |---------------------------------------------------------------------| #
# | Deyan Ginev                   #_#     | #
# | http://kwarc.info/people/dginev                            (o o)    | #
# \=========================================================ooo==U==ooo=/ #
package NNexus::Index::Template;
use warnings;
use strict;

use Mojo::DOM;
use Mojo::UserAgent;
use Mojo::UserAgent::CookieJar;
use Time::HiRes qw(sleep);
use NNexus::Morphology qw(canonicalize_url);

### EXTERNAL API
sub new {
  my ($class,%options) = @_;
  my $ua = Mojo::UserAgent->new;
  $ua->max_redirects(2)->connect_timeout(10)->request_timeout(20);
  $ua->cookie_jar(Mojo::UserAgent::CookieJar->new);
  my $visited = $options{visited}||{};
  my $queue = $options{queue}||[];

  my $self = bless {ua=>$ua,visited=>$visited,queue=>$queue}, $class;

  # Set current if we're starting up.
  my $first_url;
  if (defined $options{start}) {
    if ($options{start} eq 'default') {
      $first_url = $self->domain_root;
    } else {
      $first_url = $options{start};
    }}
  else {
    $first_url = $self->domain_root; }

  push (@{$self->{queue}}, {
      url=>canonicalize_url($first_url),
      ($options{dom} ? (dom=>$options{dom}) : ()),
      depth=>0});
  return $self;
}
sub ua {$_[0]->{ua};}

# index: Traverse a page, obtain candidate concepts and candidate further links
sub index_step {
  my ($self,%options) = @_;
  my $visited = $self->{visited};
  my $depth;

  # Grab the next job from the queue
  my $next_step = $self->next_step;
  if (ref $next_step) {
    $self->current_url($next_step->{url});
    $self->current_categories($next_step->{categories});
    $depth = $next_step->{depth} || 0;
  } else {
    # We're out of urls, last step.
    delete $self->{current_url};
  }
  # If we've visited, or we're out of urls, terminate.
  my $current_url = $self->current_url;
  return unless $current_url; # Empty return for last job
  $visited->{$current_url} = 1; # Mark visited
  # Also skip if we're over the depth limit.
  return $self->index_step if $depth > $self->depth_limit;
  return [] if $options{skip}; # We are skipping over this URL, return
  # 2.1. Prepare (or just accept) a Mojo::DOM to be analyzed
  if ($next_step->{dom}) {
    $self->current_dom($next_step->{dom});
    delete $next_step->{dom};
  } else {
    sleep($self->request_interval()); # Don't overload the server
    $self->current_dom($self->ua->get($current_url)->res->dom);
  }
  # Obtain the indexer payload
  my $payload = $self->index_page;
  # What are the candidate categories for follow-up jobs?
  my $categories = $self->candidate_categories;
  # Push all following candidate jobs to queue
  if ($depth <= $self->depth_limit) { # Don't add pointless nodes
    my $candidate_links = $self->candidate_links;
    foreach (@$candidate_links) {
      # push and shift give us breadth-first search.
      push (@{$self->{queue}}, {
        url=>canonicalize_url($_),
        categories=>$categories,
        depth=>$depth+1});
    }
  }
  # Return final list of concepts for this page
  return $payload;
}

sub next_step {
  my ($self) = @_;
  my $visited = $self->{visited};
  # Otherwise, grab the next job from the queue
  my $next_step = shift @{$self->{queue}};
  while ((ref $next_step) && ($visited->{$next_step->{url}})) {
    $next_step = shift @{$self->{queue}};
  }
  return $next_step;
}

### PULL API
# To be overloaded by concrete classes
sub depth_limit {4;}
sub domain_root {q{};} # To be overriden in the concrete classes
# TODO: Rename index_page to candidate_concepts ? Or index_links / index_categories instead?
sub index_page {[];} # To be overriden in the concrete classes
sub candidate_links {
  [];
  # TODO: Generic implementation should simply retrieve ALL s as candidate links.
}
sub candidate_categories {}
sub request_interval { 2; }
# Tests if the page is a leaf, in which case we want to skip it when should_update is 0
sub leaf_test {0;}
### SHARED METHODS
# To be directly inherited and used by concrete classes

# Getter or Setter for the current URL/DOM/Categories
sub current_url { $_[1] ? $_[0]->{current_url} = $_[1] : $_[0]->{current_url}; }
sub current_dom { $_[1] ? $_[0]->{current_dom} = $_[1] : $_[0]->{current_dom}; }
sub current_categories {$_[1] ? $_[0]->{current_categories} = $_[1] : $_[0]->{current_categories};}

1;
__END__