view release on metacpan or  search on metacpan

lib/Text/Corpus/NewYorkTimes/Document.pm  view on Meta::CPAN

      last;
    }
  }

  # pop off the lead paragraph if it repeats in the text body.
  shift @linesOfText if ($leadParagraphReoccures);

  # return the title and lines of body text.
  $Self->{body} = \@linesOfText;
  return $Self->{body};
}

=head2 C<getCategories>

 getCategories (type => 'all')

The method C<getCategories> returns the categories of C<type> assigned to the document, where
C<type> must be C<'all'>, C<'controlled'>, or C<'uncontrolled'>. The C<'uncontrolled'>
categories are those assigned to the
document by an editor without machine assistance, the C<'controlled'> categories
are those assigned with machine assistance. The type C<'all'> returns the union of the
categories from C<'controlled'> and C<'uncontrolled'>. The default is C<'all'>.

=cut

# get the categories assigned to the document.
sub getCategories
{
  my ($Self, %Parameters) = shift;

  # set the category type.
  my $type = 'a';
  $type = lc substr ($Parameters{type}, 0, 1) if (exists ($Parameters{type}) && defined ($Parameters{type}) && length ($Parameters{type}));

  # return the list of categories.
  if ($type eq 'c')
  {
    return $Self->getCategoriesControlled;
  }
  elsif ($type eq 'u')
  {
    return $Self->getCategoriesUncontrolled;
  }
  else
  {
    return $Self->getCategoriesAll;
  }
}

# get the categories assigned to the document by a human editor.
sub getCategoriesUncontrolled
{
  my $Self = shift;

  # return the categories if previously computed.
  return $Self->{categories_uncontrolled} if exists $Self->{categories_uncontrolled};

  # below is a list of the xpath expressions to get all the hand assigned categories.
  my @xpathExpressions =
  (
  '/nitf/head/docdata/identified-content/classifier[@class="indexing_service" and @type="biographical_categories"]', # 2.2.3 biographic categories
  '/nitf/head/docdata/identified-content/classifier[@class="indexing_service" and @type="descriptor"]', # 2.2.15 descriptors
  '/nitf/head/docdata/identified-content/location[@class="indexing_service"]', # 2.2.22 locations
  '/nitf/head/docdata/identified-content/classifier[@class="indexing_service" and @type="names"]', # 2.2.23 names
  '/nitf/head/docdata/identified-content/org[@class="indexing_service"]', # 2.2.34 organizations
  '/nitf/head/docdata/identified-content/person[@class="indexing_service"]', # 2.2.36 people
  '/nitf/head/docdata/identified-content/object.title[@class="indexing_service"]', # 2.2.45 titles
  );

  # get the categories.
  my @categories;
  foreach my $xpathExpression (@xpathExpressions)
  {
    my @nodeset = $Self->{xpathEngine}->findnodes($xpathExpression);
    foreach my $node (@nodeset)
    {
      push @categories, $node->textContent();
    }
  }

  # remove duplicative, then cache and return the categories.
  my %uniqueCategories = map {(lc $_, $_)} sort @categories;
  @categories = sort values %uniqueCategories;
  $Self->{categories_uncontrolled} = \@categories;
  return $Self->{categories_uncontrolled};
}

# get the categories assigned to the document by a computer and verified by a human editor.
sub getCategoriesControlled
{
  my $Self = shift;

  # return the categories if previously computed.
  return $Self->{categories_controlled} if exists $Self->{categories_controlled};

  # below is a list of the xpath expressions to get all the hand assigned categories.
  my @xpathExpressions =
  (
  '/nitf/head/docdata/identified-content/classifier[@class="online_producer" and @type="general_descriptor"]', # 2.2.17 general online descriptors
  '/nitf/head/docdata/identified-content/classifier[@class="online_producer" and @type="descriptor"]', # 2.2.26 online descriptors
  '/nitf/head/docdata/identified-content/location[@class="online_producer"]', # 2.2.29 online locations
  '/nitf/head/docdata/identified-content/org[@class="online_producer"]', # 2.2.30 online organizations
  '/nitf/head/docdata/identified-content/person[@class="online_producer"]', # 2.2.31 online people
  '/nitf/head/docdata/identified-content/object.title[@class="online_producer"]', # 2.2.33 online titles
  );

  # get the categories.
  my @categories;
  foreach my $xpathExpression (@xpathExpressions)
  {
    my @nodeset = $Self->{xpathEngine}->findnodes($xpathExpression);
    foreach my $node (@nodeset)
    {
      push @categories, $node->textContent();
    }
  }

  # remove duplicative, then cache and return the categories.
  my %uniqueCategories = map {(lc $_, $_)} sort @categories;
  @categories = sort values %uniqueCategories;
  $Self->{categories_controlled} = \@categories;
  return $Self->{categories_controlled};
}

sub getCategoriesAll
{
  my $Self = shift;

  # return the categories if previously computed.
  return $Self->{categories_all} if exists $Self->{categories_all};

  # get the controlled and uncontrolled categories.
  my $categoriesControlled = $Self->getCategoriesControlled;
  my $categoriesUncontrolled = $Self->getCategoriesUncontrolled;

  # remove duplicative, then cache and return the merged list of categories.
  my %uniqueCategories = (map {(lc $_, $_)} sort (@{$categoriesControlled}, @{$categoriesUncontrolled}));
  my @categoriesAll = sort values %uniqueCategories;
  $Self->{categories_all} = \@categoriesAll;
  return $Self->{categories_all};
}

# The method getBaseDirAndFileName returns the pair (base-directory, base-file-name) of the
# document. For example, if the document has path nyt_corpus/data/1999/11/26/1156340.xml,
# then the pair returned is ('1999/11/26', '1156340').
sub getBaseDirAndFileName
{
  my $Self = shift;
  return ($1, $2) if ($Self->{filePath} =~ m|(\d\d\d\d.\d\d.\d\d).(\d{7})\.xml$|);
  return undef;
}

=head2 C<getContent>

 getContent ()

The method C<getContent> returns the content of the document as an array reference of the text
where each item in the array is a sentence, with the first sentence being the headline or title
of the article. If the lead sentence equals the headline of the article, then the headline is
not prefixed to the list.

=cut

# get the content of the document removing the repeating lead paragraph if it exists.