view release on metacpan or  search on metacpan

lib/Alvis/NLPPlatform/UserNLPWrappers.pm  view on Meta::CPAN

# 	@Alvis::NLPPlatform::en_type = ();
	
# 	@Alvis::NLPPlatform::en_tokens_start = ();
# 	@Alvis::NLPPlatform::en_tokens_end = ();
# 	%Alvis::NLPPlatform::en_tokens_hash = ();

#     }
    
    if (    $Alvis::NLPPlatform::last_doc == 0) {
	return(1);
    }

    require Lingua::YaTeA::Corpus;
    require Lingua::YaTeA;
    my %config_yatea = Lingua::YaTeA::load_config($h_config->{'NLP_tools'}->{'YATEARC'});


    my $yatea = Lingua::YaTeA->new($config_yatea{"OPTIONS"}, \%config_yatea);

    if (defined $h_config->{'NLP_tools'}->{'YATEAOUTPUT'}) {
	print STDERR "\nYaTeA output defined is " . $h_config->{'NLP_tools'}->{'YATEAOUTPUT'} . "\n\n";
	$yatea->getOptionSet->addOption("output-path", $h_config->{'NLP_tools'}->{'YATEAOUTPUT'});
    } else {
	print STDERR "\nNo YaTeA output defined\n\n";
	$yatea->getOptionSet->addOption("output-path", $h_config->{"ALVISTMP"});
    }

    my $corpus_path = $h_config->{"TMPFILE"} . ".corpus.yatea.tmp";
    my $corpus = Lingua::YaTeA::Corpus->new($corpus_path,$yatea->getOptionSet,$yatea->getMessageSet);

    

########################################################################

    my $sentence_boundary = $yatea->getOptionSet->getSentenceBoundary;
    my $document_boundary = $yatea->getOptionSet->getDocumentBoundary;

#    $yatea->loadTestifiedTerms(\$process_counter,$corpus,$sentence_boundary,$document_boundary,$yatea->getOptionSet->MatchTypeValue,$yatea->getMessageSet,$yatea->getOptionSet->getDisplayLanguage);

    print STDERR $Lingua::YaTeA::process_counter++ . ") " . ($yatea->getMessageSet->getMessage('LOAD_CORPUS')->getContent($yatea->getOptionSet->getDisplayLanguage)) . "\n";

    $corpus->read($sentence_boundary,$document_boundary,$yatea->getFSSet,$yatea->getTestifiedTermSet,$yatea->getMessageSet,$yatea->getOptionSet->getDisplayLanguage);

    my $phrase_set = Lingua::YaTeA::PhraseSet->new;
    
    print STDERR $Lingua::YaTeA::process_counter++ . ") " . ($yatea->getMessageSet->getMessage('CHUNKING')->getContent($yatea->getOptionSet->getDisplayLanguage)) . "\n";
    $corpus->chunk($phrase_set,$sentence_boundary,$document_boundary,$yatea->getChunkingDataSet,$yatea->getFSSet,$yatea->getTagSet,$yatea->getParsingPatternSet,$yatea->getTestifiedTermSet,$yatea->getOptionSet);

    $phrase_set->sortUnparsed;
    
    print STDERR $Lingua::YaTeA::process_counter++ . ") " . ($yatea->getMessageSet->getMessage('PARSING')->getContent($yatea->getOptionSet->getDisplayLanguage)) . "\n";
    
    $phrase_set->parseProgressively($yatea->getTagSet,$yatea->getOptionSet->getParsingDirection,$yatea->getParsingPatternSet,$yatea->getChunkingDataSet,$corpus->getLexicon,$corpus->getSentenceSet,$yatea->getMessageSet,$yatea->getOptionSet->getDisplay...
    
    $phrase_set->addTermCandidates($yatea->getOptionSet);
    
    print STDERR $Lingua::YaTeA::process_counter++ . ") " . ($yatea->getMessageSet->getMessage('RESULTS')->getContent($yatea->getOptionSet->getDisplayLanguage)) . "\n";


# coments to keep
    print STDERR "\t-" . ($yatea->getMessageSet->getMessage('DISPLAY_RAW')->getContent($yatea->getOptionSet->getDisplayLanguage)) . "\'". $corpus->getOutputFileSet->getFile('debug')->getPath . "'\n";
    $phrase_set->printPhrases(FileHandle->new(">" . $corpus->getOutputFileSet->getFile('debug')->getPath));
    $phrase_set->printUnparsable($corpus->getOutputFileSet->getFile('unparsable'));


    print STDERR "\t-" . ($yatea->getMessageSet->getMessage('DISPLAY_TC_XML')->getContent($yatea->getOptionSet->getDisplayLanguage)) . "\'". $corpus->getOutputFileSet->getFile('candidates')->getPath . "'\n";
# 
    if ((exists $h_config->{"XML_OUTPUT"}->{"YATEA"}) && ($h_config->{"XML_OUTPUT"}->{"YATEA"} == 1)) {
	$phrase_set->printTermCandidatesXML("stdout",$yatea->getTagSet);
	exit;
    } else {
# 	$phrase_set->printTermCandidatesXML($corpus->getOutputFileSet->getFile("candidates"),$yatea->getTagSet);
	&storeTerms($phrase_set,$doc_hash,$yatea->getTagSet);
    }

########################################################################



########################################################################

#     }


########################################################################

    $Alvis::NLPPlatform::ALVISDEBUG || unlink $h_config->{'TMPFILE'}. ".corpus.tmp";

#    print STDERR "done - Found $Alvis::NLPPlatform::nb_relations relations, $termsfound full terms, $nb_constituents constituents.\n";
}


sub storeTerms
{
    my ($phrase_set,$doc_hash,$tagset) = @_;

    my $fh = \*STDOUT;
    
    my $term_candidate;
    my $if;
    my $pos;
    my $lf;
    my $occurrence;
    my $island;
    my $position;

    my $sem_unit;
    my $term;
    my $term_id = 1;
    my $syn_relation_id = 1;

    my $phrase_idx = $Alvis::NLPPlatform::Annotation::phrase_idx;
    my $relation_id = $Alvis::NLPPlatform::Annotation::syntactic_relation_idx;

    my %YateaTerms2AlvisSemUnits;
    my %YateaTermOcc2AlvisSemUnits;

    my $syntactic_relation_id;

    my $term_refid_head;
    my $term_refid_modifier;
    my $refid_head;

lib/Alvis/NLPPlatform/UserNLPWrappers.pm  view on Meta::CPAN

    my @tab_type;
    my @tab_string;
    my $lconst = 0;
    my $nconst = 0;
    my $phrase_id = "";
    my $csti;
    my $phrase_idx_start = $Alvis::NLPPlatform::Annotation::phrase_idx;
    require Alvis::NLPPlatform::ParseConstituents;


    my $parser = Alvis::NLPPlatform::ParseConstituents->new();

#     print STDERR $constituents;

    $parser->YYData->{CONSTITUENT_STRING} = $constituents;
    $parser->YYData->{DOC_HASH} = $doc_hash;
    $parser->YYData->{DECAL_PHRASE_IDX} = $decal_phrase_idx;
    $parser->YYData->{WORD_ID_NP_REF} =  \$word_id_np;
    $parser->YYData->{TAB_TYPE_REF} =  \@tab_type;
    $parser->YYData->{TAB_STRING_REF} =  \@tab_string;

    $parser->YYData->{LCONST_REF} =  \$lconst;
    $parser->YYData->{NCONST_REF} =  \$nconst;
;
    $parser->YYParse(yylex => \&Alvis::NLPPlatform::ParseConstituents::_Lexer, yyerror => \&Alvis::NLPPlatform::ParseConstituents::_Error);



    for($csti=1;$csti<scalar @tab_type;$csti++){
	$phrase_id = "phrase" . $Alvis::NLPPlatform::Annotation::phrase_idx;
	$doc_hash->{$phrase_id}={};
	$doc_hash->{$phrase_id}->{"id"}=$phrase_id;
	$doc_hash->{$phrase_id}->{"datatype"}="phrase";
	$doc_hash->{$phrase_id}->{"type"}=$tab_type[$csti];
	$doc_hash->{$phrase_id}->{'list_refid_components'}={};
	$doc_hash->{$phrase_id}->{'list_refid_components'}->{"datatype"}="list_refid_components";
	if (scalar(@{$tab_string[$csti]}) == 1) {
	    $doc_hash->{$phrase_id}->{'list_refid_components'}->{"refid_word"}=$tab_string[$csti];
	} else {
	    $doc_hash->{$phrase_id}->{'list_refid_components'}->{"refid_phrase"}=$tab_string[$csti];
	}
	$Alvis::NLPPlatform::Annotation::phrase_idx++;
   }
    
    print STDERR "done - Found ". ($Alvis::NLPPlatform::Annotation::phrase_idx - $phrase_idx_start) ." semantic units\n";
    push @{$doc_hash->{"log_processing1"}->{"comments"}},  "Found Terms: " . ($Alvis::NLPPlatform::Annotation::phrase_idx - $phrase_idx_start);

#    $word_count=$word_id_np-$word_count;
#    print STDERR "\nWord count for this sentence: $word_count\n";

    return $decal_phrase_idx+$csti-1;
}


sub parse_constituents_old {
    my $constituents=$_[0];
    my $tmpptr=$_[1];
    my $decal_phrase_idx=$_[1];
    my $doc_hash=$_[2];
    my $lexer;
    my $debug_mode=0;
    my $lconst=0;
    my $nconst=0;

    my @tab_nconst;
    my @tab_type;
    my @tab_string;

    my $word_count=$word_id_np;
    my $lastword="";

#    print STDERR "PHRASE DECAL: $decal_phrase_idx\n";

    my @cst_token=(
	# OPEN
	'type1','\[([A-Z]+)', sub {
	    # open constituent
	    if($lconst>0){
		$tab_string[$tab_nconst[$lconst]].="phrase".($decal_phrase_idx+$nconst+1)." ";
	    }
	    $lconst++;
	    $nconst++;
	    $tab_nconst[$lconst]=$nconst;

	    # get type
	    $tab_type[$tab_nconst[$lconst]]=$1;
	    $lexer->end('type1');
	    $lexer->start('string');

	    print STDERR "*** DEBUG *** Opened constituent $nconst with type ".$1."\n" unless ($debug_mode==0);

	},
	# CLOSE
	'type2','([A-Z]+)\]', sub {
	    # check type
	    if($1 ne $tab_type[$tab_nconst[$lconst]]){
		print STDERR "Error found at level $lconst: types don't match!\n";
		$lexer->end('ALL');
		exit 0;
	    }
	    # remove ending space
	    $tab_string[$tab_nconst[$lconst]]=~s/\s+$//sgo;
	    # close constituent
	    print STDERR "*** DEBUG *** Closing constituent $tab_nconst[$lconst]\n" unless ($debug_mode==0);
	    $lconst--;
	},
	# STRING
	'string','[^\s]+\s', sub {
	    print STDERR "*** DEBUG *** Found string '".$_[0]->text."'\n" unless ($debug_mode==0);
	    if((defined $tab_string[$tab_nconst[$lconst]])&&($tab_string[$tab_nconst[$lconst]] ne "")){
		print STDERR "*** DEBUG *** Appended to previously found string\n" unless ($debug_mode==0);
#		$tab_string[$tab_nconst[$lconst]].=$_[0]->text;
		if(($_[0]->text eq $lastword) || ($_[0]->text=~/^\./)){
		}else{
		    $tab_string[$tab_nconst[$lconst]].="word$word_id_np ";
		    $word_id_np++;
		    $lastword=$_[0]->text;
		}
	    }else{
#		$tab_string[$tab_nconst[$lconst]]=$_[0]->text;
		if(!(($_[0]->text eq $lastword)||($_[0]->text=~/^\./))){
		    $lastword=$_[0]->text;
		    $tab_string[$tab_nconst[$lconst]]="word$word_id_np ";
		    $word_id_np++;
		}else{
		}
	    }
	    $lexer->end('string');
	    $lexer->start('type2');
	},
	# MISC
	'espace','\s+', sub {
	},
	'newline','\n', sub {
	    # dump result and reset automaton memory
	},

	# ERROR HANDLER
	'ALL:ERROR' => '' => sub {
	    print STDERR "Syntax error: malformed string\n";
	    $lexer->end('ALL');
	    #return $decal_phrase_idx;
	}

	);

    Parse::Lex->exclusive('parse');
    $lexer=Parse::Lex->new(@cst_token);
#    print STDERR "\nPARSING CONSTITUENTS STRING\n$constituents\n";
    $lexer->from($constituents);
    $lexer->every(sub{});

# my $parametre;
# my $format=0;
# foreach $parametre (@ARGV){
#     if($parametre eq "--xml"){
# 	$format=1;
#     }
# }

    # create phrases (constituents)
#				$doc_hash->{"syntactic_relation$relation_id"}={};
#				$doc_hash->{"syntactic_relation$relation_id"}->{'id'}="syntactic_relation$relation_id";
#				$doc_hash->{"syntactic_relation$relation_id"}->{'datatype'}="syntactic_relation";
#				$doc_hash->{"syntactic_relation$relation_id"}->{'syntactic_relation_type'}="$relation";
#				$doc_hash->{"syntactic_relation$relation_id"}->{'refid_head'} = {};
#				$doc_hash->{"syntactic_relation$relation_id"}->{'refid_head'}->{'datatype'}="refid_head";
    my $csti=1;
#    for($csti=1;$csti<scalar @tab_type;$csti++){
#	$doc_hash->{"phrase$csti"}={};
#	$doc_hash->{"phrase$csti"}->{"id"}="phrase$csti";