view release on metacpan or  search on metacpan

CAB/Format/TT.pm  view on Meta::CPAN

		  ##-- token: field: morph-safety check (msafe|morph/safe)
		  $tok->{msafe} = $1;
		} elsif ($field =~ m/^\[(.*?moot)\/(tag|word|lemma)\]\s?(.*)$/) {
		  ##-- token: field: (moot|dmoot)/(tag|word|lemma)
		  $tok->{$1}{$2} = $3;
		} elsif ($field =~ m/^\[(.*?moot)\/analysis\]\s?(\S+)(?:\s\@\s(\S+))?\s(?:\~\s)?(.*?)(?: <([0-9\.\+\-eE]+)>)?$/) {
		  ##-- token: field: moot/analysis|dmoot/analysis
		  push(@{$tok->{$1}{analyses}}, {tag=>$2,lemma=>$3,details=>$4,prob=>$5});
		} elsif ($field =~ m/^\[(.*?moot)\/details\]\s?(\S*)(?:\s\@\s(\S+))?\s(?:\~\s)?(.*?)(?: <([0-9\.\+\-eE]+)>)?$/) {
		  ##-- token: field: moot/details|dmoot/details
		  $tok->{$1}{details} = {tag=>$2,lemma=>$3,details=>$4,prob=>$5};
		} elsif ($field =~ m/^\[((?:gn|ot)\-(?:hyper|hypo|isa|asi|syn))\]\s(\S+)$/) {
		  ##-- token: field: list field (GermaNet|OpenThesaurus hyperonyms / hyponyms)
		  push(@{$tok->{$1}}, $2);
		} elsif ($field =~ m/^\[(toka|tokpp|lang)\]\s?(.*)$/) {
		  ##-- token: field: other known list field: (toka|tokpp)
		  push(@{$tok->{$1}}, $2);
		} elsif ($field =~ m/^\[([^\]]*)\]\s?(.*)$/) {
		  ##-- token: field: unknown named field "[$name] $val", parse into $tok->{other}{$name} = \@vals
		  push(@{$tok->{other}{$1}}, $2);
		} else {
		  ##-- token: field: unnamed field
		  #$fmt->warn("parseTTString(): could not parse token field '$field' for token '$text'");
		  push(@{$tok->{other}{$fmt->{defaultFieldName}||''}}, $field);
		}
	      }
	      $tok
	    }
	  }
	  split(/\n/, $_)
	 ];
       (%sa || @$toks ? {%sa,tokens=>$toks} : qw())
     } split(/\n\n+/, $$src)
    ];

  ##-- construct & buffer document
  #$_ = bless($_,'DTA::CAB::Sentence') foreach (@$sents);
  $fmt->{doc} = bless({%doca,body=>$sents}, 'DTA::CAB::Document');
  return $fmt;
}

## $doc = $CLASS_OR_OBJECT->parseTokenizerString(\$string,\%opts)
##  + scaled-down version of parseTTString() suitable for use with dwds_tomastotath or moot/waste tokenizer output
sub parseTokenizerString {
  my ($that,$tstr,$opts) = @_;
  utf8::decode($$tstr) if (!utf8::is_utf8($$tstr));

  my $tloc = ($opts && exists($opts->{tloc}) ? $opts->{tloc}
	      : (ref($that) ? $that->{tloc}
		 : undef));
  my ($toks,%sa);
  my $sents =
    [
     map {
       %sa=qw();
       $toks=
	 [
	  map {
	    if ($_ =~ /^\%\%(.*)$/) {
	      ##-- generic line: add to '_cmts' attribute of current sentence
	      push(@{$sa{_cmts}},$1) if ($1 !~ /^\$[WS]B\$$/); ##-- generic comment, treated as sentence attribute
	      qw()
	    } elsif ($_ =~ /^$/) {
	      ##-- blank line: ignore
	      qw()
	    } elsif (/^([^\t]*)\t([0-9]+) ([0-9]+)(?:\t(.*))?$/) {
	      ##-- token
	      {text=>$1,
		 #loc=>{off=>$2,len=>$3},
		 ($tloc ? ($tloc=>"$2 $3") : qw()),
		 ($4    ? (toka=>[map {/^\[(.*)\]$/ ? $1 : $_} split(/\t/,$4)]) : qw())
	       }
	    }
	  }
	  split(/\n/, $_)
	 ];
       (%sa || @$toks ? {%sa,tokens=>$toks} : qw())
     } split(/\n\n+/, $$tstr)
    ];

  ##-- construct & buffer document
  return bless({body=>$sents}, 'DTA::CAB::Document');
}

##--------------------------------------------------------------
## Methods: Input: Generic API

## $doc = $fmt->parseDocument()
sub parseDocument { return $_[0]{doc}; }


##==============================================================================
## Methods: Output
##==============================================================================

##--------------------------------------------------------------
## Methods: Output: Generic

## $type = $fmt->mimeType()
##  + default returns text/plain
sub mimeType { return 'text/plain'; }

## $ext = $fmt->defaultExtension()
##  + returns default filename extension for this format
sub defaultExtension { return '.tt'; }

## $str = $fmt->toString()
##  + select output to byte-string
##  + flush buffered output document to byte-string

## $fmt_or_undef = $fmt->toFile($filename_or_handle, $formatLevel)
##  + select output to named file $filename.
##  + default implementation calls $fmt->toFh()

## $fmt_or_undef = $fmt->toFh($fh,$formatLevel)
##  + select output to an open filehandle $fh.
##  + default implementation calls to $fmt->formatString($formatLevel)
sub toFh {
  $_[0]->DTA::CAB::Format::toFh(@_[1..$#_]);
  $_[0]->setLayers();
  return $_[0];