view release on metacpan or  search on metacpan

CPPTokenizerWrapper.cc  view on Meta::CPAN

#include "CPPTokenizerWrapper.h"

#include "src/tokenizer.cpp"
#include "src/numbers.cpp"
#include "src/operator.cpp"
#include "src/quotes.cpp"
#include "src/structure.cpp"
#include "src/symbol.cpp"
#include "src/unknown.cpp"
#include "src/whitespace.cpp"
#include "src/word.cpp"

using namespace std;

namespace PPITokenizer {

  /***********************************************************************/
  const char* CPPTokenizerWrapper::fgTokenClasses[43] = {
    "Token", // Token_NoType = 0,
    "PPI::Token::Whitespace", // Token_WhiteSpace,
    "PPI::Token::Symbol", // Token_Symbol,
    "PPI::Token::Comment", // Token_Comment,
    "PPI::Token::Word", // Token_Word,
    "PPI::Token::DashedWord", // Token_DashedWord,
    "PPI::Token::Structure", // Token_Structure,
    "PPI::Token::Magic", // Token_Magic,
    "PPI::Token::Number", // Token_Number,
    "PPI::Token::Number::Version", // Token_Number_Version,
    "PPI::Token::Number::Float", // Token_Number_Float,
    "PPI::Token::Number::Hex", // Token_Number_Hex,
    "PPI::Token::Number::Binary", // Token_Number_Binary,
    "PPI::Token::Number::Octal", // Token_Number_Octal,
    "PPI::Token::Number::Exp", // Token_Number_Exp,
    "PPI::Token::Operator", // Token_Operator,
    "PPI::Token::Operator", // Token_Operator_Attribute,
    "PPI::Token::Unknown", // Token_Unknown,
    "PPI::Token::Quote::Single", // Token_Quote_Single,
    "PPI::Token::Quote::Double", // Token_Quote_Double,
    "PPI::Token::Quote::Interpolate", // Token_Quote_Interpolate,
    "PPI::Token::Quote::Literal", // Token_Quote_Literal,
    "PPI::Token::QuoteLike::Backtick", // Token_QuoteLike_Backtick,
    "PPI::Token::QuoteLike::Readline", // Token_QuoteLike_Readline,
    "PPI::Token::QuoteLike::Command", // Token_QuoteLike_Command,
    "PPI::Token::QuoteLike::Regexp", // Token_QuoteLike_Regexp,
    "PPI::Token::QuoteLike::Words", // Token_QuoteLike_Words,
    "PPI::Token::Regexp::Match", // Token_Regexp_Match,
    "PPI::Token::Regexp::Match", // FIXME doesn't exist in PPI Token_Regexp_Match_Bare,
    "PPI::Token::Regexp::Substitute", // Token_Regexp_Substitute,
    "PPI::Token::Regexp::Transliterate", // Token_Regexp_Transliterate,
    "PPI::Token::Cast", // Token_Cast,
    "PPI::Token::Prototype", // Token_Prototype,
    "PPI::Token::ArrayIndex", // Token_ArrayIndex,
    "PPI::Token::HereDoc", // Token_HereDoc,
    "PPI::Token::Attribute", // Token_Attribute,
    "PPI::Token::Attribute", // Doesn't exist in PPI: Token_Attribute_Parameterized, (okay to map to PPI::Token::Attribute)
    "PPI::Token::Label", // Token_Label,
    "PPI::Token::Separator", // Token_Separator,
    "PPI::Token::End", // Token_End,
    "PPI::Token::Data", // Token_Data,
    "PPI::Token::Pod", // Token_Pod,
    "PPI::Token::BOM", // Token_BOM,
  };

  const int CPPTokenizerWrapper::fgSpecialToken[43] = {
    eSimple, // Token_NoType = 0,
    eSimple, // Token_WhiteSpace,
    eSimple, // Token_Symbol,
    eSimple, // Token_Comment,
    eSimple, // Token_Word,
    eSimple, // Token_DashedWord,
    eSimple, // Token_Structure,
    eSimple, // Token_Magic,
    eSimple, // Token_Number,
    eSimple, // Token_Number_Version,
    eSimple, // Token_Number_Float,
    eSimple, // Token_Number_Hex,
    eSimple, // Token_Number_Binary,
    eSimple, // Token_Number_Octal,
    eSimple, // Token_Number_Exp,
    eSimple, // Token_Operator,
    eAttribute, // Token_Operator_Attribute,
    eSimple, // Token_Unknown,
    eQuoteSingle, // Token_Quote_Single,
    eQuoteDouble, // Token_Quote_Double,
    eExtended, // Token_Quote_Interpolate,
    eExtended, // Token_Quote_Literal,
    eBacktick, // Token_QuoteLike_Backtick,
    eExtended, // Token_QuoteLike_Readline,
    eExtended, // Token_QuoteLike_Command,
    eExtended, // Token_QuoteLike_Regexp,
    eExtended, // Token_QuoteLike_Words,
    eExtended, // Token_Regexp_Match,
    eExtended, // doesn't exist in PPI Token_Regexp_Match_Bare,
    eExtended, // Token_Regexp_Substitute,
    eExtended, // Token_Regexp_Transliterate,
    eSimple, // Token_Cast,
    eSimple, // Token_Prototype,
    eSimple, // Token_ArrayIndex,
    eHereDoc, // Token_HereDoc,
    eSimple, // Token_Attribute,
    eSimple, // Token_Attribute_Parameterized, (PPI::Token::Attribute)
    eSimple, // Token_Label,
    eSimple, // Token_Separator,
    eSimple, // Token_End,
    eSimple, // Token_Data,
    eSimple, // Token_Pod,
    eSimple, // Token_BOM,
  };



/*
 * special tokens:
 * PPI::Token::HereDoc
 * all "extended" tokens
 */

  /***********************************************************************/
  CPPTokenizerWrapper::CPPTokenizerWrapper(SV* source)
  {
    // Note: This is preprocessed to be \@lines in the Perl-land new()
    fTokenizer = new Tokenizer();
    SV* tmpSv;
    if (!SvOK(source))
      croak("Can't create PPI::XS::Tokenizer from an undefined source");
    if (SvROK(source)) { // it's a reference
      tmpSv = (SV*)SvRV(source);
      // it's an array reference
      if (SvTYPE(tmpSv) == SVt_PVAV) {
        fLines = (AV*)tmpSv;
        SvREFCNT_inc(fLines);
      }
      else
        croak("Can only create PPI::XS::Tokenizer from a string, "
              "a reference to a string or a reference to an array of lines");
    }
    else
      croak("Can only create PPI::XS::Tokenizer from a string, "
            "a reference to a string or a reference to an array of lines");
  }

  /***********************************************************************/
  CPPTokenizerWrapper::~CPPTokenizerWrapper()
  {
    SvREFCNT_dec(fLines);
    delete fTokenizer;
  }

  /***********************************************************************/
  SV*
  CPPTokenizerWrapper::get_token()
  {
    //printf("entering get token, num of lines=%d\n", av_len(fLines));
    Token* theToken = fTokenizer->pop_one_token();
    if (theToken == NULL) {
      while (av_len(fLines) >= 0) {
        SV* line = av_shift(fLines);
        if (!SvOK(line) || !SvPOK(line)) {
          SvREFCNT_dec(line); // FIXME check this
          croak("Trying to tokenize undef line");
        }

        STRLEN len;
        char* lineStr = S_stealPV(line, len);
        //cout << "!"<<lineStr<<"!" << endl;
        LineTokenizeResults res = fTokenizer->tokenizeLine(lineStr, len);