view release on metacpan or  search on metacpan

generator/parser/lexer.cpp  view on Meta::CPAN

  }
  return ret;
}

const uint index_size = 200;

KDevVarLengthArray<KDevVarLengthArray<QPair<uint, TOKEN_KIND>, 10 >, index_size > createIndicesForTokens() {
  KDevVarLengthArray<KDevVarLengthArray<QPair<uint, TOKEN_KIND>, 10 >, index_size > ret;
  ret.resize(index_size);
  #define ADD_TOKEN(string) ret[IndexedString(#string).index() % index_size].append(qMakePair(IndexedString(#string).index(), Token_ ## string));
  #define ADD_MAPPED_TOKEN(string, token) ret[IndexedString(#string).index() % index_size].append(qMakePair(IndexedString(#string).index(), token));
  ADD_TOKEN(K_DCOP);
  ADD_TOKEN(Q_OBJECT);
  ADD_MAPPED_TOKEN(Q_SIGNALS, Token_signals);
  ADD_MAPPED_TOKEN(Q_SLOTS, Token_slots);
  ADD_TOKEN(__attribute__);
  ADD_TOKEN(__typeof);
  ADD_TOKEN(and);
  ADD_TOKEN(and_eq);
  ADD_TOKEN(asm);
  ADD_TOKEN(auto);
  ADD_TOKEN(bitand);
  ADD_TOKEN(bitor);
  ADD_TOKEN(bool);
  ADD_TOKEN(break);
  ADD_TOKEN(case);
  ADD_TOKEN(catch);
  ADD_TOKEN(char);
  ADD_TOKEN(class);
  ADD_TOKEN(compl);
  ADD_TOKEN(const);
  ADD_TOKEN(const_cast);
  ADD_TOKEN(continue);
  ADD_TOKEN(default);
  ADD_TOKEN(delete);
  ADD_TOKEN(do);
  ADD_TOKEN(double);
  ADD_TOKEN(dynamic_cast);
  ADD_TOKEN(else);
  ADD_TOKEN(emit);
  ADD_TOKEN(enum);
  ADD_TOKEN(explicit);
  ADD_TOKEN(export);
  ADD_TOKEN(extern);
  ADD_TOKEN(false);
  ADD_TOKEN(float);
  ADD_TOKEN(for);
  ADD_TOKEN(friend);
  ADD_TOKEN(goto);
  ADD_TOKEN(if);
  ADD_TOKEN(incr);
  ADD_TOKEN(inline);
  ADD_TOKEN(int);
  ADD_TOKEN(k_dcop);
  ADD_TOKEN(k_dcop_signals);
  ADD_TOKEN(long);
  ADD_TOKEN(mutable);
  ADD_TOKEN(namespace);
  ADD_TOKEN(new);
  ADD_TOKEN(not);
  ADD_TOKEN(not_eq);
  ADD_TOKEN(operator);
  ADD_TOKEN(or);
  ADD_TOKEN(or_eq);
  ADD_TOKEN(private);
  ADD_TOKEN(protected);
  ADD_TOKEN(public);
  ADD_TOKEN(register);
  ADD_TOKEN(reinterpret_cast);
  ADD_TOKEN(return);
  ADD_TOKEN(short);
  ADD_TOKEN(signals);
  ADD_TOKEN(signed);
  ADD_TOKEN(sizeof);
  ADD_TOKEN(slots);
  ADD_TOKEN(static);
  ADD_TOKEN(static_cast);
  ADD_TOKEN(struct);
  ADD_TOKEN(switch);
  ADD_TOKEN(template);
  ADD_TOKEN(this);
  ADD_TOKEN(throw);
  ADD_TOKEN(true);
  ADD_TOKEN(try);
  ADD_TOKEN(typedef);
  ADD_TOKEN(typeid);
  ADD_TOKEN(typename);
  ADD_TOKEN(union);
  ADD_TOKEN(unsigned);
  ADD_TOKEN(using);
  ADD_TOKEN(virtual);
  ADD_TOKEN(void);
  ADD_TOKEN(volatile);
  ADD_TOKEN(size_t);
  ADD_TOKEN(wchar_t);
  ADD_TOKEN(while);
  ADD_TOKEN(xor);
  ADD_TOKEN(xor_eq);
  ADD_TOKEN(__qt_sig_slot__);
  return ret;
}

//A very simple lookup table: First level contains all pairs grouped by with (index % index_size), then there is a simple list
KDevVarLengthArray<KDevVarLengthArray<QPair<uint, TOKEN_KIND>, 10 >, index_size > indicesForTokens = createIndicesForTokens();

scan_fun_ptr Lexer::s_scan_table[256];
bool Lexer::s_initialized = false;

Lexer::Lexer(Control *c)
  : session(0),
    control(c),
    m_leaveSize(false)
{
}

void Lexer::tokenize(ParseSession* _session)
{
  session = _session;

  if (!s_initialized)
    initialize_scan_table();

generator/parser/lexer.cpp  view on Meta::CPAN

    }
}

void Lexer::scan_identifier_or_keyword()
{
  if(!(cursor < endCursor))
    return;
  
  //We have to merge symbols tokenized separately, they may have been contracted using ##
  SpecialCursor nextCursor(cursor);
  ++nextCursor;
  
  while(nextCursor < endCursor && (!isCharacter(*(nextCursor.current)) || isLetterOrNumber(*nextCursor.current) || characterFromIndex(*nextCursor.current) == '_')) {
    //Fortunately this shouldn't happen too often, only when ## is used within the preprocessor
    IndexedString mergedSymbol(IndexedString::fromIndex(*(cursor.current)).byteArray() + IndexedString::fromIndex(*(nextCursor.current)).byteArray());
    
    (*cursor.current) = mergedSymbol.index();
    (*nextCursor.current) = 0;
    ++nextCursor;
  }
  
  uint bucket = (*cursor.current) % index_size;
  for(int a = 0; a < indicesForTokens[bucket].size(); ++a) {
    if(indicesForTokens[bucket][a].first == *cursor.current) {
      (*session->token_stream)[index++].kind = indicesForTokens[bucket][a].second;
      ++cursor;
      return;
    }
  }

  m_leaveSize = true; //Since we may have skipped input tokens while mergin, we have to make sure that the size stays 1(the merged tokens will be empty)
  (*session->token_stream)[index].size = 1;
  (*session->token_stream)[index++].kind = Token_identifier;
  
  cursor = nextCursor;
}

void Lexer::scan_int_constant()
{
  if (*cursor == '.' && !std::isdigit(*(cursor + 1)))
    {
      scan_dot();
      return;
    }

  //const char *begin = cursor;

  while (cursor != endCursor &&  (isalnum(*cursor) || *cursor == '.'))
    ++cursor;

  //(*session->token_stream)[index].extra.symbol =
    //control->findOrInsertName((const char*) begin, cursor - begin);

  (*session->token_stream)[index++].kind = Token_number_literal;
}

void Lexer::scan_not()
{
  /*
    '!'		::= not
    '!='		::= not_equal
  */

  ++cursor;

  if (*cursor == '=')
    {
      ++cursor;
      (*session->token_stream)[index++].kind = Token_not_eq;
    }
  else
    {
      (*session->token_stream)[index++].kind = '!';
    }
}

void Lexer::scan_remainder()
{
  /*
    '%'		::= remainder
    '%='		::= remainder_equal
  */

  ++cursor;

  if (*cursor == '=')
    {
      ++cursor;
      (*session->token_stream)[index++].kind = Token_assign;
    }
  else
    {
      (*session->token_stream)[index++].kind = '%';
    }
}

void Lexer::scan_and()
{
  /*
    '&&'		::= and_and
    '&'		::= and
    '&='		::= and_equal
  */

  ++cursor;
  if (*cursor == '=')
    {
      ++cursor;
      (*session->token_stream)[index++].kind = Token_assign;
    }
  else if (*cursor == '&')
    {
      ++cursor;
      (*session->token_stream)[index++].kind = Token_and;
    }
  else
    {
      (*session->token_stream)[index++].kind = '&';
    }
}

void Lexer::scan_left_paren()
{
  ++cursor;
  (*session->token_stream)[index++].kind = '(';
}

void Lexer::scan_right_paren()
{