view release on metacpan or  search on metacpan

nametag/nametag.cpp  view on Meta::CPAN

    gazetteer_lists.clear();

    if (args.size() < 4) return cerr << "Not enough parameters to GazetteersEnhanced!" << endl, false;
    if (args.size() & 1) return cerr << "Odd number of parameters to GazetteersEnhanced!" << endl, false;

    if (args[0] == "form") match = MATCH_FORM;
    else if (args[0] == "rawlemma") match = MATCH_RAWLEMMA;
    else if (args[0] == "rawlemmas") match = MATCH_RAWLEMMAS;
    else return cerr << "First parameter of GazetteersEnhanced not one of form/rawlemma/rawlemmas!" << endl, false;

    if (args[1] == "embed_in_model") embed = EMBED_IN_MODEL;
    else if (args[1] == "out_of_model") embed = OUT_OF_MODEL;
    else return cerr << "Second parameter of GazetteersEnhanced not one of [embed_in|out_of]_model!" << endl, false;

    for (unsigned i = 2; i < args.size(); i += 2) {
      gazetteer_metas.emplace_back();
      gazetteer_metas.back().basename = args[i];
      gazetteer_metas.back().feature = *total_features + window; *total_features += TOTAL * (2 * window + 1);
      gazetteer_metas.back().entity = args[i + 1] == "NONE" ? -1 : entities.parse(args[i + 1].c_str(), true);
    }

    entity_list.clear();
    for (entity_type i = 0; i < entities.size(); i++)
      entity_list.push_back(entities.name(i));

    if (!load_gazetteer_lists(pipeline, embed == EMBED_IN_MODEL)) return false;

    return true;
  }

  virtual void load(binary_decoder& data, const nlp_pipeline& pipeline) override {
    feature_processor::load(data, pipeline);

    match = data.next_4B();
    embed = OUT_OF_MODEL;

nametag/nametag.cpp  view on Meta::CPAN

    enc.add_4B(match);

    enc.add_4B(gazetteer_metas.size());
    for (auto&& gazetteer_meta : gazetteer_metas) {
      enc.add_str(gazetteer_meta.basename);
      enc.add_4B(gazetteer_meta.feature);
      enc.add_4B(gazetteer_meta.entity);
    }

    if (embed == EMBED_IN_MODEL) {
      enc.add_4B(gazetteer_lists.size());
      for (auto&& gazetteer_list : gazetteer_lists) {
        enc.add_4B(gazetteer_list.gazetteers.size());
        for (auto&& gazetteer : gazetteer_list.gazetteers)
          enc.add_str(gazetteer);
        enc.add_4B(gazetteer_list.feature);
        enc.add_4B(gazetteer_list.entity);
        enc.add_4B(gazetteer_list.mode);
      }
    } else {

nametag/nametag.cpp  view on Meta::CPAN

      for (auto&& gazetteer : gazetteer_list.gazetteers) {
        gazetteers.push_back(gazetteer);
        if (gazetteer_types) gazetteer_types->push_back(gazetteer_list.entity);
      }
  }

 private:
  enum { MATCH_FORM = 0, MATCH_RAWLEMMA = 1, MATCH_RAWLEMMAS = 2 };
  int match;

  enum { EMBED_IN_MODEL = 0, OUT_OF_MODEL = 1 };
  int embed;

  enum { SOFT, HARD_PRE, HARD_POST, MODES_TOTAL };
  const static vector<string> basename_suffixes;

  struct gazetteer_meta_info {
    string basename;
    ner_feature feature;
    int entity;
  };