view release on metacpan or  search on metacpan

nametag/nametag.cpp  view on Meta::CPAN

        enc.add_4B(feature);
    }
  }

  virtual void process_sentence(ner_sentence& sentence, ner_feature* /*total_features*/, string& buffer) const override {
    for (unsigned i = 0; i < sentence.size; i++) {
      auto it = map.find(sentence.words[i].raw_lemma);
      if (it == map.end()) continue;

      // Apply regular gazetteer feature G + unigram gazetteer feature U
      for (auto&& feature : gazetteers_info[it->second].features) {
        apply_in_window(i, feature + G * (2*window + 1));
        apply_in_window(i, feature + U * (2*window + 1));
      }

      for (unsigned j = i + 1; gazetteers_info[it->second].prefix_of_longer && j < sentence.size; j++) {
        if (j == i + 1) buffer.assign(sentence.words[i].raw_lemma);
        buffer += ' ';
        buffer += sentence.words[j].raw_lemma;
        it = map.find(buffer);
        if (it == map.end()) break;

        // Apply regular gazetteer feature G + position specific gazetteers B, I, L
        for (auto&& feature : gazetteers_info[it->second].features)
          for (unsigned g = i; g <= j; g++) {
            apply_in_window(g, feature + G * (2*window + 1));
            apply_in_window(g, feature + (g == i ? B : g == j ? L : I) * (2*window + 1));
          }
      }
    }
  }

 private:
  struct gazetteer_info {
    vector<ner_feature> features;
    bool prefix_of_longer;
  };
  vector<gazetteer_info> gazetteers_info;
};

// GazetteersEnhanced
class gazetteers_enhanced : public feature_processor {
 public:
  enum { G = 0, U = 1, B = 2, L = 3, I = 4, TOTAL = 5 };

  virtual bool parse(int window, const vector<string>& args, entity_map& entities,
                     ner_feature* total_features, const nlp_pipeline& pipeline) override {
    if (!feature_processor::parse(window, args, entities, total_features, pipeline)) return false;

    gazetteer_metas.clear();
    gazetteer_lists.clear();

    if (args.size() < 4) return cerr << "Not enough parameters to GazetteersEnhanced!" << endl, false;
    if (args.size() & 1) return cerr << "Odd number of parameters to GazetteersEnhanced!" << endl, false;

    if (args[0] == "form") match = MATCH_FORM;
    else if (args[0] == "rawlemma") match = MATCH_RAWLEMMA;
    else if (args[0] == "rawlemmas") match = MATCH_RAWLEMMAS;
    else return cerr << "First parameter of GazetteersEnhanced not one of form/rawlemma/rawlemmas!" << endl, false;

    if (args[1] == "embed_in_model") embed = EMBED_IN_MODEL;
    else if (args[1] == "out_of_model") embed = OUT_OF_MODEL;
    else return cerr << "Second parameter of GazetteersEnhanced not one of [embed_in|out_of]_model!" << endl, false;

    for (unsigned i = 2; i < args.size(); i += 2) {
      gazetteer_metas.emplace_back();
      gazetteer_metas.back().basename = args[i];
      gazetteer_metas.back().feature = *total_features + window; *total_features += TOTAL * (2 * window + 1);
      gazetteer_metas.back().entity = args[i + 1] == "NONE" ? -1 : entities.parse(args[i + 1].c_str(), true);
    }

    entity_list.clear();
    for (entity_type i = 0; i < entities.size(); i++)
      entity_list.push_back(entities.name(i));

    if (!load_gazetteer_lists(pipeline, embed == EMBED_IN_MODEL)) return false;

    return true;
  }

  virtual void load(binary_decoder& data, const nlp_pipeline& pipeline) override {
    feature_processor::load(data, pipeline);

    match = data.next_4B();
    embed = OUT_OF_MODEL;

    gazetteer_metas.resize(data.next_4B());
    for (auto&& gazetteer_meta : gazetteer_metas) {
      data.next_str(gazetteer_meta.basename);
      gazetteer_meta.feature = data.next_4B();
      gazetteer_meta.entity = data.next_4B();
    }

    gazetteer_lists.resize(data.next_4B());
    for (auto&& gazetteer_list : gazetteer_lists) {
      gazetteer_list.gazetteers.resize(data.next_4B());
      for (auto&& gazetteer : gazetteer_list.gazetteers)
        data.next_str(gazetteer);
      gazetteer_list.feature = data.next_4B();
      gazetteer_list.entity = data.next_4B();
      gazetteer_list.mode = data.next_4B();
    }

    entity_list.resize(data.next_4B());
    for (auto&& entity : entity_list)
      data.next_str(entity);

    load_gazetteer_lists(pipeline, false);
  }

  virtual void save(binary_encoder& enc) override {
    feature_processor::save(enc);

    enc.add_4B(match);

    enc.add_4B(gazetteer_metas.size());
    for (auto&& gazetteer_meta : gazetteer_metas) {
      enc.add_str(gazetteer_meta.basename);
      enc.add_4B(gazetteer_meta.feature);
      enc.add_4B(gazetteer_meta.entity);
    }

    if (embed == EMBED_IN_MODEL) {
      enc.add_4B(gazetteer_lists.size());
      for (auto&& gazetteer_list : gazetteer_lists) {
        enc.add_4B(gazetteer_list.gazetteers.size());
        for (auto&& gazetteer : gazetteer_list.gazetteers)
          enc.add_str(gazetteer);
        enc.add_4B(gazetteer_list.feature);
        enc.add_4B(gazetteer_list.entity);
        enc.add_4B(gazetteer_list.mode);
      }
    } else {
      enc.add_4B(0);
    }

    enc.add_4B(entity_list.size());
    for (auto&& entity : entity_list)
      enc.add_str(entity);
  }

  virtual void process_sentence(ner_sentence& sentence, ner_feature* /*total_features*/, string& /*buffer*/) const override {
    vector<unsigned> nodes, new_nodes;
    vector<vector<ner_feature>> features(sentence.size);

    vector<vector<string>> recased_match_sources(sentence.size);
    for (unsigned i = 0; i < sentence.size; i++)
      recase_match_source(sentence.words[i], RECASE_ANY, recased_match_sources[i]);

    for (unsigned i = 0; i < sentence.size; i++) {
      unsigned hard_pre_length = 0, hard_pre_node = -1;
      bool hard_pre_possible = true;
      nodes.assign(1, 0);
      for (unsigned j = i; j < sentence.size && !nodes.empty(); j++) {
        new_nodes.clear();
        for (auto&& node : nodes)
          if (!gazetteers_trie[node].children.empty())
            for (auto&& match_source : recased_match_sources[j]) {
              auto range = gazetteers_trie[node].children.equal_range(match_source);
              for (auto&& it = range.first; it != range.second; it++)
                append_unless_exists(new_nodes, it->second);
            }

        hard_pre_possible = hard_pre_possible && !sentence.probabilities[j].local_filled;
        if (hard_pre_possible)
          for (auto&& node : new_nodes)
            if (gazetteers_trie[node].mode == HARD_PRE &&
                ((j - i + 1) > hard_pre_length || node < hard_pre_node))
              hard_pre_length = j - i + 1, hard_pre_node = node;

        // Fill features
        for (auto&& node : new_nodes)
          for (auto&& feature : gazetteers_trie[node].features)
            for (unsigned k = i; k <= j; k++) {
              bilou_type type = j == i ? bilou_type_U : k == i ? bilou_type_B : k == j ? bilou_type_L : bilou_type_I;
              append_unless_exists(features[k], feature + G * (2 * window + 1));
              append_unless_exists(features[k], feature + type * (2 * window + 1));
            }

        nodes.swap(new_nodes);
      }

      if (hard_pre_length)

nametag/nametag.cpp  view on Meta::CPAN

    vector<vector<string>> recased_match_sources(sentence.size);
    for (unsigned i = 0; i < sentence.size; i++)
      recase_match_source(sentence.words[i], RECASE_ANY, recased_match_sources[i]);

    buffer.clear();
    unsigned entity_until = 0;
    for (unsigned i = 0, e = 0; i < sentence.size; i++) {
      while (e < entities.size() && entities[e].start == i) {
        if (i + entities[e].length > entity_until)
          entity_until = i + entities[e].length;
        buffer.push_back(entities[e++]);
      }

      if (entity_until <= i) {
        // There is place for a possible POST gazetteer
        unsigned free_until = e < entities.size() ? entities[e].start : sentence.size;

        unsigned hard_post_length = 0, hard_post_node = -1;
        nodes.assign(1, 0);
        for (unsigned j = i; j < free_until && !nodes.empty(); j++) {
          new_nodes.clear();
          for (auto&& node : nodes)
            if (!gazetteers_trie[node].children.empty())
              for (auto&& match_source : recased_match_sources[j]) {
                auto range = gazetteers_trie[node].children.equal_range(match_source);
                for (auto&& it = range.first; it != range.second; it++)
                  append_unless_exists(new_nodes, it->second);
              }

          for (auto&& node : new_nodes)
            if (gazetteers_trie[node].mode == HARD_POST &&
                ((j - i + 1) > hard_post_length || node < hard_post_node))
              hard_post_length = j - i + 1, hard_post_node = node;

          nodes.swap(new_nodes);
        }

        if (hard_post_length) {
          buffer.emplace_back(i, hard_post_length, entity_list[gazetteers_trie[hard_post_node].entity]);
          entity_until = i + hard_post_length;
        }
      }
    }

    if (buffer.size() != entities.size())
      entities.swap(buffer);
  }

  virtual void gazetteers(vector<string>& gazetteers, vector<int>* gazetteer_types) const override {
    for (auto&& gazetteer_list : gazetteer_lists)
      for (auto&& gazetteer : gazetteer_list.gazetteers) {
        gazetteers.push_back(gazetteer);
        if (gazetteer_types) gazetteer_types->push_back(gazetteer_list.entity);
      }
  }

 private:
  enum { MATCH_FORM = 0, MATCH_RAWLEMMA = 1, MATCH_RAWLEMMAS = 2 };
  int match;

  enum { EMBED_IN_MODEL = 0, OUT_OF_MODEL = 1 };
  int embed;

  enum { SOFT, HARD_PRE, HARD_POST, MODES_TOTAL };
  const static vector<string> basename_suffixes;

  struct gazetteer_meta_info {
    string basename;
    ner_feature feature;
    int entity;
  };
  vector<gazetteer_meta_info> gazetteer_metas;

  struct gazetteer_list_info {
    vector<string> gazetteers;
    ner_feature feature;
    int entity;
    int mode;
  };
  vector<gazetteer_list_info> gazetteer_lists;

  struct gazetteer_trie_node {
    vector<ner_feature> features;
    unordered_multimap<string, unsigned> children;
    int mode = SOFT, entity = -1;
  };
  vector<gazetteer_trie_node> gazetteers_trie;

  vector<string> entity_list;

  template <class T>
  inline static void append_unless_exists(vector<T>& array, T value) {
    size_t i;
    for (i = array.size(); i; i--)
      if (array[i - 1] == value)
        break;

    if (!i)
      array.push_back(value);
  }

  bool load_gazetteer_lists(const nlp_pipeline& pipeline, bool files_must_exist) {
    string file_name, line;

    // Load raw gazetteers (maybe additional during inference)
    for (auto&& gazetteer_meta : gazetteer_metas)
      for (int mode = 0; mode < MODES_TOTAL; mode++) {
        file_name.assign(gazetteer_meta.basename).append(basename_suffixes[mode]);

        ifstream file(path_from_utf8(file_name).c_str());
        if (!file.is_open()) {
          if (mode == SOFT && files_must_exist)
            return cerr << "Cannot open gazetteers file '" << file_name << "'!" << endl, false;
          continue;
        }

        gazetteer_lists.emplace_back();
        gazetteer_lists.back().feature = gazetteer_meta.feature;
        gazetteer_lists.back().entity = gazetteer_meta.entity;
        gazetteer_lists.back().mode = mode;