librime: entry_collector.h Source File

| librime 1.2 Rime Input Method Engine, the core library

entry_collector.h
Go to the documentation of this file.
1//
2// Copyright RIME Developers
3// Distributed under the BSD License
4//
5// 2011-11-27 GONG Chen <chen.sst@gmail.com>
6//
7#ifndef RIME_ENTRY_COLLECTOR_H_
8#define RIME_ENTRY_COLLECTOR_H_
9
10#include <queue>
11#include <rime/common.h>
12#include <rime/algo/encoder.h>
13#include <rime/dict/dictionary.h>
14#include <rime/dict/vocabulary.h>
15
16namespace rime {
17
18 struct RawDictEntry {
19 RawCode raw_code;
20 string text;
21 double weight;
22};
23
24// code -> weight
25 using WeightMap = map<string, double>;
26// word -> [ { code, weight } ]
27// For the sake of memory usage, don't use word -> { code -> weight } as there
28// may be many words, but may not be many representations for a word
29 using WordMap = hash_map<string, vector<pair<string, double>>>;
30// [ (word, weight), ... ]
31 using EncodeQueue = std::queue<pair<string, string>>;
32
33class PresetVocabulary;
34class DictSettings;
35
36 class EntryCollector : public PhraseCollector {
37 public:
38 Syllabary syllabary;
39 bool build_syllabary = true;
40 vector<of<RawDictEntry>> entries;
41 size_t num_entries = 0;
42 ReverseLookupTable stems;
43
44 public:
45 EntryCollector();
46 explicit EntryCollector(Syllabary&& fixed_syllabary);
47 virtual ~EntryCollector();
48
49 void Configure(DictSettings* settings);
50 void Collect(const vector<path>& dict_files);
51
52 // export contents of table and prism to text files
53 void Dump(const path& file_path) const;
54
55 void CreateEntry(const string& word,
56 const string& code_str,
57 const string& weight_str);
58 bool TranslateWord(const string& word, vector<string>* code);
59
60 protected:
61 void LoadPresetVocabulary(DictSettings* settings);
62 // call Collect() multiple times for all required tables
63 void Collect(const path& dict_file);
64 // encode all collected entries
65 void Finish();
66
67 protected:
68 the<PresetVocabulary> preset_vocabulary;
69 the<Encoder> encoder;
70 EncodeQueue encode_queue;
71 set<string /* word */> collection;
72 WordMap words;
73 WeightMap total_weight;
74
75 private:
76 string current_dict_file;
77 size_t line_number;
78};
79
80} // namespace rime
81
82#endif // RIME_ENTRY_COLLECTOR_H_
common.h
vocabulary.h
dictionary.h
encoder.h
rime
Definition algebra.cc:12
rime::WordMap
hash_map< string, vector< pair< string, double > > > WordMap
Definition entry_collector.h:29
rime::Syllabary
set< string > Syllabary
Definition vocabulary.h:17
rime::hash_map
boost::unordered_map< Key, T > hash_map
Definition common.h:53
rime::the
std::unique_ptr< T > the
Definition common.h:58
rime::EncodeQueue
std::queue< pair< string, string > > EncodeQueue
Definition entry_collector.h:31
rime::WeightMap
map< string, double > WeightMap
Definition entry_collector.h:25
rime::ReverseLookupTable
hash_map< string, set< string > > ReverseLookupTable
Definition vocabulary.h:107
rime::RawCode
Definition encoder.h:16
rime::PhraseCollector::PhraseCollector
PhraseCollector()=default
rime::path
Definition common.h:84
rime::DictSettings
Definition dict_settings.h:16
rime::RawDictEntry
Definition entry_collector.h:18
rime::RawDictEntry::weight
double weight
Definition entry_collector.h:21
rime::RawDictEntry::text
string text
Definition entry_collector.h:20
rime::RawDictEntry::raw_code
RawCode raw_code
Definition entry_collector.h:19
rime::EntryCollector::CreateEntry
void CreateEntry(const string &word, const string &code_str, const string &weight_str)
Definition entry_collector.cc:161
rime::EntryCollector::num_entries
size_t num_entries
Definition entry_collector.h:41
rime::EntryCollector::TranslateWord
bool TranslateWord(const string &word, vector< string > *code)
Definition entry_collector.cc:223
rime::EntryCollector::Configure
void Configure(DictSettings *settings)
Definition entry_collector.cc:25
rime::EntryCollector::EntryCollector
EntryCollector()
Definition entry_collector.cc:18
rime::EntryCollector::total_weight
WeightMap total_weight
Definition entry_collector.h:73
rime::EntryCollector::encoder
the< Encoder > encoder
Definition entry_collector.h:69
rime::EntryCollector::preset_vocabulary
the< PresetVocabulary > preset_vocabulary
Definition entry_collector.h:68
rime::EntryCollector::LoadPresetVocabulary
void LoadPresetVocabulary(DictSettings *settings)
Definition entry_collector.cc:45
rime::EntryCollector::Dump
void Dump(const path &file_path) const
Definition entry_collector.cc:247
rime::EntryCollector::stems
ReverseLookupTable stems
Definition entry_collector.h:42
rime::EntryCollector::syllabary
Syllabary syllabary
Definition entry_collector.h:38
rime::EntryCollector::words
WordMap words
Definition entry_collector.h:72
rime::EntryCollector::Finish
void Finish()
Definition entry_collector.cc:134
rime::EntryCollector::collection
set< string > collection
Definition entry_collector.h:71
rime::EntryCollector::encode_queue
EncodeQueue encode_queue
Definition entry_collector.h:70
rime::EntryCollector::build_syllabary
bool build_syllabary
Definition entry_collector.h:39
rime::EntryCollector::Collect
void Collect(const vector< path > &dict_files)
Definition entry_collector.cc:38
rime::EntryCollector::entries
vector< of< RawDictEntry > > entries
Definition entry_collector.h:40
rime::PresetVocabulary
Definition preset_vocabulary.h:16