librime: table.h Source File

| librime 1.2 Rime Input Method Engine, the core library

table.h
Go to the documentation of this file.
1//
2// Copyright RIME Developers
3// Distributed under the BSD License
4//
5// 2011-07-01 GONG Chen <chen.sst@gmail.com>
6//
7
8#ifndef RIME_TABLE_H_
9#define RIME_TABLE_H_
10
11#include <cstring>
12#include <rime/common.h>
13#include <rime/dict/mapped_file.h>
14#include <rime/dict/vocabulary.h>
15#include <rime/dict/string_table.h>
16
17#define RIME_TABLE_UNION(U, V, A, a, B, b) \
18 struct U { \
19 V value; \
20 const A& a() const { \
21 return *reinterpret_cast<const A*>(this); \
22 } \
23 const B& b() const { \
24 return *reinterpret_cast<const B*>(this); \
25 } \
26 A& a() { \
27 return *reinterpret_cast<A*>(this); \
28 } \
29 B& b() { \
30 return *reinterpret_cast<B*>(this); \
31 } \
32 }
33
34namespace rime {
35
36 namespace table {
37
38// union StringType {
39// String str;
40// StringId str_id;
41// };
42RIME_TABLE_UNION(StringType, int32_t, String, str, StringId, str_id);
43
44 using Syllabary = Array<StringType>;
45
46 using Code = List<SyllableId>;
47
48 using Weight = float;
49
50 struct Entry {
51 StringType text;
52 Weight weight;
53};
54
55 struct LongEntry {
56 Code extra_code;
57 Entry entry;
58};
59
60struct PhraseIndex;
61
62 struct HeadIndexNode {
63 List<Entry> entries;
64 OffsetPtr<PhraseIndex> next_level;
65};
66
67 using HeadIndex = Array<HeadIndexNode>;
68
69 struct TrunkIndexNode {
70 SyllableId key;
71 List<Entry> entries;
72 OffsetPtr<PhraseIndex> next_level;
73};
74
75 using TrunkIndex = Array<TrunkIndexNode>;
76
77 using TailIndex = Array<LongEntry>;
78
79// union PhraseIndex {
80// TrunkIndex trunk;
81// TailIndex tail;
82// };
83 RIME_TABLE_UNION(PhraseIndex, Array<char>, TrunkIndex, trunk, TailIndex, tail);
84
85 using Index = HeadIndex;
86
87 struct Metadata {
88 static const int kFormatMaxLength = 32;
89 char format[kFormatMaxLength];
90 uint32_t dict_file_checksum;
91 uint32_t num_syllables;
92 uint32_t num_entries;
93 OffsetPtr<Syllabary> syllabary;
94 OffsetPtr<Index> index;
95 // v2
96 int32_t reserved_1;
97 int32_t reserved_2;
98 OffsetPtr<char> string_table;
99 uint32_t string_table_size;
100};
101
102} // namespace table
103
104 class TableAccessor {
105 public:
106 TableAccessor() = default;
107 TableAccessor(const Code& index_code,
108 const List<table::Entry>* entries,
109 double credibility = 0.0,
110 double quality_len = 0.0);
111 TableAccessor(const Code& index_code,
112 const Array<table::Entry>* entries,
113 double credibility = 0.0,
114 double quality_len = 0.0);
115 TableAccessor(const Code& index_code,
116 const table::TailIndex* code_map,
117 double credibility = 0.0,
118 double quality_len = 0.0);
119
120 RIME_DLL bool Next();
121
122 RIME_DLL bool exhausted() const;
123 RIME_DLL size_t remaining() const;
124 RIME_DLL const table::Entry* entry() const;
125 RIME_DLL const table::Code* extra_code() const;
126 const Code& index_code() const { return index_code_; }
127 Code code() const;
128 double credibility() const { return credibility_; }
129 double quality_len() const { return quality_len_; }
130
131 private:
132 Code index_code_;
133 const table::Entry* entries_ = nullptr;
134 const table::LongEntry* long_entries_ = nullptr;
135 size_t size_ = 0;
136 size_t cursor_ = 0;
137 double credibility_ = 0.0;
138 double quality_len_ = 0.0;
139};
140
141 using TableQueryResult = map<int, vector<TableAccessor>>;
142
143struct SyllableGraph;
144
145 class TableQuery {
146 public:
147 TableQuery(table::Index* index) : lv1_index_(index) { Reset(); }
148
149 TableAccessor Access(SyllableId syllable_id,
150 double credibility = 0.0,
151 double quality_len = 0.0) const;
152
153 // down to next level
154 bool Advance(SyllableId syllable_id,
155 double credibility = 0.0,
156 double quality_len = 0.0,
157 size_t last_pos = 0);
158
159 // up one level
160 bool Backdate();
161
162 // back to root
163 void Reset();
164
165 size_t level() const { return level_; }
166
167 double credibility_sum() const {
168 return credibility_.empty() ? 0 : credibility_.back();
169 }
170 double quality_len_sum() const {
171 return quality_len_.empty() ? 0 : quality_len_.back();
172 }
173 size_t last_pos() const { return last_pos_.empty() ? 0 : last_pos_.back(); }
174
175 protected:
176 size_t level_ = 0;
177 Code index_code_;
178 vector<double> credibility_;
179 vector<double> quality_len_;
180 vector<size_t> last_pos_;
181
182 private:
183 bool Walk(SyllableId syllable_id);
184
185 table::HeadIndex* lv1_index_ = nullptr;
186 table::TrunkIndex* lv2_index_ = nullptr;
187 table::TrunkIndex* lv3_index_ = nullptr;
188 table::TailIndex* lv4_index_ = nullptr;
189};
190
191 class Table : public MappedFile {
192 public:
193 RIME_DLL Table(const path& file_path);
194 virtual ~Table();
195
196 RIME_DLL bool Load();
197 RIME_DLL bool Save();
198 RIME_DLL bool Build(const Syllabary& syllabary,
199 const Vocabulary& vocabulary,
200 size_t num_entries,
201 uint32_t dict_file_checksum = 0);
202
203 bool GetSyllabary(Syllabary* syllabary);
204 RIME_DLL string GetSyllableById(int syllable_id);
205 RIME_DLL TableAccessor QueryWords(int syllable_id);
206 RIME_DLL TableAccessor QueryPhrases(const Code& code);
207 RIME_DLL bool Query(const SyllableGraph& syll_graph,
208 size_t start_pos,
209 TableQueryResult* result);
210 RIME_DLL string GetEntryText(const table::Entry& entry);
211
212 uint32_t dict_file_checksum() const;
213 table::Metadata* metadata() const { return metadata_; }
214
215 private:
216 table::Index* BuildIndex(const Vocabulary& vocabulary, size_t num_syllables);
217 table::HeadIndex* BuildHeadIndex(const Vocabulary& vocabulary,
218 size_t num_syllables);
219 table::TrunkIndex* BuildTrunkIndex(const Code& prefix,
220 const Vocabulary& vocabulary);
221 table::TailIndex* BuildTailIndex(const Code& prefix,
222 const Vocabulary& vocabulary);
223 bool BuildPhraseIndex(Code code,
224 const Vocabulary& vocabulary,
225 map<string, int>* index_data);
226 Array<table::Entry>* BuildEntryArray(const ShortDictEntryList& entries);
227 bool BuildEntryList(const ShortDictEntryList& src, List<table::Entry>* dest);
228 bool BuildEntry(const ShortDictEntry& dict_entry, table::Entry* entry);
229
230 string GetString(const table::StringType& x);
231 bool AddString(const string& src, table::StringType* dest, double weight);
232 bool OnBuildStart();
233 bool OnBuildFinish();
234 bool OnLoad();
235
236 protected:
237 table::Metadata* metadata_ = nullptr;
238 table::Syllabary* syllabary_ = nullptr;
239 table::Index* index_ = nullptr;
240
241 the<StringTable> string_table_;
242 the<StringTableBuilder> string_table_builder_;
243};
244
245} // namespace rime
246
247#endif // RIME_TABLE_H_
RIME_DLL
#define RIME_DLL
Definition rime_api.h:33
common.h
vocabulary.h
mapped_file.h
string_table.h
rime
Definition algebra.cc:12
rime::TableQueryResult
map< int, vector< TableAccessor > > TableQueryResult
Definition table.h:141
rime::StringId
std::uint32_t StringId
Definition string_table.h:19
rime::Syllabary
set< string > Syllabary
Definition vocabulary.h:17
rime::the
std::unique_ptr< T > the
Definition common.h:58
rime::SyllableId
int32_t SyllableId
Definition syllabifier.h:21
rime::table
Definition table.h:36
rime::table::Code
List< SyllableId > Code
Definition table.h:46
rime::table::Index
HeadIndex Index
Definition table.h:85
rime::table::TailIndex
Array< LongEntry > TailIndex
Definition table.h:77
rime::table::TrunkIndex
Array< TrunkIndexNode > TrunkIndex
Definition table.h:75
rime::table::HeadIndex
Array< HeadIndexNode > HeadIndex
Definition table.h:67
rime::table::Weight
float Weight
Definition table.h:48
rime::table::Syllabary
Array< StringType > Syllabary
Definition table.h:44
rime::table::RIME_TABLE_UNION
RIME_TABLE_UNION(StringType, int32_t, String, str, StringId, str_id)
rime::SyllableGraph
Definition syllabifier.h:39
rime::path
Definition common.h:84
rime::OffsetPtr
Definition mapped_file.h:22
rime::String
Definition mapped_file.h:53
rime::Array
Definition mapped_file.h:61
rime::List
Definition mapped_file.h:71
rime::MappedFile::file_path
const path & file_path() const
Definition mapped_file.h:121
rime::MappedFile::MappedFile
MappedFile(const path &file_path)
Definition mapped_file.cc:46
rime::table::Entry
Definition table.h:50
rime::table::Entry::text
StringType text
Definition table.h:51
rime::table::Entry::weight
Weight weight
Definition table.h:52
rime::table::LongEntry
Definition table.h:55
rime::table::LongEntry::entry
Entry entry
Definition table.h:57
rime::table::LongEntry::extra_code
Code extra_code
Definition table.h:56
rime::table::HeadIndexNode
Definition table.h:62
rime::table::HeadIndexNode::entries
List< Entry > entries
Definition table.h:63
rime::table::HeadIndexNode::next_level
OffsetPtr< PhraseIndex > next_level
Definition table.h:64
rime::table::TrunkIndexNode
Definition table.h:69
rime::table::TrunkIndexNode::entries
List< Entry > entries
Definition table.h:71
rime::table::TrunkIndexNode::key
SyllableId key
Definition table.h:70
rime::table::TrunkIndexNode::next_level
OffsetPtr< PhraseIndex > next_level
Definition table.h:72
rime::table::Metadata
Definition table.h:87
rime::table::Metadata::reserved_1
int32_t reserved_1
Definition table.h:96
rime::table::Metadata::kFormatMaxLength
static const int kFormatMaxLength
Definition table.h:88
rime::table::Metadata::dict_file_checksum
uint32_t dict_file_checksum
Definition table.h:90
rime::table::Metadata::num_entries
uint32_t num_entries
Definition table.h:92
rime::table::Metadata::string_table
OffsetPtr< char > string_table
Definition table.h:98
rime::table::Metadata::string_table_size
uint32_t string_table_size
Definition table.h:99
rime::table::Metadata::syllabary
OffsetPtr< Syllabary > syllabary
Definition table.h:93
rime::table::Metadata::num_syllables
uint32_t num_syllables
Definition table.h:91
rime::table::Metadata::reserved_2
int32_t reserved_2
Definition table.h:97
rime::table::Metadata::index
OffsetPtr< Index > index
Definition table.h:94
rime::table::Metadata::format
char format[kFormatMaxLength]
Definition table.h:89
rime::TableAccessor
Definition table.h:104
rime::TableAccessor::index_code
const Code & index_code() const
Definition table.h:126
rime::TableAccessor::extra_code
RIME_DLL const table::Code * extra_code() const
Definition table.cc:77
rime::TableAccessor::entry
RIME_DLL const table::Entry * entry() const
Definition table.cc:68
rime::TableAccessor::remaining
RIME_DLL size_t remaining() const
Definition table.cc:61
rime::TableAccessor::exhausted
RIME_DLL bool exhausted() const
Definition table.cc:54
rime::TableAccessor::credibility
double credibility() const
Definition table.h:128
rime::TableAccessor::TableAccessor
TableAccessor()=default
rime::TableAccessor::Next
RIME_DLL bool Next()
Definition table.cc:95
rime::TableAccessor::quality_len
double quality_len() const
Definition table.h:129
rime::TableAccessor::code
Code code() const
Definition table.cc:83
rime::TableQuery::quality_len_
vector< double > quality_len_
Definition table.h:179
rime::TableQuery::level_
size_t level_
Definition table.h:176
rime::TableQuery::quality_len_sum
double quality_len_sum() const
Definition table.h:170
rime::TableQuery::index_code_
Code index_code_
Definition table.h:177
rime::TableQuery::credibility_sum
double credibility_sum() const
Definition table.h:167
rime::TableQuery::Advance
bool Advance(SyllableId syllable_id, double credibility=0.0, double quality_len=0.0, size_t last_pos=0)
Definition table.cc:102
rime::TableQuery::Backdate
bool Backdate()
Definition table.cc:117
rime::TableQuery::Reset
void Reset()
Definition table.cc:130
rime::TableQuery::credibility_
vector< double > credibility_
Definition table.h:178
rime::TableQuery::last_pos
size_t last_pos() const
Definition table.h:173
rime::TableQuery::Access
TableAccessor Access(SyllableId syllable_id, double credibility=0.0, double quality_len=0.0) const
Definition table.cc:190
rime::TableQuery::last_pos_
vector< size_t > last_pos_
Definition table.h:180
rime::TableQuery::level
size_t level() const
Definition table.h:165
rime::TableQuery::TableQuery
TableQuery(table::Index *index)
Definition table.h:147
rime::Table::QueryPhrases
RIME_DLL TableAccessor QueryPhrases(const Code &code)
Definition table.cc:555
rime::Table::GetSyllabary
bool GetSyllabary(Syllabary *syllabary)
Definition table.cc:535
rime::Table::Build
RIME_DLL bool Build(const Syllabary &syllabary, const Vocabulary &vocabulary, size_t num_entries, uint32_t dict_file_checksum=0)
Definition table.cc:330
rime::Table::string_table_
the< StringTable > string_table_
Definition table.h:241
rime::Table::GetSyllableById
RIME_DLL string GetSyllableById(int syllable_id)
Definition table.cc:543
rime::Table::syllabary_
table::Syllabary * syllabary_
Definition table.h:238
rime::Table::metadata_
table::Metadata * metadata_
Definition table.h:237
rime::Table::QueryWords
RIME_DLL TableAccessor QueryWords(int syllable_id)
Definition table.cc:550
rime::Table::Save
RIME_DLL bool Save()
Definition table.cc:315
rime::Table::Table
RIME_DLL Table(const path &file_path)
Definition table.cc:265
rime::Table::GetEntryText
RIME_DLL string GetEntryText(const table::Entry &entry)
Definition table.cc:632
rime::Table::metadata
table::Metadata * metadata() const
Definition table.h:213
rime::Table::dict_file_checksum
uint32_t dict_file_checksum() const
Definition table.cc:326
rime::Table::Query
RIME_DLL bool Query(const SyllableGraph &syll_graph, size_t start_pos, TableQueryResult *result)
Definition table.cc:571
rime::Table::Load
RIME_DLL bool Load()
Definition table.cc:269
rime::Table::index_
table::Index * index_
Definition table.h:239
rime::Table::string_table_builder_
the< StringTableBuilder > string_table_builder_
Definition table.h:242
rime::Code
Definition vocabulary.h:21
rime::ShortDictEntry
Definition vocabulary.h:37
rime::ShortDictEntryList
Definition vocabulary.h:70
rime::Vocabulary
Definition vocabulary.h:100