33 memset(&bigram_table_, 0,
sizeof(bigram_table_));
38 for (
int ch1 = 0; ch1 <= bigram_table_.
max_char; ch1++) {
42 delete []char_bigram->
bigram;
54 file_name = data_file_path +
lang;
55 file_name +=
".cube.bigrams";
64 if (char_bigrams_obj ==
NULL) {
65 fprintf(stderr,
"Cube ERROR (CharBigrams::Create): could not create "
66 "character bigrams object.\n");
76 vector<string> str_vec;
79 for (
int big = 0; big < str_vec.size(); big++) {
83 if (sscanf(str_vec[big].c_str(),
"%d %x %x", &cnt, &ch1, &ch2) != 3) {
84 fprintf(stderr,
"Cube ERROR (CharBigrams::Create): invalid format "
85 "reading line: %s\n", str_vec[big].c_str());
92 if (char_bigram ==
NULL) {
93 fprintf(stderr,
"Cube ERROR (CharBigrams::Create): error allocating "
94 "additional memory for character bigram table.\n");
100 (table->
max_char + 1) *
sizeof(*char_bigram));
107 for (
int new_big = table->
max_char + 1; new_big <= ch1; new_big++) {
117 if (bigram ==
NULL) {
118 fprintf(stderr,
"Cube ERROR (CharBigrams::Create): error allocating "
119 "memory for bigram.\n");
133 new_big <= ch2; new_big++) {
152 log(
MAX(0.5, static_cast<double>(cnt)) /
156 return char_bigrams_obj;
170 if (!char_32_ptr || char_32_ptr[0] == 0) {
177 if (lower_32 && lower_32[0] != 0) {
179 cost =
MIN(cost, cost_lower);
183 if (upper_32 && upper_32[0] != 0) {
185 cost =
MIN(cost, cost_upper);
198 cost =
PairCost(
' ', char_32_ptr[0]);
199 for (c = 1; c < len; c++) {
200 cost +=
PairCost(char_32_ptr[c - 1], char_32_ptr[c]);
202 cost +=
PairCost(char_32_ptr[len - 1],
' ');
203 return static_cast<int>(cost /
static_cast<double>(len + 1));