tesseract  5.0.0
validate_myanmar.cpp
Go to the documentation of this file.
1 #include "validate_myanmar.h"
2 #include "errcode.h"
3 #include "icuerrorcode.h"
4 #include "tprintf.h"
5 #include "unicode/uchar.h" // From libicu
6 #include "unicode/uscript.h" // From libicu
7 
8 namespace tesseract {
9 
10 // Returns whether codes matches the pattern for a Myanmar Grapheme.
11 // Taken directly from the unicode table 16-3.
12 // See http://www.unicode.org/versions/Unicode9.0.0/ch16.pdf
14  const unsigned num_codes = codes_.size();
15  if (codes_used_ == num_codes) {
16  return true;
17  }
18  // Other.
19  if (IsMyanmarOther(codes_[codes_used_].second)) {
20  UseMultiCode(1);
21  return true;
22  }
23  // Kinzi.
24  if (codes_used_ + 2 < num_codes && codes_[codes_used_].second == 0x1004 &&
25  codes_[codes_used_ + 1].second == kMyanmarAsat &&
26  codes_[codes_used_ + 2].second == kMyanmarVirama) {
29  if (UseMultiCode(3)) {
30  return true;
31  }
32  }
33  // Base consonant/vowel. NOTE that since everything in Myanmar appears to be
34  // optional, except the base, this is the only place where invalid input can
35  // be detected and false returned.
36  if (IsMyanmarLetter(codes_[codes_used_].second)) {
37  if (UseMultiCode(1)) {
38  return true;
39  }
40  } else {
41  if (report_errors_) {
42  tprintf("Invalid start of Myanmar syllable:0x%x\n", codes_[codes_used_].second);
43  }
44  return false; // One of these is required.
45  }
46  if (ConsumeSubscriptIfPresent()) {
47  return true;
48  }
49  ConsumeOptionalSignsIfPresent();
50  // What we have consumed so far is a valid syllable.
51  return true;
52 }
53 
54 // TODO(rays) Doesn't use intermediate coding like the other scripts, as there
55 // is little correspondence between the content of table 16-3 and the char
56 // classes of the Indic languages. (Experts may disagree and improve!)
57 // In unicode table 16-3 there is basically a long list of optional characters,
58 // which can be coded quite easily.
59 // Unfortunately, table 16-3 doesn't include even half the Myanmar unicodes!!
60 // The table also allows sequences that still result in dotted circles!!
61 // So with a lot of guesswork the rest have been added in a reasonable place.
63  if (IsMyanmarLetter(ch)) {
64  return CharClass::kConsonant;
65  }
66  return CharClass::kOther;
67 }
68 
69 // Helper consumes/copies a virama and any subscript consonant.
70 // Returns true if the end of input is reached.
71 bool ValidateMyanmar::ConsumeSubscriptIfPresent() {
72  // Subscript consonant. It appears there can be only one.
73  const unsigned num_codes = codes_.size();
74  if (codes_used_ + 1 < num_codes && codes_[codes_used_].second == kMyanmarVirama) {
75  if (IsMyanmarLetter(codes_[codes_used_ + 1].second)) {
77  if (UseMultiCode(2)) {
78  return true;
79  }
80  }
81  }
82  return false;
83 }
84 
85 // Helper consumes/copies a series of optional signs.
86 // Returns true if the end of input is reached.
87 bool ValidateMyanmar::ConsumeOptionalSignsIfPresent() {
88  // The following characters are allowed, all optional, and in sequence.
89  // An exception is kMyanmarMedialYa, which can include kMyanmarAsat.
90  const std::vector<char32> kMedials({kMyanmarAsat, kMyanmarMedialYa, 0x103c, 0x103d, 0x103e,
91  0x105e, 0x105f, 0x1060, 0x1081, 0x1031});
92  for (char32 ch : kMedials) {
93  if (codes_[codes_used_].second == ch) {
94  if (UseMultiCode(1)) {
95  return true;
96  }
97  if (ch == kMyanmarMedialYa && codes_[codes_used_].second == kMyanmarAsat) {
98  if (UseMultiCode(1)) {
99  return true;
100  }
101  }
102  }
103  }
104  // Vowel sign i, ii, ai.
105  char32 ch = codes_[codes_used_].second;
106  if (ch == 0x102d || ch == 0x102e || ch == 0x1032) {
107  if (UseMultiCode(1)) {
108  return true;
109  }
110  }
111  // Vowel sign u, uu, and extensions.
112  ch = codes_[codes_used_].second;
113  if (ch == 0x102f || ch == 0x1030 || (0x1056 <= ch && ch <= 0x1059) || ch == 0x1062 ||
114  ch == 0x1067 || ch == 0x1068 || (0x1071 <= ch && ch <= 0x1074) ||
115  (0x1083 <= ch && ch <= 0x1086) || ch == 0x109c || ch == 0x109d) {
116  if (UseMultiCode(1)) {
117  return true;
118  }
119  }
120  // Tall aa, aa with optional asat.
121  if (codes_[codes_used_].second == 0x102b || codes_[codes_used_].second == 0x102c) {
122  if (UseMultiCode(1)) {
123  return true;
124  }
125  if (codes_[codes_used_].second == kMyanmarAsat) {
126  if (UseMultiCode(1)) {
127  return true;
128  }
129  }
130  }
131  // The following characters are allowed, all optional, and in sequence.
132  // Anusvar, Dot below, Visarga
133  const std::vector<char32> kSigns({0x1036, 0x1037, 0x1038});
134  for (char32 ch : kSigns) {
135  if (codes_[codes_used_].second == ch) {
136  if (UseMultiCode(1)) {
137  return true;
138  }
139  }
140  }
141  // Tone mark extensions.
142  ch = codes_[codes_used_].second;
143  if (ch == 0x1038 || ch == kMyanmarAsat || ch == 0x1063 || ch == 0x1064 ||
144  (0x1069 <= ch && ch <= 0x106d) || (0x1087 <= ch && ch <= 0x108d) || ch == 0x108f ||
145  ch == 0x109a || ch == 0x109b || (0xaa7b <= ch && ch <= 0xaa7d)) {
146  if (UseMultiCode(1)) {
147  return true;
148  }
149  }
150  return false;
151 }
152 
153 // Returns true if the unicode is a Myanmar "letter" including consonants
154 // and independent vowels. Although table 16-3 distinguishes between some
155 // base consonants and vowels, the extensions make no such distinction, so we
156 // put them all into a single bucket.
157 // Update MYANMAR LETTER based on following:
158 // https://unicode.org/charts/PDF/U1000.pdf - Myanmar
159 // http://unicode.org/charts/PDF/UAA60.pdf - Myanmar Extended-A
160 // http://unicode.org/charts/PDF/UA9E0.pdf - Myanmar Extended-B
161 /* static */
162 bool ValidateMyanmar::IsMyanmarLetter(char32 ch) {
163  return (0x1000 <= ch && ch <= 0x102a) || ch == 0x103f || (0x104c <= ch && ch <= 0x1055) ||
164  (0x105a <= ch && ch <= 0x105d) || ch == 0x1061 || ch == 0x1065 || ch == 0x1066 ||
165  (0x106e <= ch && ch <= 0x1070) || (0x1075 <= ch && ch <= 0x1081) || ch == 0x108e ||
166  (0xa9e0 <= ch && ch <= 0xa9e4) || (0xa9e7 <= ch && ch <= 0xa9ef) ||
167  (0xa9fa <= ch && ch <= 0xa9fe) || (0xaa60 <= ch && ch <= 0xaa6f) ||
168  (0xaa71 <= ch && ch <= 0xaa73) || ch == 0xaa7a || ch == 0xaa7e || ch == 0xaa7f;
169 }
170 
171 // Returns true if ch is a Myanmar digit or other symbol that does not take
172 // part in being a syllable eg. punctuation marks.
173 // MYANMAR DIGIT, MYANMAR SYMBOL, MYANMAR LOGOGRAM
174 // REDUPLICATION MARKS
175 /* static */
176 bool ValidateMyanmar::IsMyanmarOther(char32 ch) {
177  IcuErrorCode err;
178  UScriptCode script_code = uscript_getScript(ch, err);
179  if (script_code != USCRIPT_MYANMAR && ch != Validator::kZeroWidthJoiner &&
181  return true;
182  }
183  return (0x1040 <= ch && ch <= 0x104f) || (0x1090 <= ch && ch <= 0x1099) ||
184  (0x109e <= ch && ch <= 0x109f) || (0xa9f0 <= ch && ch <= 0xa9f9) ||
185  (ch == 0xa9e6 || ch == 0xaa70) || (0xaa74 <= ch && ch <= 0xaa79);
186 }
187 
188 } // namespace tesseract
#define ASSERT_HOST(x)
Definition: errcode.h:59
signed int char32
void tprintf(const char *format,...)
Definition: tprintf.cpp:41
signed int char32
Definition: unichar.h:51
bool ConsumeGraphemeIfValid() override
Validator::CharClass UnicodeToCharClass(char32 ch) const override
static const char32 kZeroWidthNonJoiner
Definition: validator.h:97
unsigned codes_used_
Definition: validator.h:231
bool UseMultiCode(unsigned length)
Definition: validator.h:189
std::vector< IndicPair > codes_
Definition: validator.h:225
static const char32 kZeroWidthJoiner
Definition: validator.h:98
static const char32 kMyanmarVirama
Definition: validator.h:216