tesseract  5.0.0
normstrngs.cpp
Go to the documentation of this file.
1 /**********************************************************************
2  * File: normstrngs.cpp
3  * Description: Utilities to normalize and manipulate UTF-32 and
4  * UTF-8 strings.
5  * Author: Ranjith Unnikrishnan
6  * Created: Thu July 4 2013
7  *
8  * (C) Copyright 2013, Google Inc.
9  * Licensed under the Apache License, Version 2.0 (the "License");
10  * you may not use this file except in compliance with the License.
11  * You may obtain a copy of the License at
12  * http://www.apache.org/licenses/LICENSE-2.0
13  * Unless required by applicable law or agreed to in writing, software
14  * distributed under the License is distributed on an "AS IS" BASIS,
15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16  * See the License for the specific language governing permissions and
17  * limitations under the License.
18  *
19  **********************************************************************/
20 
21 #include "normstrngs.h"
22 
23 #include <string>
24 #include <unordered_map>
25 #include <vector>
26 
27 #include <tesseract/unichar.h>
28 #include "errcode.h"
29 #include "icuerrorcode.h"
30 #include "unicode/normalizer2.h" // From libicu
31 #include "unicode/translit.h" // From libicu
32 #include "unicode/uchar.h" // From libicu
33 #include "unicode/unorm2.h" // From libicu
34 #include "unicode/uscript.h" // From libicu
35 
36 namespace tesseract {
37 
38 static bool is_hyphen_punc(const char32 ch) {
39  static const int kNumHyphenPuncUnicodes = 13;
40  static const char32 kHyphenPuncUnicodes[kNumHyphenPuncUnicodes] = {
41  '-', 0x2010, 0x2011, 0x2012, 0x2013, 0x2014, 0x2015, // hyphen..horizontal bar
42  0x207b, // superscript minus
43  0x208b, // subscript minus
44  0x2212, // minus sign
45  0xfe58, // small em dash
46  0xfe63, // small hyphen-minus
47  0xff0d, // fullwidth hyphen-minus
48  };
49  for (int kHyphenPuncUnicode : kHyphenPuncUnicodes) {
50  if (kHyphenPuncUnicode == ch) {
51  return true;
52  }
53  }
54  return false;
55 }
56 
57 static bool is_single_quote(const char32 ch) {
58  static const int kNumSingleQuoteUnicodes = 8;
59  static const char32 kSingleQuoteUnicodes[kNumSingleQuoteUnicodes] = {
60  '\'', '`',
61  0x2018, // left single quotation mark (English, others)
62  0x2019, // right single quotation mark (Danish, Finnish, Swedish, Norw.)
63  // We may have to introduce a comma set with 0x201a
64  0x201B, // single high-reveresed-9 quotation mark (PropList.txt)
65  0x2032, // prime
66  0x300C, // left corner bracket (East Asian languages)
67  0xFF07, // fullwidth apostrophe
68  };
69  for (int kSingleQuoteUnicode : kSingleQuoteUnicodes) {
70  if (kSingleQuoteUnicode == ch) {
71  return true;
72  }
73  }
74  return false;
75 }
76 
77 static bool is_double_quote(const char32 ch) {
78  static const int kNumDoubleQuoteUnicodes = 8;
79  static const char32 kDoubleQuoteUnicodes[kNumDoubleQuoteUnicodes] = {
80  '"',
81  0x201C, // left double quotation mark (English, others)
82  0x201D, // right double quotation mark (Danish, Finnish, Swedish, Norw.)
83  0x201F, // double high-reversed-9 quotation mark (PropList.txt)
84  0x2033, // double prime
85  0x301D, // reversed double prime quotation mark (East Asian langs,
86  // horiz.)
87  0x301E, // close double prime (East Asian languages written horizontally)
88  0xFF02, // fullwidth quotation mark
89  };
90  for (int kDoubleQuoteUnicode : kDoubleQuoteUnicodes) {
91  if (kDoubleQuoteUnicode == ch) {
92  return true;
93  }
94  }
95  return false;
96 }
97 
98 // Helper runs a standard unicode normalization, optional OCR normalization,
99 // and leaves the result as char32 for subsequent processing.
100 static void NormalizeUTF8ToUTF32(UnicodeNormMode u_mode, OCRNorm ocr_normalize, const char *str8,
101  std::vector<char32> *normed32) {
102  // Convert to ICU string for unicode normalization.
103  icu::UnicodeString uch_str(str8, "UTF-8");
104  IcuErrorCode error_code;
105  // Convert the enum to the new weird icu representation.
106  const char *norm_type =
107  u_mode == UnicodeNormMode::kNFKD || u_mode == UnicodeNormMode::kNFKC ? "nfkc" : "nfc";
108  UNormalization2Mode compose = u_mode == UnicodeNormMode::kNFC || u_mode == UnicodeNormMode::kNFKC
109  ? UNORM2_COMPOSE
110  : UNORM2_DECOMPOSE;
111  // Pointer to singleton does not require deletion.
112  const icu::Normalizer2 *normalizer =
113  icu::Normalizer2::getInstance(nullptr, norm_type, compose, error_code);
114  error_code.assertSuccess();
115  error_code.reset();
116  icu::UnicodeString norm_str = normalizer->normalize(uch_str, error_code);
117  error_code.assertSuccess();
118  // Convert to char32 for output. OCR normalization if required.
119  normed32->reserve(norm_str.length()); // An approximation.
120  for (int offset = 0; offset < norm_str.length(); offset = norm_str.moveIndex32(offset, 1)) {
121  char32 ch = norm_str.char32At(offset);
122  // Skip all ZWS, RTL and LTR marks.
123  if (Validator::IsZeroWidthMark(ch)) {
124  continue;
125  }
126  if (ocr_normalize == OCRNorm::kNormalize) {
127  ch = OCRNormalize(ch);
128  }
129  normed32->push_back(ch);
130  }
131 }
132 
133 // Helper removes joiners from strings that contain no letters.
134 static void StripJoiners(std::vector<char32> *str32) {
135  for (char32 ch : *str32) {
136  if (u_isalpha(ch)) {
137  return;
138  }
139  }
140  int len = 0;
141  for (char32 ch : *str32) {
143  (*str32)[len++] = ch;
144  }
145  }
146  str32->resize(len);
147 }
148 
149 // Normalizes a UTF8 string according to the given modes. Returns true on
150 // success. If false is returned, some failure or invalidity was present, and
151 // the result string is produced on a "best effort" basis.
152 bool NormalizeUTF8String(UnicodeNormMode u_mode, OCRNorm ocr_normalize,
153  GraphemeNorm grapheme_normalize, const char *str8,
154  std::string *normalized) {
155  std::vector<char32> normed32;
156  NormalizeUTF8ToUTF32(u_mode, ocr_normalize, str8, &normed32);
157  if (grapheme_normalize == GraphemeNorm::kNormalize) {
158  StripJoiners(&normed32);
159  std::vector<std::vector<char32>> graphemes;
161  normed32, &graphemes);
162  if (graphemes.empty() || graphemes[0].empty()) {
163  success = false;
164  } else if (normalized != nullptr) {
165  *normalized = UNICHAR::UTF32ToUTF8(graphemes[0]);
166  }
167  return success;
168  }
169  if (normalized != nullptr) {
170  *normalized = UNICHAR::UTF32ToUTF8(normed32);
171  }
172  return true;
173 }
174 
175 // Normalizes a UTF8 string according to the given modes and splits into
176 // graphemes according to g_mode. Returns true on success. If false is returned,
177 // some failure or invalidity was present, and the result string is produced on
178 // a "best effort" basis.
180  GraphemeNormMode g_mode, bool report_errors, const char *str8,
181  std::vector<std::string> *graphemes) {
182  std::vector<char32> normed32;
183  NormalizeUTF8ToUTF32(u_mode, ocr_normalize, str8, &normed32);
184  StripJoiners(&normed32);
185  std::vector<std::vector<char32>> graphemes32;
186  bool success = Validator::ValidateCleanAndSegment(g_mode, report_errors, normed32, &graphemes32);
187  if (g_mode != GraphemeNormMode::kSingleString && success) {
188  // If we modified the string to clean it up, the segmentation may not be
189  // correct, so check for changes and do it again.
190  std::vector<char32> cleaned32;
191  for (const auto &g : graphemes32) {
192  cleaned32.insert(cleaned32.end(), g.begin(), g.end());
193  }
194  if (cleaned32 != normed32) {
195  graphemes32.clear();
196  success = Validator::ValidateCleanAndSegment(g_mode, report_errors, cleaned32, &graphemes32);
197  }
198  }
199  graphemes->clear();
200  graphemes->reserve(graphemes32.size());
201  for (const auto &grapheme : graphemes32) {
202  graphemes->push_back(UNICHAR::UTF32ToUTF8(grapheme));
203  }
204  return success;
205 }
206 
207 // Apply just the OCR-specific normalizations and return the normalized char.
209  if (is_hyphen_punc(ch)) {
210  return '-';
211  } else if (is_single_quote(ch)) {
212  return '\'';
213  } else if (is_double_quote(ch)) {
214  return '"';
215  }
216  return ch;
217 }
218 
219 bool IsOCREquivalent(char32 ch1, char32 ch2) {
220  return OCRNormalize(ch1) == OCRNormalize(ch2);
221 }
222 
223 bool IsValidCodepoint(const char32 ch) {
224  // In the range [0, 0xD800) or [0xE000, 0x10FFFF]
225  return (static_cast<uint32_t>(ch) < 0xD800) || (ch >= 0xE000 && ch <= 0x10FFFF);
226 }
227 
228 bool IsWhitespace(const char32 ch) {
229  ASSERT_HOST_MSG(IsValidCodepoint(ch), "Invalid Unicode codepoint: 0x%x\n", ch);
230  return u_isUWhiteSpace(static_cast<UChar32>(ch));
231 }
232 
233 bool IsUTF8Whitespace(const char *text) {
234  return SpanUTF8Whitespace(text) == strlen(text);
235 }
236 
237 unsigned int SpanUTF8Whitespace(const char *text) {
238  int n_white = 0;
239  for (UNICHAR::const_iterator it = UNICHAR::begin(text, strlen(text));
240  it != UNICHAR::end(text, strlen(text)); ++it) {
241  if (!IsWhitespace(*it)) {
242  break;
243  }
244  n_white += it.utf8_len();
245  }
246  return n_white;
247 }
248 
249 unsigned int SpanUTF8NotWhitespace(const char *text) {
250  int n_notwhite = 0;
251  for (UNICHAR::const_iterator it = UNICHAR::begin(text, strlen(text));
252  it != UNICHAR::end(text, strlen(text)); ++it) {
253  if (IsWhitespace(*it)) {
254  break;
255  }
256  n_notwhite += it.utf8_len();
257  }
258  return n_notwhite;
259 }
260 
261 bool IsInterchangeValid(const char32 ch) {
262  return IsValidCodepoint(ch) && !(ch >= 0xFDD0 && ch <= 0xFDEF) && // Noncharacters.
263  !(ch >= 0xFFFE && ch <= 0xFFFF) && !(ch >= 0x1FFFE && ch <= 0x1FFFF) &&
264  !(ch >= 0x2FFFE && ch <= 0x2FFFF) && !(ch >= 0x3FFFE && ch <= 0x3FFFF) &&
265  !(ch >= 0x4FFFE && ch <= 0x4FFFF) && !(ch >= 0x5FFFE && ch <= 0x5FFFF) &&
266  !(ch >= 0x6FFFE && ch <= 0x6FFFF) && !(ch >= 0x7FFFE && ch <= 0x7FFFF) &&
267  !(ch >= 0x8FFFE && ch <= 0x8FFFF) && !(ch >= 0x9FFFE && ch <= 0x9FFFF) &&
268  !(ch >= 0xAFFFE && ch <= 0xAFFFF) && !(ch >= 0xBFFFE && ch <= 0xBFFFF) &&
269  !(ch >= 0xCFFFE && ch <= 0xCFFFF) && !(ch >= 0xDFFFE && ch <= 0xDFFFF) &&
270  !(ch >= 0xEFFFE && ch <= 0xEFFFF) && !(ch >= 0xFFFFE && ch <= 0xFFFFF) &&
271  !(ch >= 0x10FFFE && ch <= 0x10FFFF) &&
272  (!u_isISOControl(static_cast<UChar32>(ch)) || ch == '\n' || ch == '\f' || ch == '\t' ||
273  ch == '\r');
274 }
275 
277  return IsValidCodepoint(ch) && ch <= 128 &&
278  (!u_isISOControl(static_cast<UChar32>(ch)) || ch == '\n' || ch == '\f' || ch == '\t' ||
279  ch == '\r');
280 }
281 
283  // Return unchanged if not in the fullwidth-halfwidth Unicode block.
284  if (ch < 0xFF00 || ch > 0xFFEF || !IsValidCodepoint(ch)) {
285  if (ch != 0x3000) {
286  return ch;
287  }
288  }
289  // Special case for fullwidth left and right "white parentheses".
290  if (ch == 0xFF5F) {
291  return 0x2985;
292  }
293  if (ch == 0xFF60) {
294  return 0x2986;
295  }
296  // Construct a full-to-half width transliterator.
297  IcuErrorCode error_code;
298  icu::UnicodeString uch_str(static_cast<UChar32>(ch));
299  const icu::Transliterator *fulltohalf =
300  icu::Transliterator::createInstance("Fullwidth-Halfwidth", UTRANS_FORWARD, error_code);
301  error_code.assertSuccess();
302  error_code.reset();
303 
304  fulltohalf->transliterate(uch_str);
305  delete fulltohalf;
306  ASSERT_HOST(uch_str.length() != 0);
307  return uch_str[0];
308 }
309 
310 } // namespace tesseract
#define ASSERT_HOST(x)
Definition: errcode.h:59
#define ASSERT_HOST_MSG(x,...)
Definition: errcode.h:62
signed int char32
bool IsOCREquivalent(char32 ch1, char32 ch2)
Definition: normstrngs.cpp:219
GraphemeNormMode
Definition: validator.h:36
bool IsWhitespace(const char32 ch)
Definition: normstrngs.cpp:228
char32 OCRNormalize(char32 ch)
Definition: normstrngs.cpp:208
unsigned int SpanUTF8Whitespace(const char *text)
Definition: normstrngs.cpp:237
signed int char32
Definition: unichar.h:51
bool IsInterchangeValid(const char32 ch)
Definition: normstrngs.cpp:261
bool NormalizeCleanAndSegmentUTF8(UnicodeNormMode u_mode, OCRNorm ocr_normalize, GraphemeNormMode g_mode, bool report_errors, const char *str8, std::vector< std::string > *graphemes)
Definition: normstrngs.cpp:179
bool NormalizeUTF8String(UnicodeNormMode u_mode, OCRNorm ocr_normalize, GraphemeNorm grapheme_normalize, const char *str8, std::string *normalized)
Definition: normstrngs.cpp:152
bool IsInterchangeValid7BitAscii(const char32 ch)
Definition: normstrngs.cpp:276
char32 FullwidthToHalfwidth(const char32 ch)
Definition: normstrngs.cpp:282
unsigned int SpanUTF8NotWhitespace(const char *text)
Definition: normstrngs.cpp:249
bool IsValidCodepoint(const char32 ch)
Definition: normstrngs.cpp:223
bool IsUTF8Whitespace(const char *text)
Definition: normstrngs.cpp:233
static const_iterator begin(const char *utf8_str, int byte_length)
Definition: unichar.cpp:209
static std::string UTF32ToUTF8(const std::vector< char32 > &str32)
Definition: unichar.cpp:237
static const_iterator end(const char *utf8_str, int byte_length)
Definition: unichar.cpp:213
static const char32 kZeroWidthNonJoiner
Definition: validator.h:97
static bool IsZeroWidthMark(char32 ch)
Definition: validator.h:89
static const char32 kZeroWidthJoiner
Definition: validator.h:98
static bool ValidateCleanAndSegment(GraphemeNormMode g_mode, bool report_errors, const std::vector< char32 > &src, std::vector< std::vector< char32 >> *dest)
Definition: validator.cpp:40