tesseract  5.0.0
boxread.cpp
Go to the documentation of this file.
1 /**********************************************************************
2  * File: boxread.cpp
3  * Description: Read data from a box file.
4  * Author: Ray Smith
5  *
6  * (C) Copyright 2007, Google Inc.
7  ** Licensed under the Apache License, Version 2.0 (the "License");
8  ** you may not use this file except in compliance with the License.
9  ** You may obtain a copy of the License at
10  ** http://www.apache.org/licenses/LICENSE-2.0
11  ** Unless required by applicable law or agreed to in writing, software
12  ** distributed under the License is distributed on an "AS IS" BASIS,
13  ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  ** See the License for the specific language governing permissions and
15  ** limitations under the License.
16  *
17  **********************************************************************/
18 
19 #include "boxread.h"
20 
21 #include "errcode.h" // for ERRCODE, TESSEXIT
22 #include "fileerr.h" // for CANTOPENFILE
23 #include "rect.h" // for TBOX
24 #include "tprintf.h" // for tprintf
25 
26 #include <tesseract/unichar.h> // for UNICHAR
27 #include "helpers.h" // for chomp_string
28 
29 #include <climits> // for INT_MAX
30 #include <cstring> // for strchr, strcmp
31 #include <fstream> // for std::ifstream
32 #include <locale> // for std::locale::classic
33 #include <sstream> // for std::stringstream
34 #include <string> // for std::string
35 
36 namespace tesseract {
37 
38 // Special char code used to identify multi-blob labels.
39 static const char *kMultiBlobLabelCode = "WordStr";
40 
41 // Returns the box file name corresponding to the given image_filename.
42 static std::string BoxFileName(const char *image_filename) {
43  std::string box_filename = image_filename;
44  size_t length = box_filename.length();
45  std::string last = (length > 8) ? box_filename.substr(length - 8) : "";
46  if (last == ".bin.png" || last == ".nrm.png") {
47  box_filename.resize(length - 8);
48  } else {
49  size_t lastdot = box_filename.find_last_of('.');
50  if (lastdot < length) {
51  box_filename.resize(lastdot);
52  }
53  }
54  box_filename += ".box";
55  return box_filename;
56 }
57 
58 // Open the boxfile based on the given image filename.
59 FILE *OpenBoxFile(const char *fname) {
60  std::string filename = BoxFileName(fname);
61  FILE *box_file = nullptr;
62  if (!(box_file = fopen(filename.c_str(), "rb"))) {
63  CANTOPENFILE.error("read_next_box", TESSEXIT, "Can't open box file %s", filename.c_str());
64  }
65  return box_file;
66 }
67 
68 // Reads all boxes from the given filename.
69 // Reads a specific target_page number if >= 0, or all pages otherwise.
70 // Skips blanks if skip_blanks is true.
71 // The UTF-8 label of the box is put in texts, and the full box definition as
72 // a string is put in box_texts, with the corresponding page number in pages.
73 // Each of the output vectors is optional (may be nullptr).
74 // Returns false if no boxes are found.
75 bool ReadAllBoxes(int target_page, bool skip_blanks, const char *filename, std::vector<TBOX> *boxes,
76  std::vector<std::string> *texts, std::vector<std::string> *box_texts,
77  std::vector<int> *pages) {
78  std::ifstream input(BoxFileName(filename).c_str(), std::ios::in | std::ios::binary);
79  std::vector<char> box_data(std::istreambuf_iterator<char>(input), {});
80  if (box_data.empty()) {
81  return false;
82  }
83  // Convert the array of bytes to a string, so it can be used by the parser.
84  box_data.push_back('\0');
85  return ReadMemBoxes(target_page, skip_blanks, &box_data[0],
86  /*continue_on_failure*/ true, boxes, texts, box_texts, pages);
87 }
88 
89 // Reads all boxes from the string. Otherwise, as ReadAllBoxes.
90 bool ReadMemBoxes(int target_page, bool skip_blanks, const char *box_data, bool continue_on_failure,
91  std::vector<TBOX> *boxes, std::vector<std::string> *texts,
92  std::vector<std::string> *box_texts, std::vector<int> *pages) {
93  std::string box_str(box_data);
94  std::vector<std::string> lines = split(box_str, '\n');
95  if (lines.empty()) {
96  return false;
97  }
98  int num_boxes = 0;
99  for (auto &line : lines) {
100  int page = 0;
101  std::string utf8_str;
102  TBOX box;
103  if (!ParseBoxFileStr(line.c_str(), &page, utf8_str, &box)) {
104  if (continue_on_failure) {
105  continue;
106  } else {
107  return false;
108  }
109  }
110  if (skip_blanks && (utf8_str == " " || utf8_str == "\t")) {
111  continue;
112  }
113  if (target_page >= 0 && page != target_page) {
114  continue;
115  }
116  if (boxes != nullptr) {
117  boxes->push_back(box);
118  }
119  if (texts != nullptr) {
120  texts->push_back(utf8_str);
121  }
122  if (box_texts != nullptr) {
123  std::string full_text;
124  MakeBoxFileStr(utf8_str.c_str(), box, target_page, full_text);
125  box_texts->push_back(full_text);
126  }
127  if (pages != nullptr) {
128  pages->push_back(page);
129  }
130  ++num_boxes;
131  }
132  return num_boxes > 0;
133 }
134 
135 // TODO(rays) convert all uses of ReadNextBox to use the new ReadAllBoxes.
136 // Box files are used ONLY DURING TRAINING, but by both processes of
137 // creating tr files with tesseract, and unicharset_extractor.
138 // ReadNextBox factors out the code to interpret a line of a box
139 // file so that applybox and unicharset_extractor interpret the same way.
140 // This function returns the next valid box file utf8 string and coords
141 // and returns true, or false on eof (and closes the file).
142 // It ignores the utf8 file signature ByteOrderMark (U+FEFF=EF BB BF), checks
143 // for valid utf-8 and allows space or tab between fields.
144 // utf8_str is set with the unichar string, and bounding box with the box.
145 // If there are page numbers in the file, it reads them all.
146 bool ReadNextBox(int *line_number, FILE *box_file, std::string &utf8_str, TBOX *bounding_box) {
147  return ReadNextBox(-1, line_number, box_file, utf8_str, bounding_box);
148 }
149 
150 // As ReadNextBox above, but get a specific page number. (0-based)
151 // Use -1 to read any page number. Files without page number all
152 // read as if they are page 0.
153 bool ReadNextBox(int target_page, int *line_number, FILE *box_file, std::string &utf8_str,
154  TBOX *bounding_box) {
155  int page = 0;
156  char buff[kBoxReadBufSize]; // boxfile read buffer
157  char *buffptr = buff;
158 
159  while (fgets(buff, sizeof(buff) - 1, box_file)) {
160  (*line_number)++;
161 
162  buffptr = buff;
163  const auto *ubuf = reinterpret_cast<const unsigned char *>(buffptr);
164  if (ubuf[0] == 0xef && ubuf[1] == 0xbb && ubuf[2] == 0xbf) {
165  buffptr += 3; // Skip unicode file designation.
166  }
167  // Check for blank lines in box file
168  if (*buffptr == '\n' || *buffptr == '\0') {
169  continue;
170  }
171  // Skip blank boxes.
172  if (*buffptr == ' ' || *buffptr == '\t') {
173  continue;
174  }
175  if (*buffptr != '\0') {
176  if (!ParseBoxFileStr(buffptr, &page, utf8_str, bounding_box)) {
177  tprintf("Box file format error on line %i; ignored\n", *line_number);
178  continue;
179  }
180  if (target_page >= 0 && target_page != page) {
181  continue; // Not on the appropriate page.
182  }
183  return true; // Successfully read a box.
184  }
185  }
186  fclose(box_file);
187  return false; // EOF
188 }
189 
190 // Parses the given box file string into a page_number, utf8_str, and
191 // bounding_box. Returns true on a successful parse.
192 // The box file is assumed to contain box definitions, one per line, of the
193 // following format for blob-level boxes:
194 // <UTF8 str> <left> <bottom> <right> <top> <page id>
195 // and for word/line-level boxes:
196 // WordStr <left> <bottom> <right> <top> <page id> #<space-delimited word str>
197 // See applyybox.cpp for more information.
198 bool ParseBoxFileStr(const char *boxfile_str, int *page_number, std::string &utf8_str,
199  TBOX *bounding_box) {
200  *bounding_box = TBOX(); // Initialize it to empty.
201  utf8_str = "";
202  char uch[kBoxReadBufSize];
203  const char *buffptr = boxfile_str;
204  // Read the unichar without messing up on Tibetan.
205  // According to issue 253 the utf-8 surrogates 85 and A0 are treated
206  // as whitespace by sscanf, so it is more reliable to just find
207  // ascii space and tab.
208  int uch_len = 0;
209  // Skip unicode file designation, if present.
210  const auto *ubuf = reinterpret_cast<const unsigned char *>(buffptr);
211  if (ubuf[0] == 0xef && ubuf[1] == 0xbb && ubuf[2] == 0xbf) {
212  buffptr += 3;
213  }
214  // Allow a single blank as the UTF-8 string. Check for empty string and
215  // then blindly eat the first character.
216  if (*buffptr == '\0') {
217  return false;
218  }
219  do {
220  uch[uch_len++] = *buffptr++;
221  } while (*buffptr != '\0' && *buffptr != ' ' && *buffptr != '\t' &&
222  uch_len < kBoxReadBufSize - 1);
223  uch[uch_len] = '\0';
224  if (*buffptr != '\0') {
225  ++buffptr;
226  }
227  int x_min = INT_MAX;
228  int y_min = INT_MAX;
229  int x_max = INT_MIN;
230  int y_max = INT_MIN;
231  *page_number = 0;
232  std::stringstream stream(buffptr);
233  stream.imbue(std::locale::classic());
234  stream >> x_min;
235  stream >> y_min;
236  stream >> x_max;
237  stream >> y_max;
238  stream >> *page_number;
239  if (x_max < x_min || y_max < y_min) {
240  tprintf("Bad box coordinates in boxfile string! %s\n", ubuf);
241  return false;
242  }
243  // Test for long space-delimited string label.
244  if (strcmp(uch, kMultiBlobLabelCode) == 0 && (buffptr = strchr(buffptr, '#')) != nullptr) {
245  strncpy(uch, buffptr + 1, kBoxReadBufSize - 1);
246  uch[kBoxReadBufSize - 1] = '\0'; // Prevent buffer overrun.
247  chomp_string(uch);
248  uch_len = strlen(uch);
249  }
250  // Validate UTF8 by making unichars with it.
251  int used = 0;
252  while (used < uch_len) {
253  tesseract::UNICHAR ch(uch + used, uch_len - used);
254  int new_used = ch.utf8_len();
255  if (new_used == 0) {
256  tprintf("Bad UTF-8 str %s starts with 0x%02x at col %d\n", uch + used, uch[used], used + 1);
257  return false;
258  }
259  used += new_used;
260  }
261  utf8_str = uch;
262  if (x_min > x_max) {
263  std::swap(x_min, x_max);
264  }
265  if (y_min > y_max) {
266  std::swap(y_min, y_max);
267  }
268  bounding_box->set_to_given_coords(x_min, y_min, x_max, y_max);
269  return true; // Successfully read a box.
270 }
271 
272 // Creates a box file string from a unichar string, TBOX and page number.
273 void MakeBoxFileStr(const char *unichar_str, const TBOX &box, int page_num, std::string &box_str) {
274  box_str = unichar_str;
275  box_str += " " + std::to_string(box.left());
276  box_str += " " + std::to_string(box.bottom());
277  box_str += " " + std::to_string(box.right());
278  box_str += " " + std::to_string(box.top());
279  box_str += " " + std::to_string(page_num);
280 }
281 
282 } // namespace tesseract
@ TBOX
const std::vector< std::string > split(const std::string &s, char c)
Definition: helpers.h:41
LIST last(LIST var_list)
Definition: oldlist.cpp:153
bool ReadMemBoxes(int target_page, bool skip_blanks, const char *box_data, bool continue_on_failure, std::vector< TBOX > *boxes, std::vector< std::string > *texts, std::vector< std::string > *box_texts, std::vector< int > *pages)
Definition: boxread.cpp:90
bool ParseBoxFileStr(const char *boxfile_str, int *page_number, std::string &utf8_str, TBOX *bounding_box)
Definition: boxread.cpp:198
void tprintf(const char *format,...)
Definition: tprintf.cpp:41
void MakeBoxFileStr(const char *unichar_str, const TBOX &box, int page_num, std::string &box_str)
Definition: boxread.cpp:273
void chomp_string(char *str)
Definition: helpers.h:89
bool ReadAllBoxes(int target_page, bool skip_blanks, const char *filename, std::vector< TBOX > *boxes, std::vector< std::string > *texts, std::vector< std::string > *box_texts, std::vector< int > *pages)
Definition: boxread.cpp:75
const int kBoxReadBufSize
Definition: boxread.h:33
FILE * OpenBoxFile(const char *fname)
Definition: boxread.cpp:59
constexpr ERRCODE CANTOPENFILE("Can't open file")
@ TESSEXIT
Definition: errcode.h:30
bool ReadNextBox(int *line_number, FILE *box_file, std::string &utf8_str, TBOX *bounding_box)
Definition: boxread.cpp:146
int utf8_len() const
Definition: unichar.h:77
TDimension left() const
Definition: rect.h:82
void set_to_given_coords(int x_min, int y_min, int x_max, int y_max)
Definition: rect.h:282
TDimension top() const
Definition: rect.h:68
TDimension right() const
Definition: rect.h:89
TDimension bottom() const
Definition: rect.h:75
void error(const char *caller, TessErrorLogCode action, const char *format,...) const __attribute__((format(printf
Definition: errcode.cpp:38