tesseract  5.0.0
lstmtester.cpp
Go to the documentation of this file.
1 // File: lstmtester.cpp
3 // Description: Top-level line evaluation class for LSTM-based networks.
4 // Author: Ray Smith
5 //
6 // (C) Copyright 2016, Google Inc.
7 // Licensed under the Apache License, Version 2.0 (the "License");
8 // you may not use this file except in compliance with the License.
9 // You may obtain a copy of the License at
10 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
17 
18 #include "lstmtester.h"
19 #include <thread> // for std::thread
20 #include "fileio.h" // for LoadFileLinesToStrings
21 
22 namespace tesseract {
23 
24 LSTMTester::LSTMTester(int64_t max_memory) : test_data_(max_memory) {}
25 
26 // Loads a set of lstmf files that were created using the lstm.train config to
27 // tesseract into memory ready for testing. Returns false if nothing was
28 // loaded. The arg is a filename of a file that lists the filenames.
29 bool LSTMTester::LoadAllEvalData(const char *filenames_file) {
30  std::vector<std::string> filenames;
31  if (!LoadFileLinesToStrings(filenames_file, &filenames)) {
32  tprintf("Failed to load list of eval filenames from %s\n", filenames_file);
33  return false;
34  }
35  return LoadAllEvalData(filenames);
36 }
37 
38 // Loads a set of lstmf files that were created using the lstm.train config to
39 // tesseract into memory ready for testing. Returns false if nothing was
40 // loaded.
41 bool LSTMTester::LoadAllEvalData(const std::vector<std::string> &filenames) {
42  test_data_.Clear();
43  bool result = test_data_.LoadDocuments(filenames, CS_SEQUENTIAL, nullptr);
44  total_pages_ = test_data_.TotalPages();
45  return result;
46 }
47 
48 // Runs an evaluation asynchronously on the stored data and returns a string
49 // describing the results of the previous test.
50 std::string LSTMTester::RunEvalAsync(int iteration, const double *training_errors,
51  const TessdataManager &model_mgr, int training_stage) {
52  std::string result;
53  if (total_pages_ == 0) {
54  result += "No test data at iteration " + std::to_string(iteration);
55  return result;
56  }
57  if (!LockIfNotRunning()) {
58  result += "Previous test incomplete, skipping test at iteration " + std::to_string(iteration);
59  return result;
60  }
61  // Save the args.
62  std::string prev_result = test_result_;
63  test_result_ = "";
64  if (training_errors != nullptr) {
65  test_iteration_ = iteration;
66  test_training_errors_ = training_errors;
67  test_model_mgr_ = model_mgr;
68  test_training_stage_ = training_stage;
69  std::thread t(&LSTMTester::ThreadFunc, this);
70  t.detach();
71  } else {
72  UnlockRunning();
73  }
74  return prev_result;
75 }
76 
77 // Runs an evaluation synchronously on the stored data and returns a string
78 // describing the results.
79 std::string LSTMTester::RunEvalSync(int iteration, const double *training_errors,
80  const TessdataManager &model_mgr, int training_stage,
81  int verbosity) {
82  LSTMTrainer trainer;
83  trainer.InitCharSet(model_mgr);
84  TFile fp;
85  if (!model_mgr.GetComponent(TESSDATA_LSTM, &fp) || !trainer.DeSerialize(&model_mgr, &fp)) {
86  return "Deserialize failed";
87  }
88  int eval_iteration = 0;
89  double char_error = 0.0;
90  double word_error = 0.0;
91  int error_count = 0;
92  while (error_count < total_pages_) {
93  const ImageData *trainingdata = test_data_.GetPageBySerial(eval_iteration);
94  trainer.SetIteration(++eval_iteration);
95  NetworkIO fwd_outputs, targets;
96  Trainability result = trainer.PrepareForBackward(trainingdata, &fwd_outputs, &targets);
97  if (result != UNENCODABLE) {
98  char_error += trainer.NewSingleError(tesseract::ET_CHAR_ERROR);
99  word_error += trainer.NewSingleError(tesseract::ET_WORD_RECERR);
100  ++error_count;
101  if (verbosity > 1 || (verbosity > 0 && result != PERFECT)) {
102  tprintf("Truth:%s\n", trainingdata->transcription().c_str());
103  std::vector<int> ocr_labels;
104  std::vector<int> xcoords;
105  trainer.LabelsFromOutputs(fwd_outputs, &ocr_labels, &xcoords);
106  std::string ocr_text = trainer.DecodeLabels(ocr_labels);
107  tprintf("OCR :%s\n", ocr_text.c_str());
108  if (verbosity > 2 || (verbosity > 1 && result != PERFECT)) {
109  tprintf("Line BCER=%f, BWER=%f\n\n",
112  }
113  }
114  }
115  }
116  char_error *= 100.0 / total_pages_;
117  word_error *= 100.0 / total_pages_;
118  std::string result;
119  if (iteration != 0 || training_stage != 0) {
120  result += "At iteration " + std::to_string(iteration);
121  result += ", stage " + std::to_string(training_stage) + ", ";
122  }
123  result += "BCER eval=" + std::to_string(char_error);
124  result += ", BWER eval=" + std::to_string(word_error);
125  return result;
126 }
127 
128 // Helper thread function for RunEvalAsync.
129 // LockIfNotRunning must have returned true before calling ThreadFunc, and
130 // it will call UnlockRunning to release the lock after RunEvalSync completes.
131 void LSTMTester::ThreadFunc() {
132  test_result_ =
133  RunEvalSync(test_iteration_, test_training_errors_, test_model_mgr_, test_training_stage_,
134  /*verbosity*/ 0);
135  UnlockRunning();
136 }
137 
138 // Returns true if there is currently nothing running, and takes the lock
139 // if there is nothing running.
140 bool LSTMTester::LockIfNotRunning() {
141  std::lock_guard<std::mutex> lock(running_mutex_);
142  if (async_running_) {
143  return false;
144  }
145  async_running_ = true;
146  return true;
147 }
148 
149 // Releases the running lock.
150 void LSTMTester::UnlockRunning() {
151  std::lock_guard<std::mutex> lock(running_mutex_);
152  async_running_ = false;
153 }
154 
155 } // namespace tesseract
@ ET_WORD_RECERR
Definition: lstmtrainer.h:43
@ ET_CHAR_ERROR
Definition: lstmtrainer.h:44
void tprintf(const char *format,...)
Definition: tprintf.cpp:41
@ CS_SEQUENTIAL
Definition: imagedata.h:49
bool LoadFileLinesToStrings(const char *filename, std::vector< std::string > *lines)
Definition: fileio.h:32
const std::string & transcription() const
Definition: imagedata.h:104
TESS_API bool LoadDocuments(const std::vector< std::string > &filenames, CachingStrategy cache_strategy, FileReader reader)
Definition: imagedata.cpp:614
const ImageData * GetPageBySerial(int serial)
Definition: imagedata.h:317
TESS_API int TotalPages()
Definition: imagedata.cpp:659
bool GetComponent(TessdataType type, TFile *fp)
std::string DecodeLabels(const std::vector< int > &labels)
void LabelsFromOutputs(const NetworkIO &outputs, std::vector< int > *labels, std::vector< int > *xcoords)
void SetIteration(int iteration)
std::string RunEvalAsync(int iteration, const double *training_errors, const TessdataManager &model_mgr, int training_stage)
Definition: lstmtester.cpp:50
std::string RunEvalSync(int iteration, const double *training_errors, const TessdataManager &model_mgr, int training_stage, int verbosity)
Definition: lstmtester.cpp:79
LSTMTester(int64_t max_memory)
Definition: lstmtester.cpp:24
bool LoadAllEvalData(const char *filenames_file)
Definition: lstmtester.cpp:29
Trainability PrepareForBackward(const ImageData *trainingdata, NetworkIO *fwd_outputs, NetworkIO *targets)
bool InitCharSet(const std::string &traineddata_path)
Definition: lstmtrainer.h:99
double NewSingleError(ErrorTypes type) const
Definition: lstmtrainer.h:157
bool DeSerialize(const TessdataManager *mgr, TFile *fp)