API
logdump.hpp
Go to the documentation of this file.
1 /** \file logdump.hpp
2  * \brief A simple utility to dump MagAO-X binary logs to stdout.
3  *
4  * \ingroup logdump_files
5  */
6 
7 #ifndef logdump_hpp
8 #define logdump_hpp
9 
10 #include <iostream>
11 #include <cstring>
12 
13 #include <mx/ioutils/fileUtils.hpp>
14 
15 #include "../../libMagAOX/libMagAOX.hpp"
16 using namespace MagAOX::logger;
17 
18 using namespace flatlogs;
19 
20 /** \defgroup logdump logdump: MagAO-X Log Reader
21  * \brief Read a MagAO-X binary log file.
22  *
23  * <a href="../handbook/utils/logdump.html">Utility Documentation</a>
24  *
25  * \ingroup utils
26  *
27  */
28 
29 /** \defgroup logdump_files logdump Files
30  * \ingroup logdump
31  */
32 
33 /// An application to dump MagAo-X binary logs to the terminal.
34 /** \todo document this
35  * \todo add config for colors, both on/off and options to change.
36  *
37  * \ingroup logdump
38  */
39 class logdump : public mx::app::application
40 {
41 protected:
42 
43  std::string m_dir;
44  std::string m_ext;
45  std::string m_file;
46 
47  bool m_time {false};
48  bool m_jsonMode {false};
49 
50  unsigned long m_pauseTime {250}; ///When following, pause time to check for new data. msec. Default is 250 msec.
51  int m_fileCheckInterval {4}; ///When following, number of loops to wait before checking for a new file. Default is 4.
52 
53  std::vector<std::string> m_prefixes;
54 
55  size_t m_nfiles {0}; ///Number of files to dump. Default is 0, unless following then the default is 1.
56 
57  bool m_follow {false};
58 
60 
61  std::vector<eventCodeT> m_codes;
62 
63  void printLogBuff( const logPrioT & lvl,
64  const eventCodeT & ec,
65  const msgLenT & len,
66  bufferPtrT & logBuff
67  );
68  void printLogJson( const msgLenT & len,
69  bufferPtrT & logBuff
70  );
71 
72 public:
73  virtual void setupConfig();
74 
75  virtual void loadConfig();
76 
77  virtual int execute();
78 
79  virtual int gettimes(std::vector<std::string> & logs);
80 
81 };
82 
84 {
85  config.add("pauseTime","p", "pauseTime" , argType::Required, "", "pauseTime", false, "int", "When following, time in milliseconds to pause before checking for new entries.");
86  config.add("fileCheckInterval","", "fileCheckInterval" , argType::Required, "", "fileCheckInterval", false, "int", "When following, number of pause intervals between checks for new files.");
87 
88  config.add("dir","d", "dir" , argType::Required, "", "dir", false, "string", "Directory to search for logs. MagAO-X default is normally used.");
89  config.add("ext","e", "ext" , argType::Required, "", "ext", false, "string", "The file extension of log files. MagAO-X default is normally used.");
90  config.add("nfiles","n", "nfiles" , argType::Required, "", "nfiles", false, "int", "Number of log files to dump. If 0, then all matching files dumped. Default: 0, 1 if following.");
91  config.add("follow","f", "follow" , argType::True, "", "follow", false, "bool", "Follow the log, printing new entries as they appear.");
92  config.add("level","L", "level" , argType::Required, "", "level", false, "int/string", "Minimum log level to dump, either an integer or a string. -1/TELEMETRY [the default], 0/DEFAULT, 1/D1/DBG1/DEBUG2, 2/D2/DBG2/DEBUG1,3/INFO,4/WARNING,5/ERROR,6/CRITICAL,7/FATAL. Note that only the mininum unique string is required.");
93  config.add("code","C", "code" , argType::Required, "", "code", false, "int", "The event code, or vector of codes, to dump. If not specified, all codes are dumped. See logCodes.hpp for a complete list of codes.");
94  config.add("file","F", "file" , argType::Required, "", "file", false, "string", "A single file to process. If no / are found in name it will look in the specified directory (or MagAO-X default).");
95  config.add("time","T", "time" , argType::True, "", "time", false, "bool", "time span mode: prints the ISO 8601 UTC timestamps of the first and last entry, the elapsed time in seconds, and the number of records in the file as a space-delimited string");
96  config.add("json","J", "json" , argType::True, "", "json", false, "bool", "JSON mode: emits one JSON document per line for each record in the log");
97 
98 
99 }
100 
102 {
103  config(m_pauseTime, "pauseTime");
104  config(m_fileCheckInterval, "fileCheckInterval");
105 
106  //Get default log dir
107  std::string tmpstr = mx::sys::getEnv(MAGAOX_env_path);
108  if(tmpstr == "")
109  {
110  tmpstr = MAGAOX_path;
111  }
112  m_dir = tmpstr + "/" + MAGAOX_logRelPath;;
113 
114  //Now check for config option for dir
115  config(m_dir, "dir");
116 
117  m_ext = ".";
118  m_ext += MAGAOX_default_logExt;
119  config(m_ext, "ext");
120  ///\todo need to check for lack of "." and error or fix
121 
122  config(m_file, "file");
123 
124  if(m_file == "" && config.nonOptions.size() < 1)
125  {
126  std::cerr << "logdump: need application name. Try logdump -h for help.\n";
127  }
128 
129  if(m_file == "" && config.nonOptions.size() > 1)
130  {
131  std::cerr << "logdump: only one application at a time supported. Try logdump -h for help.\n";
132  }
133 
134  m_prefixes.resize(config.nonOptions.size());
135  for(size_t i=0;i<config.nonOptions.size(); ++i)
136  {
137  m_prefixes[i] = config.nonOptions[i];
138  }
139 
140  if(config.isSet("time")) m_time = true;
141  if(config.isSet("json")) m_jsonMode = true;
142 
143  config(m_follow, "follow");
144 
145  if(m_follow) m_nfiles = 1; //default to 1 if follow is set.
146  config(m_nfiles, "nfiles");
147 
148  tmpstr = "";
149  config(tmpstr, "level");
150  if(tmpstr != "")
151  {
152  m_level = logLevelFromString(tmpstr);
153  }
154 
155  config(m_codes, "code");
156 
157  std::cerr << m_codes.size() << "\n";
158 }
159 
161 {
162 
163  if(m_file == "" && m_prefixes.size() !=1 ) return -1; //error message will have been printed in loadConfig.
164 
165  std::vector<std::string> logs;
166 
167  if(m_file != "")
168  {
169  if(m_file.find('/') == std::string::npos)
170  {
171  m_file = m_dir + '/' + m_file;
172  }
173  std::cerr << "m_file: " << m_file << "\n";
174 
175  logs.push_back(m_file);
176  }
177  else
178  {
179  logs = mx::ioutils::getFileNames( m_dir, m_prefixes[0], "", m_ext);
180  }
181 
182  ///\todo if follow is set, then should nfiles default to 1 unless explicitly set?
183  if(m_nfiles == 0)
184  {
185  m_nfiles = logs.size();
186  }
187 
188  if(m_nfiles > logs.size()) m_nfiles = logs.size();
189 
190  if(m_time)
191  {
192  return gettimes(logs);
193  }
194 
195  bool firstRun = true; //for only showing latest entries on first run when following.
196 
197  for(size_t i=logs.size() - m_nfiles; i < logs.size(); ++i)
198  {
199  std::string fname = logs[i];
200  FILE * fin;
201 
202  bufferPtrT head(new char[logHeader::maxHeadSize]);
203 
204  bufferPtrT logBuff;
205 
206  fin = fopen(fname.c_str(), "rb");
207 
208  //--> get size here!!
209  off_t finSize = mx::ioutils::fileSize( fileno(fin) );
210 
211  std::cerr << fname << "\n";
212 
213  off_t totNrd = 0;
214 
215  size_t buffSz = 0;
216  while(!feof(fin)) //<--This should be an exit condition controlled by loop logic, not feof.
217  {
218  int nrd;
219 
220  ///\todo check for errors on all reads . . .
221 
222  //Read next header
223  nrd = fread( head.get(), sizeof(char), logHeader::minHeadSize, fin);
224  if(nrd == 0)
225  {
226  //If we're following and on the last log file, wait for more to show up.
227  if( m_follow == true && i == logs.size()-1)
228  {
229  int check = 0;
230  firstRun = false; //from now on we show all logs
231  while(nrd == 0)
232  {
233  std::this_thread::sleep_for( std::chrono::duration<unsigned long, std::milli>(m_pauseTime));
234  clearerr(fin);
235  nrd = fread( head.get(), sizeof(char), logHeader::minHeadSize, fin);
236  if(nrd > 0) break;
237 
238  ++check;
239  if(check >= m_fileCheckInterval)
240  {
241  //Check if a new file exists now.
242  size_t oldsz = logs.size();
243  logs = mx::ioutils::getFileNames( m_dir, m_prefixes[0], "", m_ext);
244  if(logs.size() > oldsz)
245  {
246  //new file(s) detected;
247  break;
248  }
249  check = 0;
250  }
251  }
252  }
253  else
254  {
255  break;
256  }
257  }
258 
259  //We got here without any data, probably means time to get a new file.
260  if(nrd == 0) break;
261 
262  totNrd += nrd;
263 
264  if( logHeader::msgLen0(head) == logHeader::MAX_LEN0-1)
265  {
266  //Intermediate size message, read two more bytes
267  nrd = fread( head.get() + logHeader::minHeadSize, sizeof(char), sizeof(msgLen1T), fin);
268  }
269  else if( logHeader::msgLen0(head) == logHeader::MAX_LEN0)
270  {
271  //Large size message: read 8 more bytes
272  nrd = fread( head.get() + logHeader::minHeadSize, sizeof(char), sizeof(msgLen2T), fin);
273  }
274 
275  logPrioT lvl = logHeader::logLevel(head);
276  eventCodeT ec = logHeader::eventCode(head);
277  msgLenT len = logHeader::msgLen(head);
278 
279  //Here: check if lvl, eventCode, etc, match what we want.
280  //If not, fseek and loop.
281  if(lvl > m_level)
282  {
283  fseek(fin, len, SEEK_CUR);
284  continue;
285  }
286 
287 
288  if(m_codes.size() > 0)
289  {
290  bool found = false;
291  for(size_t c = 0; c< m_codes.size(); ++c)
292  {
293  if( m_codes[c] == ec )
294  {
295  found = true;
296  break;
297  }
298  }
299 
300  if(!found)
301  {
302  fseek(fin, len, SEEK_CUR);
303  continue;
304  }
305  }
306 
307  size_t hSz = logHeader::headerSize(head);
308 
309  if( (size_t) hSz + (size_t) len > buffSz )
310  {
311  logBuff = bufferPtrT(new char[hSz + len]);
312  }
313  memcpy( logBuff.get(), head.get(), hSz);
314 
315  ///\todo what do we do if nrd not equal to expected size?
316  nrd = fread( logBuff.get() + hSz, sizeof(char), len, fin);
317  // If not following, exit loop without printing the incomplete log entry (go on to next file).cd
318  // If following, wait for it, but also be checking for new log file in case of crash
319 
320  totNrd += nrd;
321 
322  if(m_follow && firstRun && finSize > 512 && totNrd < finSize-512)
323  {
324  //firstRun = false;
325  continue;
326  }
327 
328  if (!logVerify(ec, logBuff, len))
329  {
330  std::cerr << "Log " << fname << " failed verification on code=" << ec << " at byte=" << totNrd-len-hSz <<". File possibly corrupt. Exiting." << std::endl;
331  return -1;
332  }
333 
334  if (m_jsonMode) {
335  printLogJson(len, logBuff);
336  } else {
337  printLogBuff(lvl, ec, len, logBuff);
338  }
339 
340  }
341 
342  fclose(fin);
343  }
344 
345  return 0;
346 }
347 
348 inline
349 void logdump::printLogBuff( const logPrioT & lvl,
350  const eventCodeT & ec,
351  const msgLenT & len,
352  bufferPtrT & logBuff
353  )
354 {
355  static_cast<void>(len); //be unused
356 
357  if(ec == eventCodes::GIT_STATE)
358  {
359  if(git_state::repoName(logHeader::messageBuffer(logBuff)) == "MagAOX")
360  {
361  for(int i=0;i<80;++i) std::cout << '-';
362  std::cout << "\n\t\t\t\t SOFTWARE RESTART\n";
363  for(int i=0;i<80;++i) std::cout << '-';
364  std::cout << '\n';
365  }
366 
367  }
368 
369  if(lvl < logPrio::LOG_INFO)
370  {
371  if(lvl == logPrio::LOG_EMERGENCY)
372  {
373  std::cout << "\033[104m\033[91m\033[5m\033[1m";
374  }
375 
376  if(lvl == logPrio::LOG_ALERT)
377  {
378  std::cout << "\033[101m\033[5m";
379  }
380 
381  if(lvl == logPrio::LOG_CRITICAL)
382  {
383  std::cout << "\033[41m\033[1m";
384  }
385 
386  if(lvl == logPrio::LOG_ERROR)
387  {
388  std::cout << "\033[91m\033[1m";
389  }
390 
391  if(lvl == logPrio::LOG_WARNING)
392  {
393  std::cout << "\033[93m\033[1m";
394  }
395 
396  if(lvl == logPrio::LOG_NOTICE)
397  {
398  std::cout << "\033[1m";
399  }
400 
401  }
402 
403  logStdFormat( std::cout, logBuff);
404 
405  std::cout << "\033[0m";
406  std::cout << std::endl;
407 }
408 
409 
410 inline
411 void logdump::printLogJson( const msgLenT & len,
412  bufferPtrT & logBuff
413  )
414 {
415  static_cast<void>(len); //be unused
416  logJsonFormat(std::cout, logBuff);
417  std::cout << std::endl;
418 
419 }
420 
421 
422 int logdump::gettimes(std::vector<std::string> & logs)
423 {
424  for(size_t i=logs.size() - m_nfiles; i < logs.size(); ++i)
425  {
426  std::string fname = logs[i];
427  FILE * fin;
428 
429  bufferPtrT head(new char[logHeader::maxHeadSize]);
430 
431  fin = fopen(fname.c_str(), "rb");
432 
433  //--> get size here!!
434  //off_t finSize = mx::ioutils::fileSize( fileno(fin) );
435 
436 
437  off_t totNrd = 0;
438 
439  //size_t buffSz = 0;
440 
441  //Read firs header
442 
443  int nrd;
444 
445  ///\todo check for errors on all reads . . .
446 
447  //Read next header
448  nrd = fread( head.get(), sizeof(char), logHeader::minHeadSize, fin);
449  if(nrd == 0)
450  {
451  std::cerr << "got no header\n";
452  return 0;
453  }
454 
455  if( logHeader::msgLen0(head) == logHeader::MAX_LEN0-1)
456  {
457  //Intermediate size message, read two more bytes
458  nrd = fread( head.get() + logHeader::minHeadSize, sizeof(char), sizeof(msgLen1T), fin);
459  }
460  else if( logHeader::msgLen0(head) == logHeader::MAX_LEN0)
461  {
462  //Large size message: read 8 more bytes
463  nrd = fread( head.get() + logHeader::minHeadSize, sizeof(char), sizeof(msgLen2T), fin);
464  }
465 
466  //logPrioT lvl = logHeader::logLevel(head);
467  //eventCodeT ec = logHeader::eventCode(head);
468  msgLenT len = logHeader::msgLen(head);
469  timespecX ts0 = logHeader::timespec(head);
470  //size_t hSz = logHeader::headerSize(head);
471 
472  uint32_t nRecords = 1;
473  fseek(fin, len, SEEK_CUR);
474 
475  timespecX ts;
476 
477  while(!feof(fin)) //<--This should be an exit condition controlled by loop logic, not feof.
478  {
479  int nrd;
480 
481  //Read next header
482  nrd = fread( head.get(), sizeof(char), logHeader::minHeadSize, fin);
483  if(nrd == 0)
484  {
485  break;
486  }
487  nRecords += 1;
488 
489  //We got here without any data, probably means time to get a new file.
490  if(nrd == 0) break;
491 
492  totNrd += nrd;
493 
494  if( logHeader::msgLen0(head) == logHeader::MAX_LEN0-1)
495  {
496  //Intermediate size message, read two more bytes
497  nrd = fread( head.get() + logHeader::minHeadSize, sizeof(char), sizeof(msgLen1T), fin);
498  }
499  else if( logHeader::msgLen0(head) == logHeader::MAX_LEN0)
500  {
501  //Large size message: read 8 more bytes
502  nrd = fread( head.get() + logHeader::minHeadSize, sizeof(char), sizeof(msgLen2T), fin);
503  }
504 
505  //lvl = logHeader::logLevel(head);
506  //ec = logHeader::eventCode(head);
507  len = logHeader::msgLen(head);
508  ts = logHeader::timespec(head);
509  //hSz = logHeader::headerSize(head);
510 
511  fseek(fin, len, SEEK_CUR);
512 
513 
514  }
515 
516  fclose(fin);
517 
518  double t0 = ts0.time_s + ts0.time_ns/1e9;
519  double t = ts.time_s + ts.time_ns/1e9;
520 
521  std::cout << fname << " " << ts0.ISO8601DateTimeStrX() << "Z " << ts.ISO8601DateTimeStrX() << "Z " << t-t0 << " " << nRecords << std::endl;
522  }
523 
524  return 0;
525 }
526 
527 #endif //logdump_hpp
An application to dump MagAo-X binary logs to the terminal.
Definition: logdump.hpp:40
std::string m_dir
Definition: logdump.hpp:43
std::string m_file
Definition: logdump.hpp:45
virtual int gettimes(std::vector< std::string > &logs)
Definition: logdump.hpp:422
std::vector< eventCodeT > m_codes
Definition: logdump.hpp:61
void printLogJson(const msgLenT &len, bufferPtrT &logBuff)
Definition: logdump.hpp:411
virtual int execute()
Definition: logdump.hpp:160
virtual void setupConfig()
Definition: logdump.hpp:83
virtual void loadConfig()
Definition: logdump.hpp:101
void printLogBuff(const logPrioT &lvl, const eventCodeT &ec, const msgLenT &len, bufferPtrT &logBuff)
Definition: logdump.hpp:349
std::string m_ext
Definition: logdump.hpp:44
std::vector< std::string > m_prefixes
When following, number of loops to wait before checking for a new file. Default is 4.
Definition: logdump.hpp:53
#define MAGAOX_default_logExt
The extension for MagAO-X binary log files.
Definition: defaults.hpp:22
#define MAGAOX_logRelPath
The relative path to the log directory.
Definition: paths.hpp:50
#define MAGAOX_path
The path to the MagAO-X system files.
Definition: paths.hpp:22
#define MAGAOX_env_path
Environment variable setting the MagAO-X path.
Definition: environment.hpp:20
uint16_t msgLen1T
The type used for intermediate message length.
Definition: logDefs.hpp:54
uint16_t eventCodeT
The type of an event code (16-bit unsigned int).
Definition: logDefs.hpp:40
uint64_t msgLen2T
The type used for long message length.
Definition: logDefs.hpp:62
msgLen2T msgLenT
The type used to refer to the message length, regardless of length.
Definition: logDefs.hpp:69
int8_t logPrioT
The type of the log priority code.
Definition: logDefs.hpp:21
std::shared_ptr< char > bufferPtrT
The log entry buffer smart pointer.
Definition: logHeader.hpp:58
std::ostream & cerr()
std::ostream & cout()
constexpr static logPrioT LOG_CRITICAL
The process can not continue and will shut down (fatal)
Definition: logPriority.hpp:37
constexpr static logPrioT LOG_ERROR
An error has occured which the software will attempt to correct.
Definition: logPriority.hpp:40
constexpr static logPrioT LOG_INFO
Informational. The info log level is the lowest level recorded during normal operations.
Definition: logPriority.hpp:49
constexpr static logPrioT LOG_ALERT
This should only be used if some action is required by operators to keep the system safe.
Definition: logPriority.hpp:34
constexpr static logPrioT LOG_EMERGENCY
Normal operations of the entire system should be shut down immediately.
Definition: logPriority.hpp:31
constexpr static logPrioT LOG_WARNING
A condition has occurred which may become an error, but the process continues.
Definition: logPriority.hpp:43
constexpr static logPrioT LOG_DEFAULT
Used to denote "use the default level for this log type".
Definition: logPriority.hpp:61
constexpr static logPrioT LOG_NOTICE
A normal but significant condition.
Definition: logPriority.hpp:46
logPrioT logLevelFromString(const std::string &str)
Get the log priority from a string, which might have the number or the name.
static std::string repoName(void *msgBuffer)
Access the repo name field.
Definition: git_state.hpp:85
A fixed-width timespec structure.
Definition: timespecX.hpp:35
nanosecT time_ns
Nanoseconds.
Definition: timespecX.hpp:37
secT time_s
Time since the Unix epoch.
Definition: timespecX.hpp:36
std::string ISO8601DateTimeStrX()
Get a date-time string in ISO 8601 format for timespecX.
Definition: timespecX.hpp:153