Line data Source code
1 : /** \file xrif2fits.hpp
2 : * \brief The xrif2fits class declaration and definition.
3 : *
4 : * \ingroup xrif2fits_files
5 : */
6 :
7 : #ifndef xrif2fits_hpp
8 : #define xrif2fits_hpp
9 :
10 : #include <ImageStreamIO/ImageStreamIO.h>
11 :
12 : #include <xrif/xrif.h>
13 :
14 : #include <sstream>
15 : #include <iomanip>
16 :
17 : #include <mx/ioutils/fileUtils.hpp>
18 : #include <mx/improc/eigenCube.hpp>
19 : #include <mx/improc/eigenImage.hpp>
20 :
21 : #include <mx/ioutils/fits/fitsFile.hpp>
22 :
23 : #include <mx/sys/timeUtils.hpp>
24 : using namespace mx::sys::tscomp;
25 : using namespace mx::sys::tsop;
26 :
27 : #include "../../libMagAOX/libMagAOX.hpp"
28 :
29 : #ifndef DEBUG_CRUMB
30 : #define DEBUG_CRUMB(msg) {std::cerr << msg << '(' << __FILE__ << ' ' << __LINE__ << "\n";}
31 : #endif
32 :
33 : #define ERR_INVOKED_NAME( msg ) \
34 : std::cerr << invokedName + ": " << msg << "\n at:" << __FILE__ << ' ' << __LINE__ << '\n';
35 :
36 : #define ERR_INVOKED_NAME_ERRNO( msg ) \
37 : std::cerr << invokedName + ": " << msg << "\n errno says:" << strerror( errno ) << "\n at: " << __FILE__ << ' ' \
38 : << __LINE__ << '\n'
39 :
40 : /** \defgroup xrif2fits xrif2fits: xrif-archive to FITS cube converter
41 : * \brief Read images from an xrif archive and write to FITS
42 : *
43 : * <a href="../handbook/utils/xrif2fits.html">Utility Documentation</a>
44 : *
45 : * \ingroup utils
46 : *
47 : */
48 :
49 : /** \defgroup xrif2fits_files xrif2fits Files
50 : * \ingroup xrif2fits
51 : */
52 :
53 : bool g_timeToDie = false;
54 :
55 0 : void sigTermHandler( int signum, siginfo_t *siginf, void *ucont )
56 : {
57 : // Suppress those warnings . . .
58 : static_cast<void>( signum );
59 : static_cast<void>( siginf );
60 : static_cast<void>( ucont );
61 :
62 0 : std::cerr << "\n"; // clear out the ^C char
63 :
64 0 : g_timeToDie = true;
65 0 : }
66 :
67 : /// A utility to stream MagaO-X images from xrif compressed archives to an ImageStreamIO stream.
68 : /**
69 : * \todo finish md doc for xrif2fits
70 : *
71 : * \ingroup xrif2fits
72 : */
73 : class xrif2fits : public mx::app::application
74 : {
75 : typedef XWC_DEFAULT_VERBOSITY verboseT;
76 :
77 : typedef MagAOX::file::stdFileName<verboseT> stdFileNameT;
78 :
79 : protected:
80 : /** \name Configurable Parameters
81 : * @{
82 : */
83 : std::string m_camera; /**< The INDI device name of the camera to process.
84 : Sets m_cameraHeader to `<m_camera>_header.conf` */
85 :
86 : std::string m_cameraHeader; /**< The filename of the config file containing the camera header specification.
87 : Setting this overrides the setting from m_camera.
88 : The path specified by $MagAOX_PATH/$MagAOX_CONFIG is searched,
89 : unless XRIF2FITS_CONFIGPATH is set in the environment.*/
90 :
91 : bool m_noHeader{ false }; /**< if true then no camera header is generated */
92 :
93 : std::string m_dir; /**< The directory to search for files. Can be empty if full path given in files.
94 : If files is empty, all archives in dir will be used. Defaults to `./`.*/
95 :
96 : bool m_overWriteDir {false}; ///< Overwrite an existing directory. Default is to stop if directory exists.
97 :
98 : std::vector<std::string> m_files; /**< List of files to use. If dir is not empty,
99 : it will be pre-pended to each name.*/
100 :
101 : std::vector<MagAOX::file::stdFileName<verboseT>> m_fileNames; /**< The decoded file names broken down into
102 : constituent parts */
103 :
104 : std::vector<std::string> m_logDir;
105 :
106 : std::vector<std::string> m_telDir;
107 :
108 : std::string m_outDir = "fits/";
109 :
110 : bool m_noMeta{ false };
111 :
112 : bool m_metaOnly{ false };
113 :
114 : bool m_timesOnly{ false };
115 :
116 : bool m_cubeMode{ false };
117 :
118 : logMap<verboseT> m_logs;
119 :
120 : logMap<verboseT> m_tels;
121 :
122 : protected:
123 : ///@}
124 :
125 : std::string MagAOXPath;
126 : std::string ConfigRelPath;
127 :
128 : std::vector<logMeta> m_logMetas;
129 :
130 : xrif_t m_xrif{ nullptr };
131 : xrif_t m_xrif_timing{ nullptr };
132 :
133 : public:
134 : /// c-tor
135 : /** Sets up the default config paths by reading from the environment
136 : *
137 : */
138 : xrif2fits();
139 :
140 : ~xrif2fits();
141 :
142 : virtual void setupConfig();
143 :
144 : virtual void loadConfig();
145 :
146 : virtual mx::error_t readHeaderConfig( const std::string &hcfile );
147 :
148 : virtual int execute();
149 :
150 : /// Prepare the file list and output directory
151 : /** Based on loaded configuration
152 : *
153 : * \returns mx::error_t::noerror on success
154 : * \returns error code on an error
155 : *
156 : */
157 : mx::error_t prepareFiles();
158 :
159 : template <typename dataT>
160 : int writeImages( int n, stdFileNameT &lfn );
161 :
162 : std::string format_nano( uint64_t n );
163 : };
164 :
165 : inline xrif2fits::xrif2fits()
166 : {
167 : // setup the default config path
168 : MagAOXPath = mx::sys::getEnv( MAGAOX_env_path );
169 :
170 : if(MagAOXPath == "")
171 : {
172 : MagAOXPath = MAGAOX_path;
173 : }
174 :
175 : if(MagAOXPath.size() > 0)
176 : {
177 : if(MagAOXPath.back() !='/')
178 : {
179 : MagAOXPath += '/';
180 : }
181 : }
182 :
183 : ConfigRelPath = mx::sys::getEnv( MAGAOX_env_config );
184 :
185 : if(ConfigRelPath == "")
186 : {
187 : ConfigRelPath = MAGAOX_configRelPath;
188 : }
189 :
190 : if( ConfigRelPath.size() > 0 )
191 : {
192 : if(ConfigRelPath.back() !='/')
193 : {
194 : ConfigRelPath += '/';
195 : }
196 :
197 : mx::app::application::m_configPathCLBase = MagAOXPath + ConfigRelPath + '/';
198 : }
199 :
200 : // Allow overriding the config path
201 : mx::app::application::m_configPathCLBase_env = "XRIF2FITS_CONFIGPATH";
202 : }
203 :
204 0 : inline xrif2fits::~xrif2fits()
205 : {
206 0 : if( m_xrif )
207 : {
208 0 : xrif_delete( m_xrif );
209 : }
210 :
211 0 : if( m_xrif_timing )
212 : {
213 0 : xrif_delete( m_xrif_timing );
214 : }
215 0 : }
216 :
217 0 : inline void xrif2fits::setupConfig()
218 : {
219 0 : config.add( "camera",
220 : "",
221 : "camera",
222 : argType::Required,
223 : "",
224 : "camera",
225 : false,
226 : "string",
227 : "The device name of the camera. Sets the header.camera config to <camera>_header.conf" );
228 :
229 0 : config.add( "header.camera",
230 : "",
231 : "header.camera",
232 : argType::Required,
233 : "header",
234 : "camera",
235 : false,
236 : "string",
237 : "The name of a config file defining a camera header. Overrides the default for `camera`."
238 : "Searches $MagAOX_PATH/$MagAOX_CONFIG, unless XRIF2FITS_CONFIGPATH is set in the environment." );
239 :
240 0 : config.add( "noHeader",
241 : "N",
242 : "noHeader",
243 : argType::True,
244 : "",
245 : "noHeader",
246 : false,
247 : "bool",
248 : "If true, then no camera header is generated" );
249 :
250 0 : config.add( "dir",
251 : "d",
252 : "dir",
253 : argType::Required,
254 : "",
255 : "dir",
256 : false,
257 : "string",
258 : "The directory to search for files. Can be empty if full path given in files." );
259 :
260 0 : config.add( "overwrite",
261 : "O",
262 : "overwrite",
263 : argType::True,
264 : "",
265 : "overwrite",
266 : false,
267 : "bool",
268 : "Overwrite an existing directory. Default is to stop if directory exists." );
269 :
270 0 : config.add( "files",
271 : "f",
272 : "files",
273 : argType::Required,
274 : "",
275 : "files",
276 : false,
277 : "vector<string>",
278 : "List of files to use. If dir is not empty, it will be pre-pended to each name." );
279 :
280 0 : config.add( "logdir",
281 : "l",
282 : "logdir",
283 : argType::Required,
284 : "",
285 : "logdir",
286 : false,
287 : "vector<string>",
288 : "Directories for log files." );
289 :
290 0 : config.add( "teldir",
291 : "t",
292 : "teldir",
293 : argType::Required,
294 : "",
295 : "teldir",
296 : false,
297 : "vector<string>",
298 : "Directories for telemetry files." );
299 :
300 0 : config.add( "outDir",
301 : "D",
302 : "outDir",
303 : argType::Required,
304 : "",
305 : "outDir",
306 : false,
307 : "string",
308 : "The directory in which to write output files. Default is ./fits/." );
309 :
310 0 : config.add( "metaOnly",
311 : "",
312 : "metaOnly",
313 : argType::True,
314 : "",
315 : "metaOnly",
316 : false,
317 : "bool",
318 : "If true, output only meta data, without decoding images. Default is false." );
319 :
320 0 : config.add( "time",
321 : "T",
322 : "time",
323 : argType::True,
324 : "",
325 : "time",
326 : false,
327 : "bool",
328 : "time span mode: output one line per input file in the format [filename] [start time] [end time] "
329 : "[number of frames], with ISO 8601 timestamps" );
330 :
331 0 : config.add( "noMeta",
332 : "",
333 : "noMeta",
334 : argType::True,
335 : "",
336 : "noMeta",
337 : false,
338 : "bool",
339 : "If true, the meta data file is not written (FITS headers will still be). Default is false." );
340 :
341 0 : config.add( "cubeMode",
342 : "C",
343 : "cubeMode",
344 : argType::True,
345 : "",
346 : "cubeMode",
347 : false,
348 : "bool",
349 : "If true, the archive is written as a FITS cube with minimal header. Default is false." );
350 0 : }
351 :
352 0 : inline void xrif2fits::loadConfig()
353 : {
354 0 : config( m_camera, "camera" );
355 :
356 0 : if( m_camera != "" )
357 : {
358 0 : m_cameraHeader = m_camera + "_header.conf";
359 : }
360 :
361 0 : config( m_cameraHeader, "header.camera" );
362 :
363 0 : config( m_noHeader, "noHeader" );
364 :
365 0 : config( m_dir, "dir" );
366 0 : config(m_overWriteDir, "overwrite");
367 0 : config( m_files, "files" );
368 0 : config( m_outDir, "outDir" );
369 0 : config( m_logDir, "logdir" );
370 0 : config( m_telDir, "teldir" );
371 0 : config( m_metaOnly, "metaOnly" );
372 0 : config( m_timesOnly, "time" );
373 0 : config( m_noMeta, "noMeta" );
374 0 : config( m_cubeMode, "cubeMode" );
375 :
376 0 : if( m_configPathCLBase.size() > 0 )
377 : {
378 0 : if( mx::app::application::m_configPathCLBase.back() != '/' )
379 : {
380 0 : mx::app::application::m_configPathCLBase += '/';
381 : }
382 : }
383 0 : }
384 :
385 0 : inline mx::error_t xrif2fits::readHeaderConfig( const std::string &hcfile )
386 : {
387 0 : if( hcfile == "" )
388 : {
389 0 : return mx::error_t::noerror;
390 : }
391 :
392 0 : mx::app::appConfigurator hconfig;
393 :
394 0 : hconfig.add( "include", "", "include", argType::Required, "", "include", false, "string", "" );
395 :
396 : try
397 : {
398 0 : DEBUG_CRUMB("reading: " + hcfile);
399 :
400 0 : if( hconfig.readConfig( hcfile, true ) != 0 )
401 : {
402 0 : return mx::error_report<verboseT>( mx::error_t::error, "Error reading header config: " + hcfile );
403 : }
404 : }
405 0 : catch( const std::exception &e )
406 : {
407 0 : return mx::error_report<verboseT>( mx::error_t::std_exception,
408 0 : "Exception reading header config: " + hcfile + ". " + e.what() );
409 0 : }
410 :
411 0 : std::vector<std::string> includes;
412 0 : hconfig( includes, "include" );
413 :
414 0 : for( auto &include : includes )
415 : {
416 0 : if( include.size() > 4 )
417 : {
418 0 : if( include.substr( include.size() - 5, 4 ) != ".conf" )
419 : {
420 0 : include += ".conf";
421 : }
422 : }
423 :
424 0 : DEBUG_CRUMB("reading include: " + mx::app::application::m_configPathCLBase + include);
425 :
426 0 : mx_error_check( readHeaderConfig( mx::app::application::m_configPathCLBase + include ) );
427 : }
428 :
429 0 : std::vector<std::string> devices;
430 :
431 0 : hconfig.unusedSections( devices );
432 :
433 0 : if( devices.size() == 0 && includes.size() == 0) //this allows include-only
434 : {
435 0 : return mx::error_report<verboseT>( mx::error_t::notfound, "No device sections in header config:" + hcfile );
436 : }
437 :
438 0 : for( auto &device : devices )
439 : {
440 : // Wind through all the unused targets
441 0 : for( auto it = hconfig.m_unusedConfigs.begin(); it != hconfig.m_unusedConfigs.end(); ++it )
442 : {
443 0 : if( device == it->second.section )
444 : {
445 0 : std::string eventCode = it->second.keyword;
446 :
447 : // Check if this keyword is a valid flatlogs eventCode
448 0 : flatlogs::eventCodeT ec = MagAOX::logger::eventCode( eventCode );
449 0 : if( ec != eventCodes::UNKNOWN )
450 : {
451 0 : std::vector<std::string> fields;
452 0 : hconfig.configUnused( fields, mx::app::iniFile::makeKey( device, eventCode ) );
453 :
454 0 : for( auto &field : fields )
455 : {
456 0 : std::cerr << "adding: " << device << ' ' << ec << ' ' << field << '\n';
457 0 : m_logMetas.push_back( logMetaSpec( { device, ec, field } ) );
458 : }
459 0 : }
460 0 : }
461 : }
462 : }
463 :
464 0 : return mx::error_t::noerror;
465 0 : }
466 :
467 0 : inline int xrif2fits::execute()
468 : {
469 : // Install signal handling
470 : struct sigaction act;
471 : sigset_t set;
472 :
473 0 : act.sa_sigaction = sigTermHandler;
474 0 : act.sa_flags = SA_SIGINFO;
475 0 : sigemptyset( &set );
476 0 : act.sa_mask = set;
477 :
478 0 : errno = 0;
479 0 : if( sigaction( SIGTERM, &act, 0 ) < 0 )
480 : {
481 0 : std::cerr << " (" << invokedName << "): error setting SIGTERM handler: " << strerror( errno ) << "\n";
482 0 : return -1;
483 : }
484 :
485 0 : errno = 0;
486 0 : if( sigaction( SIGQUIT, &act, 0 ) < 0 )
487 : {
488 0 : std::cerr << " (" << invokedName << "): error setting SIGQUIT handler: " << strerror( errno ) << "\n";
489 0 : return -1;
490 : }
491 :
492 0 : errno = 0;
493 0 : if( sigaction( SIGINT, &act, 0 ) < 0 )
494 : {
495 0 : std::cerr << " (" << invokedName << "): error setting SIGINT handler: " << strerror( errno ) << "\n";
496 0 : return -1;
497 : }
498 :
499 : try
500 : {
501 0 : mx::error_t errc = prepareFiles();
502 0 : if( !!errc )
503 : {
504 0 : mx::error_report<verboseT>( errc, "error from prepareFiles" );
505 0 : return -1;
506 : }
507 : }
508 0 : catch( ... )
509 : {
510 0 : std::throw_with_nested( MagAOX::xwcException( "error from prepareFiles" ) );
511 0 : }
512 :
513 : // this has to be here
514 0 : stdFileNameT &firstFile = m_fileNames[0];
515 0 : stdFileNameT &lastFile = m_fileNames.back();
516 :
517 : xrif_error_t rv;
518 0 : rv = xrif_new( &m_xrif );
519 :
520 0 : if( rv < 0 )
521 : {
522 0 : std::cerr << " (" << invokedName << "): Error allocating xrif.\n";
523 0 : return -1;
524 : }
525 :
526 0 : rv = xrif_new( &m_xrif_timing );
527 :
528 0 : if( rv < 0 )
529 : {
530 0 : std::cerr << " (" << invokedName << "): Error allocating xrif_timing.\n";
531 0 : return -1;
532 : }
533 :
534 0 : if( !m_noHeader )
535 : {
536 0 : m_logMetas.push_back( logMetaSpec( { firstFile.appName(), telem_stdcam::eventCode, "exptime" } ) );
537 :
538 : // Build list of apps, this will be automagic as part of config
539 0 : std::set<std::string> logApps;
540 :
541 0 : logApps.insert( m_fileNames[0].appName() );
542 :
543 0 : for( auto &meta : m_logMetas )
544 : {
545 0 : logApps.insert( meta.device() );
546 : }
547 :
548 0 : for( auto &app : logApps )
549 : {
550 0 : for( size_t n = 0; n < m_logDir.size(); ++n )
551 : {
552 : try
553 : {
554 0 : m_logs.loadAppToFileMap( m_logDir[n], app, ".binlog", firstFile, lastFile );
555 : }
556 0 : catch( ... )
557 : {
558 : /// for now ignore all exceptions. \todo eventually ignore only "no prior logs" etc
559 0 : }
560 : }
561 :
562 0 : for( size_t n = 0; n < m_telDir.size(); ++n )
563 : {
564 : try
565 : {
566 0 : m_tels.loadAppToFileMap( m_telDir[n], app, ".bintel", firstFile, lastFile );
567 : }
568 0 : catch( ... )
569 : {
570 : /// for now ignore all exceptions. \todo eventually ignore only "no prior logs" etc
571 0 : }
572 : }
573 :
574 0 : if( m_logs.m_appToFileMap[app].size() == 0 )
575 : {
576 0 : throw MagAOX::xwcException( "no logs found for " + app );
577 : }
578 :
579 0 : if( m_tels.m_appToFileMap[app].size() == 0 )
580 : {
581 0 : throw MagAOX::xwcException( "no telems found for " + app );
582 : }
583 : }
584 0 : }
585 :
586 : // Now de-compress and load the frames
587 : // Only decompressing the number of files needed, and only copying the number of frames needed
588 0 : for( size_t n = 0; n < m_files.size(); ++n )
589 : {
590 :
591 0 : if( g_timeToDie == true )
592 0 : break; // check before going on
593 :
594 0 : if( !m_noHeader )
595 : {
596 0 : m_tels.loadFiles( m_fileNames[n].appName(), m_fileNames[n].timestamp() );
597 : }
598 0 : if( !m_timesOnly )
599 : {
600 :
601 0 : std::cout << "******************************************************\n";
602 0 : std::cout << "* xrif2fits: decoding for " << m_fileNames[n].appName() << " (" + m_files[n] << ")\n";
603 0 : std::cout << "******************************************************\n";
604 : }
605 :
606 0 : FILE *fp_xrif = fopen( m_files[n].c_str(), "rb" );
607 0 : if( fp_xrif == nullptr )
608 : {
609 0 : std::cerr << " (" << invokedName << "): Error opening " << m_files[n] << "\n";
610 0 : std::cerr << " (" << invokedName << "): " << strerror( errno ) << "\n";
611 0 : return -1;
612 : }
613 :
614 : char header[XRIF_HEADER_SIZE];
615 :
616 0 : size_t nr = fread( header, 1, XRIF_HEADER_SIZE, fp_xrif );
617 0 : if( nr != XRIF_HEADER_SIZE )
618 : {
619 0 : std::cerr << " (" << invokedName << "): Error reading header of " << m_files[n] << "\n";
620 0 : fclose( fp_xrif );
621 0 : return -1;
622 : }
623 :
624 : uint32_t header_size;
625 0 : xrif_read_header( m_xrif, &header_size, header );
626 0 : if( !m_timesOnly )
627 : {
628 0 : std::cout << "xrif compression details:\n";
629 0 : std::cout << " difference method: " << xrif_difference_method_string( m_xrif->difference_method ) << '\n';
630 0 : std::cout << " reorder method: " << xrif_reorder_method_string( m_xrif->reorder_method ) << '\n';
631 0 : std::cout << " compression method: " << xrif_compress_method_string( m_xrif->compress_method ) << '\n';
632 0 : if( m_xrif->compress_method == XRIF_COMPRESS_LZ4 )
633 : {
634 0 : std::cout << " LZ4 acceleration: " << m_xrif->lz4_acceleration << '\n';
635 : }
636 0 : std::cout << " dimensions: " << m_xrif->width << " x " << m_xrif->height << " x " << m_xrif->depth
637 0 : << " x " << m_xrif->frames << "\n";
638 0 : std::cout << " raw size: "
639 0 : << m_xrif->width * m_xrif->height * m_xrif->depth * m_xrif->frames * m_xrif->data_size
640 0 : << " bytes\n";
641 0 : std::cout << " encoded size: " << m_xrif->compressed_size << " bytes\n";
642 0 : std::cout << " ratio: "
643 0 : << ( (double)m_xrif->compressed_size ) /
644 0 : ( m_xrif->width * m_xrif->height * m_xrif->depth * m_xrif->frames * m_xrif->data_size )
645 0 : << '\n';
646 : }
647 0 : rv = xrif_allocate_raw( m_xrif );
648 0 : if( rv != XRIF_NOERROR )
649 : {
650 0 : std::cerr << " (" << invokedName << "): Error allocating raw buffer for " << m_files[n] << "\n";
651 0 : std::cerr << "\t code: " << rv << "\n";
652 0 : return -1;
653 : }
654 :
655 0 : rv = xrif_allocate_reordered( m_xrif );
656 0 : if( rv != XRIF_NOERROR )
657 : {
658 0 : std::cerr << " (" << invokedName << "): Error allocating reordered buffer for " << m_files[n] << "\n";
659 0 : std::cerr << "\t code: " << rv << "\n";
660 0 : return -1;
661 : }
662 :
663 0 : nr = fread( m_xrif->raw_buffer, 1, m_xrif->compressed_size, fp_xrif );
664 :
665 0 : if( nr != m_xrif->compressed_size )
666 : {
667 0 : std::cerr << " (" << invokedName << "): Error reading data from " << m_files[n] << "\n";
668 0 : return -1;
669 : }
670 :
671 : // Now get timing data
672 0 : nr = fread( header, 1, XRIF_HEADER_SIZE, fp_xrif );
673 0 : if( nr != XRIF_HEADER_SIZE )
674 : {
675 0 : std::cerr << " (" << invokedName << "): Error reading timing header of " << m_files[n] << "\n";
676 0 : fclose( fp_xrif );
677 0 : return -1;
678 : }
679 :
680 0 : xrif_read_header( m_xrif_timing, &header_size, header );
681 :
682 0 : if( !m_timesOnly )
683 : {
684 0 : std::cout << "xrif timing data compression details:\n";
685 0 : std::cout << " difference method: " << xrif_difference_method_string( m_xrif_timing->difference_method )
686 0 : << '\n';
687 0 : std::cout << " reorder method: " << xrif_reorder_method_string( m_xrif_timing->reorder_method )
688 0 : << '\n';
689 0 : std::cout << " compression method: " << xrif_compress_method_string( m_xrif_timing->compress_method )
690 0 : << '\n';
691 0 : if( m_xrif_timing->compress_method == XRIF_COMPRESS_LZ4 )
692 : {
693 0 : std::cout << " LZ4 acceleration: " << m_xrif_timing->lz4_acceleration << '\n';
694 : }
695 0 : std::cout << " dimensions: " << m_xrif_timing->width << " x " << m_xrif_timing->height << " x "
696 0 : << m_xrif_timing->depth << " x " << m_xrif_timing->frames << "\n";
697 0 : std::cout << " raw size: "
698 0 : << m_xrif_timing->width * m_xrif_timing->height * m_xrif_timing->depth * m_xrif_timing->frames *
699 0 : m_xrif_timing->data_size
700 0 : << " bytes\n";
701 0 : std::cout << " encoded size: " << m_xrif_timing->compressed_size << " bytes\n";
702 0 : std::cout << " ratio: "
703 0 : << ( (double)m_xrif_timing->compressed_size ) /
704 0 : ( m_xrif_timing->width * m_xrif_timing->height * m_xrif_timing->depth *
705 0 : m_xrif_timing->frames * m_xrif_timing->data_size )
706 0 : << '\n';
707 : }
708 0 : rv = xrif_allocate_raw( m_xrif_timing );
709 0 : if( rv != XRIF_NOERROR )
710 : {
711 0 : std::cerr << " (" << invokedName << "): Error allocating raw buffer for timing data from " << m_files[n]
712 0 : << "\n";
713 0 : std::cerr << "\t code: " << rv << "\n";
714 0 : return -1;
715 : }
716 :
717 0 : rv = xrif_allocate_reordered( m_xrif_timing );
718 0 : if( rv != XRIF_NOERROR )
719 : {
720 0 : std::cerr << " (" << invokedName << "): Error allocating reordered buffer for timing data from "
721 0 : << m_files[n] << "\n";
722 0 : std::cerr << "\t code: " << rv << "\n";
723 0 : return -1;
724 : }
725 :
726 0 : nr = fread( m_xrif_timing->raw_buffer, 1, m_xrif_timing->compressed_size, fp_xrif );
727 :
728 0 : if( nr != m_xrif_timing->compressed_size )
729 : {
730 0 : std::cerr << " (" << invokedName << "): Error reading timing data from " << m_files[n] << "\n";
731 0 : return -1;
732 : }
733 :
734 0 : fclose( fp_xrif );
735 :
736 0 : if( g_timeToDie == true )
737 0 : break; // check after the long read.
738 :
739 0 : if( !m_metaOnly )
740 : {
741 0 : rv = xrif_decode( m_xrif );
742 0 : if( rv != XRIF_NOERROR )
743 : {
744 0 : std::cerr << " (" << invokedName << "): Error decoding image data from " << m_files[n] << "\n";
745 0 : std::cerr << "\t code: " << rv << "\n";
746 0 : return -1;
747 : }
748 : }
749 :
750 0 : rv = xrif_decode( m_xrif_timing );
751 0 : if( rv != XRIF_NOERROR )
752 : {
753 0 : std::cerr << " (" << invokedName << "): Error decoding timing data from " << m_files[n] << "\n";
754 0 : std::cerr << "\t code: " << rv << "\n";
755 0 : return -1;
756 : }
757 :
758 0 : if( g_timeToDie == true )
759 : {
760 0 : break; // check after the decompress.
761 : }
762 :
763 0 : if( m_timesOnly )
764 : {
765 0 : std::cout << m_files[n] << " ";
766 0 : double totalExposureTime = 0;
767 :
768 0 : for( xrif_dimension_t q = 0; q < m_xrif->frames; ++q )
769 : {
770 : timespec atime; // This is the acquisition time of the exposure
771 0 : timespec stime = { 0, 0 }; // This is the start time of the exposure, calculated as atime-exptime.
772 :
773 0 : uint64_t *curr_timing = (uint64_t *)m_xrif_timing->raw_buffer + 5 * q;
774 :
775 0 : atime.tv_sec = curr_timing[1];
776 0 : atime.tv_nsec = curr_timing[2];
777 :
778 : // We have to bootstrap the exposure time
779 0 : char *prior = nullptr;
780 0 : m_tels.getPriorLog( prior, m_fileNames[n].appName(), eventCodes::TELEM_STDCAM, atime );
781 0 : double exptime = -1;
782 0 : if( prior )
783 : {
784 0 : char *priorprior = nullptr;
785 0 : exptime = telem_stdcam::exptime( logHeader::messageBuffer( prior ) );
786 0 : stime = atime - exptime;
787 0 : m_tels.getPriorLog( priorprior, m_fileNames[n].appName(), eventCodes::TELEM_STDCAM, stime );
788 :
789 : ///\todo this needs to check for any log entries between end and start
790 0 : if( telem_stdcam::exptime( logHeader::messageBuffer( priorprior ) ) != exptime )
791 : {
792 0 : std::cerr << "Change in exposure time mid-exposure\n";
793 : }
794 : }
795 : else
796 : {
797 0 : std::cerr << "no prior\n";
798 : }
799 0 : totalExposureTime += exptime;
800 :
801 0 : std::string timestamp;
802 0 : mx::sys::timeStamp( timestamp, atime );
803 :
804 0 : std::string dateobs = mx::sys::ISO8601DateTimeStr( atime, 1 );
805 0 : if( q == 0 )
806 : {
807 0 : std::cout << dateobs << " ";
808 : }
809 0 : if( q == ( m_xrif->frames - 1 ) )
810 : {
811 0 : std::cout << dateobs << " " << totalExposureTime << " " << m_xrif->frames << "\n";
812 : }
813 0 : }
814 : }
815 : else // Normal writing
816 : {
817 0 : if( m_xrif->type_code == XRIF_TYPECODE_UINT8 )
818 : {
819 0 : if( writeImages<uint8_t>( n, m_fileNames[n] ) < 0 )
820 : {
821 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
822 0 : return -1;
823 : }
824 : }
825 0 : else if( m_xrif->type_code == XRIF_TYPECODE_INT8 )
826 : {
827 0 : if( writeImages<int8_t>( n, m_fileNames[n] ) < 0 )
828 : {
829 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
830 0 : return -1;
831 : }
832 : }
833 0 : if( m_xrif->type_code == XRIF_TYPECODE_UINT16 )
834 : {
835 0 : if( writeImages<uint16_t>( n, m_fileNames[n] ) < 0 )
836 : {
837 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
838 0 : return -1;
839 : }
840 : }
841 0 : else if( m_xrif->type_code == XRIF_TYPECODE_INT16 )
842 : {
843 0 : if( writeImages<int16_t>( n, m_fileNames[n] ) < 0 )
844 : {
845 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
846 0 : return -1;
847 : }
848 : }
849 0 : else if( m_xrif->type_code == XRIF_TYPECODE_UINT32 )
850 : {
851 0 : if( writeImages<uint32_t>( n, m_fileNames[n] ) < 0 )
852 : {
853 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
854 0 : return -1;
855 : }
856 : }
857 0 : else if( m_xrif->type_code == XRIF_TYPECODE_INT32 )
858 : {
859 0 : if( writeImages<int32_t>( n, m_fileNames[n] ) < 0 )
860 : {
861 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
862 0 : return -1;
863 : }
864 : }
865 0 : else if( m_xrif->type_code == XRIF_TYPECODE_UINT64 )
866 : {
867 0 : if( writeImages<uint32_t>( n, m_fileNames[n] ) < 0 )
868 : {
869 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
870 0 : return -1;
871 : }
872 : }
873 0 : else if( m_xrif->type_code == XRIF_TYPECODE_INT64 )
874 : {
875 0 : if( writeImages<int32_t>( n, m_fileNames[n] ) < 0 )
876 : {
877 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
878 0 : return -1;
879 : }
880 : }
881 0 : else if( m_xrif->type_code == XRIF_TYPECODE_FLOAT )
882 : {
883 0 : if( writeImages<float>( n, m_fileNames[n] ) < 0 )
884 : {
885 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
886 0 : return -1;
887 : }
888 : }
889 0 : else if( m_xrif->type_code == XRIF_TYPECODE_DOUBLE )
890 : {
891 0 : if( writeImages<float>( n, m_fileNames[n] ) < 0 )
892 : {
893 0 : ERR_INVOKED_NAME( "error writing to file: " + m_files[n] );
894 0 : return -1;
895 : }
896 : }
897 : else
898 : {
899 0 : ERR_INVOKED_NAME( "unsupported data type in file: " + m_files[n] );
900 0 : return -1;
901 : }
902 : }
903 : }
904 :
905 0 : std::cerr << " (" << invokedName << "): exited normally.\n";
906 :
907 0 : return 0;
908 : }
909 :
910 0 : inline mx::error_t xrif2fits::prepareFiles()
911 : {
912 : // If files aren't specified, we search the given directory.
913 0 : if( m_files.size() == 0 )
914 : {
915 0 : if( m_dir == "" ) // search pwd
916 : {
917 0 : m_dir = "./";
918 : }
919 :
920 0 : mx_error_check( mx::ioutils::getFileNames( m_files, m_dir, "", "", ".xrif" ) );
921 :
922 0 : for( size_t n = 0; n < m_files.size(); ++n )
923 : {
924 0 : MagAOX::file::stdFileName sfn;
925 : try
926 : {
927 0 : mx_error_check( sfn.fullName( m_files[n] ) );
928 : }
929 0 : catch( ... )
930 : {
931 0 : std::throw_with_nested( mx::exception( "From stdFileName for " + m_files[n] ) );
932 0 : }
933 :
934 : // add only if it passed
935 : try
936 : {
937 0 : m_fileNames.push_back( sfn );
938 : }
939 0 : catch( const std::bad_alloc &e )
940 : {
941 0 : std::throw_with_nested(
942 0 : mx::exception( mx::error_t::std_bad_alloc, "error adding file " + m_files[n] ) );
943 0 : }
944 0 : catch( ... )
945 : {
946 0 : std::throw_with_nested( mx::exception( "error adding file " + m_files[n] ) );
947 0 : }
948 0 : }
949 : }
950 : else // If files are specified we attach a directory to them if needed
951 : {
952 0 : if( m_dir != "" )
953 : {
954 : try
955 : {
956 0 : if( m_dir[m_dir.size() - 1] != '/' )
957 0 : m_dir += '/';
958 :
959 0 : for( size_t n = 0; n < m_files.size(); ++n )
960 : {
961 0 : m_files[n] = m_dir + m_files[n];
962 : }
963 : }
964 0 : catch( const std::bad_alloc &e )
965 : {
966 0 : std::throw_with_nested( mx::exception( mx::error_t::std_bad_alloc, "adding dir to files" ) );
967 0 : }
968 0 : catch( ... )
969 : {
970 0 : std::throw_with_nested( mx::exception("adding dir to files") );
971 0 : }
972 : }
973 :
974 0 : for( size_t n = 0; n < m_files.size(); ++n )
975 : {
976 : // since the files were specified they have to pass
977 : try
978 : {
979 0 : m_fileNames.push_back( MagAOX::file::stdFileName( m_files[n] ) );
980 : }
981 0 : catch( const std::bad_alloc &e )
982 : {
983 0 : std::throw_with_nested(
984 0 : mx::exception( mx::error_t::std_bad_alloc, "error adding file " + m_files[n] ) );
985 0 : }
986 0 : catch( ... )
987 : {
988 0 : std::throw_with_nested( mx::exception( "error adding file " + m_files[n] ) );
989 0 : }
990 : }
991 : }
992 :
993 0 : if( m_files.size() == 0 )
994 : {
995 0 : return mx::error_report<verboseT>( mx::error_t::notfound, "No xrif files found" );
996 : }
997 :
998 0 : if( m_outDir == "" )
999 : {
1000 0 : m_outDir = "./";
1001 : }
1002 : else
1003 : {
1004 : // Make sure the slash exists, then mkdir. We know size is > 0 here.
1005 0 : if( m_outDir[m_outDir.size() - 1] != '/' )
1006 : {
1007 0 : m_outDir += '/';
1008 : }
1009 :
1010 0 : if( !m_timesOnly )
1011 : {
1012 0 : if(!m_overWriteDir)
1013 : {
1014 : mx::error_t errc;
1015 0 : if(mx::ioutils::dir_exists_is(m_outDir, errc))
1016 : {
1017 0 : return mx::error_report<verboseT>(mx::error_t::eexist, "Directory " + m_outDir + " already exists.");
1018 : }
1019 :
1020 0 : if(!!errc)
1021 : {
1022 0 : return mx::error_report<verboseT>(errc, "Checking " + m_outDir);
1023 : }
1024 : }
1025 :
1026 0 : mx_error_check( mx::ioutils::createDirectories(m_outDir) );
1027 : }
1028 : }
1029 :
1030 0 : for( size_t n = 1; n < m_files.size(); ++n )
1031 : {
1032 0 : if( m_fileNames.back().appName() != m_fileNames[0].appName() )
1033 : {
1034 0 : return mx::error_report<verboseT>(mx::error_t::invalidarg, "can only operate on a single camera at a time" );
1035 : }
1036 : }
1037 :
1038 0 : if( m_camera == "" )
1039 : {
1040 0 : m_camera = m_fileNames[0].appName();
1041 0 : std::cerr << "Set camera to: " << m_camera << '\n';
1042 : }
1043 :
1044 0 : if( m_cameraHeader == "" )
1045 : {
1046 0 : m_cameraHeader = m_camera + "_header.conf";
1047 0 : std::cerr << "Set camera header to: " << m_cameraHeader << '\n';
1048 : }
1049 :
1050 0 : if( !m_noHeader )
1051 : {
1052 0 : mx::error_t errc = readHeaderConfig( mx::app::application::m_configPathCLBase + m_cameraHeader );
1053 :
1054 0 : if( !!errc )
1055 : {
1056 0 : return mx::error_report<verboseT>(
1057 0 : errc, "Error reading camera header: " + mx::app::application::m_configPathCLBase + m_cameraHeader );
1058 :
1059 : }
1060 : }
1061 :
1062 0 : return mx::error_t::noerror;
1063 : }
1064 :
1065 : template <typename dataT>
1066 0 : int xrif2fits::writeImages( int n, stdFileNameT &lfn )
1067 : {
1068 0 : mx::improc::eigenCube<dataT> tmpc(
1069 0 : reinterpret_cast<dataT *>( m_xrif->raw_buffer ), m_xrif->width, m_xrif->height, m_xrif->frames );
1070 :
1071 0 : mx::fits::fitsFile<dataT, verboseT> ff;
1072 0 : mx::fits::fitsHeader<verboseT> fh;
1073 :
1074 : // Special handling for meta output
1075 0 : logMeta exptimeMeta( logMetaSpec( lfn.appName(), telem_stdcam::eventCode, "exptime" ) );
1076 :
1077 0 : std::ofstream metaOut;
1078 :
1079 : // Print the meta-file header
1080 0 : if( !m_noMeta && !m_timesOnly )
1081 : {
1082 0 : metaOut.open( m_outDir + "meta_data.txt" );
1083 : /*metaOut << "#DATE-OBS FRAMENO ACQSEC ACQNSEC WRTSEC WRTNSEC";
1084 : metaOut << " EXPTIME";
1085 : for(size_t u=0;u<logMetas.size();++u)
1086 : {
1087 : metaOut << " " << logMetas[u].keyword() ;
1088 : }
1089 : metaOut << "\n";*/
1090 : }
1091 0 : if( m_cubeMode )
1092 : {
1093 0 : std::string outfname = m_outDir + mx::ioutils::pathStem( m_files[n] ) + ".fits";
1094 0 : ff.write( outfname, tmpc );
1095 0 : }
1096 : else
1097 : {
1098 0 : for( int q = 0; q < tmpc.planes(); ++q )
1099 : {
1100 : uint64_t cnt0;
1101 : timespec atime; // This is the acquisition time of the exposure
1102 : timespec wtime;
1103 0 : timespec stime = { 0, 0 }; // This is the start time of the exposure, calculated as atime-exptime.
1104 :
1105 0 : uint64_t *curr_timing = (uint64_t *)m_xrif_timing->raw_buffer + 5 * q;
1106 :
1107 0 : cnt0 = curr_timing[0];
1108 0 : atime.tv_sec = curr_timing[1];
1109 0 : atime.tv_nsec = curr_timing[2];
1110 0 : wtime.tv_sec = curr_timing[3];
1111 0 : wtime.tv_nsec = curr_timing[4];
1112 :
1113 0 : double exptime = -1;
1114 0 : if( !m_noHeader )
1115 : {
1116 : // We have to bootstrap the exposure time
1117 0 : char *prior = nullptr;
1118 0 : m_tels.getPriorLog( prior, lfn.appName(), eventCodes::TELEM_STDCAM, atime );
1119 :
1120 0 : if( prior )
1121 : {
1122 0 : char *priorprior = nullptr;
1123 0 : exptime = telem_stdcam::exptime( logHeader::messageBuffer( prior ) );
1124 :
1125 0 : stime = atime - exptime;
1126 0 : m_tels.getPriorLog( priorprior, lfn.appName(), eventCodes::TELEM_STDCAM, stime );
1127 :
1128 : ///\todo this needs to check for any log entries between end and start
1129 0 : if( telem_stdcam::exptime( logHeader::messageBuffer( priorprior ) ) != exptime )
1130 : {
1131 0 : std::cerr << "Change in exposure time mid-exposure\n";
1132 : }
1133 : }
1134 : else
1135 : {
1136 0 : std::cerr << "no prior\n";
1137 : }
1138 : }
1139 :
1140 : // timespecX midexp = mx::meanTimespec( atime, stime);
1141 :
1142 0 : std::string timestamp;
1143 0 : mx::sys::timeStamp( timestamp, atime );
1144 0 : std::string outfname = m_outDir + lfn.appName() + "_" + timestamp + ".fits";
1145 :
1146 0 : fh.clear();
1147 :
1148 0 : std::string dateobs = mx::sys::ISO8601DateTimeStr( atime, 1 );
1149 :
1150 0 : fh.append( "DATE-OBS", dateobs, "Date of obs. YYYY-mm-ddTHH:MM:SS" );
1151 0 : fh.append( "INSTRUME", "MagAO-X " + lfn.appName() );
1152 0 : fh.append( "CAMERA", lfn.appName() );
1153 0 : fh.append( "TELESCOP", "Magellan Clay, Las Campanas Obs." );
1154 :
1155 0 : if( !m_noMeta )
1156 : {
1157 0 : metaOut << dateobs << " " << cnt0 << " " << atime.tv_sec << " " << format_nano( atime.tv_nsec ) << " "
1158 0 : << wtime.tv_sec << " " << format_nano( wtime.tv_nsec ) << " ";
1159 : }
1160 :
1161 0 : if( exptime > -1 )
1162 : {
1163 : // First output exposure time
1164 0 : if( !m_noMeta )
1165 : {
1166 0 : metaOut << exptimeMeta.value( m_tels, stime, atime );
1167 : }
1168 :
1169 : // Then output each value in turn
1170 0 : for( size_t u = 0; u < m_logMetas.size(); ++u )
1171 : {
1172 0 : mx::fits::fitsHeaderCard<verboseT> fc = m_logMetas[u].card( m_tels, stime, atime );
1173 0 : fh.append( fc );
1174 0 : if( !m_noMeta )
1175 : {
1176 0 : metaOut << " " << m_logMetas[u].value( m_tels, stime, atime );
1177 : }
1178 : }
1179 : }
1180 :
1181 0 : fh.append( "FRAMENO", cnt0 );
1182 0 : fh.append( "ACQSEC", atime.tv_sec, "Image acq. time, seconds since Unix epoch" );
1183 0 : fh.append( "ACQNSEC", atime.tv_nsec, "Image acq. time, nanosecond component" );
1184 0 : fh.append( "WRTSEC", wtime.tv_sec, "Image write time, seconds since Unix epoch" );
1185 0 : fh.append( "WRTNSEC", wtime.tv_nsec, "Image write time, nanosecond component" );
1186 :
1187 0 : if( !m_noMeta )
1188 : {
1189 0 : metaOut << "\n";
1190 : }
1191 0 : if( !m_metaOnly )
1192 : {
1193 0 : mx::improc::eigenImage<dataT> im = tmpc.image( q );
1194 0 : ff.write( outfname, tmpc.image( q ), fh );
1195 0 : }
1196 : }
1197 : }
1198 :
1199 0 : return 0;
1200 0 : }
1201 :
1202 0 : inline std::string xrif2fits::format_nano( uint64_t n )
1203 : {
1204 0 : std::ostringstream oss;
1205 0 : oss << std::setw( 9 ) << std::setfill( '0' ) << n;
1206 0 : return oss.str();
1207 0 : };
1208 :
1209 : #endif // xrif2fits_hpp
|
