KMOS Pipeline Reference Manual  1.2.7
kmo_illumination_flat.c
00001 /* $Id: kmo_illumination_flat.c,v 1.1 2013/10/21 13:44:55 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013/10/21 13:44:55 $
00024  * $Revision: 1.1 $
00025  * $Name:  $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmo_priv_reconstruct.h"
00042 #include "kmo_priv_functions.h"
00043 #include "kmo_priv_flat.h"
00044 #include "kmo_priv_wave_cal.h"
00045 #include "kmclipm_priv_splines.h"
00046 #include "kmo_functions.h"
00047 #include "kmo_cpl_extensions.h"
00048 #include "kmo_dfs.h"
00049 #include "kmo_error.h"
00050 #include "kmo_constants.h"
00051 #include "kmo_debug.h"
00052 
00053 /*-----------------------------------------------------------------------------
00054  *                          Functions prototypes
00055  *----------------------------------------------------------------------------*/
00056 
00057 static int kmo_illumination_flat_create(cpl_plugin *);
00058 static int kmo_illumination_flat_exec(cpl_plugin *);
00059 static int kmo_illumination_flat_destroy(cpl_plugin *);
00060 static int kmo_illumination_flat(cpl_parameterlist *, cpl_frameset *);
00061 
00062 /*-----------------------------------------------------------------------------
00063  *                          Static variables
00064  *----------------------------------------------------------------------------*/
00065 
00066 static char kmo_illumination_flat_description[] =
00067 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00068 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00069 "generates the spectral calibration frame needed in this recipe. As input at\n"
00070 "least a flatfield frame is required.\n"
00071 "Contrary to kmo_illumination it doesn't use flat sky frames but rather the\n"
00072 "flatfield frames from the internal flat lamp. This recipe can be used if no\n"
00073 "acceptable flat sky frames are available.\n"
00074 "The created product, the illumination correction, can be used as input for\n"
00075 "kmo_std_star and kmo_sci_red.\n"
00076 "\n"
00077 "BASIC PARAMETERS:\n"
00078 "-----------------\n"
00079 "--imethod\n"
00080 "The interpolation method used for reconstruction.\n"
00081 "\n"
00082 "ADVANCED PARAMETERS\n"
00083 "-------------------\n"
00084 "--flux\n"
00085 "Specify if flux conservation should be applied.\n"
00086 "\n"
00087 "--neighborhoodRange\n"
00088 "Defines the range to search for neighbors during reconstruction\n"
00089 "\n"
00090 "--b_samples\n"
00091 "The number of samples in spectral direction for the reconstructed cube.\n"
00092 "Ideally this number should be greater than 2048, the detector size.\n"
00093 "\n"
00094 "--b_start\n"
00095 "--b_end\n"
00096 "Used to define manually the start and end wavelength for the reconstructed\n"
00097 "cube. By default the internally defined values are used.\n"
00098 "\n"
00099 "--cmethod\n"
00100 "Following methods of frame combination are available:\n"
00101 "   * 'ksigma' (Default)\n"
00102 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00103 "   are examined. If they deviate significantly, they will be rejected according\n"
00104 "   to the conditions:\n"
00105 "       val > mean + stdev * cpos_rej\n"
00106 "   and\n"
00107 "       val < mean - stdev * cneg_rej\n"
00108 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00109 "   parameters. In the first iteration median and percentile level are used.\n"
00110 "\n"
00111 "   * 'median'\n"
00112 "   At each pixel position the median is calculated.\n"
00113 "\n"
00114 "   * 'average'\n"
00115 "   At each pixel position the average is calculated.\n"
00116 "\n"
00117 "   * 'sum'\n"
00118 "   At each pixel position the sum is calculated.\n"
00119 "\n"
00120 "   * 'min_max'\n"
00121 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00122 "   --cmax and --cmin apply to this method.\n"
00123 "\n"
00124 "--cpos_rej\n"
00125 "--cneg_rej\n"
00126 "--citer\n"
00127 "see --cmethod='ksigma'\n"
00128 "\n"
00129 "--cmax\n"
00130 "--cmin\n"
00131 "see --cmethod='min_max'\n"
00132 "\n"
00133 "--pix_scale\n"
00134 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00135 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00136 "\n"
00137 "--suppress_extension\n"
00138 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00139 "products with the same category are produced, they will be numered consecutively\n"
00140 "starting from 0.\n"
00141 "\n"
00142 "-------------------------------------------------------------------------------\n"
00143 "  Input files:\n"
00144 "\n"
00145 "   DO                    KMOS                                                  \n"
00146 "   category              Type   Explanation                    Required #Frames\n"
00147 "   --------              -----  -----------                    -------- -------\n"
00148 "   FLAT_SKY_FLAT          F2D   Flatlamp-on exposures             Y      1-n   \n"
00149 "                                (at least 3 frames recommended)                \n"
00150 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00151 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00152 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00153 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00154 "\n"
00155 "  Output files:\n"
00156 "\n"
00157 "   DO                    KMOS\n"
00158 "   category              Type   Explanation\n"
00159 "   --------              -----  -----------\n"
00160 "   ILLUM_CORR             F2I    Illumination calibration frame   \n"
00161 "-------------------------------------------------------------------------------\n"
00162 "\n";
00163 
00164 /*-----------------------------------------------------------------------------
00165  *                              Functions code
00166  *----------------------------------------------------------------------------*/
00167 
00184 int cpl_plugin_get_info(cpl_pluginlist *list)
00185 {
00186     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00187     cpl_plugin *plugin = &recipe->interface;
00188 
00189     cpl_plugin_init(plugin,
00190                         CPL_PLUGIN_API,
00191                         KMOS_BINARY_VERSION,
00192                         CPL_PLUGIN_TYPE_RECIPE,
00193                         "kmo_illumination_flat",
00194                         "Alternative to kmo_illumination based on flatfield frames.",
00195                         kmo_illumination_flat_description,
00196                         "Alex Agudo Berbel",
00197                         "kmos-spark@mpe.mpg.de",
00198                         kmos_get_license(),
00199                         kmo_illumination_flat_create,
00200                         kmo_illumination_flat_exec,
00201                         kmo_illumination_flat_destroy);
00202 
00203     cpl_pluginlist_append(list, plugin);
00204 
00205     return 0;
00206 }
00207 
00215 static int kmo_illumination_flat_create(cpl_plugin *plugin)
00216 {
00217     cpl_recipe *recipe;
00218     cpl_parameter *p;
00219 
00220     /* Check that the plugin is part of a valid recipe */
00221     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00222         recipe = (cpl_recipe *)plugin;
00223     else
00224         return -1;
00225 
00226     /* Create the parameters list in the cpl_recipe object */
00227     recipe->parameters = cpl_parameterlist_new();
00228 
00229     /* Fill the parameters list */
00230     /* --imethod */
00231     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.imethod",
00232                                 CPL_TYPE_STRING,
00233                                 "Method to use for interpolation: "
00234                                 "[\"NN\" (nearest neighbour), "
00235                                 "\"lwNN\" (linear weighted nearest neighbor), "
00236                                 "\"swNN\" (square weighted nearest neighbor), "
00237                                 "\"MS\" (Modified Shepard's method), "
00238                                 "\"CS\" (Cubic spline)]",
00239                                 "kmos.kmo_illumination_flat",
00240                                 "CS");
00241     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00242     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00243     cpl_parameterlist_append(recipe->parameters, p);
00244 
00245     /* --neighborhoodRange */
00246     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.neighborhoodRange",
00247                                 CPL_TYPE_DOUBLE,
00248                                 "Defines the range to search for neighbors. "
00249                                 "in pixels",
00250                                 "kmos.kmo_illumination_flat",
00251                                 1.001);
00252     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00253     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00254     cpl_parameterlist_append(recipe->parameters, p);
00255 
00256     /* --flux */
00257     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.flux",
00258                                 CPL_TYPE_BOOL,
00259                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00260                                 "kmos.kmo_illumination_flat",
00261                                 FALSE);
00262     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00263     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00264     cpl_parameterlist_append(recipe->parameters, p);
00265 
00266     /* --pix_scale */
00267     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.pix_scale",
00268                                 CPL_TYPE_DOUBLE,
00269                                 "Change the pixel scale [arcsec]. "
00270                                 "Default of 0.2\" results into cubes of 14x14pix, "
00271                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00272                                 "etc.",
00273                                 "kmos.kmo_illumination_flat",
00274                                 KMOS_PIX_RESOLUTION);
00275     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00276     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00277     cpl_parameterlist_append(recipe->parameters, p);
00278 
00279     /* --suppress_extension */
00280     p = cpl_parameter_new_value("kmos.kmo_illumination_flat.suppress_extension",
00281                                 CPL_TYPE_BOOL,
00282                                 "Suppress arbitrary filename extension. "
00283                                 "(TRUE (apply) or FALSE (don't apply)",
00284                                 "kmos.kmo_illumination_flat",
00285                                 FALSE);
00286     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00287     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00288     cpl_parameterlist_append(recipe->parameters, p);
00289 
00290     // add parameters for band-definition
00291     kmo_band_pars_create(recipe->parameters,
00292                          "kmos.kmo_illumination_flat");
00293 
00294     // add parameters for combining
00295     return kmo_combine_pars_create(recipe->parameters,
00296                                    "kmos.kmo_illumination_flat",
00297                                    DEF_REJ_METHOD,
00298                                    FALSE);
00299 }
00300 
00306 static int kmo_illumination_flat_exec(cpl_plugin *plugin)
00307 {
00308     cpl_recipe  *recipe;
00309 
00310     /* Get the recipe out of the plugin */
00311     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00312         recipe = (cpl_recipe *)plugin;
00313     else return -1;
00314 
00315     return kmo_illumination_flat(recipe->parameters, recipe->frames);
00316 }
00317 
00323 static int kmo_illumination_flat_destroy(cpl_plugin *plugin)
00324 {
00325     cpl_recipe *recipe;
00326 
00327     /* Get the recipe out of the plugin */
00328     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00329         recipe = (cpl_recipe *)plugin;
00330     else return -1 ;
00331 
00332     cpl_parameterlist_delete(recipe->parameters);
00333     return 0 ;
00334 }
00335 
00350 static int kmo_illumination_flat(cpl_parameterlist *parlist, cpl_frameset *frameset)
00351 {
00352     int              ret_val                    = 0,
00353                      nr_devices                 = 0,
00354                      ifu_nr                     = 0,
00355                      nx                         = 0,
00356                      ny                         = 0,
00357                      cmax                       = 0,
00358                      cmin                       = 0,
00359                      citer                      = 0,
00360                      *bounds                    = NULL,
00361                      cnt                        = 0,
00362                      qc_max_dev_id              = 0,
00363                      qc_max_nonunif_id          = 0,
00364                      flux                       = FALSE,
00365                      background                 = FALSE,
00366                      suppress_extension         = FALSE,
00367                      mhalf                      = 3,    //width of median filter is mhalf*2 + 1
00368                      boxsize                    = 0,
00369                      i = 0, j = 0, ix = 0, iy = 0, det_nr = 0,
00370                      firstx = 0, lastx = 0, firsty = 0, lasty = 0,
00371                      xmin = 0, xmax = 0, ymin = 0, ymax = 0;
00372     const int        *punused_ifus              = NULL;
00373     float            *pbad_pix_mask             = NULL,
00374                      *pdata                     = NULL,
00375                      *pnoise                    = NULL;
00376     double           exptime                    = 0.,
00377                      cpos_rej                   = 0.0,
00378                      cneg_rej                   = 0.0,
00379                      neighborhoodRange          = 1.001,
00380                      mean_data                  = 0.0,
00381                      qc_spat_unif               = 0.0,
00382                      qc_max_dev                 = 0.0,
00383                      qc_max_nonunif             = 0.0,
00384                      tmp_stdev                  = 0.0,
00385                      tmp_mean                   = 0.0,
00386                      rotangle                   = 0.0,
00387                      tmp_rotangle               = 0.0,
00388                      rotangle_found             = 0.0,
00389                      pix_scale                  = 0.0;
00390     char             *keyword                   = NULL,
00391                      *fn_lut                    = NULL,
00392                      *suffix                    = NULL,
00393                      *fn_suffix                 = NULL,
00394                      *extname                   = NULL,
00395                      *filter                    = NULL,
00396                      content[256];
00397     const char       *method                    = NULL,
00398                      *cmethod                   = NULL,
00399                      *filter_id_l               = NULL,
00400                      *filter_id                 = NULL,
00401                      *tmp_str                   = NULL;
00402     cpl_array        *calTimestamp              = NULL,
00403                      **unused_ifus_before       = NULL,
00404                      **unused_ifus_after        = NULL;
00405     cpl_frame        *frame                     = NULL,
00406                      *xcalFrame                 = NULL,
00407                      *ycalFrame                 = NULL,
00408                      *lcalFrame                 = NULL;
00409     cpl_image        *img_in                    = NULL,
00410                      *img_dark                  = NULL,
00411                      *img_flat                  = NULL,
00412                      *combined_data             = NULL,
00413                      *xcal                      = NULL,
00414                      *ycal                      = NULL,
00415                      *lcal                      = NULL,
00416                      *bad_pix_mask              = NULL,
00417                      *data_ifu                  = NULL,
00418                      *noise_ifu                 = NULL,
00419                      **stored_data_images       = NULL,
00420                      **stored_noise_images      = NULL,
00421                      *tmp_img                   = NULL;
00422     cpl_imagelist    *cube_data                 = NULL,
00423                      *detector_in               = NULL,
00424                      **stored_data_cubes        = NULL;
00425     cpl_propertylist *main_header               = NULL,
00426                      *tmp_header                = NULL,
00427                      *sub_header                = NULL,
00428                      **stored_sub_headers       = NULL;
00429     cpl_table        *band_table                = NULL;
00430     cpl_vector       *identified_slices         = NULL,
00431                      *calAngles                 = NULL;
00432     main_fits_desc   desc_sky,
00433                      desc_xcal,
00434                      desc_ycal,
00435                      desc_lcal;
00436     gridDefinition   gd;
00437     enum kmo_frame_type fr_type;
00438 
00439     KMO_TRY
00440     {
00441         kmo_init_fits_desc(&desc_sky);
00442         kmo_init_fits_desc(&desc_xcal);
00443         kmo_init_fits_desc(&desc_ycal);
00444         kmo_init_fits_desc(&desc_lcal);
00445 
00446         /* --- check input --- */
00447         KMO_TRY_ASSURE((parlist != NULL) &&
00448                        (frameset != NULL),
00449                        CPL_ERROR_NULL_INPUT,
00450                        "Not all input data is provided!");
00451 
00452         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY_FLAT) >= 1,
00453                        CPL_ERROR_ILLEGAL_INPUT,
00454                        "One or more FLAT_SKY_FLAT frames are required!");
00455 
00456         if (cpl_frameset_count_tags(frameset, FLAT_SKY_FLAT) < 3) {
00457             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00458                                       "3 FLAT_SKY_FLAT frames!");
00459         }
00460 
00461         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00462                        CPL_ERROR_ILLEGAL_INPUT,
00463                        "Exactly one XCAL frame is required!");
00464 
00465         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00466                        CPL_ERROR_ILLEGAL_INPUT,
00467                        "Exactly one YCAL frame is required!");
00468 
00469         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00470                        CPL_ERROR_ILLEGAL_INPUT,
00471                        "Exactly one LCAL frame is required!");
00472 
00473         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00474                        CPL_ERROR_ILLEGAL_INPUT,
00475                        "Exactly one WAVE_BAND frame is required!");
00476 
00477         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination_flat") == 1,
00478                        CPL_ERROR_ILLEGAL_INPUT,
00479                        "Cannot identify RAW and CALIB frames!");
00480 
00481         /* --- get parameters --- */
00482         cpl_msg_info("", "--- Parameter setup for kmo_illumination_flat ---");
00483 
00484         KMO_TRY_EXIT_IF_NULL(
00485             method = kmo_dfs_get_parameter_string(parlist, "kmos.kmo_illumination_flat.imethod"));
00486 
00487         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00488                        (strcmp(method, "lwNN") == 0) ||
00489                        (strcmp(method, "swNN") == 0) ||
00490                        (strcmp(method, "MS") == 0) ||
00491                        (strcmp(method, "CS") == 0),
00492                        CPL_ERROR_ILLEGAL_INPUT,
00493                        "method must be either \"NN\", \"lwNN\", "
00494                        "\"swNN\", \"MS\" or \"CS\"!");
00495 
00496         KMO_TRY_EXIT_IF_ERROR(
00497             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.imethod"));
00498 
00499         neighborhoodRange = kmo_dfs_get_parameter_double(parlist, "kmos.kmo_illumination_flat.neighborhoodRange");
00500         KMO_TRY_CHECK_ERROR_STATE();
00501 
00502         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00503                        CPL_ERROR_ILLEGAL_INPUT,
00504                        "neighborhoodRange must be greater than 0.0");
00505 
00506         KMO_TRY_EXIT_IF_ERROR(
00507             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.neighborhoodRange"));
00508 
00509         flux = kmo_dfs_get_parameter_bool(parlist, "kmos.kmo_illumination_flat.flux");
00510         KMO_TRY_ASSURE((flux == 0) || (flux == 1),
00511                        CPL_ERROR_ILLEGAL_INPUT,
00512                        "flux must be either FALSE or TRUE!");
00513         KMO_TRY_EXIT_IF_ERROR(
00514             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.flux"));
00515 
00516         pix_scale = kmo_dfs_get_parameter_double(parlist, "kmos.kmo_illumination_flat.pix_scale");
00517         KMO_TRY_CHECK_ERROR_STATE();
00518         KMO_TRY_EXIT_IF_ERROR(
00519            kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.pix_scale"));
00520         KMO_TRY_ASSURE((pix_scale >= 0.01) && (pix_scale <= 0.4),
00521                        CPL_ERROR_ILLEGAL_INPUT,
00522                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00523                        "with 7x7 to 280x280 pixels)!");
00524 
00525         suppress_extension = kmo_dfs_get_parameter_bool(parlist, "kmos.kmo_illumination_flat.suppress_extension");
00526         KMO_TRY_CHECK_ERROR_STATE();
00527         KMO_TRY_EXIT_IF_ERROR(
00528             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination_flat.suppress_extension"));
00529 
00530         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00531                        CPL_ERROR_ILLEGAL_INPUT,
00532                        "suppress_extension must be TRUE or FALSE!");
00533 
00534         kmo_band_pars_load(parlist, "kmos.kmo_illumination_flat");
00535 
00536         KMO_TRY_EXIT_IF_ERROR(
00537             kmo_combine_pars_load(parlist, "kmos.kmo_illumination_flat",
00538                                   &cmethod, &cpos_rej, &cneg_rej,
00539                                   &citer, &cmin, &cmax, FALSE));
00540         cpl_msg_info("", "-------------------------------------------");
00541 
00542         // check if filter_id, grating_id and rotator offset match for all
00543         // detectors
00544         KMO_TRY_EXIT_IF_ERROR(
00545             kmo_check_frameset_setup(frameset, FLAT_SKY_FLAT, TRUE, FALSE, TRUE));
00546         KMO_TRY_EXIT_IF_ERROR(
00547             kmo_check_frame_setup(frameset, FLAT_SKY_FLAT, XCAL, TRUE, FALSE, TRUE));
00548         KMO_TRY_EXIT_IF_ERROR(
00549             kmo_check_frame_setup(frameset, XCAL, YCAL, TRUE, FALSE, TRUE));
00550         KMO_TRY_EXIT_IF_ERROR(
00551             kmo_check_frame_setup(frameset, XCAL, LCAL, TRUE, FALSE, TRUE));
00552 
00553         KMO_TRY_EXIT_IF_NULL(
00554             frame = kmo_dfs_get_frame(frameset, XCAL));
00555         KMO_TRY_EXIT_IF_NULL(
00556             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00557 
00558         KMO_TRY_EXIT_IF_ERROR(
00559             kmo_check_frame_setup_md5_xycal(frameset));
00560         KMO_TRY_EXIT_IF_ERROR(
00561             kmo_check_frame_setup_md5(frameset));
00562 
00563         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00564         cpl_msg_info("", "(grating 1, 2 & 3)");
00565 
00566         // check which IFUs are active for all frames
00567         KMO_TRY_EXIT_IF_NULL(
00568             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00569 
00570         KMO_TRY_EXIT_IF_NULL(
00571             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00572 
00573         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00574 
00575         // load desc for XCAL and check
00576         KMO_TRY_EXIT_IF_NULL(
00577             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00578         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00579         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00580                                       "be in KMOS-format!");
00581         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00582                        (desc_xcal.ex_badpix == FALSE) &&
00583                        (desc_xcal.fits_type == f2d_fits) &&
00584                        (desc_xcal.frame_type == detector_frame),
00585                        CPL_ERROR_ILLEGAL_INPUT,
00586                        "XCAL isn't in the correct format!!!");
00587         nx = desc_xcal.naxis1;
00588         ny = desc_xcal.naxis2;
00589         nr_devices = desc_xcal.nr_ext;
00590 
00591         // load desc for YCAL and check
00592         KMO_TRY_EXIT_IF_NULL(
00593             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00594         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00595         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00596                                       "be in KMOS-format!");
00597         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00598                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00599                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00600                        (desc_ycal.frame_type == desc_xcal.frame_type),
00601                        CPL_ERROR_ILLEGAL_INPUT,
00602                        "YCAL isn't in the correct format!!!");
00603         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00604                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00605                        CPL_ERROR_ILLEGAL_INPUT,
00606                        "XCAL and YCAL frame haven't same dimensions! "
00607                        "(x,y): (%d,%d) vs (%d,%d)",
00608                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00609 
00610         // load desc for LCAL and check
00611         KMO_TRY_EXIT_IF_NULL(
00612             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00613         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00614         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00615                                       "be in KMOS-format!");
00616         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00617                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00618                        (desc_lcal.frame_type == desc_xcal.frame_type),
00619                        CPL_ERROR_ILLEGAL_INPUT,
00620                        "LCAL isn't in the correct format!!!");
00621         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00622                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00623                        CPL_ERROR_ILLEGAL_INPUT,
00624                        "XCAL and LCAL frame haven't same dimensions! "
00625                        "(x,y): (%d,%d) vs (%d,%d)",
00626                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00627         KMO_TRY_EXIT_IF_NULL(
00628             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00629 
00630         // load desc for FLAT_SKY_FLAT and check
00631         nr_devices = KMOS_NR_DETECTORS;
00632         KMO_TRY_EXIT_IF_NULL(
00633             frame = kmo_dfs_get_frame(frameset, FLAT_SKY_FLAT));
00634 
00635         KMO_TRY_EXIT_IF_NULL(
00636             main_header = kmclipm_propertylist_load(
00637                                          cpl_frame_get_filename(frame), 0));
00638         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00639         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00640         kmclipm_strip_angle(&rotangle);
00641         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00642         KMO_TRY_CHECK_ERROR_STATE("EXPTIME keyword in main header "
00643                                   "missing!");
00644         cpl_propertylist_delete(main_header); main_header = NULL;
00645 
00646         cnt = 1;
00647         while (frame != NULL) {
00648             KMO_TRY_EXIT_IF_NULL(
00649                 main_header = kmclipm_propertylist_load(
00650                                              cpl_frame_get_filename(frame), 0));
00651 
00652             desc_sky = kmo_identify_fits_header(
00653                         cpl_frame_get_filename(frame));
00654             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY_FLAT frame doesn't seem to "
00655                                           "be in KMOS-format!");
00656             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00657                            (desc_sky.ex_badpix == FALSE) &&
00658                            (desc_sky.fits_type == raw_fits) &&
00659                            (desc_sky.frame_type == detector_frame),
00660                            CPL_ERROR_ILLEGAL_INPUT,
00661                            "FLAT_SKY_FLAT isn't in the correct format!!!");
00662             kmo_free_fits_desc(&desc_sky);
00663             kmo_init_fits_desc(&desc_sky);
00664 
00665             KMO_TRY_ASSURE(
00666                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00667                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE),
00668                 CPL_ERROR_ILLEGAL_INPUT,
00669                 "Arc lamps must be switched off!");
00670 
00671             KMO_TRY_ASSURE(cpl_propertylist_get_double(main_header, EXPTIME) == exptime,
00672                            CPL_ERROR_ILLEGAL_INPUT,
00673                            "EXPTIME isn't the same for all frames: (is %g and %g).",
00674                            cpl_propertylist_get_double(main_header, EXPTIME), exptime);
00675 
00676             // assert that filters have correct IDs and that all detectors of
00677             // all input frames have the same filter set
00678             for (i = 1; i <= KMOS_NR_DETECTORS; i++) {
00679                 // ESO INS FILTi ID
00680                 KMO_TRY_EXIT_IF_NULL(
00681                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00682                 KMO_TRY_EXIT_IF_NULL(
00683                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00684 
00685                 KMO_TRY_EXIT_IF_NULL(
00686                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00687                 cpl_free(keyword); keyword = NULL;
00688 
00689                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00690                                (strcmp(filter_id, "YJ") == 0) ||
00691                                (strcmp(filter_id, "H") == 0) ||
00692                                (strcmp(filter_id, "K") == 0) ||
00693                                (strcmp(filter_id, "HK") == 0),
00694                                CPL_ERROR_ILLEGAL_INPUT,
00695                                "Filter ID in primary header must be either 'IZ', "
00696                                "'YJ', 'H', 'K' or " "'HK' !");
00697 
00698                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00699                                CPL_ERROR_ILLEGAL_INPUT,
00700                                "Filter IDs must be the same for FLAT_SKY_FLAT frame"
00701                                " and lcal frame!"
00702                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00703                                i, cpl_frame_get_filename(frame),
00704                                filter_id, filter_id_l);
00705 
00706                 // ESO INS GRATi ID
00707                 KMO_TRY_EXIT_IF_NULL(
00708                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00709                 KMO_TRY_EXIT_IF_NULL(
00710                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00711 
00712                 KMO_TRY_EXIT_IF_NULL(
00713                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00714                 cpl_free(keyword); keyword = NULL;
00715 
00716                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00717                                (strcmp(filter_id, "YJ") == 0) ||
00718                                (strcmp(filter_id, "H") == 0) ||
00719                                (strcmp(filter_id, "K") == 0) ||
00720                                (strcmp(filter_id, "HK") == 0),
00721                                CPL_ERROR_ILLEGAL_INPUT,
00722                                "Grating ID in primary header must be either "
00723                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00724 
00725                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00726                                CPL_ERROR_ILLEGAL_INPUT,
00727                                "Grating IDs must be the same for FLAT_SKY_FLAT frame"
00728                                " and lcal frame!"
00729                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00730                                i, cpl_frame_get_filename(frame),
00731                                filter_id, filter_id_l);
00732 
00733                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00734                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00735                 kmclipm_strip_angle(&tmp_rotangle);
00736                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00737                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00738                         CPL_ERROR_ILLEGAL_INPUT,
00739                         "OCS ROT NAANGLE of sky flat frames differ too much: %f %f",
00740                         rotangle, tmp_rotangle);
00741             }
00742             cpl_propertylist_delete(main_header); main_header = NULL;
00743 
00744             // get next FLAT_SKY_FLAT frame
00745             frame = kmo_dfs_get_frame(frameset, NULL);
00746             KMO_TRY_CHECK_ERROR_STATE();
00747             cnt++;
00748         }
00749 
00750         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00751 
00752         if (cpl_frameset_count_tags(frameset, FLAT_SKY_FLAT) == 1) {
00753             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00754                             "since there is only one input frame! (The output "
00755                             "file won't have any noise extensions)");
00756 
00757             cmethod = "average";
00758         }
00759 
00760         KMO_TRY_EXIT_IF_NULL(
00761             frame = kmo_dfs_get_frame(frameset, FLAT_SKY_FLAT));
00762         KMO_TRY_EXIT_IF_NULL(
00763             main_header = kmo_dfs_load_primary_header(frameset, FLAT_SKY_FLAT));
00764         KMO_TRY_EXIT_IF_NULL(
00765             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
00766         KMO_TRY_EXIT_IF_NULL(
00767             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
00768         cpl_free(keyword); keyword = NULL;
00769 
00770         // setup grid definition, wavelength start and end points will be set
00771         // in the detector loop
00772         KMO_TRY_EXIT_IF_ERROR(
00773             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale, 0.));
00774 
00775         // create filename for LUT
00776         KMO_TRY_EXIT_IF_NULL(
00777             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
00778 
00779         // extract bounds
00780         KMO_TRY_EXIT_IF_NULL(
00781             tmp_header = kmo_dfs_load_primary_header(frameset, XCAL));
00782         KMO_TRY_EXIT_IF_NULL(
00783             bounds = kmclipm_extract_bounds(tmp_header));
00784         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00785 
00786         // get timestamps of xcal, ycal & lcal
00787         KMO_TRY_EXIT_IF_NULL(
00788             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
00789 
00790         // create arrays to hold reconstructed data and noise cubes and
00791         // their headers
00792         KMO_TRY_EXIT_IF_NULL(
00793             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00794                                                             sizeof(cpl_imagelist*)));
00795         KMO_TRY_EXIT_IF_NULL(
00796             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00797                                                          sizeof(cpl_image*)));
00798         KMO_TRY_EXIT_IF_NULL(
00799             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00800                                                           sizeof(cpl_image*)));
00801         KMO_TRY_EXIT_IF_NULL(
00802             stored_sub_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
00803                                                                 sizeof(cpl_propertylist*)));
00804         KMO_TRY_EXIT_IF_NULL(
00805             calAngles = cpl_vector_new(3));
00806 
00807         //
00808         // loop through all detectors
00809         //
00810         for (det_nr = 1; det_nr <= nr_devices; det_nr++) {
00811             cpl_msg_info("","Processing detector No. %d", det_nr);
00812 
00813             KMO_TRY_EXIT_IF_NULL(
00814                 detector_in = cpl_imagelist_new());
00815 
00816             // load data of det_nr of all FLAT_SKY_FLAT frames into an imagelist
00817             KMO_TRY_EXIT_IF_NULL(
00818                 img_in = kmo_dfs_load_image(frameset, FLAT_SKY_FLAT, det_nr, FALSE, TRUE, NULL));
00819 
00820             cnt = 0;
00821             while (img_in != NULL) {
00822                 cpl_imagelist_set(detector_in, img_in, cnt);
00823                 KMO_TRY_CHECK_ERROR_STATE();
00824 
00825                 /* load same extension of next FLAT_SKY_FLAT frame*/
00826                 img_in = kmo_dfs_load_image(frameset, NULL, det_nr, FALSE, TRUE, NULL);
00827                 KMO_TRY_CHECK_ERROR_STATE();
00828 
00829                 cnt++;
00830             }
00831 
00832             //
00833             // process imagelist
00834             //
00835 
00836             // combine imagelist (data only) and create noise (stdev of data)
00837             cpl_msg_info("","Combining frames...");
00838             KMO_TRY_EXIT_IF_ERROR(
00839                 kmclipm_combine_frames(detector_in,
00840                                        NULL,
00841                                        NULL,
00842                                        cmethod,
00843                                        cpos_rej,
00844                                        cneg_rej,
00845                                        citer,
00846                                        cmax,
00847                                        cmin,
00848                                        &combined_data,
00849                                        NULL,
00850                                        -1.0));
00851 
00852             if (img_dark == NULL) {
00853                 KMO_TRY_EXIT_IF_NULL(
00854                     img_dark = cpl_image_duplicate(combined_data));
00855                 KMO_TRY_EXIT_IF_ERROR(
00856                     cpl_image_multiply_scalar(img_dark, 0));
00857             }
00858 
00859             if (img_flat == NULL) {
00860                 KMO_TRY_EXIT_IF_NULL(
00861                     img_flat = cpl_image_duplicate(combined_data));
00862                 KMO_TRY_EXIT_IF_ERROR(
00863                     cpl_image_multiply_scalar(img_flat, 0));
00864                 cpl_image_add_scalar(img_flat, 1);
00865             }
00866 
00867             if (kmclipm_omit_warning_one_slice > 10) {
00868 // AA: commmented this out: Too unclear for the user, no benefit to know about this number
00869 //                cpl_msg_warning(cpl_func, "Previous warning (number of "
00870 //                                          "identified slices) occured %d times.",
00871 //                                kmclipm_omit_warning_one_slice);
00872                 kmclipm_omit_warning_one_slice = FALSE;
00873             }
00874 
00875             cpl_imagelist_delete(detector_in); detector_in = NULL;
00876 
00877             // load calibration files
00878             KMO_TRY_EXIT_IF_NULL(
00879                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
00880                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
00881 
00882             KMO_TRY_EXIT_IF_ERROR(
00883                 cpl_vector_set(calAngles, 0, rotangle_found));
00884             KMO_TRY_EXIT_IF_NULL(
00885                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
00886                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
00887             KMO_TRY_EXIT_IF_ERROR(
00888                 cpl_vector_set(calAngles, 1, rotangle_found));
00889             KMO_TRY_EXIT_IF_NULL(
00890                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
00891                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
00892             KMO_TRY_EXIT_IF_ERROR(
00893                 cpl_vector_set(calAngles, 2, rotangle_found));
00894 
00895             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
00896             KMO_TRY_EXIT_IF_NULL(
00897                 bad_pix_mask = cpl_image_duplicate(xcal));
00898 
00899             KMO_TRY_EXIT_IF_NULL(
00900                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
00901             for (ix = 0; ix < nx; ix++) {
00902                 for (iy = 0; iy < ny; iy++) {
00903                     if (isnan(pbad_pix_mask[ix+nx*iy])) {
00904                         pbad_pix_mask[ix+nx*iy] = 0.;
00905                     } else {
00906                         pbad_pix_mask[ix+nx*iy] = 1.;
00907                     }
00908                 }
00909             }
00910             KMO_TRY_CHECK_ERROR_STATE();
00911 
00912             //
00913             // reconstruct
00914             //
00915             print_warning_once_reconstruct = FALSE;
00916 
00917             char *tmp_band_method = getenv("KMO_BAND_METHOD");
00918             int band_method = 0;
00919             if (tmp_band_method != NULL) {
00920                 band_method = atoi(tmp_band_method);
00921             }
00922 
00923             // ESO INS FILTi ID
00924             KMO_TRY_EXIT_IF_NULL(
00925                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
00926                                       IFU_FILTID_POSTFIX));
00927             KMO_TRY_EXIT_IF_NULL(
00928                 filter_id = cpl_propertylist_get_string(main_header, keyword));
00929             cpl_free(keyword); keyword = NULL;
00930 
00931             KMO_TRY_EXIT_IF_NULL(
00932                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
00933             KMO_TRY_EXIT_IF_ERROR(
00934                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
00935                                              band_table));
00936             cpl_table_delete(band_table); band_table = NULL;
00937 
00938             cpl_msg_info("","Reconstructing cubes...");
00939             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
00940                 // update sub-header
00941                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + i + 1;
00942 
00943                 // load raw image and sub-header
00944                 KMO_TRY_EXIT_IF_NULL(
00945                     sub_header = kmo_dfs_load_sub_header(frameset, FLAT_SKY_FLAT,
00946                                                          det_nr, FALSE));
00947 
00948                 KMO_TRY_EXIT_IF_NULL(
00949                     punused_ifus = cpl_array_get_data_int_const(
00950                                                   unused_ifus_after[det_nr-1]));
00951 
00952                 // check if IFU is valid according to main header keywords &
00953                 // calibration files
00954                 KMO_TRY_EXIT_IF_NULL(
00955                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
00956                                           IFU_VALID_POSTFIX));
00957                 KMO_TRY_CHECK_ERROR_STATE();
00958 
00959                 // just to see if keyword exists
00960                 cpl_propertylist_get_string(main_header, keyword);
00961                 cpl_free(keyword); keyword = NULL;
00962 
00963                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
00964                     (bounds[2*(ifu_nr-1)] != -1) &&
00965                     (bounds[2*(ifu_nr-1)+1] != -1) &&
00966                     (punused_ifus[i] == 0))
00967                 {
00968                     // IFU is valid
00969                     cpl_error_reset();
00970 
00971                     // calculate WCS
00972                     KMO_TRY_EXIT_IF_ERROR(
00973                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
00974 
00975                     // reconstruct data
00976                     KMO_TRY_EXIT_IF_ERROR(
00977                         kmo_reconstruct_sci_image(ifu_nr,
00978                                                 bounds[2*(ifu_nr-1)],
00979                                                 bounds[2*(ifu_nr-1)+1],
00980                                                 combined_data,
00981                                                 NULL,
00982                                                 img_dark,
00983                                                 NULL,
00984                                                 img_flat,
00985                                                 NULL,
00986                                                 xcal,
00987                                                 ycal,
00988                                                 lcal,
00989                                                 &gd,
00990                                                 calTimestamp,
00991                                                 calAngles,
00992                                                 fn_lut,
00993                                                 &cube_data,
00994                                                 NULL,
00995                                                 flux,
00996                                                 background,
00997                                                 NULL,
00998                                                 NULL,
00999                                                 NULL));
01000                     KMO_TRY_CHECK_ERROR_STATE();
01001                 } else {
01002                     // IFU is invalid
01003                     cpl_error_reset();
01004                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01005 
01006                 // save output
01007                 KMO_TRY_EXIT_IF_NULL(
01008                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01009 
01010                 KMO_TRY_EXIT_IF_ERROR(
01011                     kmclipm_update_property_string(sub_header, EXTNAME,
01012                                                    extname,
01013                                                    "FITS extension name"));
01014 
01015                 cpl_free(extname); extname = NULL;
01016 
01017                 // store cube and sub header into array for later
01018                 stored_data_cubes[ifu_nr - 1] = cube_data;
01019                 stored_sub_headers[ifu_nr - 1] = sub_header;
01020 
01021                 cpl_image_delete(data_ifu); data_ifu = NULL;
01022                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01023                 cube_data = NULL;
01024             } // for i IFUs
01025 
01026             // free memory
01027             cpl_image_delete(combined_data); combined_data = NULL;
01028             cpl_image_delete(xcal); xcal = NULL;
01029             cpl_image_delete(ycal); ycal = NULL;
01030             cpl_image_delete(lcal); lcal = NULL;
01031             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01032         } // for nr_devices
01033 
01034         cpl_image_delete(img_dark); img_dark = NULL;
01035         cpl_image_delete(img_flat); img_flat = NULL;
01036 
01037         //
01038         // collapse cubes using rejection and apply median filtering
01039         //
01040         KMO_TRY_EXIT_IF_NULL(
01041             identified_slices = cpl_vector_new(gd.l.dim));
01042         KMO_TRY_EXIT_IF_ERROR(
01043             cpl_vector_fill(identified_slices, 1));
01044 
01045         cpl_msg_info("","Collapsing cubes...");
01046         for (j = 0; j < nr_devices; j++) {
01047             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01048                 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01049                 KMO_TRY_EXIT_IF_NULL(
01050                     punused_ifus = cpl_array_get_data_int_const(
01051                                                   unused_ifus_after[j]));
01052                 if (punused_ifus[i] == 0) {
01053                     if (stored_data_cubes[ifu_nr] != NULL) {
01054                         KMO_TRY_EXIT_IF_ERROR(
01055                             kmclipm_make_image(stored_data_cubes[ifu_nr],
01056                                                NULL,
01057                                                &stored_data_images[ifu_nr],
01058                                                &stored_noise_images[ifu_nr],
01059                                                identified_slices,
01060                                                cmethod, cpos_rej, cneg_rej,
01061                                                citer, cmax, cmin));
01062                     }
01063 
01064                     //
01065                     // apply median smoothing
01066                     //
01067 
01068                     // taking care of edges (IFUs 1-16 top/bottom, IFUs 17-24 left/right)
01069                     if (ifu_nr+1 <= 2*KMOS_IFUS_PER_DETECTOR) {
01070                         firstx = 0;
01071                         lastx = 13;
01072                         firsty = 1;
01073                         lasty = 12;
01074                     } else {
01075                         firstx = 1;
01076                         lastx= 12;
01077                         firsty = 0;
01078                         lasty = 13;
01079                     }
01080 
01081                     KMO_TRY_EXIT_IF_NULL(
01082                         tmp_img = cpl_image_duplicate(stored_data_images[ifu_nr]));
01083                     KMO_TRY_EXIT_IF_NULL(
01084                         pdata = cpl_image_get_data_float(tmp_img));
01085                     KMO_TRY_EXIT_IF_NULL(
01086                         pnoise = cpl_image_get_data_float(stored_noise_images[ifu_nr]));
01087                     nx = cpl_image_get_size_x(tmp_img);
01088                     ny = cpl_image_get_size_y(tmp_img);
01089                     KMO_TRY_CHECK_ERROR_STATE();
01090 
01091                     // median filtering
01092                     for (ix = 0; ix < nx; ix++) {
01093                         for (iy = 0; iy < ny; iy++) {
01094                             if (ix-mhalf > firstx) { xmin = ix-mhalf; } else { xmin = firstx; }
01095                             if (ix+mhalf < lastx)  { xmax = ix+mhalf; } else { xmax = lastx; }
01096                             if (iy-mhalf > firsty) { ymin = iy-mhalf; } else { ymin = firsty; }
01097                             if (iy+mhalf < lasty)  { ymax = iy+mhalf; } else { ymax = lasty; }
01098 
01099                             pdata[ix+nx*iy] = cpl_image_get_median_window(
01100                                                                 stored_data_images[ifu_nr],
01101                                                                 xmin+1, ymin+1, xmax+1, ymax+1);
01102                             KMO_TRY_CHECK_ERROR_STATE();
01103 
01104                             if (stored_noise_images[ifu_nr] != NULL) {
01105                                 boxsize = (xmax-xmin+1)*(ymax-ymin+1);
01106                                 pnoise[ix+nx*iy] /= boxsize; //sqrt(boxsize*boxsize)
01107                             }
01108 
01109                         }
01110                     }
01111 
01112                     // replace images
01113                     cpl_image_delete(stored_data_images[ifu_nr]);
01114                     stored_data_images[ifu_nr] = tmp_img;
01115                 } else {
01116                     // IFU is invalid
01117                 }
01118             } // end for (i) ifu_nr
01119         } // end for (j) det_nr
01120         cpl_vector_delete(identified_slices); identified_slices = NULL;
01121 
01122         // normalise all IFUs of a detector as a group.
01123         // Calculate mean of each IFU, add up and divide by number of successful
01124         // averaged IFUs.
01125         // Then divide all valid IFUs with mean value
01126         for (j = 0; j < nr_devices; j++) {
01127             cnt = 0;
01128             mean_data = 0;
01129             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01130                 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01131                 if (stored_data_images[ifu_nr] != NULL) {
01132                     KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[ifu_nr]) <
01133                                    cpl_image_get_size_x(stored_data_images[ifu_nr])*
01134                                    cpl_image_get_size_y(stored_data_images[ifu_nr]),
01135                                    CPL_ERROR_ILLEGAL_INPUT,
01136                                    "The collapsed image contains only invalid values!");
01137                     mean_data += cpl_image_get_mean(stored_data_images[ifu_nr]);
01138                     KMO_TRY_CHECK_ERROR_STATE();
01139                     cnt++;
01140                 }
01141 
01142             } // end for (i) ifu_nr
01143             mean_data /= cnt;
01144             if (mean_data != 0.0) {
01145                 for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01146                     ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01147                     if (stored_data_images[ifu_nr] != NULL) {
01148                         KMO_TRY_EXIT_IF_ERROR(
01149                             cpl_image_divide_scalar(stored_data_images[ifu_nr], mean_data));
01150                     }
01151                     if (stored_noise_images[ifu_nr] != NULL) {
01152                         KMO_TRY_EXIT_IF_ERROR(
01153                             cpl_image_divide_scalar(stored_noise_images[ifu_nr], mean_data));
01154                     }
01155                 } // end for (i) ifu_nr
01156             } else {
01157                 cpl_msg_warning(cpl_func, "Data couldn't be normalised (mean = 0.0)!");
01158             }
01159         } // end for (j) det_nr
01160 
01161         //
01162         // invert data and noise
01163         //
01164         double old_val  = 0.,
01165                new_val  = 0.;
01166         for (j = 0; j < nr_devices; j++) {
01167             cnt = 0;
01168             mean_data = 0;
01169             for (i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01170                 ifu_nr = j*KMOS_IFUS_PER_DETECTOR + i;
01171                 if (stored_data_images[ifu_nr] != NULL) {
01172                     // invert data
01173                     KMO_TRY_EXIT_IF_NULL(
01174                         pdata = cpl_image_get_data_float(stored_data_images[ifu_nr]));
01175                     if (stored_noise_images[ifu_nr] != NULL) {
01176                         KMO_TRY_EXIT_IF_NULL(
01177                             pnoise = cpl_image_get_data_float(stored_noise_images[ifu_nr]));
01178                     }
01179                     for (ix = 0; ix < nx; ix++) {
01180                         for (iy = 0; iy < ny; iy++) {
01181                             old_val = pdata[ix+nx*iy];
01182                             pdata[ix+nx*iy] = 1. / pdata[ix+nx*iy];
01183                             if (stored_noise_images[ifu_nr] != NULL) {
01184                                 new_val = pdata[ix+nx*iy];
01185                                 pnoise[ix+nx*iy] = sqrt(pow(new_val, 2) *
01186                                                         pow(pnoise[ix+nx*iy], 2) /
01187                                                         pow(old_val, 2));
01188                             }
01189                         }
01190                     }
01191                 }
01192             } // end for (i) ifu_nr
01193         } // end for (j) det_nr
01194 
01195         // calculate qc parameters on normalised data
01196         qc_spat_unif = 0.0;
01197         cnt = 0;
01198         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01199             if (stored_data_images[i] != NULL) {
01200                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01201                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01202 
01203                 qc_spat_unif += pow(tmp_mean-1, 2);
01204                 if (fabs(tmp_mean) > qc_max_dev) {
01205                     qc_max_dev = tmp_mean-1;
01206                     qc_max_dev_id = i+1;
01207                 }
01208                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01209                     qc_max_nonunif = tmp_stdev;
01210                     qc_max_nonunif_id = i+1;
01211                 }
01212                 KMO_TRY_CHECK_ERROR_STATE();
01213                 cnt++;
01214             }
01215         }
01216         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01217 
01218         //
01219         // save data
01220         //
01221 
01222         // update which IFUs are not used
01223         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01224 
01225         KMO_TRY_EXIT_IF_ERROR(
01226             kmo_set_unused_ifus(unused_ifus_after, main_header,
01227                                 "kmo_illumination_flat"));
01228 
01229         cpl_msg_info("","Saving data...");
01230 
01231         KMO_TRY_EXIT_IF_ERROR(
01232             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01233                                            "[adu] uniformity of illumination correction"));
01234         KMO_TRY_EXIT_IF_ERROR(
01235             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01236                                            "[adu] max. deviation from unity"));
01237         KMO_TRY_EXIT_IF_ERROR(
01238             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01239                                         "[] IFU ID with max. dev. from unity"));
01240         KMO_TRY_EXIT_IF_ERROR(
01241             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01242                                            "[adu] max. stdev of illumination corr."));
01243         KMO_TRY_EXIT_IF_ERROR(
01244             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01245                                         "[] IFU ID with max. stdev in illum. corr."));
01246 
01247         if (!suppress_extension) {
01248             KMO_TRY_EXIT_IF_NULL(
01249                 fn_suffix = cpl_sprintf("%s", suffix));
01250         } else {
01251             KMO_TRY_EXIT_IF_NULL(
01252                 fn_suffix = cpl_sprintf("%s", ""));
01253         }
01254         KMO_TRY_EXIT_IF_ERROR(
01255             kmo_dfs_save_main_header(frameset, ILLUM_CORR_FLAT, fn_suffix, frame,
01256                                      main_header, parlist, cpl_func));
01257 
01258         for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01259             KMO_TRY_EXIT_IF_ERROR(
01260                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR_FLAT, fn_suffix,
01261                                    stored_sub_headers[i], 0./0.));
01262 
01263             KMO_TRY_EXIT_IF_NULL(
01264                 tmp_str = cpl_propertylist_get_string(stored_sub_headers[i], EXTNAME));
01265             KMO_TRY_EXIT_IF_ERROR(
01266                 kmo_extname_extractor(tmp_str, &fr_type, &ifu_nr, content));
01267             KMO_TRY_EXIT_IF_NULL(
01268                 extname = kmo_extname_creator(ifu_frame, ifu_nr,
01269                                               EXT_NOISE));
01270             KMO_TRY_EXIT_IF_ERROR(
01271                 kmclipm_update_property_string(stored_sub_headers[i], EXTNAME,
01272                                                extname, "FITS extension name"));
01273             cpl_free(extname); extname = NULL;
01274 
01275             KMO_TRY_EXIT_IF_ERROR(
01276                 kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR_FLAT,
01277                                    fn_suffix, stored_sub_headers[i], 0./0.));
01278         }
01279     }
01280     KMO_CATCH
01281     {
01282         KMO_CATCH_MSG();
01283         ret_val = -1;
01284     }
01285     kmo_free_fits_desc(&desc_sky);
01286     kmo_free_fits_desc(&desc_xcal);
01287     kmo_free_fits_desc(&desc_ycal);
01288     kmo_free_fits_desc(&desc_lcal);
01289     cpl_image_delete(combined_data); combined_data = NULL;
01290     cpl_image_delete(xcal); xcal = NULL;
01291     cpl_image_delete(ycal); ycal = NULL;
01292     cpl_image_delete(lcal); lcal = NULL;
01293     cpl_image_delete(img_dark); img_dark = NULL;
01294     cpl_image_delete(img_flat); img_flat = NULL;
01295     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01296     cpl_free(bounds); bounds = NULL;
01297     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01298     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01299     cpl_free(fn_lut); fn_lut = NULL;
01300     cpl_free(suffix); suffix = NULL;
01301     cpl_free(fn_suffix); fn_suffix = NULL;
01302     cpl_free(filter); filter = NULL;
01303     if (calAngles != NULL) {
01304         cpl_vector_delete(calAngles); calAngles = NULL;
01305     }
01306     cpl_propertylist_delete(main_header); main_header = NULL;
01307     for (i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01308         if (stored_data_cubes != NULL) {
01309             cpl_imagelist_delete(stored_data_cubes[i]);
01310             stored_data_cubes[i] = NULL;
01311         }
01312         if (stored_data_images != NULL) {
01313             cpl_image_delete(stored_data_images[i]);
01314             stored_data_images[i] = NULL;
01315         }
01316         if (stored_noise_images != NULL) {
01317             cpl_image_delete(stored_noise_images[i]);
01318             stored_noise_images[i] = NULL;
01319         }
01320         if (stored_sub_headers != NULL) {
01321             cpl_propertylist_delete(stored_sub_headers[i]);
01322             stored_sub_headers[i] = NULL;
01323         }
01324     }
01325     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01326     cpl_free(stored_data_images); stored_data_images = NULL;
01327     cpl_free(stored_noise_images); stored_noise_images = NULL;
01328     cpl_free(stored_sub_headers); stored_sub_headers = NULL;
01329 
01330     return ret_val;
01331 }
01332