KMOS Pipeline Reference Manual  1.1.4
kmo_illumination.c
00001 /* $Id: kmo_illumination.c,v 1.51 2013/05/21 12:13:58 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013/05/21 12:13:58 $
00024  * $Revision: 1.51 $
00025  * $Name: HEAD $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmo_priv_reconstruct.h"
00042 #include "kmo_priv_functions.h"
00043 #include "kmo_priv_flat.h"
00044 #include "kmo_priv_wave_cal.h"
00045 #include "kmclipm_priv_splines.h"
00046 #include "kmo_functions.h"
00047 #include "kmo_cpl_extensions.h"
00048 #include "kmo_dfs.h"
00049 #include "kmo_error.h"
00050 #include "kmo_constants.h"
00051 #include "kmo_debug.h"
00052 
00053 /*-----------------------------------------------------------------------------
00054  *                          Functions prototypes
00055  *----------------------------------------------------------------------------*/
00056 
00057 static int kmo_illumination_create(cpl_plugin *);
00058 static int kmo_illumination_exec(cpl_plugin *);
00059 static int kmo_illumination_destroy(cpl_plugin *);
00060 static int kmo_illumination(cpl_parameterlist *, cpl_frameset *);
00061 
00062 /*-----------------------------------------------------------------------------
00063  *                          Static variables
00064  *----------------------------------------------------------------------------*/
00065 
00066 static char kmo_illumination_description[] =
00067 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00068 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00069 "generates the spectral calibration frame needed in this recipe. As input at\n"
00070 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n"
00071 "bration frames are required.\n"
00072 "\n"
00073 "BASIC PARAMETERS:\n"
00074 "-----------------\n"
00075 "--imethod\n"
00076 "The interpolation method used for reconstruction.\n"
00077 "\n"
00078 "--range\n"
00079 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n"
00080 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n"
00081 "\n"
00082 "ADVANCED PARAMETERS\n"
00083 "-------------------\n"
00084 "--add-all\n"
00085 "By default the first FLAT_SKY frame is omitted, since in the\n"
00086 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n"
00087 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all frames\n"
00088 "should be considered, set this parameter to TRUE.\n"
00089 "\n"
00090 "--neighborhoodRange\n"
00091 "Defines the range to search for neighbors during reconstruction\n"
00092 "\n"
00093 "--b_samples\n"
00094 "The number of samples in spectral direction for the reconstructed cube.\n"
00095 "Ideally this number should be greater than 2048, the detector size.\n"
00096 "\n"
00097 "--b_start\n"
00098 "--b_end\n"
00099 "Used to define manually the start and end wavelength for the reconstructed\n"
00100 "cube. By default the internally defined values are used.\n"
00101 "\n"
00102 "--cmethod\n"
00103 "Following methods of frame combination are available:\n"
00104 "   * 'ksigma' (Default)\n"
00105 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00106 "   are examined. If they deviate significantly, they will be rejected according\n"
00107 "   to the conditions:\n"
00108 "       val > mean + stdev * cpos_rej\n"
00109 "   and\n"
00110 "       val < mean - stdev * cneg_rej\n"
00111 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00112 "   parameters. In the first iteration median and percentile level are used.\n"
00113 "\n"
00114 "   * 'median'\n"
00115 "   At each pixel position the median is calculated.\n"
00116 "\n"
00117 "   * 'average'\n"
00118 "   At each pixel position the average is calculated.\n"
00119 "\n"
00120 "   * 'sum'\n"
00121 "   At each pixel position the sum is calculated.\n"
00122 "\n"
00123 "   * 'min_max'\n"
00124 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00125 "   --cmax and --cmin apply to this method.\n"
00126 "\n"
00127 "--cpos_rej\n"
00128 "--cneg_rej\n"
00129 "--citer\n"
00130 "see --cmethod='ksigma'\n"
00131 "\n"
00132 "--cmax\n"
00133 "--cmin\n"
00134 "see --cmethod='min_max'\n"
00135 "\n"
00136 "--pix_scale"
00137 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00138 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00139 "\n"
00140 "-------------------------------------------------------------------------------\n"
00141 "  Input files:\n"
00142 "\n"
00143 "   DO                    KMOS                                                  \n"
00144 "   category              Type   Explanation                    Required #Frames\n"
00145 "   --------              -----  -----------                    -------- -------\n"
00146 "   FLAT_SKY               F2D   Sky exposures                     Y      1-n   \n"
00147 "                                (at least 3 frames recommended)                \n"
00148 "   MASTER_DARK            F2D   Master dark                       Y       1    \n"
00149 "   MASTER_FLAT            F2D   Master flat                       Y       1    \n"
00150 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00151 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00152 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00153 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00154 "   FLAT_EDGE              F2L   Table with fitted slitlet edges   N      0,1   \n"
00155 "\n"
00156 "  Output files:\n"
00157 "\n"
00158 "   DO                    KMOS\n"
00159 "   category              Type   Explanation\n"
00160 "   --------              -----  -----------\n"
00161 "   ILLUM_CORR            F2I    Illumination calibration frame   \n"
00162 "   If FLAT_EDGE is provided: \n"
00163 "   SKYFLAT_EDGE          F2L    Frame containing parameters of fitted \n"
00164 "                                slitlets of all IFUs of all detectors\n"
00165 "-------------------------------------------------------------------------------\n"
00166 "\n";
00167 
00168 /*-----------------------------------------------------------------------------
00169  *                              Functions code
00170  *----------------------------------------------------------------------------*/
00171 
00188 int cpl_plugin_get_info(cpl_pluginlist *list)
00189 {
00190     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00191     cpl_plugin *plugin = &recipe->interface;
00192 
00193     cpl_plugin_init(plugin,
00194                         CPL_PLUGIN_API,
00195                         KMOS_BINARY_VERSION,
00196                         CPL_PLUGIN_TYPE_RECIPE,
00197                         "kmo_illumination",
00198                         "Create a calibration frame to correct spatial "
00199                         "non-uniformity of flatfield.",
00200                         kmo_illumination_description,
00201                         "Alex Agudo Berbel",
00202                         "kmos-spark@mpe.mpg.de",
00203                         kmos_get_license(),
00204                         kmo_illumination_create,
00205                         kmo_illumination_exec,
00206                         kmo_illumination_destroy);
00207 
00208     cpl_pluginlist_append(list, plugin);
00209 
00210     return 0;
00211 }
00212 
00220 static int kmo_illumination_create(cpl_plugin *plugin)
00221 {
00222     cpl_recipe *recipe;
00223     cpl_parameter *p;
00224 
00225     /* Check that the plugin is part of a valid recipe */
00226     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00227         recipe = (cpl_recipe *)plugin;
00228     else
00229         return -1;
00230 
00231     /* Create the parameters list in the cpl_recipe object */
00232     recipe->parameters = cpl_parameterlist_new();
00233 
00234     /* Fill the parameters list */
00235     /* --imethod */
00236     p = cpl_parameter_new_value("kmos.kmo_illumination.imethod",
00237                                 CPL_TYPE_STRING,
00238                                 "Method to use for interpolation: "
00239                                 "[\"NN\" (nearest neighbour), "
00240                                 "\"lwNN\" (linear weighted nearest neighbor),"
00241                                 "\"swNN\" (square weighted nearest neighbor)"
00242                                 "\"MS\" (Modified Shepard's method)"
00243                                 "\"CS\" (Cubic spline)]",
00244                                 "kmos.kmo_illumination",
00245                                 "CS");
00246     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00247     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00248     cpl_parameterlist_append(recipe->parameters, p);
00249 
00250     /* --neighborhoodRange */
00251     p = cpl_parameter_new_value("kmos.kmo_illumination.neighborhoodRange",
00252                                 CPL_TYPE_DOUBLE,
00253                                 "Defines the range to search for neighbors."
00254                                 "in pixels",
00255                                 "kmos.kmo_illumination",
00256                                 1.001);
00257     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00258     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00259     cpl_parameterlist_append(recipe->parameters, p);
00260 
00261     /* --range */
00262     p = cpl_parameter_new_value("kmos.kmo_illumination.range",
00263                                 CPL_TYPE_STRING,
00264                                 "The spectral ranges to combine when collapsing"
00265                                 "the reconstructed cubes. e.g."
00266                                 "\"x1_start,x1_end;x2_start,x2_end\" (microns)",
00267                                 "kmos.kmo_illumination",
00268                                 "");
00269     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range");
00270     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00271     cpl_parameterlist_append(recipe->parameters, p);
00272 
00273     /* --flux */
00274     p = cpl_parameter_new_value("kmos.kmo_illumination.flux",
00275                                 CPL_TYPE_BOOL,
00276                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00277                                 "kmos.kmo_illumination",
00278                                 FALSE);
00279     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00280     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00281     cpl_parameterlist_append(recipe->parameters, p);
00282 
00283     /* --add-all */
00284     p = cpl_parameter_new_value("kmos.kmo_illumination.add-all",
00285                                 CPL_TYPE_BOOL,
00286                                 "FALSE: omit 1st FLAT_SKY frame (acquisition), "
00287                                 "TRUE: don't perform any checks, add them all",
00288                                 "kmos.kmo_illumination",
00289                                 FALSE);
00290     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all");
00291     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00292     cpl_parameterlist_append(recipe->parameters, p);
00293 
00294     /* --pix_scale */
00295     p = cpl_parameter_new_value("kmos.kmo_illumination.pix_scale",
00296                                 CPL_TYPE_DOUBLE,
00297                                 "Change the pixel scale [arcsec]. "
00298                                 "Default of 0.2\" results into cubes of 14x14pix, "
00299                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00300                                 "etc.",
00301                                 "kmos.kmo_illumination",
00302                                 KMOS_PIX_RESOLUTION);
00303     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00304     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00305     cpl_parameterlist_append(recipe->parameters, p);
00306 
00307     // add parameters for band-definition
00308     kmo_band_pars_create(recipe->parameters,
00309                          "kmos.kmo_illumination");
00310 
00311     // add parameters for combining
00312     return kmo_combine_pars_create(recipe->parameters,
00313                                    "kmos.kmo_illumination",
00314                                    DEF_REJ_METHOD,
00315                                    FALSE);
00316 }
00317 
00323 static int kmo_illumination_exec(cpl_plugin *plugin)
00324 {
00325     cpl_recipe  *recipe;
00326 
00327     /* Get the recipe out of the plugin */
00328     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00329         recipe = (cpl_recipe *)plugin;
00330     else return -1;
00331 
00332     return kmo_illumination(recipe->parameters, recipe->frames);
00333 }
00334 
00340 static int kmo_illumination_destroy(cpl_plugin *plugin)
00341 {
00342     cpl_recipe *recipe;
00343 
00344     /* Get the recipe out of the plugin */
00345     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00346         recipe = (cpl_recipe *)plugin;
00347     else return -1 ;
00348 
00349     cpl_parameterlist_delete(recipe->parameters);
00350     return 0 ;
00351 }
00352 
00367 static int kmo_illumination(cpl_parameterlist *parlist, cpl_frameset *frameset)
00368 {
00369     int              ret_val                    = 0,
00370                      nr_devices                 = 0,
00371                      ifu_nr                     = 0,
00372                      nx                         = 0,
00373                      ny                         = 0,
00374                      process_noise              = FALSE,
00375                      cmax                       = 0,
00376                      cmin                       = 0,
00377                      citer                      = 0,
00378                      *bounds                    = NULL,
00379                      cnt                        = 0,
00380                      qc_max_dev_id              = 0,
00381                      qc_max_nonunif_id          = 0,
00382                      flux                       = FALSE,
00383                      background                 = FALSE,
00384                      add_all_sky                = FALSE,
00385                      same_exptime               = TRUE,
00386                      has_flat_edge              = FALSE;
00387     const int        *punused_ifus              = NULL;
00388     float            *pbad_pix_mask             = NULL;
00389     double           exptime                    = 0.,
00390                      exptime1                   = 0.,
00391                      exptime2                   = 0.,
00392                      cpos_rej                   = 0.0,
00393                      cneg_rej                   = 0.0,
00394                      neighborhoodRange          = 1.001,
00395                      mean_data                  = 0.0,
00396                      ifu_crpix                  = 0.0,
00397                      ifu_crval                  = 0.0,
00398                      ifu_cdelt                  = 0.0,
00399                      qc_spat_unif               = 0.0,
00400                      qc_max_dev                 = 0.0,
00401                      qc_max_nonunif             = 0.0,
00402                      tmp_stdev                  = 0.0,
00403                      tmp_mean                   = 0.0,
00404                      rotangle                   = 0.0,
00405                      tmp_rotangle               = 0.0,
00406                      rotangle_found             = 0.0,
00407                      pix_scale                  = 0.0;
00408     char             *keyword                   = NULL,
00409                      *fn_lut                    = NULL,
00410                      *suffix                    = NULL,
00411                      *extname                   = NULL,
00412                      *filter                    = NULL;
00413     const char       *method                    = NULL,
00414                      *cmethod                   = NULL,
00415                      *filter_id_l               = NULL,
00416                      *filter_id                 = NULL,
00417                      *ranges_txt                = NULL;
00418     cpl_array        *calTimestamp              = NULL,
00419                      **unused_ifus_before       = NULL,
00420                      **unused_ifus_after        = NULL;
00421     cpl_frame        *frame                     = NULL,
00422                      *xcalFrame                 = NULL,
00423                      *ycalFrame                 = NULL,
00424                      *lcalFrame                 = NULL;
00425     cpl_frameset     *frameset_sky              = NULL;
00426     cpl_image        *img_in                    = NULL,
00427                      *img_dark                  = NULL,
00428                      *img_dark_noise            = NULL,
00429                      *img_flat                  = NULL,
00430                      *img_flat_noise            = NULL,
00431                      *combined_data             = NULL,
00432                      *combined_noise            = NULL,
00433                      *xcal                      = NULL,
00434                      *ycal                      = NULL,
00435                      *lcal                      = NULL,
00436                      *bad_pix_mask              = NULL,
00437                      *data_ifu                  = NULL,
00438                      *noise_ifu                 = NULL,
00439                      **stored_data_images       = NULL,
00440                      **stored_noise_images      = NULL;
00441     cpl_imagelist    *cube_data                 = NULL,
00442                      *cube_noise                = NULL,
00443                      *detector_in               = NULL,
00444                      **stored_data_cubes        = NULL,
00445                      **stored_noise_cubes       = NULL;
00446     cpl_matrix       **edgepars                 = NULL;
00447     cpl_propertylist *main_header               = NULL,
00448                      *tmp_header                = NULL,
00449                      *sub_header                = NULL,
00450                      **stored_sub_data_headers  = NULL,
00451                      **stored_sub_noise_headers = NULL;
00452     cpl_table        *band_table                = NULL,
00453                      ***edge_table_sky          = NULL,
00454                      **edge_table_flat          = NULL;
00455     cpl_vector       *ranges                    = NULL,
00456                      *identified_slices         = NULL,
00457                      *calAngles                 = NULL,
00458                      **slitlet_ids              = NULL,
00459                      *shift_vec                 = NULL,
00460                      *edge_vec                  = NULL;
00461     main_fits_desc   desc_sky,
00462                      desc_dark,
00463                      desc_flat,
00464                      desc_xcal,
00465                      desc_ycal,
00466                      desc_lcal;
00467     gridDefinition   gd;
00468 
00469     KMO_TRY
00470     {
00471         kmo_init_fits_desc(&desc_sky);
00472         kmo_init_fits_desc(&desc_dark);
00473         kmo_init_fits_desc(&desc_flat);
00474         kmo_init_fits_desc(&desc_xcal);
00475         kmo_init_fits_desc(&desc_ycal);
00476         kmo_init_fits_desc(&desc_lcal);
00477 
00478         /* --- check input --- */
00479         KMO_TRY_ASSURE((parlist != NULL) &&
00480                        (frameset != NULL),
00481                        CPL_ERROR_NULL_INPUT,
00482                        "Not all input data is provided!");
00483 
00484         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY) >= 1,
00485                        CPL_ERROR_ILLEGAL_INPUT,
00486                        "One or more FLAT_SKY frames are required!");
00487 
00488         if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) {
00489             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00490                                       "3 FLAT_SKY frames!");
00491         }
00492 
00493         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_DARK) == 1,
00494                        CPL_ERROR_ILLEGAL_INPUT,
00495                        "Exactly one MASTER_DARK frame is required!");
00496 
00497         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_FLAT) == 1,
00498                        CPL_ERROR_ILLEGAL_INPUT,
00499                        "Exactly one MASTER_FLAT frame is required!");
00500 
00501         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00502                        CPL_ERROR_ILLEGAL_INPUT,
00503                        "Exactly one XCAL frame is required!");
00504 
00505         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00506                        CPL_ERROR_ILLEGAL_INPUT,
00507                        "Exactly one YCAL frame is required!");
00508 
00509         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00510                        CPL_ERROR_ILLEGAL_INPUT,
00511                        "Exactly one LCAL frame is required!");
00512 
00513         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00514                        CPL_ERROR_ILLEGAL_INPUT,
00515                        "Exactly one WAVE_BAND frame is required!");
00516 
00517         KMO_TRY_ASSURE((cpl_frameset_count_tags(frameset, FLAT_EDGE) == 1) ||
00518                        (cpl_frameset_count_tags(frameset, FLAT_EDGE) == 0),
00519                        CPL_ERROR_ILLEGAL_INPUT,
00520                        "Exactly one FLAT_EDGE frame is required!");
00521 
00522         has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE);
00523 
00524         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination") == 1,
00525                        CPL_ERROR_ILLEGAL_INPUT,
00526                        "Cannot identify RAW and CALIB frames!");
00527 
00528         /* --- get parameters --- */
00529         cpl_msg_info("", "--- Parameter setup for kmo_illumination ---");
00530 
00531         KMO_TRY_EXIT_IF_NULL(
00532             method = kmo_dfs_get_parameter_string(parlist,
00533                                               "kmos.kmo_illumination.imethod"));
00534 
00535         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00536                        (strcmp(method, "lwNN") == 0) ||
00537                        (strcmp(method, "swNN") == 0) ||
00538                        (strcmp(method, "MS") == 0) ||
00539                        (strcmp(method, "CS") == 0),
00540                        CPL_ERROR_ILLEGAL_INPUT,
00541                        "method must be either \"NN\", \"lwNN\", "
00542                        "\"swNN\", \"MS\" or \"CS\"!");
00543 
00544         KMO_TRY_EXIT_IF_ERROR(
00545             kmo_dfs_print_parameter_help(parlist,
00546                                         "kmos.kmo_illumination.imethod"));
00547 
00548         neighborhoodRange = kmo_dfs_get_parameter_double(parlist,
00549                 "kmos.kmo_illumination.neighborhoodRange");
00550         KMO_TRY_CHECK_ERROR_STATE();
00551 
00552         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00553                 CPL_ERROR_ILLEGAL_INPUT,
00554                 "neighborhoodRange must be greater than 0.0");
00555 
00556         KMO_TRY_EXIT_IF_ERROR(
00557             kmo_dfs_print_parameter_help(parlist,
00558                                     "kmos.kmo_illumination.neighborhoodRange"));
00559 
00560         ranges_txt = kmo_dfs_get_parameter_string(parlist,
00561                                                   "kmos.kmo_illumination.range");
00562         KMO_TRY_CHECK_ERROR_STATE();
00563 
00564         KMO_TRY_EXIT_IF_ERROR(
00565             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.range"));
00566 
00567         ranges = kmo_identify_ranges(ranges_txt);
00568         KMO_TRY_CHECK_ERROR_STATE();
00569 
00570         flux = kmo_dfs_get_parameter_bool(parlist,
00571                                           "kmos.kmo_illumination.flux");
00572 
00573         KMO_TRY_ASSURE((flux == 0) ||
00574                        (flux == 1),
00575                        CPL_ERROR_ILLEGAL_INPUT,
00576                        "flux must be either FALSE or TRUE!");
00577 
00578         KMO_TRY_EXIT_IF_ERROR(
00579             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.flux"));
00580 
00581         add_all_sky = kmo_dfs_get_parameter_bool(parlist,
00582                                                  "kmos.kmo_illumination.add-all");
00583 
00584         KMO_TRY_ASSURE((add_all_sky == 0) ||
00585                        (add_all_sky == 1),
00586                        CPL_ERROR_ILLEGAL_INPUT,
00587                        "add_all must be either FALSE or TRUE!");
00588 
00589         KMO_TRY_EXIT_IF_ERROR(
00590             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.add-all"));
00591 
00592         pix_scale = kmo_dfs_get_parameter_double(parlist,
00593                                         "kmos.kmo_illumination.pix_scale");
00594         KMO_TRY_CHECK_ERROR_STATE();
00595         KMO_TRY_EXIT_IF_ERROR(
00596            kmo_dfs_print_parameter_help(parlist,
00597                                        "kmos.kmo_illumination.pix_scale"));
00598         KMO_TRY_ASSURE((pix_scale >= 0.01) &&
00599                        (pix_scale <= 0.4),
00600                        CPL_ERROR_ILLEGAL_INPUT,
00601                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00602                        "with 7x7 to 280x280 pixels)!");
00603 
00604         kmo_band_pars_load(parlist, "kmos.kmo_illumination");
00605 
00606         KMO_TRY_EXIT_IF_ERROR(
00607             kmo_combine_pars_load(parlist,
00608                                   "kmos.kmo_illumination",
00609                                   &cmethod,
00610                                   &cpos_rej,
00611                                   &cneg_rej,
00612                                   &citer,
00613                                   &cmin,
00614                                   &cmax,
00615                                   FALSE));
00616         cpl_msg_info("", "-------------------------------------------");
00617 
00618         // check if filter_id, grating_id and rotator offset match for all
00619         // detectors
00620         KMO_TRY_EXIT_IF_ERROR(
00621             kmo_check_frameset_setup(frameset, FLAT_SKY,
00622                                        TRUE, FALSE, TRUE));
00623         KMO_TRY_EXIT_IF_ERROR(
00624             kmo_check_frame_setup(frameset, FLAT_SKY, XCAL,
00625                                        TRUE, FALSE, TRUE));
00626         KMO_TRY_EXIT_IF_ERROR(
00627             kmo_check_frame_setup(frameset, XCAL, YCAL,
00628                                        TRUE, FALSE, TRUE));
00629         KMO_TRY_EXIT_IF_ERROR(
00630             kmo_check_frame_setup(frameset, XCAL, LCAL,
00631                                        TRUE, FALSE, TRUE));
00632         KMO_TRY_EXIT_IF_ERROR(
00633             kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT,
00634                                        TRUE, FALSE, TRUE));
00635 
00636         KMO_TRY_EXIT_IF_NULL(
00637             frame = kmo_dfs_get_frame(frameset, XCAL));
00638         KMO_TRY_EXIT_IF_NULL(
00639             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00640 
00641         KMO_TRY_EXIT_IF_ERROR(
00642             kmo_check_frame_setup_md5_xycal(frameset));
00643         KMO_TRY_EXIT_IF_ERROR(
00644             kmo_check_frame_setup_md5(frameset));
00645 
00646         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00647         cpl_msg_info("", "(grating 1, 2 & 3)");
00648 
00649         // check which IFUs are active for all frames
00650         KMO_TRY_EXIT_IF_NULL(
00651             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00652 
00653         KMO_TRY_EXIT_IF_NULL(
00654             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00655 
00656         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00657 
00658         // load desc for MASTER_DARK and check
00659         KMO_TRY_EXIT_IF_NULL(
00660             frame = kmo_dfs_get_frame(frameset, MASTER_DARK));
00661         desc_dark = kmo_identify_fits_header(
00662                     cpl_frame_get_filename(frame));
00663         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_DARK frame doesn't seem to "
00664                                       "be in KMOS-format!");
00665         KMO_TRY_ASSURE((desc_dark.nr_ext == 2*KMOS_NR_DETECTORS) &&
00666                        (desc_dark.ex_badpix == FALSE) &&
00667                        (desc_dark.fits_type == f2d_fits) &&
00668                        (desc_dark.frame_type == detector_frame),
00669                        CPL_ERROR_ILLEGAL_INPUT,
00670                        "MASTER_DARK isn't in the correct format!!!");
00671         nx = desc_dark.naxis1;
00672         ny = desc_dark.naxis2;
00673 
00674         // load desc for MASTER_FLAT and check
00675         KMO_TRY_EXIT_IF_NULL(
00676             frame = kmo_dfs_get_frame(frameset, MASTER_FLAT));
00677         desc_flat = kmo_identify_fits_header(cpl_frame_get_filename(frame));
00678         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_FLAT frame doesn't seem to "
00679                                       "be in KMOS-format!");
00680         KMO_TRY_ASSURE((desc_flat.nr_ext % (2*KMOS_NR_DETECTORS) == 0) &&
00681                        (desc_flat.ex_badpix == FALSE) &&
00682                        (desc_flat.fits_type == f2d_fits) &&
00683                        (desc_flat.frame_type == detector_frame),
00684                        CPL_ERROR_ILLEGAL_INPUT,
00685                        "MASTER_FLAT isn't in the correct format!!!");
00686 
00687         // load desc for XCAL and check
00688         KMO_TRY_EXIT_IF_NULL(
00689             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00690         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00691         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00692                                       "be in KMOS-format!");
00693         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00694                        (desc_xcal.ex_badpix == FALSE) &&
00695                        (desc_xcal.fits_type == f2d_fits) &&
00696                        (desc_xcal.frame_type == detector_frame),
00697                        CPL_ERROR_ILLEGAL_INPUT,
00698                        "XCAL isn't in the correct format!!!");
00699         KMO_TRY_ASSURE((desc_xcal.naxis1 == nx) &&
00700                        (desc_xcal.naxis2 == ny),
00701                        CPL_ERROR_ILLEGAL_INPUT,
00702                        "MASTER_DARK and XCAL frame haven't same dimensions! "
00703                        "(x,y): (%d,%d) vs (%d,%d)",
00704                        nx, ny, desc_xcal.naxis1, desc_xcal.naxis2);
00705 
00706         nr_devices = desc_xcal.nr_ext;
00707 
00708         // load desc for YCAL and check
00709         KMO_TRY_EXIT_IF_NULL(
00710             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00711         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00712         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00713                                       "be in KMOS-format!");
00714         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00715                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00716                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00717                        (desc_ycal.frame_type == desc_xcal.frame_type),
00718                        CPL_ERROR_ILLEGAL_INPUT,
00719                        "YCAL isn't in the correct format!!!");
00720         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00721                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00722                        CPL_ERROR_ILLEGAL_INPUT,
00723                        "MASTER_DARK and YCAL frame haven't same dimensions! "
00724                        "(x,y): (%d,%d) vs (%d,%d)",
00725                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00726 
00727         // load desc for LCAL and check
00728         KMO_TRY_EXIT_IF_NULL(
00729             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00730         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00731         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00732                                       "be in KMOS-format!");
00733         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00734                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00735                        (desc_lcal.frame_type == desc_xcal.frame_type),
00736                        CPL_ERROR_ILLEGAL_INPUT,
00737                        "LCAL isn't in the correct format!!!");
00738         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00739                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00740                        CPL_ERROR_ILLEGAL_INPUT,
00741                        "MASTER_DARK and LCAL frame haven't same dimensions! "
00742                        "(x,y): (%d,%d) vs (%d,%d)",
00743                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00744         KMO_TRY_EXIT_IF_NULL(
00745             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00746 
00747         // load desc for FLAT_SKY and check
00748         nr_devices = KMOS_NR_DETECTORS;
00749         KMO_TRY_EXIT_IF_NULL(
00750             frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00751 
00752         KMO_TRY_EXIT_IF_NULL(
00753             main_header = kmclipm_propertylist_load(
00754                                          cpl_frame_get_filename(frame), 0));
00755         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00756         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00757         kmclipm_strip_angle(&rotangle);
00758         cpl_propertylist_delete(main_header); main_header = NULL;
00759 
00760         cnt = 1;
00761         while (frame != NULL) {
00762             KMO_TRY_EXIT_IF_NULL(
00763                 main_header = kmclipm_propertylist_load(
00764                                              cpl_frame_get_filename(frame), 0));
00765 
00766             desc_sky = kmo_identify_fits_header(
00767                         cpl_frame_get_filename(frame));
00768             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY frame doesn't seem to "
00769                                           "be in KMOS-format!");
00770             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00771                            (desc_sky.ex_badpix == FALSE) &&
00772                            (desc_sky.fits_type == raw_fits) &&
00773                            (desc_sky.frame_type == detector_frame),
00774                            CPL_ERROR_ILLEGAL_INPUT,
00775                            "FLAT_SKY isn't in the correct format!!!");
00776             KMO_TRY_ASSURE((desc_sky.naxis1 == nx) &&
00777                            (desc_sky.naxis2 == ny) &&
00778                            (desc_sky.nr_ext == nr_devices),
00779                            CPL_ERROR_ILLEGAL_INPUT,
00780                            "MASTER_DARK and FLAT_SKY (no. %d) frame haven't "
00781                            "same dimensions! (x,y): (%d,%d) vs (%d,%d)",
00782                            cnt, nx, ny, desc_flat.naxis1, desc_flat.naxis2);
00783             kmo_free_fits_desc(&desc_sky);
00784             kmo_init_fits_desc(&desc_sky);
00785 
00786             KMO_TRY_ASSURE(
00787                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00788                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE) &&
00789                 (kmo_check_lamp(main_header, INS_LAMP3_ST) == FALSE) &&
00790                 (kmo_check_lamp(main_header, INS_LAMP4_ST) == FALSE),
00791                 CPL_ERROR_ILLEGAL_INPUT,
00792                 "All lamps must be switched off for the FLAT_SKY frames!");
00793 
00794             // assert that filters have correct IDs and that all detectors of
00795             // all input frames have the same filter set
00796             for (int i = 1; i <= KMOS_NR_DETECTORS; i++) {
00797                 // ESO INS FILTi ID
00798                 KMO_TRY_EXIT_IF_NULL(
00799                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00800                 KMO_TRY_EXIT_IF_NULL(
00801                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00802 
00803                 KMO_TRY_EXIT_IF_NULL(
00804                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00805                 cpl_free(keyword); keyword = NULL;
00806 
00807                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00808                                (strcmp(filter_id, "YJ") == 0) ||
00809                                (strcmp(filter_id, "H") == 0) ||
00810                                (strcmp(filter_id, "K") == 0) ||
00811                                (strcmp(filter_id, "HK") == 0),
00812                                CPL_ERROR_ILLEGAL_INPUT,
00813                                "Filter ID in primary header must be either 'IZ', "
00814                                "'YJ', 'H', 'K' or " "'HK' !");
00815 
00816                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00817                                CPL_ERROR_ILLEGAL_INPUT,
00818                                "Filter IDs must be the same for FLAT_SKY frame"
00819                                " and lcal frame!"
00820                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00821                                i, cpl_frame_get_filename(frame),
00822                                filter_id, filter_id_l);
00823 
00824                 // ESO INS GRATi ID
00825                 KMO_TRY_EXIT_IF_NULL(
00826                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00827                 KMO_TRY_EXIT_IF_NULL(
00828                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00829 
00830                 KMO_TRY_EXIT_IF_NULL(
00831                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00832                 cpl_free(keyword); keyword = NULL;
00833 
00834                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00835                                (strcmp(filter_id, "YJ") == 0) ||
00836                                (strcmp(filter_id, "H") == 0) ||
00837                                (strcmp(filter_id, "K") == 0) ||
00838                                (strcmp(filter_id, "HK") == 0),
00839                                CPL_ERROR_ILLEGAL_INPUT,
00840                                "Grating ID in primary header must be either "
00841                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00842 
00843                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00844                                CPL_ERROR_ILLEGAL_INPUT,
00845                                "Grating IDs must be the same for FLAT_SKY frame"
00846                                " and lcal frame!"
00847                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00848                                i, cpl_frame_get_filename(frame),
00849                                filter_id, filter_id_l);
00850 
00851                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00852                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00853                 kmclipm_strip_angle(&tmp_rotangle);
00854                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00855                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00856                         CPL_ERROR_ILLEGAL_INPUT,
00857                         "OCS ROT NAANGLE of sky flat frames differ to much: %f %f",
00858                         rotangle, tmp_rotangle);
00859             }
00860             cpl_propertylist_delete(main_header); main_header = NULL;
00861 
00862             // get next FLAT_SKY frame
00863             frame = kmo_dfs_get_frame(frameset, NULL);
00864             KMO_TRY_CHECK_ERROR_STATE();
00865             cnt++;
00866         }
00867 
00868         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00869 
00870         //
00871         // noise will be propagated when:
00872         // MASTER_DARK and MASTER_FLAT have noise extensions and if at least
00873         // 2 FLAT_SKY frames are provided.
00874         // Otherwise noise will be ignored.
00875         //
00876         if (desc_dark.ex_noise &&
00877             desc_flat.ex_noise &&
00878             (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2)) {
00879             process_noise = TRUE;
00880         }
00881 
00882         if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00883             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00884                             "since there is only one input frame! (The output "
00885                             "file won't have any noise extensions)");
00886 
00887             cmethod = "average";
00888         }
00889 
00890         //
00891         // Check whether 1st FLAT_SKY should be omitted
00892         //
00893         KMO_TRY_EXIT_IF_NULL(
00894             frameset_sky = cpl_frameset_new());
00895 
00896         if (add_all_sky) {
00897             // just add all FLAT_SKY frames without check
00898             frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00899             while (frame != NULL) {
00900                 KMO_TRY_EXIT_IF_ERROR(
00901                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00902                 frame = kmo_dfs_get_frame(frameset, NULL);
00903             }
00904             cpl_msg_info("", "Add all FLAT_SKY without checking for acquisition frame.");
00905         } else {
00906             // check if 1st FLAT_SKY has different exposure time and whether to omit it
00907             KMO_TRY_EXIT_IF_NULL(
00908                 frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00909 
00910             if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00911                 // just one FLAT_SKY, always add
00912                 KMO_TRY_EXIT_IF_ERROR(
00913                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00914                 KMO_TRY_CHECK_ERROR_STATE();
00915             } else {
00916                 // several FLAT_SKY frames, check exptime
00917 
00918                 // get exptime 1
00919                 KMO_TRY_EXIT_IF_NULL(
00920                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00921                 exptime1 = cpl_propertylist_get_double(main_header, EXPTIME);
00922                 KMO_TRY_CHECK_ERROR_STATE();
00923                 cpl_propertylist_delete(main_header); main_header = NULL;
00924 
00925                 // get exptime 2
00926                 frame = kmo_dfs_get_frame(frameset, NULL);
00927                 KMO_TRY_EXIT_IF_NULL(
00928                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00929                 exptime2 = cpl_propertylist_get_double(main_header, EXPTIME);
00930                 KMO_TRY_CHECK_ERROR_STATE();
00931                 cpl_propertylist_delete(main_header); main_header = NULL;
00932 
00933                 // loop remaining frames
00934                 same_exptime = TRUE;
00935                 frame = kmo_dfs_get_frame(frameset, NULL);
00936                 while (same_exptime && (frame != NULL)) {
00937                     KMO_TRY_EXIT_IF_NULL(
00938                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00939                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00940                     KMO_TRY_CHECK_ERROR_STATE();
00941                     cpl_propertylist_delete(main_header); main_header = NULL;
00942                     if (fabs(exptime-exptime2) > 0.01) {
00943                         // not same
00944                         same_exptime = FALSE;
00945                     }
00946                     frame = kmo_dfs_get_frame(frameset, NULL);
00947                 }
00948 
00949                 if (same_exptime) {
00950                     // frame [2,n] have same exptime, add them
00951                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00952                     KMO_TRY_EXIT_IF_NULL(
00953                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00954                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00955                     KMO_TRY_CHECK_ERROR_STATE();
00956                     cpl_propertylist_delete(main_header); main_header = NULL;
00957                     cpl_msg_info("", "Omit FLAT_SKY: %s with EXPTIME of %g sec (acquisition), other frame(s) have EXPTIME of %g sec", cpl_frame_get_filename(frame), exptime, exptime2);
00958                     frame = kmo_dfs_get_frame(frameset, NULL);
00959                     while (frame != NULL) {
00960                         KMO_TRY_EXIT_IF_ERROR(
00961                             cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00962                         frame = kmo_dfs_get_frame(frameset, NULL);
00963                     }
00964                     if (fabs(exptime1-exptime2) < 0.01) {
00965                         cpl_msg_warning("", "The 1st FLAT_SKY has the same exposure time as the following ones. "
00966                                             "It has anyway been omitted since we assume it is an acquisition frame. "
00967                                             "If you want to add it anyway call this recipe with the --add-all parameter");
00968                     }
00969                 } else {
00970                     cpl_msg_error("", "The exposure times of the FLAT_SKY frames don't match!");
00971                     cpl_msg_error("", "We assume that the 1st frame is an acquisition frame and would be omitted.");
00972                     cpl_msg_error("", "The following frames should have the same exposure time if they originate from the same template.");
00973                     cpl_msg_error("", "If you want to reduce them anyway call this recipe with the --add-all parameter");
00974                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00975                     while (frame != NULL) {
00976                         KMO_TRY_EXIT_IF_NULL(
00977                             main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00978                         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00979                         KMO_TRY_CHECK_ERROR_STATE();
00980                         cpl_propertylist_delete(main_header); main_header = NULL;
00981                         cpl_msg_error("", "FLAT_SKY: %s, EXPTIME: %g", cpl_frame_get_filename(frame), exptime);
00982                         frame = kmo_dfs_get_frame(frameset, NULL);
00983                     }
00984                     cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
00985                     KMO_TRY_CHECK_ERROR_STATE();
00986                 }
00987             }
00988         }
00989 
00990         KMO_TRY_EXIT_IF_NULL(
00991             frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY));
00992         KMO_TRY_EXIT_IF_NULL(
00993             main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY));
00994         KMO_TRY_EXIT_IF_NULL(
00995             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
00996         KMO_TRY_EXIT_IF_NULL(
00997             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
00998         cpl_free(keyword); keyword = NULL;
00999 
01000         //
01001         // set default band-specific ranges for collapsing
01002         //
01003         if (ranges == NULL) {
01004             if (strcmp(filter, "IZ") == 0) {
01005                 ranges_txt = "0.81,1.05";
01006             } else if (strcmp(filter, "YJ") == 0) {
01007                 ranges_txt = "1.025,1.3";
01008             } else if (strcmp(filter, "H") == 0) {
01009                 ranges_txt = "1.5,1.7";
01010             } else if (strcmp(filter, "K") == 0) {
01011                 ranges_txt = "2.1,2.35";
01012             } else if (strcmp(filter, "HK") == 0) {
01013                 ranges_txt = "1.5,1.7;2.1,2.35";
01014 //                ranges_txt = "1.5,1.7";
01015             } else {
01016                 KMO_TRY_ASSURE(1 == 0,
01017                                CPL_ERROR_ILLEGAL_INPUT,
01018                                "We really shouldn't get here...");
01019             }
01020             cpl_msg_info("", "Spectral range to collapse has been set to %s um for this band.", ranges_txt);
01021             ranges = kmo_identify_ranges(ranges_txt);
01022             KMO_TRY_CHECK_ERROR_STATE();
01023         }
01024 
01025         // setup grid definition, wavelength start and end points will be set
01026         // in the detector loop
01027         KMO_TRY_EXIT_IF_ERROR(
01028             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale));
01029 
01030         // create filename for LUT
01031         KMO_TRY_EXIT_IF_NULL(
01032             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
01033 
01034         // extract bounds
01035         KMO_TRY_EXIT_IF_NULL(
01036             tmp_header = kmclipm_propertylist_load(
01037                                          cpl_frame_get_filename(xcalFrame), 0));
01038         KMO_TRY_EXIT_IF_NULL(
01039             bounds = kmclipm_extract_bounds(tmp_header));
01040         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
01041 
01042         // get timestamps of xcal, ycal & lcal
01043         KMO_TRY_EXIT_IF_NULL(
01044             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
01045 
01046         // create arrays to hold reconstructed data and noise cubes and
01047         // their headers
01048         KMO_TRY_EXIT_IF_NULL(
01049             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01050                                                             sizeof(cpl_imagelist*)));
01051         KMO_TRY_EXIT_IF_NULL(
01052             stored_noise_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01053                                                              sizeof(cpl_imagelist*)));
01054         KMO_TRY_EXIT_IF_NULL(
01055             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01056                                                          sizeof(cpl_image*)));
01057         KMO_TRY_EXIT_IF_NULL(
01058             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01059                                                           sizeof(cpl_image*)));
01060         KMO_TRY_EXIT_IF_NULL(
01061             stored_sub_data_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01062                                                                      sizeof(cpl_propertylist*)));
01063         KMO_TRY_EXIT_IF_NULL(
01064             stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01065                                                                       sizeof(cpl_propertylist*)));
01066         KMO_TRY_EXIT_IF_NULL(
01067             edge_table_sky = (cpl_table***)cpl_calloc(KMOS_NR_DETECTORS,
01068                                                       sizeof(cpl_table**)));
01069         KMO_TRY_EXIT_IF_NULL(
01070             edge_table_flat = (cpl_table**)cpl_calloc(KMOS_IFUS_PER_DETECTOR,
01071                                                       sizeof(cpl_table*)));
01072         KMO_TRY_EXIT_IF_NULL(
01073             calAngles = cpl_vector_new(3));
01074 
01075         //
01076         // loop through all detectors
01077         //
01078         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01079             cpl_msg_info("","Processing detector No. %d", det_nr);
01080 
01081             KMO_TRY_EXIT_IF_NULL(
01082                 detector_in = cpl_imagelist_new());
01083 
01084             // load data of det_nr of all FLAT_SKY frames into an imagelist
01085             KMO_TRY_EXIT_IF_NULL(
01086                 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, TRUE, NULL));
01087 
01088             cnt = 0;
01089             while (img_in != NULL) {
01090                 cpl_imagelist_set(detector_in, img_in, cnt);
01091                 KMO_TRY_CHECK_ERROR_STATE();
01092 
01093                 /* load same extension of next FLAT_SKY frame*/
01094                 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, TRUE, NULL);
01095                 KMO_TRY_CHECK_ERROR_STATE();
01096 
01097                 cnt++;
01098             }
01099 
01100             //
01101             // process imagelist
01102             //
01103 
01104             // combine imagelist (data only) and create noise (stdev of data)
01105             cpl_msg_info("","Combining frames...");
01106             if (process_noise) {
01107                 KMO_TRY_EXIT_IF_ERROR(
01108                     kmclipm_combine_frames(detector_in,
01109                                            NULL,
01110                                            NULL,
01111                                            cmethod,
01112                                            cpos_rej,
01113                                            cneg_rej,
01114                                            citer,
01115                                            cmax,
01116                                            cmin,
01117                                            &combined_data,
01118                                            &combined_noise,
01119                                            -1.0));
01120             } else {
01121                 KMO_TRY_EXIT_IF_ERROR(
01122                     kmclipm_combine_frames(detector_in,
01123                                            NULL,
01124                                            NULL,
01125                                            cmethod,
01126                                            cpos_rej,
01127                                            cneg_rej,
01128                                            citer,
01129                                            cmax,
01130                                            cmin,
01131                                            &combined_data,
01132                                            NULL,
01133                                            -1.0));
01134             }
01135 
01136             if (kmclipm_omit_warning_one_slice > 10) {
01137                 cpl_msg_warning(cpl_func, "Previous warning (number of "
01138                                           "identified slices) occured %d times.",
01139                                 kmclipm_omit_warning_one_slice);
01140                 kmclipm_omit_warning_one_slice = FALSE;
01141             }
01142 
01143             cpl_imagelist_delete(detector_in); detector_in = NULL;
01144 
01145             // load calibration files
01146             KMO_TRY_EXIT_IF_NULL(
01147                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
01148                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01149 
01150             KMO_TRY_EXIT_IF_ERROR(
01151                 cpl_vector_set(calAngles, 0, rotangle_found));
01152             KMO_TRY_EXIT_IF_NULL(
01153                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
01154                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01155             KMO_TRY_EXIT_IF_ERROR(
01156                 cpl_vector_set(calAngles, 1, rotangle_found));
01157             KMO_TRY_EXIT_IF_NULL(
01158                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
01159                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01160             KMO_TRY_EXIT_IF_ERROR(
01161                 cpl_vector_set(calAngles, 2, rotangle_found));
01162 
01163             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
01164             KMO_TRY_EXIT_IF_NULL(
01165                 bad_pix_mask = cpl_image_duplicate(xcal));
01166 
01167             KMO_TRY_EXIT_IF_NULL(
01168                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
01169             for (int x = 0; x < nx; x++) {
01170                 for (int y = 0; y < ny; y++) {
01171                     if (isnan(pbad_pix_mask[x+nx*y])) {
01172                         pbad_pix_mask[x+nx*y] = 0.;
01173                     } else {
01174                         pbad_pix_mask[x+nx*y] = 1.;
01175                     }
01176                 }
01177             }
01178             KMO_TRY_CHECK_ERROR_STATE();
01179 
01180             //
01181             // calculate SKYFLAT_EDGE
01182             //
01183             if (has_flat_edge) {
01184                 // get edge-edgepars from FLAT_SKY
01185                 KMO_TRY_EXIT_IF_ERROR(
01186                     kmo_calc_edgepars(combined_data,
01187                                       unused_ifus_after[det_nr-1],
01188                                       bad_pix_mask,
01189                                       det_nr,
01190                                       &slitlet_ids,
01191                                       &edgepars));
01192                 KMO_TRY_CHECK_ERROR_STATE();
01193 
01194                 // copy edgepars to table for saving later on
01195                 KMO_TRY_EXIT_IF_NULL(
01196                     edge_table_sky[det_nr-1] = kmo_edgepars_to_table(slitlet_ids, edgepars));
01197 
01198                 if (edgepars != NULL) {
01199                     for (int i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01200                         cpl_matrix_delete(edgepars[i]); edgepars[i] = NULL;
01201                     }
01202                     cpl_free(edgepars); edgepars = NULL;
01203                 }
01204                 if (slitlet_ids != NULL) {
01205                     for (int i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01206                         cpl_vector_delete(slitlet_ids[i]); slitlet_ids[i] = NULL;
01207                     }
01208                     cpl_free(slitlet_ids); slitlet_ids = NULL;
01209                 }
01210                 KMO_TRY_CHECK_ERROR_STATE();
01211 
01212                 //
01213                 // correlate FLAT_EDGE and SKYFLAT_EDGE
01214                 //
01215 
01216                 // load flat_edge from MASTER_FLAT
01217                 KMO_TRY_EXIT_IF_NULL(
01218                     frame = kmo_dfs_get_frame(frameset, FLAT_EDGE));
01219                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01220                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01221                     KMO_TRY_EXIT_IF_NULL(
01222                         edge_table_flat[j] = kmclipm_cal_table_load(cpl_frame_get_filename(frame),
01223                                                                     ifu_nr, rotangle, 0, &tmp_rotangle));
01224                 }
01225 
01226                 //
01227                 // calculate shift value
01228                 //
01229 
01230                 KMO_TRY_EXIT_IF_NULL(
01231                     shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR));
01232                 KMO_TRY_EXIT_IF_NULL(
01233                     edge_vec = cpl_vector_new(2*KMOS_SLITLET_X));
01234 
01235                 // get shift values for each IFU by comparing all edge parameters,
01236                 // rejecting and applying median
01237                 int row = 1024; // middle of frame
01238                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01239                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01240                     for (int edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) {
01241                         double flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat[j], edgeNr, row);
01242                         double skyval  = kmo_calc_fitted_slitlet_edge(edge_table_sky[det_nr-1][j], edgeNr, row);
01243                         cpl_vector_set(edge_vec, edgeNr, flatval-skyval);
01244                     }
01245 
01246                     // reject deviating edge-differences
01247                     kmclipm_vector *kv = NULL;
01248                     KMO_TRY_EXIT_IF_NULL(
01249                         kv = kmclipm_vector_create(cpl_vector_duplicate(edge_vec)));
01250                     kmclipm_reject_deviant(kv, 3, 3, NULL, NULL);
01251 
01252                     // set shift value for each IFU
01253                     cpl_vector_set(shift_vec, j, kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC));
01254                     kmclipm_vector_delete(kv); kv = NULL;
01255                 }
01256                 cpl_vector_delete(edge_vec); edge_vec = NULL;
01257                 KMO_TRY_CHECK_ERROR_STATE();
01258 
01259                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01260                     cpl_table_delete(edge_table_flat[j]);
01261                     edge_table_flat[j] = NULL;
01262                 }
01263 
01264                 // take median of all IFU-shift-values
01265                 double shift_val = -cpl_vector_get_median(shift_vec);
01266                 cpl_vector_delete(shift_vec); shift_vec = NULL;
01267 
01268                 cpl_msg_info("", "Shift detector %d by %g pixels.", det_nr, shift_val);
01269 
01270                 int     xdim                = cpl_image_get_size_x(combined_data),
01271                         ydim                = cpl_image_get_size_y(combined_data);
01272                 double  *array_in           = cpl_calloc(xdim, sizeof(double)),
01273                         *array_out          = NULL;
01274                 float   *pcombined_data     = cpl_image_get_data_float(combined_data),
01275                         *pcombined_noise    = NULL;
01276     //            float   *tmpArray           = cpl_calloc(xdim, sizeof(float));
01277                 if (process_noise) {
01278                     pcombined_noise = cpl_image_get_data_float(combined_noise);
01279                 }
01280 
01281                 for (int iy = 0; iy < ydim; iy++) {
01282                     // cubic spline
01283                     for (int ix = 0; ix < xdim; ix++) {
01284                         array_in[ix] = pcombined_data[ix+iy*xdim];
01285                     }
01286                     array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01287                                                     xdim, shift_val, 1.0,
01288                                                     NATURAL);
01289                     for (int ix = 0; ix < xdim; ix++) {
01290                       pcombined_data[ix+iy*xdim] = array_out[ix];
01291                     }
01292                     cpl_free(array_out);
01293 
01294     //                // linear
01295     //                for (int ix = 1; ix < xdim; ix++) {
01296     //                    tmpArray[ix-1] = (pcombined_data[ix+iy*xdim]-pcombined_data[(ix-1)+iy*xdim])*shift_val +
01297     //                                     pcombined_data[(ix-1)+iy*xdim];
01298     //                }
01299     //                for (int ix = 1; ix < xdim; ix++) {
01300     //                    pcombined_data[ix+iy*xdim] = tmpArray[ix];
01301     //                }
01302 
01303                     if (process_noise) {
01304                         // cubic spline
01305                         for (int ix = 0; ix < xdim; ix++) {
01306                             array_in[ix] = pcombined_noise[ix+iy*xdim];
01307                         }
01308                         array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01309                                                         xdim, shift_val, 1.0,
01310                                                         NATURAL);
01311                         for (int ix = 0; ix < xdim; ix++) {
01312                           pcombined_noise[ix+iy*xdim] = array_out[ix];
01313                         }
01314                         cpl_free(array_out);
01315 
01316     //                    // linear
01317     //                    for (int ix = 1; ix < xdim; ix++) {
01318     //                        tmpArray[ix-1] = (pcombined_noise[ix+iy*xdim]-pcombined_noise[(ix-1)+iy*xdim])*shift_val +
01319     //                                         pcombined_noise[(ix-1)+iy*xdim];
01320     //                    }
01321     //                    for (int ix = 1; ix < xdim; ix++) {
01322     //                        pcombined_noise[ix+iy*xdim] = tmpArray[ix];
01323     //                    }
01324                     }
01325                 }
01326                 cpl_free(array_in); array_in = NULL;
01327             }
01328             //
01329             // reconstruct
01330             //
01331             // load MASTER_DARK and MASTER_FLAT
01332             KMO_TRY_EXIT_IF_NULL(
01333                 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK,
01334                                               det_nr, FALSE, FALSE, NULL));
01335 
01336             if (process_noise) {
01337                 KMO_TRY_EXIT_IF_NULL(
01338                     img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK,
01339                                                         det_nr, TRUE, FALSE, NULL));
01340             }
01341 
01342             KMO_TRY_EXIT_IF_NULL(
01343                 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE,
01344                                                   rotangle, FALSE, NULL,
01345                                                   &rotangle_found, -1, 0, 0));
01346 
01347             if (process_noise) {
01348                 KMO_TRY_EXIT_IF_NULL(
01349                     img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, TRUE,
01350                                                             rotangle, FALSE, NULL,
01351                                                             &rotangle_found, -1, 0, 0));
01352             }
01353 
01354             char *tmp_band_method = getenv("KMO_BAND_METHOD");
01355             int band_method = 0;
01356             if (tmp_band_method != NULL) {
01357                 band_method = atoi(tmp_band_method);
01358             }
01359 
01360             // ESO INS FILTi ID
01361             KMO_TRY_EXIT_IF_NULL(
01362                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
01363                                       IFU_FILTID_POSTFIX));
01364             KMO_TRY_EXIT_IF_NULL(
01365                 filter_id = cpl_propertylist_get_string(main_header, keyword));
01366             cpl_free(keyword); keyword = NULL;
01367 
01368             KMO_TRY_EXIT_IF_NULL(
01369                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
01370             KMO_TRY_EXIT_IF_ERROR(
01371                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
01372                                              band_table));
01373             cpl_table_delete(band_table); band_table = NULL;
01374 
01375             cpl_msg_info("","Reconstructing cubes...");
01376             for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01377                 // update sub-header
01378                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01379 
01380                 // load raw image and sub-header
01381                 KMO_TRY_EXIT_IF_NULL(
01382                     sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY,
01383                                                          det_nr, FALSE));
01384 
01385                 KMO_TRY_EXIT_IF_NULL(
01386                     punused_ifus = cpl_array_get_data_int_const(
01387                                                   unused_ifus_after[det_nr-1]));
01388 
01389                 // check if IFU is valid according to main header keywords &
01390                 // calibration files
01391                 KMO_TRY_EXIT_IF_NULL(
01392                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
01393                                           IFU_VALID_POSTFIX));
01394                 KMO_TRY_CHECK_ERROR_STATE();
01395                 ranges_txt = cpl_propertylist_get_string(main_header, keyword);
01396                 cpl_free(keyword); keyword = NULL;
01397 
01398                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
01399                     (bounds[2*(ifu_nr-1)] != -1) &&
01400                     (bounds[2*(ifu_nr-1)+1] != -1) &&
01401                     (punused_ifus[j] == 0))
01402                 {
01403                     // IFU is valid
01404                     cpl_error_reset();
01405 
01406                     // calculate WCS
01407                     KMO_TRY_EXIT_IF_ERROR(
01408                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
01409 
01410                     // reconstruct data
01411                     KMO_TRY_EXIT_IF_ERROR(
01412                         kmo_reconstruct_sci_image(ifu_nr,
01413                                                 bounds[2*(ifu_nr-1)],
01414                                                 bounds[2*(ifu_nr-1)+1],
01415                                                 combined_data,
01416                                                 combined_noise,
01417                                                 img_dark,
01418                                                 img_dark_noise,
01419                                                 img_flat,
01420                                                 img_flat_noise,
01421                                                 xcal,
01422                                                 ycal,
01423                                                 lcal,
01424                                                 &gd,
01425                                                 calTimestamp,
01426                                                 calAngles,
01427                                                 fn_lut,
01428                                                 &cube_data,
01429                                                 &cube_noise,
01430                                                 flux,
01431                                                 background));
01432                     KMO_TRY_CHECK_ERROR_STATE();
01433                 } else {
01434                     // IFU is invalid
01435                     cpl_error_reset();
01436                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01437 
01438                 // save output
01439                 KMO_TRY_EXIT_IF_NULL(
01440                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01441 
01442                 KMO_TRY_EXIT_IF_ERROR(
01443                     kmclipm_update_property_string(sub_header, EXTNAME,
01444                                                    extname,
01445                                                    "FITS extension name"));
01446 
01447                 cpl_free(extname); extname = NULL;
01448 
01449                 // store cube and sub header into array for later
01450                 stored_data_cubes[ifu_nr - 1] = cube_data;
01451                 stored_sub_data_headers[ifu_nr - 1] = sub_header;
01452 
01453                 if (process_noise) {
01454                     KMO_TRY_EXIT_IF_NULL(
01455                         sub_header = cpl_propertylist_duplicate(
01456                                            stored_sub_data_headers[ifu_nr - 1]));
01457                     KMO_TRY_EXIT_IF_NULL(
01458                         extname = kmo_extname_creator(ifu_frame, ifu_nr,
01459                                                       EXT_NOISE));
01460 
01461                     KMO_TRY_EXIT_IF_ERROR(
01462                         kmclipm_update_property_string(sub_header,
01463                                                 EXTNAME,
01464                                                 extname,
01465                                                 "FITS extension name"));
01466 
01467                     cpl_free(extname); extname = NULL;
01468 
01469                     stored_noise_cubes[ifu_nr - 1] = cube_noise;
01470                     stored_sub_noise_headers[ifu_nr - 1] = sub_header;
01471                 }
01472                 cpl_image_delete(data_ifu); data_ifu = NULL;
01473                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01474                 cube_data = NULL;
01475                 cube_noise = NULL;
01476             } // for j IFUs
01477 
01478             // free memory
01479             cpl_image_delete(combined_data); combined_data = NULL;
01480             cpl_image_delete(combined_noise); combined_noise = NULL;
01481             cpl_image_delete(xcal); xcal = NULL;
01482             cpl_image_delete(ycal); ycal = NULL;
01483             cpl_image_delete(lcal); lcal = NULL;
01484             cpl_image_delete(img_dark); img_dark = NULL;
01485             cpl_image_delete(img_flat); img_flat = NULL;
01486             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01487             if (process_noise) {
01488                 cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01489                 cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01490             }
01491         } // for nr_devices
01492 
01493         cpl_free(edge_table_flat); edge_table_flat = NULL;
01494 
01495         // collapse cubes using rejection
01496         cpl_msg_info("","Collapsing cubes...");
01497         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01498             for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01499                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01500 
01501                 KMO_TRY_EXIT_IF_NULL(
01502                     punused_ifus = cpl_array_get_data_int_const(
01503                                                   unused_ifus_after[det_nr-1]));
01504                 if (punused_ifus[j] == 0) {
01505                     if (stored_sub_data_headers[ifu_nr-1] != NULL) {
01506                     // IFU is valid
01507                     ifu_crpix = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01508                                                             CRPIX3);
01509                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01510                                    "CRPIX3 keyword in FITS-header is missing!");
01511 
01512                     ifu_crval = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01513                                                             CRVAL3);
01514                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01515                                    "CRVAL3 keyword in FITS-header is missing!");
01516 
01517                     ifu_cdelt = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01518                                                             CDELT3);
01519                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01520                                    "CDELT3 keyword in FITS-header is missing!");
01521 
01522                     KMO_TRY_EXIT_IF_NULL(
01523                         identified_slices = kmo_identify_slices(ranges,
01524                                                                 ifu_crpix,
01525                                                                 ifu_crval,
01526                                                                 ifu_cdelt,
01527                                                                 gd.l.dim));
01528                     }/* else {
01529                         KMO_TRY_EXIT_IF_NULL(
01530                             identified_slices = cpl_vector_new(gd.l.dim));
01531                         cpl_vector_fill(identified_slices, 1.);
01532                     }*/
01533 
01534                     if (stored_data_cubes[ifu_nr-1] != NULL) {
01535                         KMO_TRY_EXIT_IF_ERROR(
01536                             kmclipm_make_image(stored_data_cubes[ifu_nr-1],
01537                                                stored_noise_cubes[ifu_nr-1],
01538                                                &stored_data_images[ifu_nr-1],
01539                                                &stored_noise_images[ifu_nr-1],
01540                                                identified_slices,
01541                                                cmethod, cpos_rej, cneg_rej,
01542                                                citer, cmax, cmin));
01543                     }
01544                     cpl_vector_delete(identified_slices); identified_slices = NULL;
01545                 } else {
01546                     // IFU is invalid
01547                 }
01548             }
01549         }
01550 
01551         // normalise all IFUs as a group.
01552         // Calculate mean of each IFU, add up and divide by number of successful
01553         // averaged IFUs.
01554         // Then divide all valid IFUs with mean value
01555         cnt = 0;
01556         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01557             if (stored_data_images[i] != NULL) {
01558                 KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[i]) <
01559                                cpl_image_get_size_x(stored_data_images[i])*
01560                                cpl_image_get_size_y(stored_data_images[i]),
01561                                CPL_ERROR_ILLEGAL_INPUT,
01562                                "The collapsed, dark-subtracted image contains "
01563                                "only invalid values! Probably the provided "
01564                                "FLAT_SKY frames are exactly the same as the "
01565                                "frames used for MASTER_DARK calculation.");
01566 
01567                 mean_data += cpl_image_get_mean(stored_data_images[i]);
01568                 KMO_TRY_CHECK_ERROR_STATE();
01569                 cnt++;
01570             }
01571 
01572         }
01573         mean_data /= cnt;
01574 
01575         if (mean_data != 0.0) {
01576             for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01577                 if (stored_data_images[i] != NULL) {
01578                     KMO_TRY_EXIT_IF_ERROR(
01579                         cpl_image_divide_scalar(stored_data_images[i],
01580                                                 mean_data));
01581                 }
01582             }
01583         } else {
01584             cpl_msg_warning(cpl_func, "Data couldn't be normalised "
01585                                       "(mean = 0.0)!");
01586         }
01587 
01588         if (process_noise) {
01589             if (mean_data != 0.0) {
01590                 for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01591                     if (stored_noise_images[i] != NULL) {
01592                         KMO_TRY_EXIT_IF_ERROR(
01593                             cpl_image_divide_scalar(stored_noise_images[i],
01594                                                     mean_data));
01595                     }
01596                 }
01597             } else {
01598                 cpl_msg_warning(cpl_func, "Noise couldn't be normalised "
01599                                           "(mean = 0.0)!");
01600             }
01601         }
01602 
01603         // calculate qc parameters on normalised data
01604         qc_spat_unif = 0.0;
01605         cnt = 0;
01606         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01607             if (stored_data_images[i] != NULL) {
01608                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01609                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01610 
01611                 qc_spat_unif += pow(tmp_mean-1, 2);
01612                 if (fabs(tmp_mean) > qc_max_dev) {
01613                     qc_max_dev = tmp_mean-1;
01614                     qc_max_dev_id = i+1;
01615                 }
01616                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01617                     qc_max_nonunif = tmp_stdev;
01618                     qc_max_nonunif_id = i+1;
01619                 }
01620                 KMO_TRY_CHECK_ERROR_STATE();
01621                 cnt++;
01622             }
01623         }
01624         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01625 
01626         //
01627         // save data
01628         //
01629 
01630         // update which IFUs are not used
01631         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01632 
01633         KMO_TRY_EXIT_IF_ERROR(
01634             kmo_set_unused_ifus(unused_ifus_after, main_header,
01635                                 "kmo_illumination"));
01636 
01637         cpl_msg_info("","Saving data...");
01638 
01639         KMO_TRY_EXIT_IF_ERROR(
01640             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01641                                            "[adu] uniformity of illumination correction"));
01642         KMO_TRY_EXIT_IF_ERROR(
01643             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01644                                            "[adu] max. deviation from unity"));
01645         KMO_TRY_EXIT_IF_ERROR(
01646             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01647                                         "[] IFU ID with max. dev. from unity"));
01648         KMO_TRY_EXIT_IF_ERROR(
01649             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01650                                            "[adu] max. stdev of illumination corr."));
01651         KMO_TRY_EXIT_IF_ERROR(
01652             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01653                                         "[] IFU ID with max. stdev in illum. corr."));
01654 
01655         KMO_TRY_EXIT_IF_ERROR(
01656             kmo_dfs_save_main_header(frameset, ILLUM_CORR, suffix, frame,
01657                                      main_header, parlist, cpl_func));
01658 
01659         KMO_TRY_EXIT_IF_ERROR(
01660             kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, suffix, frame,
01661                                      main_header, parlist, cpl_func));
01662 
01663         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01664             KMO_TRY_EXIT_IF_ERROR(
01665                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, suffix,
01666                                    stored_sub_data_headers[i], 0./0.));
01667 
01668             if (process_noise) {
01669                 KMO_TRY_EXIT_IF_ERROR(
01670                     kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR,
01671                                        suffix, stored_sub_noise_headers[i], 0./0.));
01672             }
01673         }
01674 
01675         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01676             for (int ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) {
01677                 KMO_TRY_EXIT_IF_ERROR(
01678                     kmclipm_update_property_int(stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01679                                                 CAL_IFU_NR,
01680                                                 ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR,
01681                                                 "IFU Number {1..24}"));
01682                 KMO_TRY_EXIT_IF_ERROR(
01683                     kmclipm_update_property_double(
01684                                                 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01685                                                 CAL_ROTANGLE,
01686                                                 rotangle_found,
01687                                                 "[deg] Rotator relative to nasmyth"));
01688                 if (has_flat_edge) {
01689                     // save edge-parameters as product
01690                     KMO_TRY_EXIT_IF_ERROR(
01691                         kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], SKYFLAT_EDGE, suffix,
01692                                            stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]));
01693                 }
01694             }
01695         }
01696     }
01697     KMO_CATCH
01698     {
01699         KMO_CATCH_MSG();
01700         ret_val = -1;
01701     }
01702     kmo_free_fits_desc(&desc_sky);
01703     kmo_free_fits_desc(&desc_dark);
01704     kmo_free_fits_desc(&desc_flat);
01705     kmo_free_fits_desc(&desc_xcal);
01706     kmo_free_fits_desc(&desc_ycal);
01707     kmo_free_fits_desc(&desc_lcal);
01708     cpl_image_delete(combined_data); combined_data = NULL;
01709     cpl_image_delete(combined_noise); combined_noise = NULL;
01710     cpl_image_delete(xcal); xcal = NULL;
01711     cpl_image_delete(ycal); ycal = NULL;
01712     cpl_image_delete(lcal); lcal = NULL;
01713     cpl_image_delete(img_dark); img_dark = NULL;
01714     cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01715     cpl_image_delete(img_flat); img_flat = NULL;
01716     cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01717     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01718     cpl_free(bounds); bounds = NULL;
01719     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01720     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01721     cpl_free(fn_lut); fn_lut = NULL;
01722     cpl_free(suffix); suffix = NULL;
01723     cpl_frameset_delete(frameset_sky); frameset_sky = NULL;
01724     cpl_vector_delete(ranges); ranges = NULL;
01725     cpl_free(filter); filter = NULL;
01726     if (calAngles != NULL) {
01727         cpl_vector_delete(calAngles); calAngles = NULL;
01728     }
01729     cpl_propertylist_delete(main_header); main_header = NULL;
01730     for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01731         if (stored_data_cubes != NULL) {
01732             cpl_imagelist_delete(stored_data_cubes[i]);
01733             stored_data_cubes[i] = NULL;
01734         }
01735         if (stored_noise_cubes != NULL) {
01736             cpl_imagelist_delete(stored_noise_cubes[i]);
01737             stored_noise_cubes[i] = NULL;
01738         }
01739         if (stored_data_images != NULL) {
01740             cpl_image_delete(stored_data_images[i]);
01741             stored_data_images[i] = NULL;
01742         }
01743         if (stored_noise_images != NULL) {
01744             cpl_image_delete(stored_noise_images[i]);
01745             stored_noise_images[i] = NULL;
01746         }
01747         if (stored_sub_data_headers != NULL) {
01748             cpl_propertylist_delete(stored_sub_data_headers[i]);
01749             stored_sub_data_headers[i] = NULL;
01750         }
01751         if (stored_sub_noise_headers != NULL) {
01752             cpl_propertylist_delete(stored_sub_noise_headers[i]);
01753             stored_sub_noise_headers[i] = NULL;
01754         }
01755     }
01756     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01757     cpl_free(stored_noise_cubes); stored_noise_cubes = NULL;
01758     cpl_free(stored_data_images); stored_data_images = NULL;
01759     cpl_free(stored_noise_images); stored_noise_images = NULL;
01760     cpl_free(stored_sub_data_headers); stored_sub_data_headers = NULL;
01761     cpl_free(stored_sub_noise_headers); stored_sub_noise_headers = NULL;
01762     if (edge_table_sky != NULL) {
01763         for (int i = 0; i < KMOS_NR_DETECTORS; i++) {
01764             if (edge_table_sky[i] != NULL) {
01765                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01766                     cpl_table_delete(edge_table_sky[i][j]);
01767                     edge_table_sky[i][j] = NULL;
01768                 }
01769                 cpl_free(edge_table_sky[i]); edge_table_sky[i] = NULL;
01770             }
01771         }
01772         cpl_free(edge_table_sky); edge_table_sky = NULL;
01773     }
01774     if (edge_table_flat != NULL) {
01775         for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01776             cpl_table_delete(edge_table_flat[j]);
01777             edge_table_flat[j] = NULL;
01778         }
01779         cpl_free(edge_table_flat); edge_table_flat = NULL;
01780     }
01781 
01782     return ret_val;
01783 }
01784