KMOS Pipeline Reference Manual  1.1.2
kmo_illumination.c
00001 /* $Id: kmo_illumination.c,v 1.46 2013/05/07 08:35:45 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013/05/07 08:35:45 $
00024  * $Revision: 1.46 $
00025  * $Name: HEAD $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmo_priv_reconstruct.h"
00042 #include "kmo_priv_functions.h"
00043 #include "kmo_priv_flat.h"
00044 #include "kmo_priv_wave_cal.h"
00045 #include "kmclipm_priv_splines.h"
00046 #include "kmo_functions.h"
00047 #include "kmo_cpl_extensions.h"
00048 #include "kmo_dfs.h"
00049 #include "kmo_error.h"
00050 #include "kmo_constants.h"
00051 #include "kmo_debug.h"
00052 
00053 /*-----------------------------------------------------------------------------
00054  *                          Functions prototypes
00055  *----------------------------------------------------------------------------*/
00056 
00057 static int kmo_illumination_create(cpl_plugin *);
00058 static int kmo_illumination_exec(cpl_plugin *);
00059 static int kmo_illumination_destroy(cpl_plugin *);
00060 static int kmo_illumination(cpl_parameterlist *, cpl_frameset *);
00061 
00062 /*-----------------------------------------------------------------------------
00063  *                          Static variables
00064  *----------------------------------------------------------------------------*/
00065 
00066 static char kmo_illumination_description[] =
00067 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00068 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00069 "generates the spectral calibration frame needed in this recipe. As input at\n"
00070 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n"
00071 "bration frames are required.\n"
00072 "\n"
00073 "BASIC PARAMETERS:\n"
00074 "-----------------\n"
00075 "--imethod\n"
00076 "The interpolation method used for reconstruction.\n"
00077 "\n"
00078 "--range\n"
00079 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n"
00080 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n"
00081 "\n"
00082 "ADVANCED PARAMETERS\n"
00083 "-------------------\n"
00084 "--add-all\n"
00085 "By default the first FLAT_SKY frame is omitted, since in the\n"
00086 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n"
00087 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all frames\n"
00088 "should be considered, set this parameter to TRUE.\n"
00089 "\n"
00090 "--neighborhoodRange\n"
00091 "Defines the range to search for neighbors during reconstruction\n"
00092 "\n"
00093 "--b_samples\n"
00094 "The number of samples in spectral direction for the reconstructed cube.\n"
00095 "Ideally this number should be greater than 2048, the detector size.\n"
00096 "\n"
00097 "--b_start\n"
00098 "--b_end\n"
00099 "Used to define manually the start and end wavelength for the reconstructed\n"
00100 "cube. By default the internally defined values are used.\n"
00101 "\n"
00102 "--cmethod\n"
00103 "Following methods of frame combination are available:\n"
00104 "   * 'ksigma' (Default)\n"
00105 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00106 "   are examined. If they deviate significantly, they will be rejected according\n"
00107 "   to the conditions:\n"
00108 "       val > mean + stdev * cpos_rej\n"
00109 "   and\n"
00110 "       val < mean - stdev * cneg_rej\n"
00111 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00112 "   parameters. In the first iteration median and percentile level are used.\n"
00113 "\n"
00114 "   * 'median'\n"
00115 "   At each pixel position the median is calculated.\n"
00116 "\n"
00117 "   * 'average'\n"
00118 "   At each pixel position the average is calculated.\n"
00119 "\n"
00120 "   * 'sum'\n"
00121 "   At each pixel position the sum is calculated.\n"
00122 "\n"
00123 "   * 'min_max'\n"
00124 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00125 "   --cmax and --cmin apply to this method.\n"
00126 "\n"
00127 "--cpos_rej\n"
00128 "--cneg_rej\n"
00129 "--citer\n"
00130 "see --cmethod='ksigma'\n"
00131 "\n"
00132 "--cmax\n"
00133 "--cmin\n"
00134 "see --cmethod='min_max'\n"
00135 "\n"
00136 "-------------------------------------------------------------------------------\n"
00137 "  Input files:\n"
00138 "\n"
00139 "   DO                    KMOS                                                  \n"
00140 "   category              Type   Explanation                    Required #Frames\n"
00141 "   --------              -----  -----------                    -------- -------\n"
00142 "   FLAT_SKY               F2D   Sky exposures                     Y      1-n   \n"
00143 "                                (at least 3 frames recommended)                \n"
00144 "   MASTER_DARK            F2D   Master dark                       Y       1    \n"
00145 "   MASTER_FLAT            F2D   Master flat                       Y       1    \n"
00146 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00147 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00148 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00149 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00150 "   FLAT_EDGE              F2L   Table with fitted slitlet edges   N      0,1   \n"
00151 "\n"
00152 "  Output files:\n"
00153 "\n"
00154 "   DO                    KMOS\n"
00155 "   category              Type   Explanation\n"
00156 "   --------              -----  -----------\n"
00157 "   ILLUM_CORR            F2I    Illumination calibration frame   \n"
00158 "   If FLAT_EDGE is provided: \n"
00159 "   SKYFLAT_EDGE          F2L    Frame containing parameters of fitted \n"
00160 "                                slitlets of all IFUs of all detectors\n"
00161 "-------------------------------------------------------------------------------\n"
00162 "\n";
00163 
00164 /*-----------------------------------------------------------------------------
00165  *                              Functions code
00166  *----------------------------------------------------------------------------*/
00167 
00184 int cpl_plugin_get_info(cpl_pluginlist *list)
00185 {
00186     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00187     cpl_plugin *plugin = &recipe->interface;
00188 
00189     cpl_plugin_init(plugin,
00190                         CPL_PLUGIN_API,
00191                         KMOS_BINARY_VERSION,
00192                         CPL_PLUGIN_TYPE_RECIPE,
00193                         "kmo_illumination",
00194                         "Create a calibration frame to correct spatial "
00195                         "non-uniformity of flatfield.",
00196                         kmo_illumination_description,
00197                         "Alex Agudo Berbel",
00198                         "agudo@mpe.mpg.de",
00199                         kmos_get_license(),
00200                         kmo_illumination_create,
00201                         kmo_illumination_exec,
00202                         kmo_illumination_destroy);
00203 
00204     cpl_pluginlist_append(list, plugin);
00205 
00206     return 0;
00207 }
00208 
00216 static int kmo_illumination_create(cpl_plugin *plugin)
00217 {
00218     cpl_recipe *recipe;
00219     cpl_parameter *p;
00220 
00221     /* Check that the plugin is part of a valid recipe */
00222     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00223         recipe = (cpl_recipe *)plugin;
00224     else
00225         return -1;
00226 
00227     /* Create the parameters list in the cpl_recipe object */
00228     recipe->parameters = cpl_parameterlist_new();
00229 
00230     /* Fill the parameters list */
00231     /* --imethod */
00232     p = cpl_parameter_new_value("kmos.kmo_illumination.imethod",
00233                                 CPL_TYPE_STRING,
00234                                 "Method to use for interpolation: "
00235                                 "[\"NN\" (nearest neighbour), "
00236                                 "\"lwNN\" (linear weighted nearest neighbor),"
00237                                 "\"swNN\" (square weighted nearest neighbor)"
00238                                 "\"MS\" (Modified Shepard's method)"
00239                                 "\"CS\" (Cubic spline)]",
00240                                 "kmos.kmo_illumination",
00241                                 "CS");
00242     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00243     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00244     cpl_parameterlist_append(recipe->parameters, p);
00245 
00246     /* --neighborhoodRange */
00247     p = cpl_parameter_new_value("kmos.kmo_illumination.neighborhoodRange",
00248                                 CPL_TYPE_DOUBLE,
00249                                 "Defines the range to search for neighbors."
00250                                 "in pixels",
00251                                 "kmos.kmo_illumination",
00252                                 1.001);
00253     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00254     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00255     cpl_parameterlist_append(recipe->parameters, p);
00256 
00257     /* --range */
00258     p = cpl_parameter_new_value("kmos.kmo_illumination.range",
00259                                 CPL_TYPE_STRING,
00260                                 "The spectral ranges to combine when collapsing"
00261                                 "the reconstructed cubes. e.g."
00262                                 "\"x1_start,x1_end;x2_start,x2_end\" (microns)",
00263                                 "kmos.kmo_illumination",
00264                                 "");
00265     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range");
00266     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00267     cpl_parameterlist_append(recipe->parameters, p);
00268 
00269     /* --flux */
00270     p = cpl_parameter_new_value("kmos.kmo_illumination.flux",
00271                                 CPL_TYPE_BOOL,
00272                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00273                                 "kmos.kmo_illumination",
00274                                 FALSE);
00275     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00276     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00277     cpl_parameterlist_append(recipe->parameters, p);
00278 
00279     /* --add-all */
00280     p = cpl_parameter_new_value("kmos.kmo_illumination.add-all",
00281                                 CPL_TYPE_BOOL,
00282                                 "FALSE: omit 1st FLAT_SKY frame (acquisition), "
00283                                 "TRUE: don't perform any checks, add them all",
00284                                 "kmos.kmo_illumination",
00285                                 FALSE);
00286     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all");
00287     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00288     cpl_parameterlist_append(recipe->parameters, p);
00289 
00290     // add parameters for band-definition
00291     kmo_band_pars_create(recipe->parameters,
00292                          "kmos.kmo_illumination");
00293 
00294     // add parameters for combining
00295     return kmo_combine_pars_create(recipe->parameters,
00296                                    "kmos.kmo_illumination",
00297                                    DEF_REJ_METHOD,
00298                                    FALSE);
00299 }
00300 
00306 static int kmo_illumination_exec(cpl_plugin *plugin)
00307 {
00308     cpl_recipe  *recipe;
00309 
00310     /* Get the recipe out of the plugin */
00311     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00312         recipe = (cpl_recipe *)plugin;
00313     else return -1;
00314 
00315     return kmo_illumination(recipe->parameters, recipe->frames);
00316 }
00317 
00323 static int kmo_illumination_destroy(cpl_plugin *plugin)
00324 {
00325     cpl_recipe *recipe;
00326 
00327     /* Get the recipe out of the plugin */
00328     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00329         recipe = (cpl_recipe *)plugin;
00330     else return -1 ;
00331 
00332     cpl_parameterlist_delete(recipe->parameters);
00333     return 0 ;
00334 }
00335 
00350 static int kmo_illumination(cpl_parameterlist *parlist, cpl_frameset *frameset)
00351 {
00352     int              ret_val                    = 0,
00353                      nr_devices                 = 0,
00354                      ifu_nr                     = 0,
00355                      nx                         = 0,
00356                      ny                         = 0,
00357                      process_noise              = FALSE,
00358                      cmax                       = 0,
00359                      cmin                       = 0,
00360                      citer                      = 0,
00361                      *bounds                    = NULL,
00362                      cnt                        = 0,
00363                      qc_max_dev_id              = 0,
00364                      qc_max_nonunif_id          = 0,
00365                      flux                       = FALSE,
00366                      background                 = FALSE,
00367                      add_all_sky                = FALSE,
00368                      same_exptime               = TRUE,
00369                      has_flat_edge              = FALSE;
00370     const int        *punused_ifus              = NULL;
00371     float            *pbad_pix_mask             = NULL;
00372     double           exptime                    = 0.,
00373                      exptime1                   = 0.,
00374                      exptime2                   = 0.,
00375                      cpos_rej                   = 0.0,
00376                      cneg_rej                   = 0.0,
00377                      neighborhoodRange          = 1.001,
00378                      mean_data                  = 0.0,
00379                      ifu_crpix                  = 0.0,
00380                      ifu_crval                  = 0.0,
00381                      ifu_cdelt                  = 0.0,
00382                      qc_spat_unif               = 0.0,
00383                      qc_max_dev                 = 0.0,
00384                      qc_max_nonunif             = 0.0,
00385                      tmp_stdev                  = 0.0,
00386                      tmp_mean                   = 0.0,
00387                      rotangle                   = 0.0,
00388                      tmp_rotangle               = 0.0,
00389                      rotangle_found             = 0.0;
00390     char             *keyword                   = NULL,
00391                      *fn_lut                    = NULL,
00392                      *suffix                    = NULL,
00393                      *extname                   = NULL,
00394                      *filter                    = NULL;
00395     const char       *method                    = NULL,
00396                      *cmethod                   = NULL,
00397                      *filter_id_l               = NULL,
00398                      *filter_id                 = NULL,
00399                      *ranges_txt                = NULL;
00400     cpl_array        *calTimestamp              = NULL,
00401                      **unused_ifus_before       = NULL,
00402                      **unused_ifus_after        = NULL;
00403     cpl_frame        *frame                     = NULL,
00404                      *xcalFrame                 = NULL,
00405                      *ycalFrame                 = NULL,
00406                      *lcalFrame                 = NULL;
00407     cpl_frameset     *frameset_sky              = NULL;
00408     cpl_image        *img_in                    = NULL,
00409                      *img_dark                  = NULL,
00410                      *img_dark_noise            = NULL,
00411                      *img_flat                  = NULL,
00412                      *img_flat_noise            = NULL,
00413                      *combined_data             = NULL,
00414                      *combined_noise            = NULL,
00415                      *xcal                      = NULL,
00416                      *ycal                      = NULL,
00417                      *lcal                      = NULL,
00418                      *bad_pix_mask              = NULL,
00419                      *data_ifu                  = NULL,
00420                      *noise_ifu                 = NULL,
00421                      **stored_data_images       = NULL,
00422                      **stored_noise_images      = NULL;
00423     cpl_imagelist    *cube_data                 = NULL,
00424                      *cube_noise                = NULL,
00425                      *detector_in               = NULL,
00426                      **stored_data_cubes        = NULL,
00427                      **stored_noise_cubes       = NULL;
00428     cpl_matrix       **edgepars                 = NULL;
00429     cpl_propertylist *main_header               = NULL,
00430                      *tmp_header                = NULL,
00431                      *sub_header                = NULL,
00432                      **stored_sub_data_headers  = NULL,
00433                      **stored_sub_noise_headers = NULL;
00434     cpl_table        *band_table                = NULL,
00435                      ***edge_table_sky          = NULL,
00436                      **edge_table_flat          = NULL;
00437     cpl_vector       *ranges                    = NULL,
00438                      *identified_slices         = NULL,
00439                      *calAngles                 = NULL,
00440                      **slitlet_ids              = NULL,
00441                      *shift_vec                 = NULL,
00442                      *edge_vec                  = NULL;
00443     main_fits_desc   desc_sky,
00444                      desc_dark,
00445                      desc_flat,
00446                      desc_xcal,
00447                      desc_ycal,
00448                      desc_lcal;
00449     gridDefinition   gd;
00450 
00451     KMO_TRY
00452     {
00453         kmo_init_fits_desc(&desc_sky);
00454         kmo_init_fits_desc(&desc_dark);
00455         kmo_init_fits_desc(&desc_flat);
00456         kmo_init_fits_desc(&desc_xcal);
00457         kmo_init_fits_desc(&desc_ycal);
00458         kmo_init_fits_desc(&desc_lcal);
00459 
00460         /* --- check input --- */
00461         KMO_TRY_ASSURE((parlist != NULL) &&
00462                        (frameset != NULL),
00463                        CPL_ERROR_NULL_INPUT,
00464                        "Not all input data is provided!");
00465 
00466         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY) >= 1,
00467                        CPL_ERROR_ILLEGAL_INPUT,
00468                        "One or more FLAT_SKY frames are required!");
00469 
00470         if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) {
00471             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00472                                       "3 FLAT_SKY frames!");
00473         }
00474 
00475         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_DARK) == 1,
00476                        CPL_ERROR_ILLEGAL_INPUT,
00477                        "Exactly one MASTER_DARK frame is required!");
00478 
00479         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_FLAT) == 1,
00480                        CPL_ERROR_ILLEGAL_INPUT,
00481                        "Exactly one MASTER_FLAT frame is required!");
00482 
00483         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00484                        CPL_ERROR_ILLEGAL_INPUT,
00485                        "Exactly one XCAL frame is required!");
00486 
00487         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00488                        CPL_ERROR_ILLEGAL_INPUT,
00489                        "Exactly one YCAL frame is required!");
00490 
00491         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00492                        CPL_ERROR_ILLEGAL_INPUT,
00493                        "Exactly one LCAL frame is required!");
00494 
00495         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00496                        CPL_ERROR_ILLEGAL_INPUT,
00497                        "Exactly one WAVE_BAND frame is required!");
00498 
00499         KMO_TRY_ASSURE((cpl_frameset_count_tags(frameset, FLAT_EDGE) == 1) ||
00500                        (cpl_frameset_count_tags(frameset, FLAT_EDGE) == 0),
00501                        CPL_ERROR_ILLEGAL_INPUT,
00502                        "Exactly one FLAT_EDGE frame is required!");
00503 
00504         has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE);
00505 
00506         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination") == 1,
00507                        CPL_ERROR_ILLEGAL_INPUT,
00508                        "Cannot identify RAW and CALIB frames!");
00509 
00510         /* --- get parameters --- */
00511         cpl_msg_info("", "--- Parameter setup for kmo_illumination ---");
00512 
00513         KMO_TRY_EXIT_IF_NULL(
00514             method = kmo_dfs_get_parameter_string(parlist,
00515                                               "kmos.kmo_illumination.imethod"));
00516 
00517         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00518                        (strcmp(method, "lwNN") == 0) ||
00519                        (strcmp(method, "swNN") == 0) ||
00520                        (strcmp(method, "MS") == 0) ||
00521                        (strcmp(method, "CS") == 0),
00522                        CPL_ERROR_ILLEGAL_INPUT,
00523                        "method must be either \"NN\", \"lwNN\", "
00524                        "\"swNN\", \"MS\" or \"CS\"!");
00525 
00526         KMO_TRY_EXIT_IF_ERROR(
00527             kmo_dfs_print_parameter_help(parlist,
00528                                         "kmos.kmo_illumination.imethod"));
00529 
00530         neighborhoodRange = kmo_dfs_get_parameter_double(parlist,
00531                 "kmos.kmo_illumination.neighborhoodRange");
00532         KMO_TRY_CHECK_ERROR_STATE();
00533 
00534         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00535                 CPL_ERROR_ILLEGAL_INPUT,
00536                 "neighborhoodRange must be greater than 0.0");
00537 
00538         KMO_TRY_EXIT_IF_ERROR(
00539             kmo_dfs_print_parameter_help(parlist,
00540                                     "kmos.kmo_illumination.neighborhoodRange"));
00541 
00542         ranges_txt = kmo_dfs_get_parameter_string(parlist,
00543                                                   "kmos.kmo_illumination.range");
00544         KMO_TRY_CHECK_ERROR_STATE();
00545 
00546         KMO_TRY_EXIT_IF_ERROR(
00547             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.range"));
00548 
00549         ranges = kmo_identify_ranges(ranges_txt);
00550         KMO_TRY_CHECK_ERROR_STATE();
00551 
00552         flux = kmo_dfs_get_parameter_bool(parlist,
00553                                           "kmos.kmo_illumination.flux");
00554 
00555         KMO_TRY_ASSURE((flux == 0) ||
00556                        (flux == 1),
00557                        CPL_ERROR_ILLEGAL_INPUT,
00558                        "flux must be either FALSE or TRUE!");
00559 
00560         KMO_TRY_EXIT_IF_ERROR(
00561             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.flux"));
00562 
00563         add_all_sky = kmo_dfs_get_parameter_bool(parlist,
00564                                                  "kmos.kmo_illumination.add-all");
00565 
00566         KMO_TRY_ASSURE((add_all_sky == 0) ||
00567                        (add_all_sky == 1),
00568                        CPL_ERROR_ILLEGAL_INPUT,
00569                        "add_all must be either FALSE or TRUE!");
00570 
00571         KMO_TRY_EXIT_IF_ERROR(
00572             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.add-all"));
00573 
00574         kmo_band_pars_load(parlist, "kmos.kmo_illumination");
00575 
00576         KMO_TRY_EXIT_IF_ERROR(
00577             kmo_combine_pars_load(parlist,
00578                                   "kmos.kmo_illumination",
00579                                   &cmethod,
00580                                   &cpos_rej,
00581                                   &cneg_rej,
00582                                   &citer,
00583                                   &cmin,
00584                                   &cmax,
00585                                   FALSE));
00586         cpl_msg_info("", "-------------------------------------------");
00587 
00588         // check if filter_id, grating_id and rotator offset match for all
00589         // detectors
00590         KMO_TRY_EXIT_IF_ERROR(
00591             kmo_check_frameset_setup(frameset, FLAT_SKY,
00592                                        TRUE, FALSE, TRUE));
00593         KMO_TRY_EXIT_IF_ERROR(
00594             kmo_check_frame_setup(frameset, FLAT_SKY, XCAL,
00595                                        TRUE, FALSE, TRUE));
00596         KMO_TRY_EXIT_IF_ERROR(
00597             kmo_check_frame_setup(frameset, XCAL, YCAL,
00598                                        TRUE, FALSE, TRUE));
00599         KMO_TRY_EXIT_IF_ERROR(
00600             kmo_check_frame_setup(frameset, XCAL, LCAL,
00601                                        TRUE, FALSE, TRUE));
00602         KMO_TRY_EXIT_IF_ERROR(
00603             kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT,
00604                                        TRUE, FALSE, TRUE));
00605 
00606         KMO_TRY_EXIT_IF_NULL(
00607             frame = kmo_dfs_get_frame(frameset, XCAL));
00608         KMO_TRY_EXIT_IF_NULL(
00609             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00610 
00611         KMO_TRY_EXIT_IF_ERROR(
00612             kmo_check_frame_setup_md5_xycal(frameset));
00613         KMO_TRY_EXIT_IF_ERROR(
00614             kmo_check_frame_setup_md5(frameset));
00615 
00616         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00617         cpl_msg_info("", "(grating 1, 2 & 3)");
00618 
00619         // check which IFUs are active for all frames
00620         KMO_TRY_EXIT_IF_NULL(
00621             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00622 
00623         KMO_TRY_EXIT_IF_NULL(
00624             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00625 
00626         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00627 
00628         // load desc for MASTER_DARK and check
00629         KMO_TRY_EXIT_IF_NULL(
00630             frame = kmo_dfs_get_frame(frameset, MASTER_DARK));
00631         desc_dark = kmo_identify_fits_header(
00632                     cpl_frame_get_filename(frame));
00633         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_DARK frame doesn't seem to "
00634                                       "be in KMOS-format!");
00635         KMO_TRY_ASSURE((desc_dark.nr_ext == 6) &&
00636                        (desc_dark.ex_badpix == FALSE) &&
00637                        (desc_dark.fits_type == f2d_fits) &&
00638                        (desc_dark.frame_type == detector_frame),
00639                        CPL_ERROR_ILLEGAL_INPUT,
00640                        "MASTER_DARK isn't in the correct format!!!");
00641         nx = desc_dark.naxis1;
00642         ny = desc_dark.naxis2;
00643 
00644         // load desc for MASTER_FLAT and check
00645         KMO_TRY_EXIT_IF_NULL(
00646             frame = kmo_dfs_get_frame(frameset, MASTER_FLAT));
00647         desc_flat = kmo_identify_fits_header(cpl_frame_get_filename(frame));
00648         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_FLAT frame doesn't seem to "
00649                                       "be in KMOS-format!");
00650         KMO_TRY_ASSURE((desc_flat.nr_ext % 6 == 0) &&
00651                        (desc_flat.ex_badpix == FALSE) &&
00652                        (desc_flat.fits_type == f2d_fits) &&
00653                        (desc_flat.frame_type == detector_frame),
00654                        CPL_ERROR_ILLEGAL_INPUT,
00655                        "MASTER_FLAT isn't in the correct format!!!");
00656 
00657         // load desc for XCAL and check
00658         KMO_TRY_EXIT_IF_NULL(
00659             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00660         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00661         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00662                                       "be in KMOS-format!");
00663         KMO_TRY_ASSURE((desc_xcal.nr_ext % 3 == 0) &&
00664                        (desc_xcal.ex_badpix == FALSE) &&
00665                        (desc_xcal.fits_type == f2d_fits) &&
00666                        (desc_xcal.frame_type == detector_frame),
00667                        CPL_ERROR_ILLEGAL_INPUT,
00668                        "XCAL isn't in the correct format!!!");
00669         KMO_TRY_ASSURE((desc_xcal.naxis1 == nx) &&
00670                        (desc_xcal.naxis2 == ny),
00671                        CPL_ERROR_ILLEGAL_INPUT,
00672                        "MASTER_DARK and XCAL frame haven't same dimensions! "
00673                        "(x,y): (%d,%d) vs (%d,%d)",
00674                        nx, ny, desc_xcal.naxis1, desc_xcal.naxis2);
00675 
00676         nr_devices = desc_xcal.nr_ext;
00677 
00678         // load desc for YCAL and check
00679         KMO_TRY_EXIT_IF_NULL(
00680             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00681         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00682         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00683                                       "be in KMOS-format!");
00684         KMO_TRY_ASSURE((desc_ycal.nr_ext % 3 == 0) &&
00685                        (desc_ycal.ex_badpix == FALSE) &&
00686                        (desc_ycal.fits_type == f2d_fits) &&
00687                        (desc_ycal.frame_type == detector_frame),
00688                        CPL_ERROR_ILLEGAL_INPUT,
00689                        "YCAL isn't in the correct format!!!");
00690         KMO_TRY_ASSURE((desc_ycal.naxis1 == nx) &&
00691                        (desc_ycal.naxis2 == ny) &&
00692                        (desc_ycal.nr_ext == nr_devices),
00693                        CPL_ERROR_ILLEGAL_INPUT,
00694                        "MASTER_DARK and YCAL frame haven't same dimensions! "
00695                        "(x,y): (%d,%d) vs (%d,%d)",
00696                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00697 
00698         // load desc for LCAL and check
00699         KMO_TRY_EXIT_IF_NULL(
00700             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00701         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00702         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00703                                       "be in KMOS-format!");
00704         KMO_TRY_ASSURE((desc_lcal.nr_ext % 3 == 0) &&
00705                        (desc_lcal.ex_badpix == FALSE) &&
00706                        (desc_lcal.fits_type == f2d_fits) &&
00707                        (desc_lcal.frame_type == detector_frame),
00708                        CPL_ERROR_ILLEGAL_INPUT,
00709                        "LCAL isn't in the correct format!!!");
00710         KMO_TRY_ASSURE((desc_lcal.naxis1 == nx) &&
00711                        (desc_lcal.naxis2 == ny) &&
00712                        (desc_lcal.nr_ext == nr_devices),
00713                        CPL_ERROR_ILLEGAL_INPUT,
00714                        "MASTER_DARK and LCAL frame haven't same dimensions! "
00715                        "(x,y): (%d,%d) vs (%d,%d)",
00716                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00717         KMO_TRY_EXIT_IF_NULL(
00718             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00719 
00720         // load desc for FLAT_SKY and check
00721         nr_devices = KMOS_NR_DETECTORS;
00722         KMO_TRY_EXIT_IF_NULL(
00723             frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00724 
00725         KMO_TRY_EXIT_IF_NULL(
00726             main_header = kmclipm_propertylist_load(
00727                                          cpl_frame_get_filename(frame), 0));
00728         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00729         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00730         kmclipm_strip_angle(&rotangle);
00731         cpl_propertylist_delete(main_header); main_header = NULL;
00732 
00733         cnt = 1;
00734         while (frame != NULL) {
00735             KMO_TRY_EXIT_IF_NULL(
00736                 main_header = kmclipm_propertylist_load(
00737                                              cpl_frame_get_filename(frame), 0));
00738 
00739             desc_sky = kmo_identify_fits_header(
00740                         cpl_frame_get_filename(frame));
00741             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY frame doesn't seem to "
00742                                           "be in KMOS-format!");
00743             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00744                            (desc_sky.ex_badpix == FALSE) &&
00745                            (desc_sky.fits_type == raw_fits) &&
00746                            (desc_sky.frame_type == detector_frame),
00747                            CPL_ERROR_ILLEGAL_INPUT,
00748                            "FLAT_SKY isn't in the correct format!!!");
00749             KMO_TRY_ASSURE((desc_sky.naxis1 == nx) &&
00750                            (desc_sky.naxis2 == ny) &&
00751                            (desc_sky.nr_ext == nr_devices),
00752                            CPL_ERROR_ILLEGAL_INPUT,
00753                            "MASTER_DARK and FLAT_SKY (no. %d) frame haven't "
00754                            "same dimensions! (x,y): (%d,%d) vs (%d,%d)",
00755                            cnt, nx, ny, desc_flat.naxis1, desc_flat.naxis2);
00756             kmo_free_fits_desc(&desc_sky);
00757             kmo_init_fits_desc(&desc_sky);
00758 
00759             KMO_TRY_ASSURE(
00760                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00761                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE) &&
00762                 (kmo_check_lamp(main_header, INS_LAMP3_ST) == FALSE) &&
00763                 (kmo_check_lamp(main_header, INS_LAMP4_ST) == FALSE),
00764                 CPL_ERROR_ILLEGAL_INPUT,
00765                 "All lamps must be switched off for the FLAT_SKY frames!");
00766 
00767             // assert that filters have correct IDs and that all detectors of
00768             // all input frames have the same filter set
00769             for (int i = 1; i <= KMOS_NR_DETECTORS; i++) {
00770                 // ESO INS FILTi ID
00771                 KMO_TRY_EXIT_IF_NULL(
00772                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00773                 KMO_TRY_EXIT_IF_NULL(
00774                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00775 
00776                 KMO_TRY_EXIT_IF_NULL(
00777                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00778                 cpl_free(keyword); keyword = NULL;
00779 
00780                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00781                                (strcmp(filter_id, "YJ") == 0) ||
00782                                (strcmp(filter_id, "H") == 0) ||
00783                                (strcmp(filter_id, "K") == 0) ||
00784                                (strcmp(filter_id, "HK") == 0),
00785                                CPL_ERROR_ILLEGAL_INPUT,
00786                                "Filter ID in primary header must be either 'IZ', "
00787                                "'YJ', 'H', 'K' or " "'HK' !");
00788 
00789                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00790                                CPL_ERROR_ILLEGAL_INPUT,
00791                                "Filter IDs must be the same for FLAT_SKY frame"
00792                                " and lcal frame!"
00793                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00794                                i, cpl_frame_get_filename(frame),
00795                                filter_id, filter_id_l);
00796 
00797                 // ESO INS GRATi ID
00798                 KMO_TRY_EXIT_IF_NULL(
00799                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00800                 KMO_TRY_EXIT_IF_NULL(
00801                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00802 
00803                 KMO_TRY_EXIT_IF_NULL(
00804                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00805                 cpl_free(keyword); keyword = NULL;
00806 
00807                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00808                                (strcmp(filter_id, "YJ") == 0) ||
00809                                (strcmp(filter_id, "H") == 0) ||
00810                                (strcmp(filter_id, "K") == 0) ||
00811                                (strcmp(filter_id, "HK") == 0),
00812                                CPL_ERROR_ILLEGAL_INPUT,
00813                                "Grating ID in primary header must be either "
00814                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00815 
00816                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00817                                CPL_ERROR_ILLEGAL_INPUT,
00818                                "Grating IDs must be the same for FLAT_SKY frame"
00819                                " and lcal frame!"
00820                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00821                                i, cpl_frame_get_filename(frame),
00822                                filter_id, filter_id_l);
00823 
00824                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00825                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00826                 kmclipm_strip_angle(&tmp_rotangle);
00827                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00828                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00829                         CPL_ERROR_ILLEGAL_INPUT,
00830                         "OCS ROT NAANGLE of sky flat frames differ to much: %f %f",
00831                         rotangle, tmp_rotangle);
00832             }
00833             cpl_propertylist_delete(main_header); main_header = NULL;
00834 
00835             // get next FLAT_SKY frame
00836             frame = kmo_dfs_get_frame(frameset, NULL);
00837             KMO_TRY_CHECK_ERROR_STATE();
00838             cnt++;
00839         }
00840 
00841         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00842 
00843         //
00844         // noise will be propagated when:
00845         // MASTER_DARK and MASTER_FLAT have noise extensions and if at least
00846         // 2 FLAT_SKY frames are provided.
00847         // Otherwise noise will be ignored.
00848         //
00849         if (desc_dark.ex_noise &&
00850             desc_flat.ex_noise &&
00851             (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2)) {
00852             process_noise = TRUE;
00853         }
00854 
00855         if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00856             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00857                             "since there is only one input frame! (The output "
00858                             "file won't have any noise extensions)");
00859 
00860             cmethod = "average";
00861         }
00862 
00863         //
00864         // Check whether 1st FLAT_SKY should be omitted
00865         //
00866         KMO_TRY_EXIT_IF_NULL(
00867             frameset_sky = cpl_frameset_new());
00868 
00869         if (add_all_sky) {
00870             // just add all FLAT_SKY frames without check
00871             frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00872             while (frame != NULL) {
00873                 KMO_TRY_EXIT_IF_ERROR(
00874                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00875                 frame = kmo_dfs_get_frame(frameset, NULL);
00876             }
00877             cpl_msg_info("", "Add all FLAT_SKY without checking for acquisition frame.");
00878         } else {
00879             // check if 1st FLAT_SKY has different exposure time and whether to omit it
00880             KMO_TRY_EXIT_IF_NULL(
00881                 frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00882 
00883             if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00884                 // just one FLAT_SKY, always add
00885                 KMO_TRY_EXIT_IF_ERROR(
00886                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00887                 KMO_TRY_CHECK_ERROR_STATE();
00888             } else {
00889                 // several FLAT_SKY frames, check exptime
00890 
00891                 // get exptime 1
00892                 KMO_TRY_EXIT_IF_NULL(
00893                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00894                 exptime1 = cpl_propertylist_get_double(main_header, EXPTIME);
00895                 KMO_TRY_CHECK_ERROR_STATE();
00896                 cpl_propertylist_delete(main_header); main_header = NULL;
00897 
00898                 // get exptime 2
00899                 frame = kmo_dfs_get_frame(frameset, NULL);
00900                 KMO_TRY_EXIT_IF_NULL(
00901                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00902                 exptime2 = cpl_propertylist_get_double(main_header, EXPTIME);
00903                 KMO_TRY_CHECK_ERROR_STATE();
00904                 cpl_propertylist_delete(main_header); main_header = NULL;
00905 
00906                 // loop remaining frames
00907                 same_exptime = TRUE;
00908                 frame = kmo_dfs_get_frame(frameset, NULL);
00909                 while (same_exptime && (frame != NULL)) {
00910                     KMO_TRY_EXIT_IF_NULL(
00911                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00912                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00913                     KMO_TRY_CHECK_ERROR_STATE();
00914                     cpl_propertylist_delete(main_header); main_header = NULL;
00915                     if (fabs(exptime-exptime2) > 0.01) {
00916                         // not same
00917                         same_exptime = FALSE;
00918                     }
00919                     frame = kmo_dfs_get_frame(frameset, NULL);
00920                 }
00921 
00922                 if (same_exptime) {
00923                     // frame [2,n] have same exptime, add them
00924                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00925                     KMO_TRY_EXIT_IF_NULL(
00926                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00927                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00928                     KMO_TRY_CHECK_ERROR_STATE();
00929                     cpl_propertylist_delete(main_header); main_header = NULL;
00930                     cpl_msg_info("", "Omit FLAT_SKY: %s with EXPTIME of %g sec (acquisition), other frame(s) have EXPTIME of %g sec", cpl_frame_get_filename(frame), exptime, exptime2);
00931                     frame = kmo_dfs_get_frame(frameset, NULL);
00932                     while (frame != NULL) {
00933                         KMO_TRY_EXIT_IF_ERROR(
00934                             cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00935                         frame = kmo_dfs_get_frame(frameset, NULL);
00936                     }
00937                     if (fabs(exptime1-exptime2) < 0.01) {
00938                         cpl_msg_warning("", "The 1st FLAT_SKY has the same exposure time as the following ones. "
00939                                             "It has anyway been omitted since we assume it is an acquisition frame. "
00940                                             "If you want to add it anyway call this recipe with the --add-all parameter");
00941                     }
00942                 } else {
00943                     cpl_msg_error("", "The exposure times of the FLAT_SKY frames don't match!");
00944                     cpl_msg_error("", "We assume that the 1st frame is an acquisition frame and would be omitted.");
00945                     cpl_msg_error("", "The following frames should have the same exposure time if they originate from the same template.");
00946                     cpl_msg_error("", "If you want to reduce them anyway call this recipe with the --add-all parameter");
00947                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00948                     while (frame != NULL) {
00949                         KMO_TRY_EXIT_IF_NULL(
00950                             main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00951                         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00952                         KMO_TRY_CHECK_ERROR_STATE();
00953                         cpl_propertylist_delete(main_header); main_header = NULL;
00954                         cpl_msg_error("", "FLAT_SKY: %s, EXPTIME: %g", cpl_frame_get_filename(frame), exptime);
00955                         frame = kmo_dfs_get_frame(frameset, NULL);
00956                     }
00957                     cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
00958                     KMO_TRY_CHECK_ERROR_STATE();
00959                 }
00960             }
00961         }
00962 
00963         KMO_TRY_EXIT_IF_NULL(
00964             frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY));
00965         KMO_TRY_EXIT_IF_NULL(
00966             main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY));
00967         KMO_TRY_EXIT_IF_NULL(
00968             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
00969         KMO_TRY_EXIT_IF_NULL(
00970             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
00971         cpl_free(keyword); keyword = NULL;
00972 
00973         //
00974         // set default band-specific ranges for collapsing
00975         //
00976         if (ranges == NULL) {
00977             if (strcmp(filter, "IZ") == 0) {
00978                 ranges_txt = "0.81,1.05";
00979             } else if (strcmp(filter, "YJ") == 0) {
00980                 ranges_txt = "1.025,1.3";
00981             } else if (strcmp(filter, "H") == 0) {
00982                 ranges_txt = "1.5,1.7";
00983             } else if (strcmp(filter, "K") == 0) {
00984                 ranges_txt = "2.1,2.35";
00985             } else if (strcmp(filter, "HK") == 0) {
00986                 ranges_txt = "1.5,1.7;2.1,2.35";
00987 //                ranges_txt = "1.5,1.7";
00988             } else {
00989                 KMO_TRY_ASSURE(1 == 0,
00990                                CPL_ERROR_ILLEGAL_INPUT,
00991                                "We really shouldn't get here...");
00992             }
00993             cpl_msg_info("", "Spectral range to collapse has been set to %s um for this band.", ranges_txt);
00994             ranges = kmo_identify_ranges(ranges_txt);
00995             KMO_TRY_CHECK_ERROR_STATE();
00996         }
00997 
00998         // setup grid definition, wavelength start and end points will be set
00999         // in the detector loop
01000         KMO_TRY_EXIT_IF_ERROR(
01001             kmclipm_setup_grid(&gd, method, neighborhoodRange, KMOS_PIX_RESOLUTION));
01002 
01003         // create filename for LUT
01004         KMO_TRY_EXIT_IF_NULL(
01005             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
01006 
01007         // extract bounds
01008         KMO_TRY_EXIT_IF_NULL(
01009             tmp_header = kmclipm_propertylist_load(
01010                                          cpl_frame_get_filename(xcalFrame), 0));
01011         KMO_TRY_EXIT_IF_NULL(
01012             bounds = kmclipm_extract_bounds(tmp_header));
01013         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
01014 
01015         // get timestamps of xcal, ycal & lcal
01016         KMO_TRY_EXIT_IF_NULL(
01017             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
01018 
01019         // create arrays to hold reconstructed data and noise cubes and
01020         // their headers
01021         KMO_TRY_EXIT_IF_NULL(
01022             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01023                                                             sizeof(cpl_imagelist*)));
01024         KMO_TRY_EXIT_IF_NULL(
01025             stored_noise_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01026                                                              sizeof(cpl_imagelist*)));
01027         KMO_TRY_EXIT_IF_NULL(
01028             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01029                                                          sizeof(cpl_image*)));
01030         KMO_TRY_EXIT_IF_NULL(
01031             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01032                                                           sizeof(cpl_image*)));
01033         KMO_TRY_EXIT_IF_NULL(
01034             stored_sub_data_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01035                                                                      sizeof(cpl_propertylist*)));
01036         KMO_TRY_EXIT_IF_NULL(
01037             stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01038                                                                       sizeof(cpl_propertylist*)));
01039         KMO_TRY_EXIT_IF_NULL(
01040             edge_table_sky = (cpl_table***)cpl_calloc(KMOS_NR_DETECTORS,
01041                                                       sizeof(cpl_table**)));
01042         KMO_TRY_EXIT_IF_NULL(
01043             edge_table_flat = (cpl_table**)cpl_calloc(KMOS_IFUS_PER_DETECTOR,
01044                                                       sizeof(cpl_table*)));
01045         KMO_TRY_EXIT_IF_NULL(
01046             calAngles = cpl_vector_new(3));
01047 
01048         //
01049         // loop through all detectors
01050         //
01051         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01052             cpl_msg_info("","Processing detector No. %d", det_nr);
01053 
01054             KMO_TRY_EXIT_IF_NULL(
01055                 detector_in = cpl_imagelist_new());
01056 
01057             // load data of det_nr of all FLAT_SKY frames into an imagelist
01058             KMO_TRY_EXIT_IF_NULL(
01059                 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, TRUE, NULL));
01060 
01061             cnt = 0;
01062             while (img_in != NULL) {
01063                 cpl_imagelist_set(detector_in, img_in, cnt);
01064                 KMO_TRY_CHECK_ERROR_STATE();
01065 
01066                 /* load same extension of next FLAT_SKY frame*/
01067                 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, TRUE, NULL);
01068                 KMO_TRY_CHECK_ERROR_STATE();
01069 
01070                 cnt++;
01071             }
01072 
01073             //
01074             // process imagelist
01075             //
01076 
01077             // combine imagelist (data only) and create noise (stdev of data)
01078             cpl_msg_info("","Combining frames...");
01079             if (process_noise) {
01080                 KMO_TRY_EXIT_IF_ERROR(
01081                     kmclipm_combine_frames(detector_in,
01082                                            NULL,
01083                                            NULL,
01084                                            cmethod,
01085                                            cpos_rej,
01086                                            cneg_rej,
01087                                            citer,
01088                                            cmax,
01089                                            cmin,
01090                                            &combined_data,
01091                                            &combined_noise,
01092                                            -1.0));
01093             } else {
01094                 KMO_TRY_EXIT_IF_ERROR(
01095                     kmclipm_combine_frames(detector_in,
01096                                            NULL,
01097                                            NULL,
01098                                            cmethod,
01099                                            cpos_rej,
01100                                            cneg_rej,
01101                                            citer,
01102                                            cmax,
01103                                            cmin,
01104                                            &combined_data,
01105                                            NULL,
01106                                            -1.0));
01107             }
01108 
01109             if (kmclipm_omit_warning_one_slice > 10) {
01110                 cpl_msg_warning(cpl_func, "Previous warning (number of "
01111                                           "identified slices) occured %d times.",
01112                                 kmclipm_omit_warning_one_slice);
01113                 kmclipm_omit_warning_one_slice = FALSE;
01114             }
01115 
01116             cpl_imagelist_delete(detector_in); detector_in = NULL;
01117 
01118             // load calibration files
01119             KMO_TRY_EXIT_IF_NULL(
01120                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
01121                                               FALSE, NULL, &rotangle_found));
01122 
01123             KMO_TRY_EXIT_IF_ERROR(
01124                 cpl_vector_set(calAngles, 0, rotangle_found));
01125             KMO_TRY_EXIT_IF_NULL(
01126                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
01127                                               FALSE, NULL, &rotangle_found));
01128             KMO_TRY_EXIT_IF_ERROR(
01129                 cpl_vector_set(calAngles, 1, rotangle_found));
01130             KMO_TRY_EXIT_IF_NULL(
01131                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
01132                                               FALSE, NULL, &rotangle_found));
01133             KMO_TRY_EXIT_IF_ERROR(
01134                 cpl_vector_set(calAngles, 2, rotangle_found));
01135 
01136             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
01137             KMO_TRY_EXIT_IF_NULL(
01138                 bad_pix_mask = cpl_image_duplicate(xcal));
01139 
01140             KMO_TRY_EXIT_IF_NULL(
01141                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
01142             for (int x = 0; x < nx; x++) {
01143                 for (int y = 0; y < ny; y++) {
01144                     if (isnan(pbad_pix_mask[x+nx*y])) {
01145                         pbad_pix_mask[x+nx*y] = 0.;
01146                     } else {
01147                         pbad_pix_mask[x+nx*y] = 1.;
01148                     }
01149                 }
01150             }
01151             KMO_TRY_CHECK_ERROR_STATE();
01152 
01153             //
01154             // calculate SKYFLAT_EDGE
01155             //
01156             if (has_flat_edge) {
01157                 // get edge-edgepars from FLAT_SKY
01158                 KMO_TRY_EXIT_IF_ERROR(
01159                     kmo_calc_edgepars(combined_data,
01160                                       unused_ifus_after[det_nr-1],
01161                                       bad_pix_mask,
01162                                       det_nr,
01163                                       &slitlet_ids,
01164                                       &edgepars));
01165                 KMO_TRY_CHECK_ERROR_STATE();
01166 
01167                 // copy edgepars to table for saving later on
01168                 KMO_TRY_EXIT_IF_NULL(
01169                     edge_table_sky[det_nr-1] = kmo_edgepars_to_table(slitlet_ids, edgepars));
01170 
01171                 if (edgepars != NULL) {
01172                     for (int i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01173                         cpl_matrix_delete(edgepars[i]); edgepars[i] = NULL;
01174                     }
01175                     cpl_free(edgepars); edgepars = NULL;
01176                 }
01177                 if (slitlet_ids != NULL) {
01178                     for (int i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01179                         cpl_vector_delete(slitlet_ids[i]); slitlet_ids[i] = NULL;
01180                     }
01181                     cpl_free(slitlet_ids); slitlet_ids = NULL;
01182                 }
01183                 KMO_TRY_CHECK_ERROR_STATE();
01184 
01185                 //
01186                 // correlate FLAT_EDGE and SKYFLAT_EDGE
01187                 //
01188 
01189                 // load flat_edge from MASTER_FLAT
01190                 KMO_TRY_EXIT_IF_NULL(
01191                     frame = kmo_dfs_get_frame(frameset, FLAT_EDGE));
01192                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01193                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01194                     KMO_TRY_EXIT_IF_NULL(
01195                         edge_table_flat[j] = kmclipm_cal_table_load(cpl_frame_get_filename(frame),
01196                                                                     ifu_nr, rotangle, 0, &tmp_rotangle));
01197                 }
01198 
01199                 //
01200                 // calculate shift value
01201                 //
01202 
01203                 KMO_TRY_EXIT_IF_NULL(
01204                     shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR));
01205                 KMO_TRY_EXIT_IF_NULL(
01206                     edge_vec = cpl_vector_new(2*KMOS_SLITLET_X));
01207 
01208                 // get shift values for each IFU by comparing all edge parameters,
01209                 // rejecting and applying median
01210                 int row = 1024; // middle of frame
01211                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01212                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01213                     for (int edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) {
01214                         double flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat[j], edgeNr, row);
01215                         double skyval  = kmo_calc_fitted_slitlet_edge(edge_table_sky[det_nr-1][j], edgeNr, row);
01216                         cpl_vector_set(edge_vec, edgeNr, flatval-skyval);
01217                     }
01218 
01219                     // reject deviating edge-differences
01220                     kmclipm_vector *kv = NULL;
01221                     KMO_TRY_EXIT_IF_NULL(
01222                         kv = kmclipm_vector_create(cpl_vector_duplicate(edge_vec)));
01223                     kmclipm_reject_deviant(kv, 3, 3, NULL, NULL);
01224 
01225                     // set shift value for each IFU
01226                     cpl_vector_set(shift_vec, j, kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC));
01227                     kmclipm_vector_delete(kv); kv = NULL;
01228                 }
01229                 cpl_vector_delete(edge_vec); edge_vec = NULL;
01230                 KMO_TRY_CHECK_ERROR_STATE();
01231 
01232                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01233                     cpl_table_delete(edge_table_flat[j]);
01234                     edge_table_flat[j] = NULL;
01235                 }
01236 
01237                 // take median of all IFU-shift-values
01238                 double shift_val = -cpl_vector_get_median(shift_vec);
01239                 cpl_vector_delete(shift_vec); shift_vec = NULL;
01240 
01241                 cpl_msg_info("", "Shift detector %d by %g pixels.", det_nr, shift_val);
01242 
01243                 int     xdim                = cpl_image_get_size_x(combined_data),
01244                         ydim                = cpl_image_get_size_y(combined_data);
01245                 double  *array_in           = cpl_calloc(xdim, sizeof(double)),
01246                         *array_out          = NULL;
01247                 float   *pcombined_data     = cpl_image_get_data_float(combined_data),
01248                         *pcombined_noise    = NULL;
01249     //            float   *tmpArray           = cpl_calloc(xdim, sizeof(float));
01250                 if (process_noise) {
01251                     pcombined_noise = cpl_image_get_data_float(combined_noise);
01252                 }
01253 
01254                 for (int iy = 0; iy < ydim; iy++) {
01255                     // cubic spline
01256                     for (int ix = 0; ix < xdim; ix++) {
01257                         array_in[ix] = pcombined_data[ix+iy*xdim];
01258                     }
01259                     array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01260                                                     xdim, shift_val, 1.0,
01261                                                     NATURAL);
01262                     for (int ix = 0; ix < xdim; ix++) {
01263                       pcombined_data[ix+iy*xdim] = array_out[ix];
01264                     }
01265                     cpl_free(array_out);
01266 
01267     //                // linear
01268     //                for (int ix = 1; ix < xdim; ix++) {
01269     //                    tmpArray[ix-1] = (pcombined_data[ix+iy*xdim]-pcombined_data[(ix-1)+iy*xdim])*shift_val +
01270     //                                     pcombined_data[(ix-1)+iy*xdim];
01271     //                }
01272     //                for (int ix = 1; ix < xdim; ix++) {
01273     //                    pcombined_data[ix+iy*xdim] = tmpArray[ix];
01274     //                }
01275 
01276                     if (process_noise) {
01277                         // cubic spline
01278                         for (int ix = 0; ix < xdim; ix++) {
01279                             array_in[ix] = pcombined_noise[ix+iy*xdim];
01280                         }
01281                         array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01282                                                         xdim, shift_val, 1.0,
01283                                                         NATURAL);
01284                         for (int ix = 0; ix < xdim; ix++) {
01285                           pcombined_noise[ix+iy*xdim] = array_out[ix];
01286                         }
01287                         cpl_free(array_out);
01288 
01289     //                    // linear
01290     //                    for (int ix = 1; ix < xdim; ix++) {
01291     //                        tmpArray[ix-1] = (pcombined_noise[ix+iy*xdim]-pcombined_noise[(ix-1)+iy*xdim])*shift_val +
01292     //                                         pcombined_noise[(ix-1)+iy*xdim];
01293     //                    }
01294     //                    for (int ix = 1; ix < xdim; ix++) {
01295     //                        pcombined_noise[ix+iy*xdim] = tmpArray[ix];
01296     //                    }
01297                     }
01298                 }
01299                 cpl_free(array_in); array_in = NULL;
01300             }
01301             //
01302             // reconstruct
01303             //
01304             // load MASTER_DARK and MASTER_FLAT
01305             KMO_TRY_EXIT_IF_NULL(
01306                 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK,
01307                                               det_nr, FALSE, FALSE, NULL));
01308 
01309             if (process_noise) {
01310                 KMO_TRY_EXIT_IF_NULL(
01311                     img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK,
01312                                                         det_nr, TRUE, FALSE, NULL));
01313             }
01314 
01315             KMO_TRY_EXIT_IF_NULL(
01316                 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE,
01317                                                   rotangle, FALSE, NULL, &rotangle_found));
01318 
01319             if (process_noise) {
01320                 KMO_TRY_EXIT_IF_NULL(
01321                     img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, TRUE,
01322                                                             rotangle, FALSE, NULL, &rotangle_found));
01323             }
01324 
01325             char *tmp_band_method = getenv("KMO_BAND_METHOD");
01326             int band_method = 0;
01327             if (tmp_band_method != NULL) {
01328                 band_method = atoi(tmp_band_method);
01329             }
01330 
01331             // ESO INS FILTi ID
01332             KMO_TRY_EXIT_IF_NULL(
01333                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
01334                                       IFU_FILTID_POSTFIX));
01335             KMO_TRY_EXIT_IF_NULL(
01336                 filter_id = cpl_propertylist_get_string(main_header, keyword));
01337             cpl_free(keyword); keyword = NULL;
01338 
01339             KMO_TRY_EXIT_IF_NULL(
01340                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
01341             KMO_TRY_EXIT_IF_ERROR(
01342                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
01343                                              band_table));
01344             cpl_table_delete(band_table); band_table = NULL;
01345 
01346             cpl_msg_info("","Reconstructing cubes...");
01347             for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01348                 // update sub-header
01349                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01350 
01351                 // load raw image and sub-header
01352                 KMO_TRY_EXIT_IF_NULL(
01353                     sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY,
01354                                                          det_nr, FALSE));
01355 
01356                 KMO_TRY_EXIT_IF_NULL(
01357                     punused_ifus = cpl_array_get_data_int_const(
01358                                                   unused_ifus_after[det_nr-1]));
01359 
01360                 // check if IFU is valid according to main header keywords &
01361                 // calibration files
01362                 KMO_TRY_EXIT_IF_NULL(
01363                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
01364                                           IFU_VALID_POSTFIX));
01365                 KMO_TRY_CHECK_ERROR_STATE();
01366                 ranges_txt = cpl_propertylist_get_string(main_header, keyword);
01367                 cpl_free(keyword); keyword = NULL;
01368 
01369                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
01370                     (bounds[2*(ifu_nr-1)] != -1) &&
01371                     (bounds[2*(ifu_nr-1)+1] != -1) &&
01372                     (punused_ifus[j] == 0))
01373                 {
01374                     // IFU is valid
01375                     cpl_error_reset();
01376 
01377                     // calculate WCS
01378                     KMO_TRY_EXIT_IF_ERROR(
01379                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
01380 
01381                     // reconstruct data
01382                     KMO_TRY_EXIT_IF_ERROR(
01383                         kmo_reconstruct_sci_image(ifu_nr,
01384                                                 bounds[2*(ifu_nr-1)],
01385                                                 bounds[2*(ifu_nr-1)+1],
01386                                                 combined_data,
01387                                                 combined_noise,
01388                                                 img_dark,
01389                                                 img_dark_noise,
01390                                                 img_flat,
01391                                                 img_flat_noise,
01392                                                 xcal,
01393                                                 ycal,
01394                                                 lcal,
01395                                                 &gd,
01396                                                 calTimestamp,
01397                                                 calAngles,
01398                                                 fn_lut,
01399                                                 &cube_data,
01400                                                 &cube_noise,
01401                                                 flux,
01402                                                 background));
01403                     KMO_TRY_CHECK_ERROR_STATE();
01404                 } else {
01405                     // IFU is invalid
01406                     cpl_error_reset();
01407                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01408 
01409                 // save output
01410                 KMO_TRY_EXIT_IF_NULL(
01411                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01412 
01413                 KMO_TRY_EXIT_IF_ERROR(
01414                     kmclipm_update_property_string(sub_header, EXTNAME,
01415                                                    extname,
01416                                                    "FITS extension name"));
01417 
01418                 cpl_free(extname); extname = NULL;
01419 
01420                 // store cube and sub header into array for later
01421                 stored_data_cubes[ifu_nr - 1] = cube_data;
01422                 stored_sub_data_headers[ifu_nr - 1] = sub_header;
01423 
01424                 if (process_noise) {
01425                     KMO_TRY_EXIT_IF_NULL(
01426                         sub_header = cpl_propertylist_duplicate(
01427                                            stored_sub_data_headers[ifu_nr - 1]));
01428                     KMO_TRY_EXIT_IF_NULL(
01429                         extname = kmo_extname_creator(ifu_frame, ifu_nr,
01430                                                       EXT_NOISE));
01431 
01432                     KMO_TRY_EXIT_IF_ERROR(
01433                         kmclipm_update_property_string(sub_header,
01434                                                 EXTNAME,
01435                                                 extname,
01436                                                 "FITS extension name"));
01437 
01438                     cpl_free(extname); extname = NULL;
01439 
01440                     stored_noise_cubes[ifu_nr - 1] = cube_noise;
01441                     stored_sub_noise_headers[ifu_nr - 1] = sub_header;
01442                 }
01443                 cpl_image_delete(data_ifu); data_ifu = NULL;
01444                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01445                 cube_data = NULL;
01446                 cube_noise = NULL;
01447             } // for j IFUs
01448 
01449             // free memory
01450             cpl_image_delete(combined_data); combined_data = NULL;
01451             cpl_image_delete(combined_noise); combined_noise = NULL;
01452             cpl_image_delete(xcal); xcal = NULL;
01453             cpl_image_delete(ycal); ycal = NULL;
01454             cpl_image_delete(lcal); lcal = NULL;
01455             cpl_image_delete(img_dark); img_dark = NULL;
01456             cpl_image_delete(img_flat); img_flat = NULL;
01457             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01458             if (process_noise) {
01459                 cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01460                 cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01461             }
01462         } // for nr_devices
01463 
01464         cpl_free(edge_table_flat); edge_table_flat = NULL;
01465 
01466         // collapse cubes using rejection
01467         cpl_msg_info("","Collapsing cubes...");
01468         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01469             for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01470                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01471 
01472                 KMO_TRY_EXIT_IF_NULL(
01473                     punused_ifus = cpl_array_get_data_int_const(
01474                                                   unused_ifus_after[det_nr-1]));
01475                 if (punused_ifus[j] == 0) {
01476                     if (stored_sub_data_headers[ifu_nr-1] != NULL) {
01477                     // IFU is valid
01478                     ifu_crpix = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01479                                                             CRPIX3);
01480                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01481                                    "CRPIX3 keyword in FITS-header is missing!");
01482 
01483                     ifu_crval = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01484                                                             CRVAL3);
01485                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01486                                    "CRVAL3 keyword in FITS-header is missing!");
01487 
01488                     ifu_cdelt = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01489                                                             CDELT3);
01490                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01491                                    "CDELT3 keyword in FITS-header is missing!");
01492 
01493                     KMO_TRY_EXIT_IF_NULL(
01494                         identified_slices = kmo_identify_slices(ranges,
01495                                                                 ifu_crpix,
01496                                                                 ifu_crval,
01497                                                                 ifu_cdelt,
01498                                                                 gd.l.dim));
01499                     }/* else {
01500                         KMO_TRY_EXIT_IF_NULL(
01501                             identified_slices = cpl_vector_new(gd.l.dim));
01502                         cpl_vector_fill(identified_slices, 1.);
01503                     }*/
01504 
01505                     if (stored_data_cubes[ifu_nr-1] != NULL) {
01506                         KMO_TRY_EXIT_IF_ERROR(
01507                             kmclipm_make_image(stored_data_cubes[ifu_nr-1],
01508                                                stored_noise_cubes[ifu_nr-1],
01509                                                &stored_data_images[ifu_nr-1],
01510                                                &stored_noise_images[ifu_nr-1],
01511                                                identified_slices,
01512                                                cmethod, cpos_rej, cneg_rej,
01513                                                citer, cmax, cmin));
01514                     }
01515                     cpl_vector_delete(identified_slices); identified_slices = NULL;
01516                 } else {
01517                     // IFU is invalid
01518                 }
01519             }
01520         }
01521 
01522         // normalise all IFUs as a group.
01523         // Calculate mean of each IFU, add up and divide by number of successful
01524         // averaged IFUs.
01525         // Then divide all valid IFUs with mean value
01526         cnt = 0;
01527         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01528             if (stored_data_images[i] != NULL) {
01529                 KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[i]) <
01530                                cpl_image_get_size_x(stored_data_images[i])*
01531                                cpl_image_get_size_y(stored_data_images[i]),
01532                                CPL_ERROR_ILLEGAL_INPUT,
01533                                "The collapsed, dark-subtracted image contains "
01534                                "only invalid values! Probably the provided "
01535                                "FLAT_SKY frames are exactly the same as the "
01536                                "frames used for MASTER_DARK calculation.");
01537 
01538                 mean_data += cpl_image_get_mean(stored_data_images[i]);
01539                 KMO_TRY_CHECK_ERROR_STATE();
01540                 cnt++;
01541             }
01542 
01543         }
01544         mean_data /= cnt;
01545 
01546         if (mean_data != 0.0) {
01547             for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01548                 if (stored_data_images[i] != NULL) {
01549                     KMO_TRY_EXIT_IF_ERROR(
01550                         cpl_image_divide_scalar(stored_data_images[i],
01551                                                 mean_data));
01552                 }
01553             }
01554         } else {
01555             cpl_msg_warning(cpl_func, "Data couldn't be normalised "
01556                                       "(mean = 0.0)!");
01557         }
01558 
01559         if (process_noise) {
01560             if (mean_data != 0.0) {
01561                 for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01562                     if (stored_noise_images[i] != NULL) {
01563                         KMO_TRY_EXIT_IF_ERROR(
01564                             cpl_image_divide_scalar(stored_noise_images[i],
01565                                                     mean_data));
01566                     }
01567                 }
01568             } else {
01569                 cpl_msg_warning(cpl_func, "Noise couldn't be normalised "
01570                                           "(mean = 0.0)!");
01571             }
01572         }
01573 
01574         // calculate qc parameters on normalised data
01575         qc_spat_unif = 0.0;
01576         cnt = 0;
01577         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01578             if (stored_data_images[i] != NULL) {
01579                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01580                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01581 
01582                 qc_spat_unif += pow(tmp_mean-1, 2);
01583                 if (fabs(tmp_mean) > qc_max_dev) {
01584                     qc_max_dev = tmp_mean-1;
01585                     qc_max_dev_id = i+1;
01586                 }
01587                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01588                     qc_max_nonunif = tmp_stdev;
01589                     qc_max_nonunif_id = i+1;
01590                 }
01591                 KMO_TRY_CHECK_ERROR_STATE();
01592                 cnt++;
01593             }
01594         }
01595         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01596 
01597         //
01598         // save data
01599         //
01600 
01601         // update which IFUs are not used
01602         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01603 
01604         KMO_TRY_EXIT_IF_ERROR(
01605             kmo_set_unused_ifus(unused_ifus_after, main_header,
01606                                 "kmo_illumination"));
01607 
01608         cpl_msg_info("","Saving data...");
01609 
01610         KMO_TRY_EXIT_IF_ERROR(
01611             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01612                                            "[adu] uniformity of illumination correction"));
01613         KMO_TRY_EXIT_IF_ERROR(
01614             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01615                                            "[adu] max. deviation from unity"));
01616         KMO_TRY_EXIT_IF_ERROR(
01617             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01618                                         "[] IFU ID with max. dev. from unity"));
01619         KMO_TRY_EXIT_IF_ERROR(
01620             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01621                                            "[adu] max. stdev of illumination corr."));
01622         KMO_TRY_EXIT_IF_ERROR(
01623             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01624                                         "[] IFU ID with max. stdev in illum. corr."));
01625 
01626         KMO_TRY_EXIT_IF_ERROR(
01627             kmo_dfs_save_main_header(frameset, ILLUM_CORR, suffix, frame,
01628                                      main_header, parlist, cpl_func));
01629 
01630         KMO_TRY_EXIT_IF_ERROR(
01631             kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, suffix, frame,
01632                                      main_header, parlist, cpl_func));
01633 
01634         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01635             KMO_TRY_EXIT_IF_ERROR(
01636                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, suffix,
01637                                    stored_sub_data_headers[i], 0./0.));
01638 
01639             if (process_noise) {
01640                 KMO_TRY_EXIT_IF_ERROR(
01641                     kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR,
01642                                        suffix, stored_sub_noise_headers[i], 0./0.));
01643             }
01644         }
01645 
01646         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01647             for (int ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) {
01648                 KMO_TRY_EXIT_IF_ERROR(
01649                     kmclipm_update_property_int(stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01650                                                 CAL_IFU_NR,
01651                                                 ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR,
01652                                                 "IFU Number {1..24}"));
01653                 KMO_TRY_EXIT_IF_ERROR(
01654                     kmclipm_update_property_double(
01655                                                 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01656                                                 CAL_ROTANGLE,
01657                                                 rotangle_found,
01658                                                 "[deg] Rotator relative to nasmyth"));
01659                 if (has_flat_edge) {
01660                     // save edge-parameters as product
01661                     KMO_TRY_EXIT_IF_ERROR(
01662                         kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], SKYFLAT_EDGE, suffix,
01663                                            stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]));
01664                 }
01665             }
01666         }
01667     }
01668     KMO_CATCH
01669     {
01670         KMO_CATCH_MSG();
01671         ret_val = -1;
01672     }
01673     kmo_free_fits_desc(&desc_sky);
01674     kmo_free_fits_desc(&desc_dark);
01675     kmo_free_fits_desc(&desc_flat);
01676     kmo_free_fits_desc(&desc_xcal);
01677     kmo_free_fits_desc(&desc_ycal);
01678     kmo_free_fits_desc(&desc_lcal);
01679     cpl_image_delete(combined_data); combined_data = NULL;
01680     cpl_image_delete(combined_noise); combined_noise = NULL;
01681     cpl_image_delete(xcal); xcal = NULL;
01682     cpl_image_delete(ycal); ycal = NULL;
01683     cpl_image_delete(lcal); lcal = NULL;
01684     cpl_image_delete(img_dark); img_dark = NULL;
01685     cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01686     cpl_image_delete(img_flat); img_flat = NULL;
01687     cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01688     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01689     cpl_free(bounds); bounds = NULL;
01690     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01691     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01692     cpl_free(fn_lut); fn_lut = NULL;
01693     cpl_free(suffix); suffix = NULL;
01694     cpl_frameset_delete(frameset_sky); frameset_sky = NULL;
01695     cpl_vector_delete(ranges); ranges = NULL;
01696     cpl_free(filter); filter = NULL;
01697     if (calAngles != NULL) {
01698         cpl_vector_delete(calAngles); calAngles = NULL;
01699     }
01700     cpl_propertylist_delete(main_header); main_header = NULL;
01701     for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01702         if (stored_data_cubes != NULL) {
01703             cpl_imagelist_delete(stored_data_cubes[i]);
01704             stored_data_cubes[i] = NULL;
01705         }
01706         if (stored_noise_cubes != NULL) {
01707             cpl_imagelist_delete(stored_noise_cubes[i]);
01708             stored_noise_cubes[i] = NULL;
01709         }
01710         if (stored_data_images != NULL) {
01711             cpl_image_delete(stored_data_images[i]);
01712             stored_data_images[i] = NULL;
01713         }
01714         if (stored_noise_images != NULL) {
01715             cpl_image_delete(stored_noise_images[i]);
01716             stored_noise_images[i] = NULL;
01717         }
01718         if (stored_sub_data_headers != NULL) {
01719             cpl_propertylist_delete(stored_sub_data_headers[i]);
01720             stored_sub_data_headers[i] = NULL;
01721         }
01722         if (stored_sub_noise_headers != NULL) {
01723             cpl_propertylist_delete(stored_sub_noise_headers[i]);
01724             stored_sub_noise_headers[i] = NULL;
01725         }
01726     }
01727     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01728     cpl_free(stored_noise_cubes); stored_noise_cubes = NULL;
01729     cpl_free(stored_data_images); stored_data_images = NULL;
01730     cpl_free(stored_noise_images); stored_noise_images = NULL;
01731     cpl_free(stored_sub_data_headers); stored_sub_data_headers = NULL;
01732     cpl_free(stored_sub_noise_headers); stored_sub_noise_headers = NULL;
01733     if (edge_table_sky != NULL) {
01734         for (int i = 0; i < KMOS_NR_DETECTORS; i++) {
01735             if (edge_table_sky[i] != NULL) {
01736                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01737                     cpl_table_delete(edge_table_sky[i][j]);
01738                     edge_table_sky[i][j] = NULL;
01739                 }
01740                 cpl_free(edge_table_sky[i]); edge_table_sky[i] = NULL;
01741             }
01742         }
01743         cpl_free(edge_table_sky); edge_table_sky = NULL;
01744     }
01745     if (edge_table_flat != NULL) {
01746         for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01747             cpl_table_delete(edge_table_flat[j]);
01748             edge_table_flat[j] = NULL;
01749         }
01750         cpl_free(edge_table_flat); edge_table_flat = NULL;
01751     }
01752 
01753     return ret_val;
01754 }
01755