KMOS Pipeline Reference Manual  1.1.5
kmo_illumination.c
00001 /* $Id: kmo_illumination.c,v 1.52 2013/06/03 15:00:23 aagudo Exp $
00002  *
00003  * This file is part of the KMOS Pipeline
00004  * Copyright (C) 2002,2003 European Southern Observatory
00005  *
00006  * This program is free software; you can redistribute it and/or modify
00007  * it under the terms of the GNU General Public License as published by
00008  * the Free Software Foundation; either version 2 of the License, or
00009  * (at your option) any later version.
00010  *
00011  * This program is distributed in the hope that it will be useful,
00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
00014  * GNU General Public License for more details.
00015  *
00016  * You should have received a copy of the GNU General Public License
00017  * along with this program; if not, write to the Free Software
00018  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00019  */
00020 
00021 /*
00022  * $Author: aagudo $
00023  * $Date: 2013/06/03 15:00:23 $
00024  * $Revision: 1.52 $
00025  * $Name: HEAD $
00026  */
00027 
00028 #ifdef HAVE_CONFIG_H
00029 #include <config.h>
00030 #endif
00031 
00032 /*-----------------------------------------------------------------------------
00033  *                              Includes
00034  *----------------------------------------------------------------------------*/
00035 
00036 #include <math.h>
00037 #include <string.h>
00038 
00039 #include <cpl.h>
00040 
00041 #include "kmo_priv_reconstruct.h"
00042 #include "kmo_priv_functions.h"
00043 #include "kmo_priv_flat.h"
00044 #include "kmo_priv_wave_cal.h"
00045 #include "kmclipm_priv_splines.h"
00046 #include "kmo_functions.h"
00047 #include "kmo_cpl_extensions.h"
00048 #include "kmo_dfs.h"
00049 #include "kmo_error.h"
00050 #include "kmo_constants.h"
00051 #include "kmo_debug.h"
00052 
00053 /*-----------------------------------------------------------------------------
00054  *                          Functions prototypes
00055  *----------------------------------------------------------------------------*/
00056 
00057 static int kmo_illumination_create(cpl_plugin *);
00058 static int kmo_illumination_exec(cpl_plugin *);
00059 static int kmo_illumination_destroy(cpl_plugin *);
00060 static int kmo_illumination(cpl_parameterlist *, cpl_frameset *);
00061 
00062 /*-----------------------------------------------------------------------------
00063  *                          Static variables
00064  *----------------------------------------------------------------------------*/
00065 
00066 static char kmo_illumination_description[] =
00067 "This recipe creates the spatial non-uniformity calibration frame needed for\n"
00068 "all three detectors. It must be called after the kmo_wave_cal-recipe, which\n"
00069 "generates the spectral calibration frame needed in this recipe. As input at\n"
00070 "least a sky, a master dark, a master flat and the spatial and spectral cali-\n"
00071 "bration frames are required.\n"
00072 "\n"
00073 "BASIC PARAMETERS:\n"
00074 "-----------------\n"
00075 "--imethod\n"
00076 "The interpolation method used for reconstruction.\n"
00077 "\n"
00078 "--range\n"
00079 "The spectral ranges to combine when collapsing the reconstructed cubes. e.g.\n"
00080 "\"x1_start,x1_end;x2_start,x2_end\" (microns)\n"
00081 "\n"
00082 "ADVANCED PARAMETERS\n"
00083 "-------------------\n"
00084 "--add-all\n"
00085 "By default the first FLAT_SKY frame is omitted, since in the\n"
00086 "KMOS_spec_cal_skyflat template this is an acquisition frame to estimate the\n"
00087 "needed exposure time for the subsequent FLAT_SKY frames. If anyway all frames\n"
00088 "should be considered, set this parameter to TRUE.\n"
00089 "\n"
00090 "--neighborhoodRange\n"
00091 "Defines the range to search for neighbors during reconstruction\n"
00092 "\n"
00093 "--b_samples\n"
00094 "The number of samples in spectral direction for the reconstructed cube.\n"
00095 "Ideally this number should be greater than 2048, the detector size.\n"
00096 "\n"
00097 "--b_start\n"
00098 "--b_end\n"
00099 "Used to define manually the start and end wavelength for the reconstructed\n"
00100 "cube. By default the internally defined values are used.\n"
00101 "\n"
00102 "--cmethod\n"
00103 "Following methods of frame combination are available:\n"
00104 "   * 'ksigma' (Default)\n"
00105 "   An iterative sigma clipping. For each position all pixels in the spectrum\n"
00106 "   are examined. If they deviate significantly, they will be rejected according\n"
00107 "   to the conditions:\n"
00108 "       val > mean + stdev * cpos_rej\n"
00109 "   and\n"
00110 "       val < mean - stdev * cneg_rej\n"
00111 "   where --cpos_rej, --cneg_rej and --citer are the corresponding configuration\n"
00112 "   parameters. In the first iteration median and percentile level are used.\n"
00113 "\n"
00114 "   * 'median'\n"
00115 "   At each pixel position the median is calculated.\n"
00116 "\n"
00117 "   * 'average'\n"
00118 "   At each pixel position the average is calculated.\n"
00119 "\n"
00120 "   * 'sum'\n"
00121 "   At each pixel position the sum is calculated.\n"
00122 "\n"
00123 "   * 'min_max'\n"
00124 "   The specified number of minimum and maximum pixel values will be rejected.\n"
00125 "   --cmax and --cmin apply to this method.\n"
00126 "\n"
00127 "--cpos_rej\n"
00128 "--cneg_rej\n"
00129 "--citer\n"
00130 "see --cmethod='ksigma'\n"
00131 "\n"
00132 "--cmax\n"
00133 "--cmin\n"
00134 "see --cmethod='min_max'\n"
00135 "\n"
00136 "--pix_scale"
00137 "Change the pixel scale [arcsec]. Default of 0.2\" results into cubes of\n"
00138 "14x14pix, a scale of 0.1\" results into cubes of 28x28pix, etc.\n"
00139 "\n"
00140 "--suppress_extension\n"
00141 "If set to TRUE, the arbitrary filename extensions are supressed. If multiple\n"
00142 "products with the same category are produced, they will be numered consecutively\n"
00143 "starting from 0.\n"
00144 "\n"
00145 "-------------------------------------------------------------------------------\n"
00146 "  Input files:\n"
00147 "\n"
00148 "   DO                    KMOS                                                  \n"
00149 "   category              Type   Explanation                    Required #Frames\n"
00150 "   --------              -----  -----------                    -------- -------\n"
00151 "   FLAT_SKY               F2D   Sky exposures                     Y      1-n   \n"
00152 "                                (at least 3 frames recommended)                \n"
00153 "   MASTER_DARK            F2D   Master dark                       Y       1    \n"
00154 "   MASTER_FLAT            F2D   Master flat                       Y       1    \n"
00155 "   XCAL                   F2D   x calibration frame               Y       1    \n"
00156 "   YCAL                   F2D   y calibration frame               Y       1    \n"
00157 "   LCAL                   F2D   Wavelength calib. frame           Y       1    \n"
00158 "   WAVE_BAND              F2L   Table with start-/end-wavelengths Y       1    \n"
00159 "   FLAT_EDGE              F2L   Table with fitted slitlet edges   N      0,1   \n"
00160 "\n"
00161 "  Output files:\n"
00162 "\n"
00163 "   DO                    KMOS\n"
00164 "   category              Type   Explanation\n"
00165 "   --------              -----  -----------\n"
00166 "   ILLUM_CORR            F2I    Illumination calibration frame   \n"
00167 "   If FLAT_EDGE is provided: \n"
00168 "   SKYFLAT_EDGE          F2L    Frame containing parameters of fitted \n"
00169 "                                slitlets of all IFUs of all detectors\n"
00170 "-------------------------------------------------------------------------------\n"
00171 "\n";
00172 
00173 /*-----------------------------------------------------------------------------
00174  *                              Functions code
00175  *----------------------------------------------------------------------------*/
00176 
00193 int cpl_plugin_get_info(cpl_pluginlist *list)
00194 {
00195     cpl_recipe *recipe = cpl_calloc(1, sizeof *recipe);
00196     cpl_plugin *plugin = &recipe->interface;
00197 
00198     cpl_plugin_init(plugin,
00199                         CPL_PLUGIN_API,
00200                         KMOS_BINARY_VERSION,
00201                         CPL_PLUGIN_TYPE_RECIPE,
00202                         "kmo_illumination",
00203                         "Create a calibration frame to correct spatial "
00204                         "non-uniformity of flatfield.",
00205                         kmo_illumination_description,
00206                         "Alex Agudo Berbel",
00207                         "kmos-spark@mpe.mpg.de",
00208                         kmos_get_license(),
00209                         kmo_illumination_create,
00210                         kmo_illumination_exec,
00211                         kmo_illumination_destroy);
00212 
00213     cpl_pluginlist_append(list, plugin);
00214 
00215     return 0;
00216 }
00217 
00225 static int kmo_illumination_create(cpl_plugin *plugin)
00226 {
00227     cpl_recipe *recipe;
00228     cpl_parameter *p;
00229 
00230     /* Check that the plugin is part of a valid recipe */
00231     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00232         recipe = (cpl_recipe *)plugin;
00233     else
00234         return -1;
00235 
00236     /* Create the parameters list in the cpl_recipe object */
00237     recipe->parameters = cpl_parameterlist_new();
00238 
00239     /* Fill the parameters list */
00240     /* --imethod */
00241     p = cpl_parameter_new_value("kmos.kmo_illumination.imethod",
00242                                 CPL_TYPE_STRING,
00243                                 "Method to use for interpolation: "
00244                                 "[\"NN\" (nearest neighbour), "
00245                                 "\"lwNN\" (linear weighted nearest neighbor),"
00246                                 "\"swNN\" (square weighted nearest neighbor)"
00247                                 "\"MS\" (Modified Shepard's method)"
00248                                 "\"CS\" (Cubic spline)]",
00249                                 "kmos.kmo_illumination",
00250                                 "CS");
00251     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "imethod");
00252     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00253     cpl_parameterlist_append(recipe->parameters, p);
00254 
00255     /* --neighborhoodRange */
00256     p = cpl_parameter_new_value("kmos.kmo_illumination.neighborhoodRange",
00257                                 CPL_TYPE_DOUBLE,
00258                                 "Defines the range to search for neighbors."
00259                                 "in pixels",
00260                                 "kmos.kmo_illumination",
00261                                 1.001);
00262     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "neighborhoodRange");
00263     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00264     cpl_parameterlist_append(recipe->parameters, p);
00265 
00266     /* --range */
00267     p = cpl_parameter_new_value("kmos.kmo_illumination.range",
00268                                 CPL_TYPE_STRING,
00269                                 "The spectral ranges to combine when collapsing"
00270                                 "the reconstructed cubes. e.g."
00271                                 "\"x1_start,x1_end;x2_start,x2_end\" (microns)",
00272                                 "kmos.kmo_illumination",
00273                                 "");
00274     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "range");
00275     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00276     cpl_parameterlist_append(recipe->parameters, p);
00277 
00278     /* --flux */
00279     p = cpl_parameter_new_value("kmos.kmo_illumination.flux",
00280                                 CPL_TYPE_BOOL,
00281                                 "TRUE: Apply flux conservation. FALSE: otherwise",
00282                                 "kmos.kmo_illumination",
00283                                 FALSE);
00284     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "flux");
00285     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00286     cpl_parameterlist_append(recipe->parameters, p);
00287 
00288     /* --add-all */
00289     p = cpl_parameter_new_value("kmos.kmo_illumination.add-all",
00290                                 CPL_TYPE_BOOL,
00291                                 "FALSE: omit 1st FLAT_SKY frame (acquisition), "
00292                                 "TRUE: don't perform any checks, add them all",
00293                                 "kmos.kmo_illumination",
00294                                 FALSE);
00295     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "add-all");
00296     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00297     cpl_parameterlist_append(recipe->parameters, p);
00298 
00299     /* --pix_scale */
00300     p = cpl_parameter_new_value("kmos.kmo_illumination.pix_scale",
00301                                 CPL_TYPE_DOUBLE,
00302                                 "Change the pixel scale [arcsec]. "
00303                                 "Default of 0.2\" results into cubes of 14x14pix, "
00304                                 "a scale of 0.1\" results into cubes of 28x28pix, "
00305                                 "etc.",
00306                                 "kmos.kmo_illumination",
00307                                 KMOS_PIX_RESOLUTION);
00308     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "pix_scale");
00309     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00310     cpl_parameterlist_append(recipe->parameters, p);
00311 
00312     /* --suppress_extension */
00313     p = cpl_parameter_new_value("kmos.kmo_illumination.suppress_extension",
00314                                 CPL_TYPE_BOOL,
00315                                 "Suppress arbitrary filename extension."
00316                                 "(TRUE (apply) or FALSE (don't apply)",
00317                                 "kmos.kmo_illumination",
00318                                 FALSE);
00319     cpl_parameter_set_alias(p, CPL_PARAMETER_MODE_CLI, "suppress_extension");
00320     cpl_parameter_disable(p, CPL_PARAMETER_MODE_ENV);
00321     cpl_parameterlist_append(recipe->parameters, p);
00322 
00323     // add parameters for band-definition
00324     kmo_band_pars_create(recipe->parameters,
00325                          "kmos.kmo_illumination");
00326 
00327     // add parameters for combining
00328     return kmo_combine_pars_create(recipe->parameters,
00329                                    "kmos.kmo_illumination",
00330                                    DEF_REJ_METHOD,
00331                                    FALSE);
00332 }
00333 
00339 static int kmo_illumination_exec(cpl_plugin *plugin)
00340 {
00341     cpl_recipe  *recipe;
00342 
00343     /* Get the recipe out of the plugin */
00344     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00345         recipe = (cpl_recipe *)plugin;
00346     else return -1;
00347 
00348     return kmo_illumination(recipe->parameters, recipe->frames);
00349 }
00350 
00356 static int kmo_illumination_destroy(cpl_plugin *plugin)
00357 {
00358     cpl_recipe *recipe;
00359 
00360     /* Get the recipe out of the plugin */
00361     if (cpl_plugin_get_type(plugin) == CPL_PLUGIN_TYPE_RECIPE) 
00362         recipe = (cpl_recipe *)plugin;
00363     else return -1 ;
00364 
00365     cpl_parameterlist_delete(recipe->parameters);
00366     return 0 ;
00367 }
00368 
00383 static int kmo_illumination(cpl_parameterlist *parlist, cpl_frameset *frameset)
00384 {
00385     int              ret_val                    = 0,
00386                      nr_devices                 = 0,
00387                      ifu_nr                     = 0,
00388                      nx                         = 0,
00389                      ny                         = 0,
00390                      process_noise              = FALSE,
00391                      cmax                       = 0,
00392                      cmin                       = 0,
00393                      citer                      = 0,
00394                      *bounds                    = NULL,
00395                      cnt                        = 0,
00396                      qc_max_dev_id              = 0,
00397                      qc_max_nonunif_id          = 0,
00398                      flux                       = FALSE,
00399                      background                 = FALSE,
00400                      add_all_sky                = FALSE,
00401                      same_exptime               = TRUE,
00402                      suppress_extension         = FALSE,
00403                      has_flat_edge              = FALSE;
00404     const int        *punused_ifus              = NULL;
00405     float            *pbad_pix_mask             = NULL;
00406     double           exptime                    = 0.,
00407                      exptime1                   = 0.,
00408                      exptime2                   = 0.,
00409                      cpos_rej                   = 0.0,
00410                      cneg_rej                   = 0.0,
00411                      neighborhoodRange          = 1.001,
00412                      mean_data                  = 0.0,
00413                      ifu_crpix                  = 0.0,
00414                      ifu_crval                  = 0.0,
00415                      ifu_cdelt                  = 0.0,
00416                      qc_spat_unif               = 0.0,
00417                      qc_max_dev                 = 0.0,
00418                      qc_max_nonunif             = 0.0,
00419                      tmp_stdev                  = 0.0,
00420                      tmp_mean                   = 0.0,
00421                      rotangle                   = 0.0,
00422                      tmp_rotangle               = 0.0,
00423                      rotangle_found             = 0.0,
00424                      pix_scale                  = 0.0;
00425     char             *keyword                   = NULL,
00426                      *fn_lut                    = NULL,
00427                      *suffix                    = NULL,
00428                      *fn_suffix                 = NULL,
00429                      *extname                   = NULL,
00430                      *filter                    = NULL;
00431     const char       *method                    = NULL,
00432                      *cmethod                   = NULL,
00433                      *filter_id_l               = NULL,
00434                      *filter_id                 = NULL,
00435                      *ranges_txt                = NULL;
00436     cpl_array        *calTimestamp              = NULL,
00437                      **unused_ifus_before       = NULL,
00438                      **unused_ifus_after        = NULL;
00439     cpl_frame        *frame                     = NULL,
00440                      *xcalFrame                 = NULL,
00441                      *ycalFrame                 = NULL,
00442                      *lcalFrame                 = NULL;
00443     cpl_frameset     *frameset_sky              = NULL;
00444     cpl_image        *img_in                    = NULL,
00445                      *img_dark                  = NULL,
00446                      *img_dark_noise            = NULL,
00447                      *img_flat                  = NULL,
00448                      *img_flat_noise            = NULL,
00449                      *combined_data             = NULL,
00450                      *combined_noise            = NULL,
00451                      *xcal                      = NULL,
00452                      *ycal                      = NULL,
00453                      *lcal                      = NULL,
00454                      *bad_pix_mask              = NULL,
00455                      *data_ifu                  = NULL,
00456                      *noise_ifu                 = NULL,
00457                      **stored_data_images       = NULL,
00458                      **stored_noise_images      = NULL;
00459     cpl_imagelist    *cube_data                 = NULL,
00460                      *cube_noise                = NULL,
00461                      *detector_in               = NULL,
00462                      **stored_data_cubes        = NULL,
00463                      **stored_noise_cubes       = NULL;
00464     cpl_matrix       **edgepars                 = NULL;
00465     cpl_propertylist *main_header               = NULL,
00466                      *tmp_header                = NULL,
00467                      *sub_header                = NULL,
00468                      **stored_sub_data_headers  = NULL,
00469                      **stored_sub_noise_headers = NULL;
00470     cpl_table        *band_table                = NULL,
00471                      ***edge_table_sky          = NULL,
00472                      **edge_table_flat          = NULL;
00473     cpl_vector       *ranges                    = NULL,
00474                      *identified_slices         = NULL,
00475                      *calAngles                 = NULL,
00476                      **slitlet_ids              = NULL,
00477                      *shift_vec                 = NULL,
00478                      *edge_vec                  = NULL;
00479     main_fits_desc   desc_sky,
00480                      desc_dark,
00481                      desc_flat,
00482                      desc_xcal,
00483                      desc_ycal,
00484                      desc_lcal;
00485     gridDefinition   gd;
00486 
00487     KMO_TRY
00488     {
00489         kmo_init_fits_desc(&desc_sky);
00490         kmo_init_fits_desc(&desc_dark);
00491         kmo_init_fits_desc(&desc_flat);
00492         kmo_init_fits_desc(&desc_xcal);
00493         kmo_init_fits_desc(&desc_ycal);
00494         kmo_init_fits_desc(&desc_lcal);
00495 
00496         /* --- check input --- */
00497         KMO_TRY_ASSURE((parlist != NULL) &&
00498                        (frameset != NULL),
00499                        CPL_ERROR_NULL_INPUT,
00500                        "Not all input data is provided!");
00501 
00502         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, FLAT_SKY) >= 1,
00503                        CPL_ERROR_ILLEGAL_INPUT,
00504                        "One or more FLAT_SKY frames are required!");
00505 
00506         if (cpl_frameset_count_tags(frameset, FLAT_SKY) < 3) {
00507             cpl_msg_warning(cpl_func, "It is recommended to provide at least "
00508                                       "3 FLAT_SKY frames!");
00509         }
00510 
00511         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_DARK) == 1,
00512                        CPL_ERROR_ILLEGAL_INPUT,
00513                        "Exactly one MASTER_DARK frame is required!");
00514 
00515         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, MASTER_FLAT) == 1,
00516                        CPL_ERROR_ILLEGAL_INPUT,
00517                        "Exactly one MASTER_FLAT frame is required!");
00518 
00519         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, XCAL) == 1,
00520                        CPL_ERROR_ILLEGAL_INPUT,
00521                        "Exactly one XCAL frame is required!");
00522 
00523         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, YCAL) == 1,
00524                        CPL_ERROR_ILLEGAL_INPUT,
00525                        "Exactly one YCAL frame is required!");
00526 
00527         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, LCAL) == 1,
00528                        CPL_ERROR_ILLEGAL_INPUT,
00529                        "Exactly one LCAL frame is required!");
00530 
00531         KMO_TRY_ASSURE(cpl_frameset_count_tags(frameset, WAVE_BAND) == 1,
00532                        CPL_ERROR_ILLEGAL_INPUT,
00533                        "Exactly one WAVE_BAND frame is required!");
00534 
00535         KMO_TRY_ASSURE((cpl_frameset_count_tags(frameset, FLAT_EDGE) == 1) ||
00536                        (cpl_frameset_count_tags(frameset, FLAT_EDGE) == 0),
00537                        CPL_ERROR_ILLEGAL_INPUT,
00538                        "Exactly one FLAT_EDGE frame is required!");
00539 
00540         has_flat_edge = cpl_frameset_count_tags(frameset, FLAT_EDGE);
00541 
00542         KMO_TRY_ASSURE(kmo_dfs_set_groups(frameset, "kmo_illumination") == 1,
00543                        CPL_ERROR_ILLEGAL_INPUT,
00544                        "Cannot identify RAW and CALIB frames!");
00545 
00546         /* --- get parameters --- */
00547         cpl_msg_info("", "--- Parameter setup for kmo_illumination ---");
00548 
00549         KMO_TRY_EXIT_IF_NULL(
00550             method = kmo_dfs_get_parameter_string(parlist,
00551                                               "kmos.kmo_illumination.imethod"));
00552 
00553         KMO_TRY_ASSURE((strcmp(method, "NN") == 0) ||
00554                        (strcmp(method, "lwNN") == 0) ||
00555                        (strcmp(method, "swNN") == 0) ||
00556                        (strcmp(method, "MS") == 0) ||
00557                        (strcmp(method, "CS") == 0),
00558                        CPL_ERROR_ILLEGAL_INPUT,
00559                        "method must be either \"NN\", \"lwNN\", "
00560                        "\"swNN\", \"MS\" or \"CS\"!");
00561 
00562         KMO_TRY_EXIT_IF_ERROR(
00563             kmo_dfs_print_parameter_help(parlist,
00564                                         "kmos.kmo_illumination.imethod"));
00565 
00566         neighborhoodRange = kmo_dfs_get_parameter_double(parlist,
00567                 "kmos.kmo_illumination.neighborhoodRange");
00568         KMO_TRY_CHECK_ERROR_STATE();
00569 
00570         KMO_TRY_ASSURE(neighborhoodRange > 0.0,
00571                 CPL_ERROR_ILLEGAL_INPUT,
00572                 "neighborhoodRange must be greater than 0.0");
00573 
00574         KMO_TRY_EXIT_IF_ERROR(
00575             kmo_dfs_print_parameter_help(parlist,
00576                                     "kmos.kmo_illumination.neighborhoodRange"));
00577 
00578         ranges_txt = kmo_dfs_get_parameter_string(parlist,
00579                                                   "kmos.kmo_illumination.range");
00580         KMO_TRY_CHECK_ERROR_STATE();
00581 
00582         KMO_TRY_EXIT_IF_ERROR(
00583             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.range"));
00584 
00585         ranges = kmo_identify_ranges(ranges_txt);
00586         KMO_TRY_CHECK_ERROR_STATE();
00587 
00588         flux = kmo_dfs_get_parameter_bool(parlist,
00589                                           "kmos.kmo_illumination.flux");
00590 
00591         KMO_TRY_ASSURE((flux == 0) ||
00592                        (flux == 1),
00593                        CPL_ERROR_ILLEGAL_INPUT,
00594                        "flux must be either FALSE or TRUE!");
00595 
00596         KMO_TRY_EXIT_IF_ERROR(
00597             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.flux"));
00598 
00599         add_all_sky = kmo_dfs_get_parameter_bool(parlist,
00600                                                  "kmos.kmo_illumination.add-all");
00601 
00602         KMO_TRY_ASSURE((add_all_sky == 0) ||
00603                        (add_all_sky == 1),
00604                        CPL_ERROR_ILLEGAL_INPUT,
00605                        "add_all must be either FALSE or TRUE!");
00606 
00607         KMO_TRY_EXIT_IF_ERROR(
00608             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.add-all"));
00609 
00610         pix_scale = kmo_dfs_get_parameter_double(parlist,
00611                                         "kmos.kmo_illumination.pix_scale");
00612         KMO_TRY_CHECK_ERROR_STATE();
00613         KMO_TRY_EXIT_IF_ERROR(
00614            kmo_dfs_print_parameter_help(parlist,
00615                                        "kmos.kmo_illumination.pix_scale"));
00616         KMO_TRY_ASSURE((pix_scale >= 0.01) &&
00617                        (pix_scale <= 0.4),
00618                        CPL_ERROR_ILLEGAL_INPUT,
00619                        "pix_scale must be between 0.01 and 0.4 (results in cubes "
00620                        "with 7x7 to 280x280 pixels)!");
00621 
00622         suppress_extension = kmo_dfs_get_parameter_bool(parlist,
00623                                           "kmos.kmo_illumination.suppress_extension");
00624         KMO_TRY_CHECK_ERROR_STATE();
00625         KMO_TRY_EXIT_IF_ERROR(
00626             kmo_dfs_print_parameter_help(parlist, "kmos.kmo_illumination.suppress_extension"));
00627 
00628         KMO_TRY_ASSURE((suppress_extension == TRUE) || (suppress_extension == FALSE),
00629                        CPL_ERROR_ILLEGAL_INPUT,
00630                        "suppress_extension must be TRUE or FALSE!");
00631 
00632         kmo_band_pars_load(parlist, "kmos.kmo_illumination");
00633 
00634         KMO_TRY_EXIT_IF_ERROR(
00635             kmo_combine_pars_load(parlist,
00636                                   "kmos.kmo_illumination",
00637                                   &cmethod,
00638                                   &cpos_rej,
00639                                   &cneg_rej,
00640                                   &citer,
00641                                   &cmin,
00642                                   &cmax,
00643                                   FALSE));
00644         cpl_msg_info("", "-------------------------------------------");
00645 
00646         // check if filter_id, grating_id and rotator offset match for all
00647         // detectors
00648         KMO_TRY_EXIT_IF_ERROR(
00649             kmo_check_frameset_setup(frameset, FLAT_SKY,
00650                                        TRUE, FALSE, TRUE));
00651         KMO_TRY_EXIT_IF_ERROR(
00652             kmo_check_frame_setup(frameset, FLAT_SKY, XCAL,
00653                                        TRUE, FALSE, TRUE));
00654         KMO_TRY_EXIT_IF_ERROR(
00655             kmo_check_frame_setup(frameset, XCAL, YCAL,
00656                                        TRUE, FALSE, TRUE));
00657         KMO_TRY_EXIT_IF_ERROR(
00658             kmo_check_frame_setup(frameset, XCAL, LCAL,
00659                                        TRUE, FALSE, TRUE));
00660         KMO_TRY_EXIT_IF_ERROR(
00661             kmo_check_frame_setup(frameset, XCAL, MASTER_FLAT,
00662                                        TRUE, FALSE, TRUE));
00663 
00664         KMO_TRY_EXIT_IF_NULL(
00665             frame = kmo_dfs_get_frame(frameset, XCAL));
00666         KMO_TRY_EXIT_IF_NULL(
00667             suffix = kmo_dfs_get_suffix(frame, TRUE, FALSE));
00668 
00669         KMO_TRY_EXIT_IF_ERROR(
00670             kmo_check_frame_setup_md5_xycal(frameset));
00671         KMO_TRY_EXIT_IF_ERROR(
00672             kmo_check_frame_setup_md5(frameset));
00673 
00674         cpl_msg_info("", "Detected instrument setup:   %s", suffix+1);
00675         cpl_msg_info("", "(grating 1, 2 & 3)");
00676 
00677         // check which IFUs are active for all frames
00678         KMO_TRY_EXIT_IF_NULL(
00679             unused_ifus_before = kmo_get_unused_ifus(frameset, 0, 0));
00680 
00681         KMO_TRY_EXIT_IF_NULL(
00682             unused_ifus_after = kmo_duplicate_unused_ifus(unused_ifus_before));
00683 
00684         kmo_print_unused_ifus(unused_ifus_before, FALSE);
00685 
00686         // load desc for MASTER_DARK and check
00687         KMO_TRY_EXIT_IF_NULL(
00688             frame = kmo_dfs_get_frame(frameset, MASTER_DARK));
00689         desc_dark = kmo_identify_fits_header(
00690                     cpl_frame_get_filename(frame));
00691         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_DARK frame doesn't seem to "
00692                                       "be in KMOS-format!");
00693         KMO_TRY_ASSURE((desc_dark.nr_ext == 2*KMOS_NR_DETECTORS) &&
00694                        (desc_dark.ex_badpix == FALSE) &&
00695                        (desc_dark.fits_type == f2d_fits) &&
00696                        (desc_dark.frame_type == detector_frame),
00697                        CPL_ERROR_ILLEGAL_INPUT,
00698                        "MASTER_DARK isn't in the correct format!!!");
00699         nx = desc_dark.naxis1;
00700         ny = desc_dark.naxis2;
00701 
00702         // load desc for MASTER_FLAT and check
00703         KMO_TRY_EXIT_IF_NULL(
00704             frame = kmo_dfs_get_frame(frameset, MASTER_FLAT));
00705         desc_flat = kmo_identify_fits_header(cpl_frame_get_filename(frame));
00706         KMO_TRY_CHECK_ERROR_STATE_MSG("MASTER_FLAT frame doesn't seem to "
00707                                       "be in KMOS-format!");
00708         KMO_TRY_ASSURE((desc_flat.nr_ext % (2*KMOS_NR_DETECTORS) == 0) &&
00709                        (desc_flat.ex_badpix == FALSE) &&
00710                        (desc_flat.fits_type == f2d_fits) &&
00711                        (desc_flat.frame_type == detector_frame),
00712                        CPL_ERROR_ILLEGAL_INPUT,
00713                        "MASTER_FLAT isn't in the correct format!!!");
00714 
00715         // load desc for XCAL and check
00716         KMO_TRY_EXIT_IF_NULL(
00717             xcalFrame = kmo_dfs_get_frame(frameset, XCAL));
00718         desc_xcal = kmo_identify_fits_header(cpl_frame_get_filename(xcalFrame));
00719         KMO_TRY_CHECK_ERROR_STATE_MSG("XCAL frame doesn't seem to "
00720                                       "be in KMOS-format!");
00721         KMO_TRY_ASSURE((desc_xcal.nr_ext % KMOS_NR_DETECTORS == 0) &&
00722                        (desc_xcal.ex_badpix == FALSE) &&
00723                        (desc_xcal.fits_type == f2d_fits) &&
00724                        (desc_xcal.frame_type == detector_frame),
00725                        CPL_ERROR_ILLEGAL_INPUT,
00726                        "XCAL isn't in the correct format!!!");
00727         KMO_TRY_ASSURE((desc_xcal.naxis1 == nx) &&
00728                        (desc_xcal.naxis2 == ny),
00729                        CPL_ERROR_ILLEGAL_INPUT,
00730                        "MASTER_DARK and XCAL frame haven't same dimensions! "
00731                        "(x,y): (%d,%d) vs (%d,%d)",
00732                        nx, ny, desc_xcal.naxis1, desc_xcal.naxis2);
00733 
00734         nr_devices = desc_xcal.nr_ext;
00735 
00736         // load desc for YCAL and check
00737         KMO_TRY_EXIT_IF_NULL(
00738             ycalFrame = kmo_dfs_get_frame(frameset, YCAL));
00739         desc_ycal = kmo_identify_fits_header(cpl_frame_get_filename(ycalFrame));
00740         KMO_TRY_CHECK_ERROR_STATE_MSG("YCAL frame doesn't seem to "
00741                                       "be in KMOS-format!");
00742         KMO_TRY_ASSURE((desc_ycal.nr_ext == desc_xcal.nr_ext) &&
00743                        (desc_ycal.ex_badpix == desc_xcal.ex_badpix) &&
00744                        (desc_ycal.fits_type == desc_xcal.fits_type) &&
00745                        (desc_ycal.frame_type == desc_xcal.frame_type),
00746                        CPL_ERROR_ILLEGAL_INPUT,
00747                        "YCAL isn't in the correct format!!!");
00748         KMO_TRY_ASSURE((desc_ycal.naxis1 == desc_xcal.naxis1) &&
00749                        (desc_ycal.naxis2 == desc_xcal.naxis2),
00750                        CPL_ERROR_ILLEGAL_INPUT,
00751                        "MASTER_DARK and YCAL frame haven't same dimensions! "
00752                        "(x,y): (%d,%d) vs (%d,%d)",
00753                        nx, ny, desc_ycal.naxis1, desc_ycal.naxis2);
00754 
00755         // load desc for LCAL and check
00756         KMO_TRY_EXIT_IF_NULL(
00757             lcalFrame = kmo_dfs_get_frame(frameset, LCAL));
00758         desc_lcal = kmo_identify_fits_header(cpl_frame_get_filename(lcalFrame));
00759         KMO_TRY_CHECK_ERROR_STATE_MSG("LCAL frame doesn't seem to "
00760                                       "be in KMOS-format!");
00761         KMO_TRY_ASSURE((desc_lcal.ex_badpix == desc_xcal.ex_badpix) &&
00762                        (desc_lcal.fits_type == desc_xcal.fits_type) &&
00763                        (desc_lcal.frame_type == desc_xcal.frame_type),
00764                        CPL_ERROR_ILLEGAL_INPUT,
00765                        "LCAL isn't in the correct format!!!");
00766         KMO_TRY_ASSURE((desc_lcal.naxis1 == desc_xcal.naxis1) &&
00767                        (desc_lcal.naxis2 == desc_xcal.naxis2),
00768                        CPL_ERROR_ILLEGAL_INPUT,
00769                        "MASTER_DARK and LCAL frame haven't same dimensions! "
00770                        "(x,y): (%d,%d) vs (%d,%d)",
00771                        nx, ny, desc_lcal.naxis1, desc_lcal.naxis2);
00772         KMO_TRY_EXIT_IF_NULL(
00773             tmp_header = kmo_dfs_load_primary_header(frameset, LCAL));
00774 
00775         // load desc for FLAT_SKY and check
00776         nr_devices = KMOS_NR_DETECTORS;
00777         KMO_TRY_EXIT_IF_NULL(
00778             frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00779 
00780         KMO_TRY_EXIT_IF_NULL(
00781             main_header = kmclipm_propertylist_load(
00782                                          cpl_frame_get_filename(frame), 0));
00783         rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00784         KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00785         kmclipm_strip_angle(&rotangle);
00786         cpl_propertylist_delete(main_header); main_header = NULL;
00787 
00788         cnt = 1;
00789         while (frame != NULL) {
00790             KMO_TRY_EXIT_IF_NULL(
00791                 main_header = kmclipm_propertylist_load(
00792                                              cpl_frame_get_filename(frame), 0));
00793 
00794             desc_sky = kmo_identify_fits_header(
00795                         cpl_frame_get_filename(frame));
00796             KMO_TRY_CHECK_ERROR_STATE_MSG("FLAT_SKY frame doesn't seem to "
00797                                           "be in KMOS-format!");
00798             KMO_TRY_ASSURE((desc_sky.nr_ext == 3) &&
00799                            (desc_sky.ex_badpix == FALSE) &&
00800                            (desc_sky.fits_type == raw_fits) &&
00801                            (desc_sky.frame_type == detector_frame),
00802                            CPL_ERROR_ILLEGAL_INPUT,
00803                            "FLAT_SKY isn't in the correct format!!!");
00804             KMO_TRY_ASSURE((desc_sky.naxis1 == nx) &&
00805                            (desc_sky.naxis2 == ny) &&
00806                            (desc_sky.nr_ext == nr_devices),
00807                            CPL_ERROR_ILLEGAL_INPUT,
00808                            "MASTER_DARK and FLAT_SKY (no. %d) frame haven't "
00809                            "same dimensions! (x,y): (%d,%d) vs (%d,%d)",
00810                            cnt, nx, ny, desc_flat.naxis1, desc_flat.naxis2);
00811             kmo_free_fits_desc(&desc_sky);
00812             kmo_init_fits_desc(&desc_sky);
00813 
00814             KMO_TRY_ASSURE(
00815                 (kmo_check_lamp(main_header, INS_LAMP1_ST) == FALSE) &&
00816                 (kmo_check_lamp(main_header, INS_LAMP2_ST) == FALSE) &&
00817                 (kmo_check_lamp(main_header, INS_LAMP3_ST) == FALSE) &&
00818                 (kmo_check_lamp(main_header, INS_LAMP4_ST) == FALSE),
00819                 CPL_ERROR_ILLEGAL_INPUT,
00820                 "All lamps must be switched off for the FLAT_SKY frames!");
00821 
00822             // assert that filters have correct IDs and that all detectors of
00823             // all input frames have the same filter set
00824             for (int i = 1; i <= KMOS_NR_DETECTORS; i++) {
00825                 // ESO INS FILTi ID
00826                 KMO_TRY_EXIT_IF_NULL(
00827                     keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, i, IFU_FILTID_POSTFIX));
00828                 KMO_TRY_EXIT_IF_NULL(
00829                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00830 
00831                 KMO_TRY_EXIT_IF_NULL(
00832                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00833                 cpl_free(keyword); keyword = NULL;
00834 
00835                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00836                                (strcmp(filter_id, "YJ") == 0) ||
00837                                (strcmp(filter_id, "H") == 0) ||
00838                                (strcmp(filter_id, "K") == 0) ||
00839                                (strcmp(filter_id, "HK") == 0),
00840                                CPL_ERROR_ILLEGAL_INPUT,
00841                                "Filter ID in primary header must be either 'IZ', "
00842                                "'YJ', 'H', 'K' or " "'HK' !");
00843 
00844                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00845                                CPL_ERROR_ILLEGAL_INPUT,
00846                                "Filter IDs must be the same for FLAT_SKY frame"
00847                                " and lcal frame!"
00848                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00849                                i, cpl_frame_get_filename(frame),
00850                                filter_id, filter_id_l);
00851 
00852                 // ESO INS GRATi ID
00853                 KMO_TRY_EXIT_IF_NULL(
00854                     keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, i, IFU_GRATID_POSTFIX));
00855                 KMO_TRY_EXIT_IF_NULL(
00856                     filter_id = cpl_propertylist_get_string(main_header, keyword));
00857 
00858                 KMO_TRY_EXIT_IF_NULL(
00859                     filter_id_l = cpl_propertylist_get_string(tmp_header, keyword));
00860                 cpl_free(keyword); keyword = NULL;
00861 
00862                 KMO_TRY_ASSURE((strcmp(filter_id, "IZ") == 0) ||
00863                                (strcmp(filter_id, "YJ") == 0) ||
00864                                (strcmp(filter_id, "H") == 0) ||
00865                                (strcmp(filter_id, "K") == 0) ||
00866                                (strcmp(filter_id, "HK") == 0),
00867                                CPL_ERROR_ILLEGAL_INPUT,
00868                                "Grating ID in primary header must be either "
00869                                "'IZ', 'YJ', 'H', 'K' or " "'HK' !");
00870 
00871                 KMO_TRY_ASSURE(strcmp(filter_id, filter_id_l) == 0,
00872                                CPL_ERROR_ILLEGAL_INPUT,
00873                                "Grating IDs must be the same for FLAT_SKY frame"
00874                                " and lcal frame!"
00875                                "Detector No.: %d\n%s: %s\nLCAL: %s\n",
00876                                i, cpl_frame_get_filename(frame),
00877                                filter_id, filter_id_l);
00878 
00879                 tmp_rotangle = cpl_propertylist_get_double(main_header, ROTANGLE);
00880                 KMO_TRY_CHECK_ERROR_STATE_MSG("Cannot retrieve ROTANGLE FITS keyword from sky frame!");
00881                 kmclipm_strip_angle(&tmp_rotangle);
00882                 KMO_TRY_ASSURE((abs(rotangle - tmp_rotangle) < 10.0) ||
00883                                (abs(rotangle - tmp_rotangle) > 360.-10.) ,
00884                         CPL_ERROR_ILLEGAL_INPUT,
00885                         "OCS ROT NAANGLE of sky flat frames differ to much: %f %f",
00886                         rotangle, tmp_rotangle);
00887             }
00888             cpl_propertylist_delete(main_header); main_header = NULL;
00889 
00890             // get next FLAT_SKY frame
00891             frame = kmo_dfs_get_frame(frameset, NULL);
00892             KMO_TRY_CHECK_ERROR_STATE();
00893             cnt++;
00894         }
00895 
00896         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
00897 
00898         //
00899         // noise will be propagated when:
00900         // MASTER_DARK and MASTER_FLAT have noise extensions and if at least
00901         // 2 FLAT_SKY frames are provided.
00902         // Otherwise noise will be ignored.
00903         //
00904         if (desc_dark.ex_noise &&
00905             desc_flat.ex_noise &&
00906             (cpl_frameset_count_tags(frameset, FLAT_SKY) >= 2)) {
00907             process_noise = TRUE;
00908         }
00909 
00910         if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00911             cpl_msg_warning(cpl_func, "cmethod is changed to 'average' "
00912                             "since there is only one input frame! (The output "
00913                             "file won't have any noise extensions)");
00914 
00915             cmethod = "average";
00916         }
00917 
00918         //
00919         // Check whether 1st FLAT_SKY should be omitted
00920         //
00921         KMO_TRY_EXIT_IF_NULL(
00922             frameset_sky = cpl_frameset_new());
00923 
00924         if (add_all_sky) {
00925             // just add all FLAT_SKY frames without check
00926             frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00927             while (frame != NULL) {
00928                 KMO_TRY_EXIT_IF_ERROR(
00929                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00930                 frame = kmo_dfs_get_frame(frameset, NULL);
00931             }
00932             cpl_msg_info("", "Add all FLAT_SKY without checking for acquisition frame.");
00933         } else {
00934             // check if 1st FLAT_SKY has different exposure time and whether to omit it
00935             KMO_TRY_EXIT_IF_NULL(
00936                 frame = kmo_dfs_get_frame(frameset, FLAT_SKY));
00937 
00938             if (cpl_frameset_count_tags(frameset, FLAT_SKY) == 1) {
00939                 // just one FLAT_SKY, always add
00940                 KMO_TRY_EXIT_IF_ERROR(
00941                     cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00942                 KMO_TRY_CHECK_ERROR_STATE();
00943             } else {
00944                 // several FLAT_SKY frames, check exptime
00945 
00946                 // get exptime 1
00947                 KMO_TRY_EXIT_IF_NULL(
00948                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00949                 exptime1 = cpl_propertylist_get_double(main_header, EXPTIME);
00950                 KMO_TRY_CHECK_ERROR_STATE();
00951                 cpl_propertylist_delete(main_header); main_header = NULL;
00952 
00953                 // get exptime 2
00954                 frame = kmo_dfs_get_frame(frameset, NULL);
00955                 KMO_TRY_EXIT_IF_NULL(
00956                     main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00957                 exptime2 = cpl_propertylist_get_double(main_header, EXPTIME);
00958                 KMO_TRY_CHECK_ERROR_STATE();
00959                 cpl_propertylist_delete(main_header); main_header = NULL;
00960 
00961                 // loop remaining frames
00962                 same_exptime = TRUE;
00963                 frame = kmo_dfs_get_frame(frameset, NULL);
00964                 while (same_exptime && (frame != NULL)) {
00965                     KMO_TRY_EXIT_IF_NULL(
00966                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00967                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00968                     KMO_TRY_CHECK_ERROR_STATE();
00969                     cpl_propertylist_delete(main_header); main_header = NULL;
00970                     if (fabs(exptime-exptime2) > 0.01) {
00971                         // not same
00972                         same_exptime = FALSE;
00973                     }
00974                     frame = kmo_dfs_get_frame(frameset, NULL);
00975                 }
00976 
00977                 if (same_exptime) {
00978                     // frame [2,n] have same exptime, add them
00979                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
00980                     KMO_TRY_EXIT_IF_NULL(
00981                         main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
00982                     exptime = cpl_propertylist_get_double(main_header, EXPTIME);
00983                     KMO_TRY_CHECK_ERROR_STATE();
00984                     cpl_propertylist_delete(main_header); main_header = NULL;
00985                     cpl_msg_info("", "Omit FLAT_SKY: %s with EXPTIME of %g sec (acquisition), other frame(s) have EXPTIME of %g sec", cpl_frame_get_filename(frame), exptime, exptime2);
00986                     frame = kmo_dfs_get_frame(frameset, NULL);
00987                     while (frame != NULL) {
00988                         KMO_TRY_EXIT_IF_ERROR(
00989                             cpl_frameset_insert(frameset_sky, cpl_frame_duplicate(frame)));
00990                         frame = kmo_dfs_get_frame(frameset, NULL);
00991                     }
00992                     if (fabs(exptime1-exptime2) < 0.01) {
00993                         cpl_msg_warning("", "The 1st FLAT_SKY has the same exposure time as the following ones. "
00994                                             "It has anyway been omitted since we assume it is an acquisition frame. "
00995                                             "If you want to add it anyway call this recipe with the --add-all parameter");
00996                     }
00997                 } else {
00998                     cpl_msg_error("", "The exposure times of the FLAT_SKY frames don't match!");
00999                     cpl_msg_error("", "We assume that the 1st frame is an acquisition frame and would be omitted.");
01000                     cpl_msg_error("", "The following frames should have the same exposure time if they originate from the same template.");
01001                     cpl_msg_error("", "If you want to reduce them anyway call this recipe with the --add-all parameter");
01002                     frame = kmo_dfs_get_frame(frameset, FLAT_SKY);
01003                     while (frame != NULL) {
01004                         KMO_TRY_EXIT_IF_NULL(
01005                             main_header = kmclipm_propertylist_load(cpl_frame_get_filename(frame), 0));
01006                         exptime = cpl_propertylist_get_double(main_header, EXPTIME);
01007                         KMO_TRY_CHECK_ERROR_STATE();
01008                         cpl_propertylist_delete(main_header); main_header = NULL;
01009                         cpl_msg_error("", "FLAT_SKY: %s, EXPTIME: %g", cpl_frame_get_filename(frame), exptime);
01010                         frame = kmo_dfs_get_frame(frameset, NULL);
01011                     }
01012                     cpl_error_set(cpl_func, CPL_ERROR_ILLEGAL_INPUT);
01013                     KMO_TRY_CHECK_ERROR_STATE();
01014                 }
01015             }
01016         }
01017 
01018         KMO_TRY_EXIT_IF_NULL(
01019             frame = kmo_dfs_get_frame(frameset_sky, FLAT_SKY));
01020         KMO_TRY_EXIT_IF_NULL(
01021             main_header = kmo_dfs_load_primary_header(frameset_sky, FLAT_SKY));
01022         KMO_TRY_EXIT_IF_NULL(
01023             keyword = cpl_sprintf("%s%d%s", IFU_GRATID_PREFIX, 1, IFU_GRATID_POSTFIX));
01024         KMO_TRY_EXIT_IF_NULL(
01025             filter = cpl_sprintf("%s", cpl_propertylist_get_string(main_header, keyword)));
01026         cpl_free(keyword); keyword = NULL;
01027 
01028         //
01029         // set default band-specific ranges for collapsing
01030         //
01031         if (ranges == NULL) {
01032             if (strcmp(filter, "IZ") == 0) {
01033                 ranges_txt = "0.81,1.05";
01034             } else if (strcmp(filter, "YJ") == 0) {
01035                 ranges_txt = "1.025,1.3";
01036             } else if (strcmp(filter, "H") == 0) {
01037                 ranges_txt = "1.5,1.7";
01038             } else if (strcmp(filter, "K") == 0) {
01039                 ranges_txt = "2.1,2.35";
01040             } else if (strcmp(filter, "HK") == 0) {
01041                 ranges_txt = "1.5,1.7;2.1,2.35";
01042 //                ranges_txt = "1.5,1.7";
01043             } else {
01044                 KMO_TRY_ASSURE(1 == 0,
01045                                CPL_ERROR_ILLEGAL_INPUT,
01046                                "We really shouldn't get here...");
01047             }
01048             cpl_msg_info("", "Spectral range to collapse has been set to %s um for this band.", ranges_txt);
01049             ranges = kmo_identify_ranges(ranges_txt);
01050             KMO_TRY_CHECK_ERROR_STATE();
01051         }
01052 
01053         // setup grid definition, wavelength start and end points will be set
01054         // in the detector loop
01055         KMO_TRY_EXIT_IF_ERROR(
01056             kmclipm_setup_grid(&gd, method, neighborhoodRange, pix_scale));
01057 
01058         // create filename for LUT
01059         KMO_TRY_EXIT_IF_NULL(
01060             fn_lut = cpl_sprintf("%s%s", "lut", suffix));
01061 
01062         // extract bounds
01063         KMO_TRY_EXIT_IF_NULL(
01064             tmp_header = kmclipm_propertylist_load(
01065                                          cpl_frame_get_filename(xcalFrame), 0));
01066         KMO_TRY_EXIT_IF_NULL(
01067             bounds = kmclipm_extract_bounds(tmp_header));
01068         cpl_propertylist_delete(tmp_header); tmp_header = NULL;
01069 
01070         // get timestamps of xcal, ycal & lcal
01071         KMO_TRY_EXIT_IF_NULL(
01072             calTimestamp = kmo_get_timestamps(xcalFrame, ycalFrame, lcalFrame));
01073 
01074         // create arrays to hold reconstructed data and noise cubes and
01075         // their headers
01076         KMO_TRY_EXIT_IF_NULL(
01077             stored_data_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01078                                                             sizeof(cpl_imagelist*)));
01079         KMO_TRY_EXIT_IF_NULL(
01080             stored_noise_cubes = (cpl_imagelist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01081                                                              sizeof(cpl_imagelist*)));
01082         KMO_TRY_EXIT_IF_NULL(
01083             stored_data_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01084                                                          sizeof(cpl_image*)));
01085         KMO_TRY_EXIT_IF_NULL(
01086             stored_noise_images = (cpl_image**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01087                                                           sizeof(cpl_image*)));
01088         KMO_TRY_EXIT_IF_NULL(
01089             stored_sub_data_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01090                                                                      sizeof(cpl_propertylist*)));
01091         KMO_TRY_EXIT_IF_NULL(
01092             stored_sub_noise_headers = (cpl_propertylist**)cpl_calloc(nr_devices * KMOS_IFUS_PER_DETECTOR,
01093                                                                       sizeof(cpl_propertylist*)));
01094         KMO_TRY_EXIT_IF_NULL(
01095             edge_table_sky = (cpl_table***)cpl_calloc(KMOS_NR_DETECTORS,
01096                                                       sizeof(cpl_table**)));
01097         KMO_TRY_EXIT_IF_NULL(
01098             edge_table_flat = (cpl_table**)cpl_calloc(KMOS_IFUS_PER_DETECTOR,
01099                                                       sizeof(cpl_table*)));
01100         KMO_TRY_EXIT_IF_NULL(
01101             calAngles = cpl_vector_new(3));
01102 
01103         //
01104         // loop through all detectors
01105         //
01106         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01107             cpl_msg_info("","Processing detector No. %d", det_nr);
01108 
01109             KMO_TRY_EXIT_IF_NULL(
01110                 detector_in = cpl_imagelist_new());
01111 
01112             // load data of det_nr of all FLAT_SKY frames into an imagelist
01113             KMO_TRY_EXIT_IF_NULL(
01114                 img_in = kmo_dfs_load_image(frameset_sky, FLAT_SKY, det_nr, FALSE, TRUE, NULL));
01115 
01116             cnt = 0;
01117             while (img_in != NULL) {
01118                 cpl_imagelist_set(detector_in, img_in, cnt);
01119                 KMO_TRY_CHECK_ERROR_STATE();
01120 
01121                 /* load same extension of next FLAT_SKY frame*/
01122                 img_in = kmo_dfs_load_image(frameset_sky, NULL, det_nr, FALSE, TRUE, NULL);
01123                 KMO_TRY_CHECK_ERROR_STATE();
01124 
01125                 cnt++;
01126             }
01127 
01128             //
01129             // process imagelist
01130             //
01131 
01132             // combine imagelist (data only) and create noise (stdev of data)
01133             cpl_msg_info("","Combining frames...");
01134             if (process_noise) {
01135                 KMO_TRY_EXIT_IF_ERROR(
01136                     kmclipm_combine_frames(detector_in,
01137                                            NULL,
01138                                            NULL,
01139                                            cmethod,
01140                                            cpos_rej,
01141                                            cneg_rej,
01142                                            citer,
01143                                            cmax,
01144                                            cmin,
01145                                            &combined_data,
01146                                            &combined_noise,
01147                                            -1.0));
01148             } else {
01149                 KMO_TRY_EXIT_IF_ERROR(
01150                     kmclipm_combine_frames(detector_in,
01151                                            NULL,
01152                                            NULL,
01153                                            cmethod,
01154                                            cpos_rej,
01155                                            cneg_rej,
01156                                            citer,
01157                                            cmax,
01158                                            cmin,
01159                                            &combined_data,
01160                                            NULL,
01161                                            -1.0));
01162             }
01163 
01164             if (kmclipm_omit_warning_one_slice > 10) {
01165                 cpl_msg_warning(cpl_func, "Previous warning (number of "
01166                                           "identified slices) occured %d times.",
01167                                 kmclipm_omit_warning_one_slice);
01168                 kmclipm_omit_warning_one_slice = FALSE;
01169             }
01170 
01171             cpl_imagelist_delete(detector_in); detector_in = NULL;
01172 
01173             // load calibration files
01174             KMO_TRY_EXIT_IF_NULL(
01175                 xcal = kmo_dfs_load_cal_image(frameset, XCAL, det_nr, FALSE, rotangle,
01176                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01177 
01178             KMO_TRY_EXIT_IF_ERROR(
01179                 cpl_vector_set(calAngles, 0, rotangle_found));
01180             KMO_TRY_EXIT_IF_NULL(
01181                 ycal = kmo_dfs_load_cal_image(frameset, YCAL, det_nr, FALSE, rotangle,
01182                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01183             KMO_TRY_EXIT_IF_ERROR(
01184                 cpl_vector_set(calAngles, 1, rotangle_found));
01185             KMO_TRY_EXIT_IF_NULL(
01186                 lcal = kmo_dfs_load_cal_image(frameset, LCAL, det_nr, FALSE, rotangle,
01187                                               FALSE, NULL, &rotangle_found, -1, 0, 0));
01188             KMO_TRY_EXIT_IF_ERROR(
01189                 cpl_vector_set(calAngles, 2, rotangle_found));
01190 
01191             // load bad pixel mask from XCAL and set NaNs to 0 and all other values to 1
01192             KMO_TRY_EXIT_IF_NULL(
01193                 bad_pix_mask = cpl_image_duplicate(xcal));
01194 
01195             KMO_TRY_EXIT_IF_NULL(
01196                 pbad_pix_mask = cpl_image_get_data_float(bad_pix_mask));
01197             for (int x = 0; x < nx; x++) {
01198                 for (int y = 0; y < ny; y++) {
01199                     if (isnan(pbad_pix_mask[x+nx*y])) {
01200                         pbad_pix_mask[x+nx*y] = 0.;
01201                     } else {
01202                         pbad_pix_mask[x+nx*y] = 1.;
01203                     }
01204                 }
01205             }
01206             KMO_TRY_CHECK_ERROR_STATE();
01207 
01208             //
01209             // calculate SKYFLAT_EDGE
01210             //
01211             if (has_flat_edge) {
01212                 // get edge-edgepars from FLAT_SKY
01213                 KMO_TRY_EXIT_IF_ERROR(
01214                     kmo_calc_edgepars(combined_data,
01215                                       unused_ifus_after[det_nr-1],
01216                                       bad_pix_mask,
01217                                       det_nr,
01218                                       &slitlet_ids,
01219                                       &edgepars));
01220                 KMO_TRY_CHECK_ERROR_STATE();
01221 
01222                 // copy edgepars to table for saving later on
01223                 KMO_TRY_EXIT_IF_NULL(
01224                     edge_table_sky[det_nr-1] = kmo_edgepars_to_table(slitlet_ids, edgepars));
01225 
01226                 if (edgepars != NULL) {
01227                     for (int i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01228                         cpl_matrix_delete(edgepars[i]); edgepars[i] = NULL;
01229                     }
01230                     cpl_free(edgepars); edgepars = NULL;
01231                 }
01232                 if (slitlet_ids != NULL) {
01233                     for (int i = 0; i < KMOS_IFUS_PER_DETECTOR; i++) {
01234                         cpl_vector_delete(slitlet_ids[i]); slitlet_ids[i] = NULL;
01235                     }
01236                     cpl_free(slitlet_ids); slitlet_ids = NULL;
01237                 }
01238                 KMO_TRY_CHECK_ERROR_STATE();
01239 
01240                 //
01241                 // correlate FLAT_EDGE and SKYFLAT_EDGE
01242                 //
01243 
01244                 // load flat_edge from MASTER_FLAT
01245                 KMO_TRY_EXIT_IF_NULL(
01246                     frame = kmo_dfs_get_frame(frameset, FLAT_EDGE));
01247                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01248                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01249                     KMO_TRY_EXIT_IF_NULL(
01250                         edge_table_flat[j] = kmclipm_cal_table_load(cpl_frame_get_filename(frame),
01251                                                                     ifu_nr, rotangle, 0, &tmp_rotangle));
01252                 }
01253 
01254                 //
01255                 // calculate shift value
01256                 //
01257 
01258                 KMO_TRY_EXIT_IF_NULL(
01259                     shift_vec = cpl_vector_new(KMOS_IFUS_PER_DETECTOR));
01260                 KMO_TRY_EXIT_IF_NULL(
01261                     edge_vec = cpl_vector_new(2*KMOS_SLITLET_X));
01262 
01263                 // get shift values for each IFU by comparing all edge parameters,
01264                 // rejecting and applying median
01265                 int row = 1024; // middle of frame
01266                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01267                     ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01268                     for (int edgeNr = 0; edgeNr < 2*KMOS_SLITLET_X; edgeNr++) {
01269                         double flatval = kmo_calc_fitted_slitlet_edge(edge_table_flat[j], edgeNr, row);
01270                         double skyval  = kmo_calc_fitted_slitlet_edge(edge_table_sky[det_nr-1][j], edgeNr, row);
01271                         cpl_vector_set(edge_vec, edgeNr, flatval-skyval);
01272                     }
01273 
01274                     // reject deviating edge-differences
01275                     kmclipm_vector *kv = NULL;
01276                     KMO_TRY_EXIT_IF_NULL(
01277                         kv = kmclipm_vector_create(cpl_vector_duplicate(edge_vec)));
01278                     kmclipm_reject_deviant(kv, 3, 3, NULL, NULL);
01279 
01280                     // set shift value for each IFU
01281                     cpl_vector_set(shift_vec, j, kmclipm_vector_get_median(kv, KMCLIPM_ARITHMETIC));
01282                     kmclipm_vector_delete(kv); kv = NULL;
01283                 }
01284                 cpl_vector_delete(edge_vec); edge_vec = NULL;
01285                 KMO_TRY_CHECK_ERROR_STATE();
01286 
01287                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01288                     cpl_table_delete(edge_table_flat[j]);
01289                     edge_table_flat[j] = NULL;
01290                 }
01291 
01292                 // take median of all IFU-shift-values
01293                 double shift_val = -cpl_vector_get_median(shift_vec);
01294                 cpl_vector_delete(shift_vec); shift_vec = NULL;
01295 
01296                 cpl_msg_info("", "Shift detector %d by %g pixels.", det_nr, shift_val);
01297 
01298                 int     xdim                = cpl_image_get_size_x(combined_data),
01299                         ydim                = cpl_image_get_size_y(combined_data);
01300                 double  *array_in           = cpl_calloc(xdim, sizeof(double)),
01301                         *array_out          = NULL;
01302                 float   *pcombined_data     = cpl_image_get_data_float(combined_data),
01303                         *pcombined_noise    = NULL;
01304     //            float   *tmpArray           = cpl_calloc(xdim, sizeof(float));
01305                 if (process_noise) {
01306                     pcombined_noise = cpl_image_get_data_float(combined_noise);
01307                 }
01308 
01309                 for (int iy = 0; iy < ydim; iy++) {
01310                     // cubic spline
01311                     for (int ix = 0; ix < xdim; ix++) {
01312                         array_in[ix] = pcombined_data[ix+iy*xdim];
01313                     }
01314                     array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01315                                                     xdim, shift_val, 1.0,
01316                                                     NATURAL);
01317                     for (int ix = 0; ix < xdim; ix++) {
01318                       pcombined_data[ix+iy*xdim] = array_out[ix];
01319                     }
01320                     cpl_free(array_out);
01321 
01322     //                // linear
01323     //                for (int ix = 1; ix < xdim; ix++) {
01324     //                    tmpArray[ix-1] = (pcombined_data[ix+iy*xdim]-pcombined_data[(ix-1)+iy*xdim])*shift_val +
01325     //                                     pcombined_data[(ix-1)+iy*xdim];
01326     //                }
01327     //                for (int ix = 1; ix < xdim; ix++) {
01328     //                    pcombined_data[ix+iy*xdim] = tmpArray[ix];
01329     //                }
01330 
01331                     if (process_noise) {
01332                         // cubic spline
01333                         for (int ix = 0; ix < xdim; ix++) {
01334                             array_in[ix] = pcombined_noise[ix+iy*xdim];
01335                         }
01336                         array_out = cubicspline_reg_reg(xdim, 0., 1., array_in,
01337                                                         xdim, shift_val, 1.0,
01338                                                         NATURAL);
01339                         for (int ix = 0; ix < xdim; ix++) {
01340                           pcombined_noise[ix+iy*xdim] = array_out[ix];
01341                         }
01342                         cpl_free(array_out);
01343 
01344     //                    // linear
01345     //                    for (int ix = 1; ix < xdim; ix++) {
01346     //                        tmpArray[ix-1] = (pcombined_noise[ix+iy*xdim]-pcombined_noise[(ix-1)+iy*xdim])*shift_val +
01347     //                                         pcombined_noise[(ix-1)+iy*xdim];
01348     //                    }
01349     //                    for (int ix = 1; ix < xdim; ix++) {
01350     //                        pcombined_noise[ix+iy*xdim] = tmpArray[ix];
01351     //                    }
01352                     }
01353                 }
01354                 cpl_free(array_in); array_in = NULL;
01355             }
01356             //
01357             // reconstruct
01358             //
01359             // load MASTER_DARK and MASTER_FLAT
01360             KMO_TRY_EXIT_IF_NULL(
01361                 img_dark = kmo_dfs_load_image(frameset, MASTER_DARK,
01362                                               det_nr, FALSE, FALSE, NULL));
01363 
01364             if (process_noise) {
01365                 KMO_TRY_EXIT_IF_NULL(
01366                     img_dark_noise = kmo_dfs_load_image(frameset, MASTER_DARK,
01367                                                         det_nr, TRUE, FALSE, NULL));
01368             }
01369 
01370             KMO_TRY_EXIT_IF_NULL(
01371                 img_flat = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, FALSE,
01372                                                   rotangle, FALSE, NULL,
01373                                                   &rotangle_found, -1, 0, 0));
01374 
01375             if (process_noise) {
01376                 KMO_TRY_EXIT_IF_NULL(
01377                     img_flat_noise = kmo_dfs_load_cal_image(frameset, MASTER_FLAT, det_nr, TRUE,
01378                                                             rotangle, FALSE, NULL,
01379                                                             &rotangle_found, -1, 0, 0));
01380             }
01381 
01382             char *tmp_band_method = getenv("KMO_BAND_METHOD");
01383             int band_method = 0;
01384             if (tmp_band_method != NULL) {
01385                 band_method = atoi(tmp_band_method);
01386             }
01387 
01388             // ESO INS FILTi ID
01389             KMO_TRY_EXIT_IF_NULL(
01390                 keyword = cpl_sprintf("%s%d%s", IFU_FILTID_PREFIX, det_nr,
01391                                       IFU_FILTID_POSTFIX));
01392             KMO_TRY_EXIT_IF_NULL(
01393                 filter_id = cpl_propertylist_get_string(main_header, keyword));
01394             cpl_free(keyword); keyword = NULL;
01395 
01396             KMO_TRY_EXIT_IF_NULL(
01397                 band_table = kmo_dfs_load_table(frameset, WAVE_BAND, 1, 0));
01398             KMO_TRY_EXIT_IF_ERROR(
01399                 kmclipm_setup_grid_band_lcal(&gd, lcal, filter_id, band_method,
01400                                              band_table));
01401             cpl_table_delete(band_table); band_table = NULL;
01402 
01403             cpl_msg_info("","Reconstructing cubes...");
01404             for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01405                 // update sub-header
01406                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01407 
01408                 // load raw image and sub-header
01409                 KMO_TRY_EXIT_IF_NULL(
01410                     sub_header = kmo_dfs_load_sub_header(frameset_sky, FLAT_SKY,
01411                                                          det_nr, FALSE));
01412 
01413                 KMO_TRY_EXIT_IF_NULL(
01414                     punused_ifus = cpl_array_get_data_int_const(
01415                                                   unused_ifus_after[det_nr-1]));
01416 
01417                 // check if IFU is valid according to main header keywords &
01418                 // calibration files
01419                 KMO_TRY_EXIT_IF_NULL(
01420                     keyword = cpl_sprintf("%s%d%s", IFU_VALID_PREFIX, ifu_nr,
01421                                           IFU_VALID_POSTFIX));
01422                 KMO_TRY_CHECK_ERROR_STATE();
01423                 ranges_txt = cpl_propertylist_get_string(main_header, keyword);
01424                 cpl_free(keyword); keyword = NULL;
01425 
01426                 if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) &&
01427                     (bounds[2*(ifu_nr-1)] != -1) &&
01428                     (bounds[2*(ifu_nr-1)+1] != -1) &&
01429                     (punused_ifus[j] == 0))
01430                 {
01431                     // IFU is valid
01432                     cpl_error_reset();
01433 
01434                     // calculate WCS
01435                     KMO_TRY_EXIT_IF_ERROR(
01436                         kmo_calc_wcs_gd(main_header, sub_header, ifu_nr, gd));
01437 
01438                     // reconstruct data
01439                     KMO_TRY_EXIT_IF_ERROR(
01440                         kmo_reconstruct_sci_image(ifu_nr,
01441                                                 bounds[2*(ifu_nr-1)],
01442                                                 bounds[2*(ifu_nr-1)+1],
01443                                                 combined_data,
01444                                                 combined_noise,
01445                                                 img_dark,
01446                                                 img_dark_noise,
01447                                                 img_flat,
01448                                                 img_flat_noise,
01449                                                 xcal,
01450                                                 ycal,
01451                                                 lcal,
01452                                                 &gd,
01453                                                 calTimestamp,
01454                                                 calAngles,
01455                                                 fn_lut,
01456                                                 &cube_data,
01457                                                 &cube_noise,
01458                                                 flux,
01459                                                 background));
01460                     KMO_TRY_CHECK_ERROR_STATE();
01461                 } else {
01462                     // IFU is invalid
01463                     cpl_error_reset();
01464                 } // if ((cpl_error_get_code() == CPL_ERROR_DATA_NOT_FOUND) ...
01465 
01466                 // save output
01467                 KMO_TRY_EXIT_IF_NULL(
01468                     extname = kmo_extname_creator(ifu_frame, ifu_nr, EXT_DATA));
01469 
01470                 KMO_TRY_EXIT_IF_ERROR(
01471                     kmclipm_update_property_string(sub_header, EXTNAME,
01472                                                    extname,
01473                                                    "FITS extension name"));
01474 
01475                 cpl_free(extname); extname = NULL;
01476 
01477                 // store cube and sub header into array for later
01478                 stored_data_cubes[ifu_nr - 1] = cube_data;
01479                 stored_sub_data_headers[ifu_nr - 1] = sub_header;
01480 
01481                 if (process_noise) {
01482                     KMO_TRY_EXIT_IF_NULL(
01483                         sub_header = cpl_propertylist_duplicate(
01484                                            stored_sub_data_headers[ifu_nr - 1]));
01485                     KMO_TRY_EXIT_IF_NULL(
01486                         extname = kmo_extname_creator(ifu_frame, ifu_nr,
01487                                                       EXT_NOISE));
01488 
01489                     KMO_TRY_EXIT_IF_ERROR(
01490                         kmclipm_update_property_string(sub_header,
01491                                                 EXTNAME,
01492                                                 extname,
01493                                                 "FITS extension name"));
01494 
01495                     cpl_free(extname); extname = NULL;
01496 
01497                     stored_noise_cubes[ifu_nr - 1] = cube_noise;
01498                     stored_sub_noise_headers[ifu_nr - 1] = sub_header;
01499                 }
01500                 cpl_image_delete(data_ifu); data_ifu = NULL;
01501                 cpl_image_delete(noise_ifu); noise_ifu = NULL;
01502                 cube_data = NULL;
01503                 cube_noise = NULL;
01504             } // for j IFUs
01505 
01506             // free memory
01507             cpl_image_delete(combined_data); combined_data = NULL;
01508             cpl_image_delete(combined_noise); combined_noise = NULL;
01509             cpl_image_delete(xcal); xcal = NULL;
01510             cpl_image_delete(ycal); ycal = NULL;
01511             cpl_image_delete(lcal); lcal = NULL;
01512             cpl_image_delete(img_dark); img_dark = NULL;
01513             cpl_image_delete(img_flat); img_flat = NULL;
01514             cpl_image_delete(bad_pix_mask); bad_pix_mask = NULL;
01515             if (process_noise) {
01516                 cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01517                 cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01518             }
01519         } // for nr_devices
01520 
01521         cpl_free(edge_table_flat); edge_table_flat = NULL;
01522 
01523         // collapse cubes using rejection
01524         cpl_msg_info("","Collapsing cubes...");
01525         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01526             for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01527                 ifu_nr = (det_nr-1)*KMOS_IFUS_PER_DETECTOR + j + 1;
01528 
01529                 KMO_TRY_EXIT_IF_NULL(
01530                     punused_ifus = cpl_array_get_data_int_const(
01531                                                   unused_ifus_after[det_nr-1]));
01532                 if (punused_ifus[j] == 0) {
01533                     if (stored_sub_data_headers[ifu_nr-1] != NULL) {
01534                     // IFU is valid
01535                     ifu_crpix = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01536                                                             CRPIX3);
01537                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01538                                    "CRPIX3 keyword in FITS-header is missing!");
01539 
01540                     ifu_crval = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01541                                                             CRVAL3);
01542                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01543                                    "CRVAL3 keyword in FITS-header is missing!");
01544 
01545                     ifu_cdelt = cpl_propertylist_get_double(stored_sub_data_headers[ifu_nr-1],
01546                                                             CDELT3);
01547                     KMO_TRY_CHECK_ERROR_STATE_MSG(
01548                                    "CDELT3 keyword in FITS-header is missing!");
01549 
01550                     KMO_TRY_EXIT_IF_NULL(
01551                         identified_slices = kmo_identify_slices(ranges,
01552                                                                 ifu_crpix,
01553                                                                 ifu_crval,
01554                                                                 ifu_cdelt,
01555                                                                 gd.l.dim));
01556                     }/* else {
01557                         KMO_TRY_EXIT_IF_NULL(
01558                             identified_slices = cpl_vector_new(gd.l.dim));
01559                         cpl_vector_fill(identified_slices, 1.);
01560                     }*/
01561 
01562                     if (stored_data_cubes[ifu_nr-1] != NULL) {
01563                         KMO_TRY_EXIT_IF_ERROR(
01564                             kmclipm_make_image(stored_data_cubes[ifu_nr-1],
01565                                                stored_noise_cubes[ifu_nr-1],
01566                                                &stored_data_images[ifu_nr-1],
01567                                                &stored_noise_images[ifu_nr-1],
01568                                                identified_slices,
01569                                                cmethod, cpos_rej, cneg_rej,
01570                                                citer, cmax, cmin));
01571                     }
01572                     cpl_vector_delete(identified_slices); identified_slices = NULL;
01573                 } else {
01574                     // IFU is invalid
01575                 }
01576             }
01577         }
01578 
01579         // normalise all IFUs as a group.
01580         // Calculate mean of each IFU, add up and divide by number of successful
01581         // averaged IFUs.
01582         // Then divide all valid IFUs with mean value
01583         cnt = 0;
01584         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01585             if (stored_data_images[i] != NULL) {
01586                 KMO_TRY_ASSURE(cpl_image_count_rejected(stored_data_images[i]) <
01587                                cpl_image_get_size_x(stored_data_images[i])*
01588                                cpl_image_get_size_y(stored_data_images[i]),
01589                                CPL_ERROR_ILLEGAL_INPUT,
01590                                "The collapsed, dark-subtracted image contains "
01591                                "only invalid values! Probably the provided "
01592                                "FLAT_SKY frames are exactly the same as the "
01593                                "frames used for MASTER_DARK calculation.");
01594 
01595                 mean_data += cpl_image_get_mean(stored_data_images[i]);
01596                 KMO_TRY_CHECK_ERROR_STATE();
01597                 cnt++;
01598             }
01599 
01600         }
01601         mean_data /= cnt;
01602 
01603         if (mean_data != 0.0) {
01604             for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01605                 if (stored_data_images[i] != NULL) {
01606                     KMO_TRY_EXIT_IF_ERROR(
01607                         cpl_image_divide_scalar(stored_data_images[i],
01608                                                 mean_data));
01609                 }
01610             }
01611         } else {
01612             cpl_msg_warning(cpl_func, "Data couldn't be normalised "
01613                                       "(mean = 0.0)!");
01614         }
01615 
01616         if (process_noise) {
01617             if (mean_data != 0.0) {
01618                 for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01619                     if (stored_noise_images[i] != NULL) {
01620                         KMO_TRY_EXIT_IF_ERROR(
01621                             cpl_image_divide_scalar(stored_noise_images[i],
01622                                                     mean_data));
01623                     }
01624                 }
01625             } else {
01626                 cpl_msg_warning(cpl_func, "Noise couldn't be normalised "
01627                                           "(mean = 0.0)!");
01628             }
01629         }
01630 
01631         // calculate qc parameters on normalised data
01632         qc_spat_unif = 0.0;
01633         cnt = 0;
01634         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01635             if (stored_data_images[i] != NULL) {
01636                 tmp_mean = cpl_image_get_mean(stored_data_images[i]);
01637                 tmp_stdev = cpl_image_get_stdev (stored_data_images[i]);
01638 
01639                 qc_spat_unif += pow(tmp_mean-1, 2);
01640                 if (fabs(tmp_mean) > qc_max_dev) {
01641                     qc_max_dev = tmp_mean-1;
01642                     qc_max_dev_id = i+1;
01643                 }
01644                 if (fabs(tmp_stdev) > qc_max_nonunif) {
01645                     qc_max_nonunif = tmp_stdev;
01646                     qc_max_nonunif_id = i+1;
01647                 }
01648                 KMO_TRY_CHECK_ERROR_STATE();
01649                 cnt++;
01650             }
01651         }
01652         qc_spat_unif = sqrt(qc_spat_unif / cnt);
01653 
01654         //
01655         // save data
01656         //
01657 
01658         // update which IFUs are not used
01659         kmo_print_unused_ifus(unused_ifus_after, TRUE);
01660 
01661         KMO_TRY_EXIT_IF_ERROR(
01662             kmo_set_unused_ifus(unused_ifus_after, main_header,
01663                                 "kmo_illumination"));
01664 
01665         cpl_msg_info("","Saving data...");
01666 
01667         KMO_TRY_EXIT_IF_ERROR(
01668             kmclipm_update_property_double(main_header, QC_SPAT_UNIF, qc_spat_unif,
01669                                            "[adu] uniformity of illumination correction"));
01670         KMO_TRY_EXIT_IF_ERROR(
01671             kmclipm_update_property_double(main_header, QC_SPAT_MAX_DEV, qc_max_dev,
01672                                            "[adu] max. deviation from unity"));
01673         KMO_TRY_EXIT_IF_ERROR(
01674             kmclipm_update_property_int(main_header, QC_SPAT_MAX_DEV_ID, qc_max_dev_id,
01675                                         "[] IFU ID with max. dev. from unity"));
01676         KMO_TRY_EXIT_IF_ERROR(
01677             kmclipm_update_property_double(main_header, QC_SPAT_MAX_NONUNIF, qc_max_nonunif,
01678                                            "[adu] max. stdev of illumination corr."));
01679         KMO_TRY_EXIT_IF_ERROR(
01680             kmclipm_update_property_int(main_header, QC_SPAT_MAX_NONUNIF_ID, qc_max_nonunif_id,
01681                                         "[] IFU ID with max. stdev in illum. corr."));
01682 
01683         if (!suppress_extension) {
01684             KMO_TRY_EXIT_IF_NULL(
01685                 fn_suffix = cpl_sprintf("%s", suffix));
01686         } else {
01687             KMO_TRY_EXIT_IF_NULL(
01688                 fn_suffix = cpl_sprintf("%s", ""));
01689         }
01690         KMO_TRY_EXIT_IF_ERROR(
01691             kmo_dfs_save_main_header(frameset, ILLUM_CORR, fn_suffix, frame,
01692                                      main_header, parlist, cpl_func));
01693 
01694         KMO_TRY_EXIT_IF_ERROR(
01695             kmo_dfs_save_main_header(frameset, SKYFLAT_EDGE, fn_suffix, frame,
01696                                      main_header, parlist, cpl_func));
01697 
01698         for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01699             KMO_TRY_EXIT_IF_ERROR(
01700                 kmo_dfs_save_image(stored_data_images[i], ILLUM_CORR, fn_suffix,
01701                                    stored_sub_data_headers[i], 0./0.));
01702 
01703             if (process_noise) {
01704                 KMO_TRY_EXIT_IF_ERROR(
01705                     kmo_dfs_save_image(stored_noise_images[i], ILLUM_CORR,
01706                                        fn_suffix, stored_sub_noise_headers[i], 0./0.));
01707             }
01708         }
01709 
01710         for (int det_nr = 1; det_nr <= nr_devices; det_nr++) {
01711             for (int ifu_nr = 0; ifu_nr < KMOS_IFUS_PER_DETECTOR; ifu_nr++) {
01712                 KMO_TRY_EXIT_IF_ERROR(
01713                     kmclipm_update_property_int(stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01714                                                 CAL_IFU_NR,
01715                                                 ifu_nr+1+(det_nr-1)*KMOS_IFUS_PER_DETECTOR,
01716                                                 "IFU Number {1..24}"));
01717                 KMO_TRY_EXIT_IF_ERROR(
01718                     kmclipm_update_property_double(
01719                                                 stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr],
01720                                                 CAL_ROTANGLE,
01721                                                 rotangle_found,
01722                                                 "[deg] Rotator relative to nasmyth"));
01723                 if (has_flat_edge) {
01724                     // save edge-parameters as product
01725                     KMO_TRY_EXIT_IF_ERROR(
01726                         kmo_dfs_save_table(edge_table_sky[det_nr-1][ifu_nr], SKYFLAT_EDGE, fn_suffix,
01727                                            stored_sub_data_headers[(det_nr-1)*KMOS_IFUS_PER_DETECTOR+ifu_nr]));
01728                 }
01729             }
01730         }
01731     }
01732     KMO_CATCH
01733     {
01734         KMO_CATCH_MSG();
01735         ret_val = -1;
01736     }
01737     kmo_free_fits_desc(&desc_sky);
01738     kmo_free_fits_desc(&desc_dark);
01739     kmo_free_fits_desc(&desc_flat);
01740     kmo_free_fits_desc(&desc_xcal);
01741     kmo_free_fits_desc(&desc_ycal);
01742     kmo_free_fits_desc(&desc_lcal);
01743     cpl_image_delete(combined_data); combined_data = NULL;
01744     cpl_image_delete(combined_noise); combined_noise = NULL;
01745     cpl_image_delete(xcal); xcal = NULL;
01746     cpl_image_delete(ycal); ycal = NULL;
01747     cpl_image_delete(lcal); lcal = NULL;
01748     cpl_image_delete(img_dark); img_dark = NULL;
01749     cpl_image_delete(img_dark_noise); img_dark_noise = NULL;
01750     cpl_image_delete(img_flat); img_flat = NULL;
01751     cpl_image_delete(img_flat_noise); img_flat_noise = NULL;
01752     cpl_array_delete(calTimestamp); calTimestamp = NULL;
01753     cpl_free(bounds); bounds = NULL;
01754     kmo_free_unused_ifus(unused_ifus_before); unused_ifus_before = NULL;
01755     kmo_free_unused_ifus(unused_ifus_after); unused_ifus_after = NULL;
01756     cpl_free(fn_lut); fn_lut = NULL;
01757     cpl_free(suffix); suffix = NULL;
01758     cpl_free(fn_suffix); fn_suffix = NULL;
01759     cpl_frameset_delete(frameset_sky); frameset_sky = NULL;
01760     cpl_vector_delete(ranges); ranges = NULL;
01761     cpl_free(filter); filter = NULL;
01762     if (calAngles != NULL) {
01763         cpl_vector_delete(calAngles); calAngles = NULL;
01764     }
01765     cpl_propertylist_delete(main_header); main_header = NULL;
01766     for (int i = 0; i < nr_devices * KMOS_IFUS_PER_DETECTOR; i++) {
01767         if (stored_data_cubes != NULL) {
01768             cpl_imagelist_delete(stored_data_cubes[i]);
01769             stored_data_cubes[i] = NULL;
01770         }
01771         if (stored_noise_cubes != NULL) {
01772             cpl_imagelist_delete(stored_noise_cubes[i]);
01773             stored_noise_cubes[i] = NULL;
01774         }
01775         if (stored_data_images != NULL) {
01776             cpl_image_delete(stored_data_images[i]);
01777             stored_data_images[i] = NULL;
01778         }
01779         if (stored_noise_images != NULL) {
01780             cpl_image_delete(stored_noise_images[i]);
01781             stored_noise_images[i] = NULL;
01782         }
01783         if (stored_sub_data_headers != NULL) {
01784             cpl_propertylist_delete(stored_sub_data_headers[i]);
01785             stored_sub_data_headers[i] = NULL;
01786         }
01787         if (stored_sub_noise_headers != NULL) {
01788             cpl_propertylist_delete(stored_sub_noise_headers[i]);
01789             stored_sub_noise_headers[i] = NULL;
01790         }
01791     }
01792     cpl_free(stored_data_cubes); stored_data_cubes = NULL;
01793     cpl_free(stored_noise_cubes); stored_noise_cubes = NULL;
01794     cpl_free(stored_data_images); stored_data_images = NULL;
01795     cpl_free(stored_noise_images); stored_noise_images = NULL;
01796     cpl_free(stored_sub_data_headers); stored_sub_data_headers = NULL;
01797     cpl_free(stored_sub_noise_headers); stored_sub_noise_headers = NULL;
01798     if (edge_table_sky != NULL) {
01799         for (int i = 0; i < KMOS_NR_DETECTORS; i++) {
01800             if (edge_table_sky[i] != NULL) {
01801                 for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01802                     cpl_table_delete(edge_table_sky[i][j]);
01803                     edge_table_sky[i][j] = NULL;
01804                 }
01805                 cpl_free(edge_table_sky[i]); edge_table_sky[i] = NULL;
01806             }
01807         }
01808         cpl_free(edge_table_sky); edge_table_sky = NULL;
01809     }
01810     if (edge_table_flat != NULL) {
01811         for (int j = 0; j < KMOS_IFUS_PER_DETECTOR; j++) {
01812             cpl_table_delete(edge_table_flat[j]);
01813             edge_table_flat[j] = NULL;
01814         }
01815         cpl_free(edge_table_flat); edge_table_flat = NULL;
01816     }
01817 
01818     return ret_val;
01819 }
01820